From 0a11e20a740a1a6be51f4ee4ceee27bc486423ec Mon Sep 17 00:00:00 2001 From: cdos-rla Date: Mon, 11 Jul 2022 12:05:35 -0600 Subject: [PATCH] version 2.3.70.1 --- .../java/us/us/freeandfair/corla/Main.java | 654 +++++++++ .../us/us/freeandfair/corla/asm/ASMEvent.java | 71 + .../corla/asm/ASMEventToEndpointRelation.java | 147 +++ .../us/us/freeandfair/corla/asm/ASMState.java | 68 + .../freeandfair/corla/asm/ASMTransition.java | 171 +++ .../corla/asm/ASMTransitionFunction.java | 291 ++++ .../freeandfair/corla/asm/ASMUtilities.java | 136 ++ .../corla/asm/AbstractStateMachine.java | 303 +++++ .../corla/asm/AuditBoardDashboardASM.java | 63 + .../corla/asm/CountyDashboardASM.java | 62 + .../corla/asm/DoSDashboardASM.java | 63 + .../us/us/freeandfair/corla/asm/Event.java | 22 + .../corla/asm/PersistentASMState.java | 313 +++++ .../us/us/freeandfair/corla/asm/UIEvent.java | 30 + .../corla/asm/UIToASMEventRelation.java | 126 ++ .../corla/auth/AbstractAuthentication.java | 439 +++++++ .../corla/auth/AuthenticationInterface.java | 281 ++++ .../corla/auth/AuthenticationResult.java | 57 + .../corla/auth/AuthenticationStage.java | 30 + .../corla/auth/AuthenticationStatus.java | 76 ++ .../corla/auth/DatabaseAuthentication.java | 93 ++ .../corla/controller/AuditReport.java | 223 ++++ .../corla/controller/BallotSelection.java | 583 +++++++++ .../controller/ComparisonAuditController.java | 691 ++++++++++ .../corla/controller/ContestCounter.java | 214 +++ .../controller/DeleteFileController.java | 162 +++ .../controller/ImportFileController.java | 235 ++++ .../freeandfair/corla/crypto/HashChecker.java | 101 ++ .../crypto/PseudoRandomNumberGenerator.java | 194 +++ .../corla/csv/BallotManifestParser.java | 45 + .../corla/csv/CVRExportParser.java | 45 + .../csv/ColoradoBallotManifestParser.java | 233 ++++ .../corla/csv/ContestNameParser.java | 319 +++++ .../corla/csv/DominionCVRExportParser.java | 769 +++++++++++ .../us/us/freeandfair/corla/csv/Result.java | 49 + .../corla/endpoint/ACVRDownload.java | 98 ++ .../corla/endpoint/ACVRDownloadByCounty.java | 129 ++ .../corla/endpoint/ACVRUpload.java | 203 +++ .../AbstractAuditBoardDashboardEndpoint.java | 54 + .../AbstractCountyDashboardEndpoint.java | 54 + .../AbstractDoSDashboardEndpoint.java | 53 + .../corla/endpoint/AbstractEndpoint.java | 714 ++++++++++ .../corla/endpoint/AppInfoEndpoint.java | 100 ++ .../endpoint/AuditBoardDashboardASMState.java | 67 + .../corla/endpoint/AuditBoardSignIn.java | 168 +++ .../corla/endpoint/AuditBoardSignOut.java | 93 ++ .../endpoint/AuditInvestigationReport.java | 98 ++ .../endpoint/AuthenticateAdministrator.java | 115 ++ .../AuthenticateCountyAdministrator.java | 85 ++ .../AuthenticateStateAdministrator.java | 87 ++ .../endpoint/BallotManifestDownload.java | 83 ++ .../BallotManifestDownloadByCounty.java | 114 ++ .../corla/endpoint/BallotManifestImport.java | 211 +++ .../corla/endpoint/BallotNotFound.java | 193 +++ .../corla/endpoint/CORSFilter.java | 126 ++ .../corla/endpoint/CVRDownload.java | 97 ++ .../corla/endpoint/CVRDownloadByCounty.java | 127 ++ .../corla/endpoint/CVRDownloadByID.java | 83 ++ .../corla/endpoint/CVRExportImport.java | 147 +++ .../corla/endpoint/CVRToAuditDownload.java | 317 +++++ .../corla/endpoint/CVRToAuditList.java | 214 +++ .../corla/endpoint/ContestDownload.java | 107 ++ .../endpoint/ContestDownloadByCounty.java | 138 ++ .../corla/endpoint/ContestDownloadByID.java | 82 ++ .../endpoint/CountyDashboardASMState.java | 67 + .../endpoint/CountyDashboardRefresh.java | 87 ++ .../corla/endpoint/CountyReportDownload.java | 175 +++ .../corla/endpoint/DeleteFile.java | 101 ++ .../corla/endpoint/DoSDashboardASMState.java | 67 + .../corla/endpoint/DoSDashboardRefresh.java | 95 ++ .../freeandfair/corla/endpoint/Endpoint.java | 95 ++ .../corla/endpoint/FileDownload.java | 135 ++ .../corla/endpoint/FileUpload.java | 489 +++++++ .../corla/endpoint/IndicateHandCount.java | 175 +++ .../endpoint/IntermediateAuditReport.java | 98 ++ .../corla/endpoint/PublishAuditReport.java | 123 ++ .../corla/endpoint/PublishDataToAudit.java | 58 + .../corla/endpoint/ReportBallotsToAudit.java | 58 + .../corla/endpoint/ResetAudit.java | 105 ++ .../corla/endpoint/ResetDatabase.java | 126 ++ .../RiskLimitForComparisonAudits.java | 105 ++ .../us/freeandfair/corla/endpoint/Root.java | 52 + .../endpoint/SelectContestsForAudit.java | 133 ++ .../corla/endpoint/SetAuditBoardCount.java | 174 +++ .../corla/endpoint/SetContestNames.java | 187 +++ .../corla/endpoint/SetRandomSeed.java | 137 ++ .../corla/endpoint/SignOffAuditRound.java | 409 ++++++ .../corla/endpoint/StartAuditRound.java | 534 ++++++++ .../corla/endpoint/StateReportDownload.java | 130 ++ .../corla/endpoint/Unauthenticate.java | 92 ++ .../corla/endpoint/UpdateAuditInfo.java | 246 ++++ .../freeandfair/corla/json/AppInfoJSON.java | 40 + ...ditInvestigationReportInfoJsonAdapter.java | 110 ++ .../corla/json/CVRContestInfoJsonAdapter.java | 187 +++ .../corla/json/CVRToAuditResponse.java | 280 ++++ .../corla/json/CanonicalUpdate.java | 29 + .../corla/json/ContestJsonAdapter.java | 192 +++ .../corla/json/ContestToAuditJsonAdapter.java | 149 +++ .../json/CountyDashboardRefreshResponse.java | 456 +++++++ .../json/DoSDashboardRefreshResponse.java | 254 ++++ .../corla/json/FreeAndFairNamingStrategy.java | 41 + .../corla/json/InstantTypeAdapter.java | 70 + .../IntermediateAuditReportJsonAdapter.java | 99 ++ .../us/us/freeandfair/corla/json/Result.java | 41 + .../corla/json/ServerASMResponse.java | 50 + .../corla/json/SubmittedAuditCVR.java | 102 ++ .../corla/json/SubmittedAuditRoundStart.java | 87 ++ .../corla/json/SubmittedBallotNotFound.java | 85 ++ .../corla/json/SubmittedCredentials.java | 78 ++ .../corla/json/TwoFactorResponse.java | 43 + .../corla/json/UploadedFileDTO.java | 58 + .../corla/json/VersionExclusionStrategy.java | 45 + .../us/us/freeandfair/corla/math/Audit.java | 249 ++++ .../corla/model/Administrator.java | 260 ++++ .../freeandfair/corla/model/AuditBoard.java | 147 +++ .../us/freeandfair/corla/model/AuditInfo.java | 322 +++++ .../model/AuditInvestigationReportInfo.java | 146 +++ .../freeandfair/corla/model/AuditReason.java | 68 + .../corla/model/AuditSelection.java | 44 + .../freeandfair/corla/model/AuditStatus.java | 27 + .../us/freeandfair/corla/model/AuditType.java | 22 + .../corla/model/BallotManifestInfo.java | 374 ++++++ .../freeandfair/corla/model/CVRAuditInfo.java | 319 +++++ .../corla/model/CVRContestInfo.java | 200 +++ .../corla/model/CastVoteRecord.java | 728 ++++++++++ .../us/us/freeandfair/corla/model/Choice.java | 159 +++ .../corla/model/ComparisonAudit.java | 1166 +++++++++++++++++ .../us/freeandfair/corla/model/Contest.java | 320 +++++ .../corla/model/ContestResult.java | 462 +++++++ .../corla/model/ContestToAudit.java | 158 +++ .../us/us/freeandfair/corla/model/County.java | 192 +++ .../model/CountyContestComparisonAudit.java | 947 +++++++++++++ .../corla/model/CountyContestResult.java | 624 +++++++++ .../corla/model/CountyDashboard.java | 1022 +++++++++++++++ .../freeandfair/corla/model/DoSDashboard.java | 324 +++++ .../us/freeandfair/corla/model/Elector.java | 139 ++ .../freeandfair/corla/model/ImportStatus.java | 128 ++ .../model/IntermediateAuditReportInfo.java | 127 ++ .../us/freeandfair/corla/model/LogEntry.java | 331 +++++ .../us/us/freeandfair/corla/model/Round.java | 579 ++++++++ .../us/freeandfair/corla/model/Tribute.java | 117 ++ .../freeandfair/corla/model/UploadedFile.java | 335 +++++ .../persistence/AuditReasonSetConverter.java | 68 + .../AuditSelectionIntegerMapConverter.java | 69 + .../BallotSequenceAssignmentConverter.java | 68 + .../CountyCanonicalContestsMapConverter.java | 60 + .../persistence/ElectorListConverter.java | 73 ++ .../FreeAndFairNamingStrategy.java | 86 ++ .../persistence/IntegerListConverter.java | 64 + .../persistence/LongIntegerMapConverter.java | 54 + .../corla/persistence/LongListConverter.java | 65 + .../corla/persistence/Persistence.java | 753 +++++++++++ .../corla/persistence/PersistentEntity.java | 38 + .../corla/persistence/ResultConverter.java | 53 + .../persistence/SignatoriesConverter.java | 66 + .../persistence/StringListConverter.java | 74 ++ .../corla/persistence/StringSetConverter.java | 65 + .../corla/query/AdministratorQueries.java | 77 ++ .../query/BallotManifestInfoQueries.java | 261 ++++ .../corla/query/CastVoteRecordQueries.java | 661 ++++++++++ .../corla/query/ComparisonAuditQueries.java | 113 ++ .../corla/query/ContestQueries.java | 119 ++ .../corla/query/ContestResultQueries.java | 50 + .../CountyContestComparisonAuditQueries.java | 74 ++ .../query/CountyContestResultQueries.java | 186 +++ .../corla/query/CountyQueries.java | 105 ++ .../corla/query/DatabaseResetQueries.java | 96 ++ .../corla/query/ExportQueries.java | 275 ++++ .../corla/query/LogEntryQueries.java | 87 ++ .../query/PersistentASMStateQueries.java | 92 ++ .../corla/query/TributeQueries.java | 35 + .../corla/query/UploadedFileQueries.java | 205 +++ .../corla/report/CountyReport.java | 941 +++++++++++++ .../freeandfair/corla/report/ReportRows.java | 507 +++++++ .../freeandfair/corla/report/StateReport.java | 781 +++++++++++ .../corla/report/WorkbookWriter.java | 116 ++ .../corla/util/BallotAssignment.java | 68 + .../corla/util/BallotSequencer.java | 78 ++ .../corla/util/DBExceptionUtil.java | 91 ++ .../corla/util/EqualsHashcodeHelper.java | 62 + .../corla/util/ExponentialBackoffHelper.java | 56 + .../us/freeandfair/corla/util/FileHelper.java | 57 + .../corla/util/NaturalOrderComparator.java | 125 ++ .../us/us/freeandfair/corla/util/Pair.java | 94 ++ .../corla/util/PhantomBallots.java | 122 ++ .../freeandfair/corla/util/PrettyPrinter.java | 45 + .../us/freeandfair/corla/util/SetCreator.java | 42 + .../freeandfair/corla/util/SparkHelper.java | 85 ++ .../corla/util/SuppressFBWarnings.java | 35 + .../corla/util/UploadedFileStreamer.java | 126 ++ 190 files changed, 35824 insertions(+) create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/Main.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/ASMEvent.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/ASMEventToEndpointRelation.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/ASMState.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/ASMTransition.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/ASMTransitionFunction.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/ASMUtilities.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/AbstractStateMachine.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/AuditBoardDashboardASM.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/CountyDashboardASM.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/DoSDashboardASM.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/Event.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/PersistentASMState.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/UIEvent.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/UIToASMEventRelation.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/auth/AbstractAuthentication.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/auth/AuthenticationInterface.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/auth/AuthenticationResult.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/auth/AuthenticationStage.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/auth/AuthenticationStatus.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/auth/DatabaseAuthentication.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/controller/AuditReport.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/controller/BallotSelection.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/controller/ComparisonAuditController.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/controller/ContestCounter.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/controller/DeleteFileController.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/controller/ImportFileController.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/crypto/HashChecker.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/crypto/PseudoRandomNumberGenerator.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/csv/BallotManifestParser.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/csv/CVRExportParser.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/csv/ColoradoBallotManifestParser.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/csv/ContestNameParser.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/csv/DominionCVRExportParser.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/csv/Result.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ACVRDownload.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ACVRDownloadByCounty.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ACVRUpload.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AbstractAuditBoardDashboardEndpoint.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AbstractCountyDashboardEndpoint.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AbstractDoSDashboardEndpoint.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AbstractEndpoint.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AppInfoEndpoint.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AuditBoardDashboardASMState.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AuditBoardSignIn.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AuditBoardSignOut.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AuditInvestigationReport.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AuthenticateAdministrator.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AuthenticateCountyAdministrator.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AuthenticateStateAdministrator.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/BallotManifestDownload.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/BallotManifestDownloadByCounty.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/BallotManifestImport.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/BallotNotFound.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CORSFilter.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CVRDownload.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CVRDownloadByCounty.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CVRDownloadByID.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CVRExportImport.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CVRToAuditDownload.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CVRToAuditList.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ContestDownload.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ContestDownloadByCounty.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ContestDownloadByID.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CountyDashboardASMState.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CountyDashboardRefresh.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CountyReportDownload.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/DeleteFile.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/DoSDashboardASMState.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/DoSDashboardRefresh.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/Endpoint.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/FileDownload.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/FileUpload.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/IndicateHandCount.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/IntermediateAuditReport.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/PublishAuditReport.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/PublishDataToAudit.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ReportBallotsToAudit.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ResetAudit.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ResetDatabase.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/RiskLimitForComparisonAudits.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/Root.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/SelectContestsForAudit.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/SetAuditBoardCount.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/SetContestNames.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/SetRandomSeed.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/SignOffAuditRound.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/StartAuditRound.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/StateReportDownload.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/Unauthenticate.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/UpdateAuditInfo.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/AppInfoJSON.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/AuditInvestigationReportInfoJsonAdapter.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/CVRContestInfoJsonAdapter.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/CVRToAuditResponse.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/CanonicalUpdate.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/ContestJsonAdapter.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/ContestToAuditJsonAdapter.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/CountyDashboardRefreshResponse.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/DoSDashboardRefreshResponse.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/FreeAndFairNamingStrategy.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/InstantTypeAdapter.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/IntermediateAuditReportJsonAdapter.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/Result.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/ServerASMResponse.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/SubmittedAuditCVR.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/SubmittedAuditRoundStart.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/SubmittedBallotNotFound.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/SubmittedCredentials.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/TwoFactorResponse.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/UploadedFileDTO.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/VersionExclusionStrategy.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/math/Audit.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/Administrator.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/AuditBoard.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/AuditInfo.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/AuditInvestigationReportInfo.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/AuditReason.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/AuditSelection.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/AuditStatus.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/AuditType.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/BallotManifestInfo.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/CVRAuditInfo.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/CVRContestInfo.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/CastVoteRecord.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/Choice.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/ComparisonAudit.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/Contest.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/ContestResult.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/ContestToAudit.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/County.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/CountyContestComparisonAudit.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/CountyContestResult.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/CountyDashboard.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/DoSDashboard.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/Elector.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/ImportStatus.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/IntermediateAuditReportInfo.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/LogEntry.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/Round.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/Tribute.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/UploadedFile.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/AuditReasonSetConverter.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/AuditSelectionIntegerMapConverter.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/BallotSequenceAssignmentConverter.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/CountyCanonicalContestsMapConverter.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/ElectorListConverter.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/FreeAndFairNamingStrategy.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/IntegerListConverter.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/LongIntegerMapConverter.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/LongListConverter.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/Persistence.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/PersistentEntity.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/ResultConverter.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/SignatoriesConverter.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/StringListConverter.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/StringSetConverter.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/AdministratorQueries.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/BallotManifestInfoQueries.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/CastVoteRecordQueries.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/ComparisonAuditQueries.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/ContestQueries.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/ContestResultQueries.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/CountyContestComparisonAuditQueries.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/CountyContestResultQueries.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/CountyQueries.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/DatabaseResetQueries.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/ExportQueries.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/LogEntryQueries.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/PersistentASMStateQueries.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/TributeQueries.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/UploadedFileQueries.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/report/CountyReport.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/report/ReportRows.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/report/StateReport.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/report/WorkbookWriter.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/BallotAssignment.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/BallotSequencer.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/DBExceptionUtil.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/EqualsHashcodeHelper.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/ExponentialBackoffHelper.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/FileHelper.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/NaturalOrderComparator.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/Pair.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/PhantomBallots.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/PrettyPrinter.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/SetCreator.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/SparkHelper.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/SuppressFBWarnings.java create mode 100644 server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/UploadedFileStreamer.java diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/Main.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/Main.java new file mode 100644 index 00000000..7b4bb8ab --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/Main.java @@ -0,0 +1,654 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 19, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla; + +import static spark.Spark.*; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Paths; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.List; +import java.util.Properties; +import java.util.Scanner; + +import javax.persistence.PersistenceException; + +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; +import org.apache.maven.model.Model; +import org.apache.maven.model.io.xpp3.MavenXpp3Reader; +import org.codehaus.plexus.util.xml.pull.XmlPullParserException; +import org.eclipse.jetty.http.HttpStatus; +import org.eclipse.jetty.server.Server; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +import spark.Request; +import spark.Response; +import spark.Service; +import spark.embeddedserver.EmbeddedServers; +import spark.embeddedserver.jetty.EmbeddedJettyServer; +import spark.embeddedserver.jetty.JettyHandler; +import spark.http.matching.MatcherFilter; +import spark.route.Routes; +import spark.staticfiles.StaticFilesConfiguration; + +import us.freeandfair.corla.asm.AbstractStateMachine; +import us.freeandfair.corla.asm.AuditBoardDashboardASM; +import us.freeandfair.corla.asm.CountyDashboardASM; +import us.freeandfair.corla.asm.DoSDashboardASM; +import us.freeandfair.corla.asm.PersistentASMState; +import us.freeandfair.corla.auth.AuthenticationInterface; +import us.freeandfair.corla.endpoint.CORSFilter; +import us.freeandfair.corla.endpoint.Endpoint; +import us.freeandfair.corla.json.FreeAndFairNamingStrategy; +import us.freeandfair.corla.json.InstantTypeAdapter; +import us.freeandfair.corla.json.VersionExclusionStrategy; +import us.freeandfair.corla.model.County; +import us.freeandfair.corla.model.CountyDashboard; +import us.freeandfair.corla.model.DoSDashboard; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.query.PersistentASMStateQueries; +import us.freeandfair.corla.util.SuppressFBWarnings; + +/** + * The main executable for the ColoradoRLA server. + * + * @author Daniel M. Zimmerman + * @author Joseph R. Kiniry + * @version 1.0.0 + */ +@SuppressWarnings({"PMD.GodClass", "PMD.ExcessiveImports"}) +@SuppressFBWarnings("ST_WRITE_TO_STATIC_FROM_INSTANCE_METHOD") +public final class Main { + /** + * The path to the default properties resource. + */ + public static final String DEFAULT_PROPERTIES = + "us/freeandfair/corla/default.properties"; + + /** + * The path to the resource containing the list of endpoint classes. + */ + public static final String ENDPOINT_CLASSES = + "us/freeandfair/corla/endpoint/endpoint_classes"; + + /** + * The name of the property that specifies county IDs. + */ + public static final String COUNTY_IDS = "county_ids"; + + /** + * The name of the logger. + */ + public static final String LOGGER_NAME = "corla"; + + /** + * The default HTTP port number (can be overridden by properties). + */ + public static final int DEFAULT_HTTP_PORT = 8888; + + /** + * The default HTTPS port number (can be overridden by properties). + */ + public static final int DEFAULT_HTTPS_PORT = 8889; + + /** + * The minimum valid port number. + */ + public static final int MIN_PORT = 1024; + + /** + * The maximum valid port number. + */ + public static final int MAX_PORT = 65535; + + /** + * The logger. + */ + public static final Logger LOGGER = LogManager.getLogger(LOGGER_NAME); + + /** + * The Gson object to use for translation to and from JSON; since + * Gson is thread-safe, we only need one for the system. Note that + * any custom Gson serializers/deserializers we use must also be + * thread-safe. + */ + // @review kiniry Should we configure Gson to serialize nulls via + // serializeNulls() as well? This will, of course, cost more in + // bandwidth, but the tradeoff is completeness and clarity of wire + // format. Perhaps we should just bandwidth and performance + // benchmark with and without serializeNulls() and + // setPrettyPrinting()? + public static final Gson GSON = + new GsonBuilder(). + registerTypeAdapter(Instant.class, new InstantTypeAdapter()). + setFieldNamingStrategy(new FreeAndFairNamingStrategy()). + setExclusionStrategies(new VersionExclusionStrategy()). + setPrettyPrinting().create(); + + /** + * The version string. + */ + public static final String VERSION; + + /** + * Which authentication subsystem implementation are we to use? + */ + private static AuthenticationInterface static_authentication; + + /** + * The properties loaded from the properties file. + */ + @SuppressWarnings("PMD.AssignmentToNonFinalStatic") + private static Properties static_properties; + + // Version Initializer + + static { + final String pom_location = + "/META-INF/maven/us.freeandfair.production/corla-server/pom.xml"; + final File pom = new File("pom.xml"); + String version = "UNKNOWN"; + InputStream pom_stream = null; + + if (pom.exists()) { + try { + pom_stream = new FileInputStream(pom); + } catch (final FileNotFoundException e) { + // this can't happen because we tested that the file existed + } + } else { + pom_stream = Main.class.getResourceAsStream(pom_location); + } + + if (pom_stream != null) { + try (InputStreamReader isr = new InputStreamReader(pom_stream, "UTF-8")) { + final MavenXpp3Reader reader = new MavenXpp3Reader(); + final Model model = reader.read(isr); + version = model.getVersion(); + } catch (final IOException | XmlPullParserException e) { + LOGGER.info("could not obtain version number: " + e); + } + } + VERSION = version; + } + + // Constructors + + /** + * Constructs a new ColoradoRLA server with the specified properties. + * + * @param the_properties The properties. + */ + public Main(final Properties the_properties) { + static_properties = the_properties; + } + + // Instance Methods + + /** + * @return the version string of the system. + */ + public static String version() { + return VERSION; + } + + /** + * @return the implementation of `AuthenticationInterface` demanded by + * the system's properties file and loaded at startup. + */ + public static AuthenticationInterface authentication() { + return static_authentication; + } + + /** + * @return a read-only view of the properties in use by the system at runtime. + */ + public static Properties properties() { + return new Properties(static_properties); + } + + /** + * Creates a default set of properties. + * + * @return The default properties. + */ + public static Properties defaultProperties() { + final Properties properties = new Properties(); + try { + properties.load(ClassLoader.getSystemResourceAsStream(DEFAULT_PROPERTIES)); + } catch (final IOException e) { + throw new IllegalStateException + ("Error loading default properties file, aborting.", e); + } + return properties; + } + + /** + * Setup the authentication subsystem according to the property setting + * `authentication_class` in the system's properties file. + */ + private void setupAuthentication() { + String authentication_class = null; + try { + // classload and attach to the authentication field the appropriate + // implementation of `AuthenticationInterface`. + authentication_class = static_properties.getProperty("authentication_class"); + if (authentication_class == null) { + authentication_class = "us.freeandfair.corla.auth.DatabaseAuthentication"; + } + static_authentication = (AuthenticationInterface) + Class.forName(authentication_class).newInstance(); + LOGGER.info("Loaded authentication subsystem `" + authentication_class + "'"); + static_authentication.setLogger(LOGGER); + static_authentication.setGSON(GSON); + final String authentication_server = + static_properties.getProperty("authentication_server", "localhost"); + static_authentication.setAuthenticationServerName(authentication_server); + LOGGER.info("Initialized authentication subsystem `" + authentication_class + "'"); + } catch (final ClassNotFoundException | + IllegalAccessException | InstantiationException e) { + LOGGER.fatal("Authentication class '" + authentication_class + "' not found."); + LOGGER.fatal("Check the value of `authentication_class` in your RLA Tool " + + "system properties."); + } + } + + /** + * Parse a port number from properties. + * + * @param the_property The name of the property. + * @param the_default The default port number. + */ + private int parsePortNumber(final String the_property, final int the_default) { + int result = the_default; + + try { + final int prop_port = + Integer.parseInt(static_properties.getProperty(the_property, + String.valueOf(the_default))); + if (MIN_PORT <= prop_port && prop_port < MAX_PORT) { + result = prop_port; + } else { + LOGGER.info("invalid port number in property " + the_property + + ", using default " + the_default); + } + } catch (final NumberFormatException e) { + LOGGER.info("could not read port number property " + the_property + + ", using default " + the_default); + } + + return result; + } + + /** + * Redirect a request from HTTP to HTTPS. + * + * @param the_request The request. + * @param the_response The response. + * @param the_port The HTTPS port. + */ + private void httpsRedirect(final Request the_request, final Response the_response, + final int the_port) { + try { + final URL request_url = new URL(the_request.url()); + final URL redirect_url = new URL("https", request_url.getHost(), + the_port, request_url.getFile()); + the_response.redirect(redirect_url.toString()); + } catch (final MalformedURLException e) { + // this should probably never happen, since we're getting the original + // URL from a legitimate request + the_response.status(HttpStatus.BAD_REQUEST_400); + } + } + + /** + * Activate the endpoints. + */ + private void activateEndpoints() { + final List endpoints = new ArrayList<>(); + try (InputStream endpoint_stream = + ClassLoader.getSystemResourceAsStream(ENDPOINT_CLASSES)) { + if (endpoint_stream == null) { + Main.LOGGER.error("could not load list of entity classes"); + } else { + final Scanner scanner = new Scanner(endpoint_stream, "UTF-8"); + while (scanner.hasNextLine()) { + final String endpoint_class = scanner.nextLine(); + final Endpoint endpoint = + (Endpoint) Class.forName(endpoint_class).newInstance(); + endpoints.add(endpoint); + Main.LOGGER.info("added endpoint class " + endpoint_class); + } + scanner.close(); + } + } catch (final IOException e) { + Main.LOGGER.error("error reading list of endpoint classes: " + e); + } catch (final ClassNotFoundException | InstantiationException | + IllegalAccessException | ClassCastException e) { + Main.LOGGER.error("invalid endpoint class specified: " + e); + } + + for (final Endpoint e : endpoints) { + final CORSFilter cors_and_before = + new CORSFilter(static_properties, (the_request, the_response) -> + e.before(the_request, the_response)); + before(e.endpointName(), cors_and_before); + after(e.endpointName(), (the_request, the_response) -> + e.after(the_request, the_response)); + afterAfter(e.endpointName(), (the_request, the_response) -> + e.afterAfter(the_request, the_response)); + switch (e.endpointType()) { + case GET: + get(e.endpointName(), (the_request, the_response) -> + e.endpoint(the_request, the_response)); + break; + + case PUT: + put(e.endpointName(), (the_request, the_response) -> + e.endpoint(the_request, the_response)); + break; + + case POST: + post(e.endpointName(), (the_request, the_response) -> + e.endpoint(the_request, the_response)); + break; + + default: + } + } + } + + /** + * Restores an ASM's state or persists it in the database. + * + * @param the_asm The ASM. + * @param the_state The persistent state to restore, or null to persist + * the state to the database. + * @exception PersistenceException if the state cannot be persisted. + */ + private void restoreOrPersistState(final AbstractStateMachine the_asm, + final PersistentASMState the_state) + throws PersistenceException { + if (the_state == null) { + // there is no such state in the database, so persist one + Main.LOGGER.debug("no state found for " + the_asm + + ", persisting one"); + final PersistentASMState new_state = PersistentASMState.stateFor(the_asm); + Persistence.saveOrUpdate(new_state); + } else { + Main.LOGGER.debug(the_asm + " state found in db: " + the_state); + } + } + + /** + * Initializes the ASMs. Each one for which no state exists in the database + * has its state persisted in the database. + * + * @param the_counties The counties to initialize ASMs for. + * @exception PersistenceException if we can't initialize the ASMs. + */ + private void initializeASMsAndDashboards(final List the_counties) + throws PersistenceException { + // first, check the DoS dashboard + final PersistentASMState dos_state = + PersistentASMStateQueries.get(DoSDashboardASM.class, DoSDashboardASM.IDENTITY); + restoreOrPersistState(new DoSDashboardASM(), dos_state); + + DoSDashboard dosdb = Persistence.getByID(DoSDashboard.ID, DoSDashboard.class); + if (dosdb == null) { + dosdb = new DoSDashboard(); + Persistence.saveOrUpdate(dosdb); + } + + // next, iterate over the counties and check the county and + // audit board dashboards + for (final County c : the_counties) { + final String asm_id = String.valueOf(c.id()); + final PersistentASMState county_state = + PersistentASMStateQueries.get(CountyDashboardASM.class, asm_id); + restoreOrPersistState(new CountyDashboardASM(asm_id), county_state); + + final PersistentASMState audit_state = + PersistentASMStateQueries.get(AuditBoardDashboardASM.class, asm_id); + restoreOrPersistState(new AuditBoardDashboardASM(asm_id), audit_state); + + CountyDashboard cdb = Persistence.getByID(c.id(), CountyDashboard.class); + if (cdb == null) { + cdb = new CountyDashboard(c); + Persistence.saveOrUpdate(cdb); + } + } + } + + /** + * Initializes the counties in the database using information from the + * county properties. + * + * @return the counties. + */ + private List initializeCounties() { + final Properties properties = new Properties(); + try { + properties.load(ClassLoader. + getSystemResourceAsStream(static_properties.getProperty(COUNTY_IDS))); + } catch (final IOException e) { + throw new IllegalStateException("Error loading county IDs, aborting.", e); + } + final List result = new ArrayList(); + try { + for (final String s : properties.stringPropertyNames()) { + // if the property name is an integer, we assume it's a county ID + try { + final Long id = Long.valueOf(s); + final String name = properties.getProperty(s); + County county = Persistence.getByID(id, County.class); + if (county == null) { + county = new County(name, id); + } else if (!county.name().equals(name)) { + // update the county's name while preserving the rest of its info + Main.LOGGER.info("Updating " + county.name() + " county name to " + name); + final County new_county = new County(name, id); + new_county.setID(county.id()); + county = new_county; + } + Persistence.saveOrUpdate(county); + result.add(county); + } catch (final NumberFormatException e) { + // we skip this property because it wasn't numeric + } + } + } catch (final PersistenceException e) { + throw new IllegalStateException("Error loading county IDs, aborting.", e); + } + return result; + } + + /** + * Generates a string representation of a Properties object, including all + * properties (even default ones). + * + * @param the_properties The Properties object. + * @return the string representation. + */ + private static String propertiesString(final Properties the_properties) { + final StringBuilder sb = new StringBuilder(); + + sb.append('{'); + final Enumeration property_names = the_properties.propertyNames(); + boolean not_first = false; + while (property_names.hasMoreElements()) { + final Object prop = property_names.nextElement(); + if (not_first) { + sb.append(", "); + } else { + not_first = true; + } + sb.append(prop.toString()); + sb.append('='); + sb.append(the_properties.getProperty(prop.toString())); + } + sb.append('}'); + + return sb.toString(); + } + + /** + * Starts a ColoradoRLA server. + */ + public void start() { + LOGGER.info("starting server version " + VERSION + " with properties: " + + propertiesString(static_properties)); + + // provide properties to the persistence engine + Persistence.setProperties(static_properties); + + if (Persistence.beginTransaction()) { + initializeASMsAndDashboards(initializeCounties()); + try { + Persistence.commitTransaction(); + } catch (final PersistenceException e) { + throw new IllegalStateException("could not initialize data in database", e); + } + } else { + LOGGER.error("could not open database connection"); + return; + } + + // secure the session cookies by adding an embedded server handler + EmbeddedServers.add(EmbeddedServers.Identifiers.JETTY, + (final Routes the_route_matcher, + final StaticFilesConfiguration the_static_files_config, + final boolean the_has_multiple_handler) -> { + final MatcherFilter matcher_filter = + new MatcherFilter(the_route_matcher, the_static_files_config, + false, the_has_multiple_handler); + matcher_filter.init(null); + + final JettyHandler handler = new JettyHandler(matcher_filter); + handler.getSessionCookieConfig().setHttpOnly(true); + // secure cookies don't work if we're not using HTTPS + // handler.getSessionCookieConfig().setSecure(true); + + return new EmbeddedJettyServer((int the_max_threads, + int the_min_threads, + int the_thread_timeout) -> { + return new Server(); + }, handler); + }); + + // get the port numbers from properties + final int http_port = parsePortNumber("http_port", DEFAULT_HTTP_PORT); + final int https_port = parsePortNumber("https_port", DEFAULT_HTTPS_PORT); + + // get key store information from properties, if applicable + String keystore_path = static_properties.getProperty("keystore", null); + if (keystore_path != null && !(new File(keystore_path).exists())) { + // the keystore property isn't an absolute or relative pathname that exists, so + // let's try to load it as a resource + final URL keystore_url = Main.class.getResource(keystore_path); + if (keystore_url != null) { + try { + keystore_path = Paths.get(keystore_url.toURI()).toString(); + } catch (final URISyntaxException e) { + // keystore_path stays null + } + } + } + + // if we have a keystore, everything is on SSL except the redirect; otherwise, + // everything is in plaintext + + if (keystore_path == null) { + port(http_port); + } else { + port(https_port); + final String keystore_password = + static_properties.getProperty("keystore_password", null); + secure(keystore_path, keystore_password, null, null); + + // redirect everything + final Service redirect = Service.ignite(); + redirect.port(http_port); + redirect.before((the_request, the_response) -> + httpsRedirect(the_request, the_response, https_port)); + } + + // authentication subsystem + setupAuthentication(); + + // static files location + staticFileLocation("/us/freeandfair/corla/static"); + + // start the endpoints + activateEndpoints(); + } + + + /** + * The main method. Starts the server using the specified properties + * file. + * + * @param the_args Command line arguments. Only the first one is + * considered, and it is interpreted as the path to a properties + * file. If no arguments are supplied, default properties are + * used. If the specified properties file cannot be loaded, the + * server does not start. + */ + public static void main(final String... the_args) { + // set headless mode - this prevents Apache POI from starting a GUI when + // generating Excel files + System.setProperty("java.awt.headless", "true"); + + final Properties default_properties = defaultProperties(); + Properties properties = new Properties(default_properties); + if (the_args.length > 0) { + final File file = new File(the_args[0]); + try { + LOGGER.info("attempting to load properties from " + file); + properties.load(new FileInputStream(file)); + } catch (final IOException e) { + // could not load properties that way, let's try XML + try { + LOGGER.info("load failed, attempting to load XML properties from " + file); + properties = new Properties(default_properties); + properties.loadFromXML(new FileInputStream(file)); + } catch (final IOException ex) { + // could not load properties that way either, let's abort + LOGGER.error("could not load properties, exiting"); + return; + } + } + } else { + LOGGER.info("no property file specified, using default properties"); + } + + final Main main = new Main(properties); + try { + main.start(); + } catch (final IllegalStateException e) { + LOGGER.error("unable to run: " + e); + } + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/ASMEvent.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/ASMEvent.java new file mode 100644 index 00000000..2d77ea1d --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/ASMEvent.java @@ -0,0 +1,71 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 8, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.asm; + +/** + * The events of the Abstract State Machine (ASM) of the Colorado RLA Tool. + * @trace asm.asm_event + * @author Joseph R. Kiniry + * @version 1.0.0 + */ +public interface ASMEvent extends Event { + /** + * The Department of State Dashboard's events. + * @trace asm.department_of_state_dashboard_event + */ + enum DoSDashboardEvent implements ASMEvent { + PARTIAL_AUDIT_INFO_EVENT, // public inbound event + COMPLETE_AUDIT_INFO_EVENT, // public inbound event + DOS_START_ROUND_EVENT, // public inbound event + DOS_ROUND_COMPLETE_EVENT, // private internal event + AUDIT_EVENT, // private internal event + DOS_COUNTY_AUDIT_COMPLETE_EVENT, // private internal event + DOS_AUDIT_COMPLETE_EVENT, // private internal event + PUBLISH_AUDIT_REPORT_EVENT // public inbound event + } + + /** + * The County Dashboard's events. + * @trace asm.county_dashboard_event + */ + enum CountyDashboardEvent implements ASMEvent { + IMPORT_BALLOT_MANIFEST_EVENT, // public inbound event + IMPORT_CVRS_EVENT, // public inbound event + DELETE_BALLOT_MANIFEST_EVENT, // public inbound event + DELETE_CVRS_EVENT, // public inbound event + CVR_IMPORT_SUCCESS_EVENT, // private internal event + CVR_IMPORT_FAILURE_EVENT, // private internal event + COUNTY_START_AUDIT_EVENT, // private internal event + COUNTY_AUDIT_COMPLETE_EVENT // private internal event + } + + /** + * The Audit Board Dashboard's events. + * @trace asm.audit_board_dashboard_event + */ + enum AuditBoardDashboardEvent implements ASMEvent { + COUNTY_DEADLINE_MISSED_EVENT, // private internal event + NO_CONTESTS_TO_AUDIT_EVENT, // private internal event + REPORT_MARKINGS_EVENT, // public inbound event + REPORT_BALLOT_NOT_FOUND_EVENT, // public inbound event + SUBMIT_AUDIT_INVESTIGATION_REPORT_EVENT, // public inbound event + SUBMIT_INTERMEDIATE_AUDIT_REPORT_EVENT, // public inbound event + SIGN_OUT_AUDIT_BOARD_EVENT, // public inbound event + SIGN_IN_AUDIT_BOARD_EVENT, // public inbound event + ROUND_START_EVENT, // private internal event + ROUND_COMPLETE_EVENT, // private internal event + ROUND_SIGN_OFF_EVENT, // public inbound event + RISK_LIMIT_ACHIEVED_EVENT, // private internal event + ABORT_AUDIT_EVENT, // public inbound event + BALLOTS_EXHAUSTED_EVENT // private internal event + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/ASMEventToEndpointRelation.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/ASMEventToEndpointRelation.java new file mode 100644 index 00000000..990b8cc2 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/ASMEventToEndpointRelation.java @@ -0,0 +1,147 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 8, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.asm; + +import static us.freeandfair.corla.asm.ASMEvent.AuditBoardDashboardEvent.*; +import static us.freeandfair.corla.asm.ASMEvent.CountyDashboardEvent.*; +import static us.freeandfair.corla.asm.ASMEvent.DoSDashboardEvent.*; + +import java.util.HashSet; +import java.util.Set; + +import us.freeandfair.corla.endpoint.Endpoint; +import us.freeandfair.corla.util.Pair; + +/** + * @description The mapping between ASM events and server endpoints. + * @trace asm.ui_to_asm_event_relation + * @author Joseph R. Kiniry + * @version 1.0.0 + * @todo kiniry Introduce AbstractRelation parent class. + * @todo dmz use an entity instead of Pair<> to enable persistence + */ +@SuppressWarnings("PMD.TooManyStaticImports") +public class ASMEventToEndpointRelation { + /** + * A constant encoding that we have not yet implemented a particular + * endpoint. + */ + public static final String UNIMPLEMENTED = "UNIMPLEMENTED"; + + /** + * The relation encoded via a set of pairs. + */ + private final Set> my_relation = + new HashSet>(); + + /** + * Create an instance of this relation, which contains the full set + * of public ASM events and Endpoints. + * @design kiniry This should probably be refactored as a singleton. + */ + public ASMEventToEndpointRelation() { + addDoSDashboardPairs(); + addCountyDashboardPairs(); + addAuditBoardDashboardPairs(); + } + + private void addDoSDashboardPairs() { + // All Department of State Dashboard pairs. + my_relation.add(new Pair( + PARTIAL_AUDIT_INFO_EVENT, + UNIMPLEMENTED)); + my_relation.add(new Pair( + DOS_START_ROUND_EVENT, + UNIMPLEMENTED)); + my_relation.add(new Pair( + PUBLISH_AUDIT_REPORT_EVENT, + UNIMPLEMENTED)); + } + + private void addCountyDashboardPairs() { + // All County Dashboard pairs. + my_relation.add(new Pair( + IMPORT_BALLOT_MANIFEST_EVENT, + "BallotManifestUpload")); + my_relation.add(new Pair( + IMPORT_CVRS_EVENT, + UNIMPLEMENTED)); + my_relation.add(new Pair( + COUNTY_START_AUDIT_EVENT, + UNIMPLEMENTED)); + } + + private void addAuditBoardDashboardPairs() { + // All Audit Board Dashboard pairs. + my_relation.add(new Pair( + REPORT_MARKINGS_EVENT, + UNIMPLEMENTED)); + my_relation.add(new Pair( + REPORT_BALLOT_NOT_FOUND_EVENT, + UNIMPLEMENTED)); + my_relation.add(new Pair( + SUBMIT_AUDIT_INVESTIGATION_REPORT_EVENT, + UNIMPLEMENTED)); + my_relation.add(new Pair( + SUBMIT_INTERMEDIATE_AUDIT_REPORT_EVENT, + UNIMPLEMENTED)); + } + + /** + * Is a_pair a member of this relation? + * @param a_pair the UIEvent/ASMEvent pair to check. + */ + public boolean member(final ASMEvent an_ae, final Endpoint an_e) { + return my_relation.contains(new Pair(an_ae, an_e)); + } + + // @todo kiniry Do we need these arrows anymore, especially given + // that relations are not 1-1? + + /** + * Follow the relation from left to right. + * @param a_ae the ASM event to lookup. + * @return the endpoints corresponding to 'a_ae', or null if no such + * endpoints exists. + */ + public Set rightArrow(final ASMEvent a_ae) { + // iterate over all elements in the map and, for each one whose + // left element matches a_ae, include the right element in the + // resulting set. + final Set result = new HashSet(); + for (final Pair p : my_relation) { + if (p.first().equals(a_ae)) { + result.add(p.second()); + } + } + return result; + } + + /** + * Follow the relation from right to left. + * @param an_endpoint the endpoint to lookup. + * @return the ASM events corresponding to 'an_endpoint', or null if + * no such events exists. + */ + public Set leftArrow(final String an_endpoint) { + // iterate over all elements in the map and, for each one whose + // right element matches an_ae, include the left element in the + // resulting set. + final Set result = new HashSet(); + for (final Pair p : my_relation) { + if (p.second().equals(an_endpoint)) { + result.add(p.first()); + } + } + return result; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/ASMState.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/ASMState.java new file mode 100644 index 00000000..af966286 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/ASMState.java @@ -0,0 +1,68 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 8, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.asm; + +/** + * The states of the Abstract State Machine (ASM) of the Colorado RLA Tool. + * @trace asm.asm_state + * @author Joseph R. Kiniry + * @version 1.0.0 + */ +public interface ASMState { + /** + * The Department of State Dashboard's states. + * @trace asm.department_of_state_dashboard_state + */ + enum DoSDashboardState implements ASMState { + DOS_INITIAL_STATE, + PARTIAL_AUDIT_INFO_SET, + COMPLETE_AUDIT_INFO_SET, + RANDOM_SEED_PUBLISHED, + DOS_AUDIT_ONGOING, + DOS_ROUND_COMPLETE, + DOS_AUDIT_COMPLETE, + AUDIT_RESULTS_PUBLISHED + } + + /** + * The County Dashboard's states. + * @trace asm.county_dashboard_state + */ + enum CountyDashboardState implements ASMState { + COUNTY_INITIAL_STATE, + BALLOT_MANIFEST_OK, + CVRS_IMPORTING, + CVRS_OK, + BALLOT_MANIFEST_OK_AND_CVRS_IMPORTING, + BALLOT_MANIFEST_AND_CVRS_OK, + COUNTY_AUDIT_UNDERWAY, + COUNTY_AUDIT_COMPLETE, + DEADLINE_MISSED + } + + /** + * The Audit Board Dashboard's states. + * @trace asm.audit_board_dashboard_state + */ + enum AuditBoardDashboardState implements ASMState { + AUDIT_INITIAL_STATE, + WAITING_FOR_ROUND_START, + WAITING_FOR_ROUND_START_NO_AUDIT_BOARD, + ROUND_IN_PROGRESS, + ROUND_IN_PROGRESS_NO_AUDIT_BOARD, + WAITING_FOR_ROUND_SIGN_OFF, + WAITING_FOR_ROUND_SIGN_OFF_NO_AUDIT_BOARD, + AUDIT_COMPLETE, + UNABLE_TO_AUDIT, + AUDIT_ABORTED + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/ASMTransition.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/ASMTransition.java new file mode 100644 index 00000000..38a3f8fb --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/ASMTransition.java @@ -0,0 +1,171 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 10, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.asm; + +import static us.freeandfair.corla.util.EqualsHashcodeHelper.nullableEquals; + +import java.io.Serializable; +import java.util.HashSet; +import java.util.Set; + +import us.freeandfair.corla.util.SetCreator; + +/** + * A single transition of an abstract state machine. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public class ASMTransition implements Serializable { + /** + * The serialVersionUID. + */ + private static final long serialVersionUID = 1L; + + /** + * The start state for this transition. + */ + private final Set my_start_states = new HashSet(); + + /** + * The set of events for this transition. + */ + private final Set my_events = new HashSet(); + + /** + * The end state for this transition. + */ + private final ASMState my_end_state; + + /** + * Constructs an ASMTransition with the specified start state, + * event, and end state. + * + * @param the_start_state The start state. + * @param the_event The event. + * @param the_end_state The end state. + */ + public ASMTransition(final ASMState the_start_state, + final ASMEvent the_event, + final ASMState the_end_state) { + this(SetCreator.setOf(the_start_state), + SetCreator.setOf(the_event), + the_end_state); + } + + /** + * Constructs an ASMTransition with the specified set of start states, + * event, and end state. + * + * @param the_start_states The start states. + * @param the_event The event. + * @param the_end_state The end state. + */ + public ASMTransition(final Set the_start_states, + final ASMEvent the_event, + final ASMState the_end_state) { + this(the_start_states, + SetCreator.setOf(the_event), + the_end_state); + } + + /** + * Constructs an ASMTransition with the specified start state, + * set of events, and end state. + * + * @param the_start_state The start state. + * @param the_events The events. + * @param the_end_state The end state. + */ + public ASMTransition(final ASMState the_start_state, + final Set the_events, + final ASMState the_end_state) { + this(SetCreator.setOf(the_start_state), + the_events, + the_end_state); + } + + /** + * Constructs an ASMTransition with the specified start states, + * set of events, and end state. + * + * @param the_start_states The start states. + * @param the_events The events. + * @param the_end_state The end state. + */ + public ASMTransition(final Set the_start_states, + final Set the_events, + final ASMState the_end_state) { + my_start_states.addAll(the_start_states); + my_events.addAll(the_events); + my_end_state = the_end_state; + } + + /** + * @return the start state. + */ + public Set startStates() { + return my_start_states; + } + + /** + * @return the events. + */ + public Set events() { + return my_events; + } + + /** + * @return the end state. + */ + public ASMState endState() { + return my_end_state; + } + + /** + * @return a String representation of this ASMTransition + */ + @Override + public String toString() { + return "ASMTransition [start=" + my_start_states + + ", events=" + my_events + ", end=" + + my_end_state + "]"; + } + + /** + * Compare this object with another for equivalence. + * + * @param the_other The other object. + * @return true if the objects are equivalent, false otherwise. + */ + @Override + public boolean equals(final Object the_other) { + boolean result = true; + if (the_other instanceof ASMTransition) { + final ASMTransition other_transition = (ASMTransition) the_other; + result &= nullableEquals(other_transition.startStates(), startStates()); + result &= nullableEquals(other_transition.events(), events()); + result &= nullableEquals(other_transition.endState(), endState()); + } else { + result = false; + } + return result; + } + + /** + * @return a hash code for this object. + */ + @Override + public int hashCode() { + return startStates().hashCode(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/ASMTransitionFunction.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/ASMTransitionFunction.java new file mode 100644 index 00000000..4af9c709 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/ASMTransitionFunction.java @@ -0,0 +1,291 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 8, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.asm; + +import static us.freeandfair.corla.asm.ASMEvent.AuditBoardDashboardEvent.*; +import static us.freeandfair.corla.asm.ASMEvent.CountyDashboardEvent.*; +import static us.freeandfair.corla.asm.ASMEvent.DoSDashboardEvent.*; +import static us.freeandfair.corla.asm.ASMState.AuditBoardDashboardState.*; +import static us.freeandfair.corla.asm.ASMState.CountyDashboardState.*; +import static us.freeandfair.corla.asm.ASMState.DoSDashboardState.*; + +import us.freeandfair.corla.util.SetCreator; + +/** + * The generic idea of an ASM transition function. + * @trace asm.asm_transition_function + * @author Joseph R. Kiniry + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings({"PMD.TooManyStaticImports", "PMD.AvoidDuplicateLiterals"}) +public interface ASMTransitionFunction { + /** + * The Department of State Dashboard's transition function. + * @trace asm.dos_dashboard_next_state + */ + enum DoSDashboardTransitionFunction implements ASMTransitionFunction { + A(new ASMTransition(SetCreator.setOf(DOS_INITIAL_STATE, + PARTIAL_AUDIT_INFO_SET, + COMPLETE_AUDIT_INFO_SET), + PARTIAL_AUDIT_INFO_EVENT, + PARTIAL_AUDIT_INFO_SET)), + B(new ASMTransition(SetCreator.setOf(DOS_INITIAL_STATE, + PARTIAL_AUDIT_INFO_SET, + COMPLETE_AUDIT_INFO_SET), + COMPLETE_AUDIT_INFO_EVENT, + COMPLETE_AUDIT_INFO_SET)), + D(new ASMTransition(COMPLETE_AUDIT_INFO_SET, + DOS_START_ROUND_EVENT, + DOS_AUDIT_ONGOING)), + E(new ASMTransition(DOS_AUDIT_ONGOING, + SetCreator.setOf(AUDIT_EVENT, + DOS_COUNTY_AUDIT_COMPLETE_EVENT, + DOS_START_ROUND_EVENT), + DOS_AUDIT_ONGOING)), + F(new ASMTransition(DOS_AUDIT_ONGOING, + DOS_ROUND_COMPLETE_EVENT, + DOS_ROUND_COMPLETE)), + G(new ASMTransition(DOS_ROUND_COMPLETE, + DOS_START_ROUND_EVENT, + DOS_AUDIT_ONGOING)), + H(new ASMTransition(SetCreator.setOf(DOS_AUDIT_ONGOING, + DOS_ROUND_COMPLETE), + DOS_AUDIT_COMPLETE_EVENT, + DOS_AUDIT_COMPLETE)), + I(new ASMTransition(DOS_AUDIT_COMPLETE, + PUBLISH_AUDIT_REPORT_EVENT, + AUDIT_RESULTS_PUBLISHED)); + + /** + * A single transition. + */ + @SuppressWarnings("PMD.ConstantsInInterface") + private final transient ASMTransition my_transition; + + /** + * Create a transition. + * @param the_pair the (current state, event) pair. + * @param the_state the state transitioned to when the pair is witnessed. + */ + DoSDashboardTransitionFunction(final ASMTransition the_transition) { + my_transition = the_transition; + } + + /** + * @return the pair encoding this enumeration. + */ + public ASMTransition value() { + return my_transition; + } + } + + /** + * The County Board Dashboard's transition function. + * @trace asm.county_dashboard_next_state + */ + enum CountyDashboardTransitionFunction implements ASMTransitionFunction { + A(new ASMTransition(COUNTY_INITIAL_STATE, + IMPORT_BALLOT_MANIFEST_EVENT, + BALLOT_MANIFEST_OK)), + B(new ASMTransition(COUNTY_INITIAL_STATE, + IMPORT_CVRS_EVENT, + CVRS_IMPORTING)), + C(new ASMTransition(CVRS_IMPORTING, + CVR_IMPORT_SUCCESS_EVENT, + CVRS_OK)), + D(new ASMTransition(CVRS_IMPORTING, + CVR_IMPORT_FAILURE_EVENT, + COUNTY_INITIAL_STATE)), + E(new ASMTransition(BALLOT_MANIFEST_OK, + IMPORT_CVRS_EVENT, + BALLOT_MANIFEST_OK_AND_CVRS_IMPORTING)), + F(new ASMTransition(BALLOT_MANIFEST_OK_AND_CVRS_IMPORTING, + CVR_IMPORT_SUCCESS_EVENT, + BALLOT_MANIFEST_AND_CVRS_OK)), + G(new ASMTransition(BALLOT_MANIFEST_OK_AND_CVRS_IMPORTING, + CVR_IMPORT_FAILURE_EVENT, + BALLOT_MANIFEST_OK)), + H(new ASMTransition(BALLOT_MANIFEST_OK, + IMPORT_BALLOT_MANIFEST_EVENT, + BALLOT_MANIFEST_OK)), + H2(new ASMTransition(BALLOT_MANIFEST_OK, + DELETE_CVRS_EVENT, + BALLOT_MANIFEST_OK)), + I(new ASMTransition(CVRS_OK, + IMPORT_BALLOT_MANIFEST_EVENT, + BALLOT_MANIFEST_AND_CVRS_OK)), + J(new ASMTransition(CVRS_OK, + IMPORT_CVRS_EVENT, + CVRS_IMPORTING)), + K(new ASMTransition(BALLOT_MANIFEST_AND_CVRS_OK, + IMPORT_BALLOT_MANIFEST_EVENT, + BALLOT_MANIFEST_AND_CVRS_OK)), + K1(new ASMTransition(BALLOT_MANIFEST_AND_CVRS_OK, + DELETE_BALLOT_MANIFEST_EVENT, + CVRS_OK)), + K2(new ASMTransition(BALLOT_MANIFEST_AND_CVRS_OK, + DELETE_CVRS_EVENT, + BALLOT_MANIFEST_OK)), + L(new ASMTransition(BALLOT_MANIFEST_AND_CVRS_OK, + IMPORT_CVRS_EVENT, + BALLOT_MANIFEST_OK_AND_CVRS_IMPORTING)), + M(new ASMTransition(BALLOT_MANIFEST_AND_CVRS_OK, + COUNTY_START_AUDIT_EVENT, + COUNTY_AUDIT_UNDERWAY)), + N(new ASMTransition(COUNTY_AUDIT_UNDERWAY, + COUNTY_AUDIT_COMPLETE_EVENT, + COUNTY_AUDIT_COMPLETE)), + O(new ASMTransition(SetCreator.setOf(COUNTY_INITIAL_STATE, + BALLOT_MANIFEST_OK, + CVRS_OK, + CVRS_IMPORTING, + BALLOT_MANIFEST_OK_AND_CVRS_IMPORTING), + COUNTY_START_AUDIT_EVENT, + DEADLINE_MISSED)); + + /** + * A single transition. + */ + @SuppressWarnings("PMD.ConstantsInInterface") + private final transient ASMTransition my_transition; + + /** + * Create a transition. + * @param the_pair the (current state, event) pair. + * @param the_state the state transitioned to when the pair is witnessed. + */ + CountyDashboardTransitionFunction(final ASMTransition the_transition) { + my_transition = the_transition; + } + + /** + * @return the pair encoding this enumeration. + */ + public ASMTransition value() { + return my_transition; + } + } + + /** + * The Audit Board Dashboard's transition function. + * @trace asm.audit_board_dashboard_next_state + */ + enum AuditBoardDashboardTransitionFunction implements ASMTransitionFunction { + A(new ASMTransition(SetCreator.setOf(AUDIT_INITIAL_STATE, + WAITING_FOR_ROUND_START_NO_AUDIT_BOARD), + ROUND_START_EVENT, + ROUND_IN_PROGRESS_NO_AUDIT_BOARD)), + B(new ASMTransition(SetCreator.setOf(AUDIT_INITIAL_STATE, + WAITING_FOR_ROUND_START_NO_AUDIT_BOARD), + SIGN_IN_AUDIT_BOARD_EVENT, + WAITING_FOR_ROUND_START)), + C(new ASMTransition(AUDIT_INITIAL_STATE, + SetCreator.setOf(NO_CONTESTS_TO_AUDIT_EVENT, + RISK_LIMIT_ACHIEVED_EVENT), + AUDIT_COMPLETE)), + D(new ASMTransition(AUDIT_INITIAL_STATE, + COUNTY_DEADLINE_MISSED_EVENT, + UNABLE_TO_AUDIT)), + F(new ASMTransition(WAITING_FOR_ROUND_START, + ROUND_START_EVENT, + ROUND_IN_PROGRESS)), + G(new ASMTransition(WAITING_FOR_ROUND_START, + SIGN_OUT_AUDIT_BOARD_EVENT, + WAITING_FOR_ROUND_START_NO_AUDIT_BOARD)), + H(new ASMTransition(WAITING_FOR_ROUND_START, + RISK_LIMIT_ACHIEVED_EVENT, + AUDIT_COMPLETE)), + M(new ASMTransition(ROUND_IN_PROGRESS, + SetCreator.setOf(REPORT_MARKINGS_EVENT, + REPORT_BALLOT_NOT_FOUND_EVENT, + SUBMIT_AUDIT_INVESTIGATION_REPORT_EVENT), + ROUND_IN_PROGRESS)), + N(new ASMTransition(ROUND_IN_PROGRESS, + SIGN_OUT_AUDIT_BOARD_EVENT, + ROUND_IN_PROGRESS_NO_AUDIT_BOARD)), + O(new ASMTransition(ROUND_IN_PROGRESS, + ROUND_COMPLETE_EVENT, + WAITING_FOR_ROUND_SIGN_OFF)), + + // this can happen if there are no ballots to audit in the first round + O1(new ASMTransition(AUDIT_INITIAL_STATE, + ROUND_COMPLETE_EVENT, + WAITING_FOR_ROUND_SIGN_OFF)), + + // this can happen if there are no ballots to audit in subsequent rounds + O2(new ASMTransition(WAITING_FOR_ROUND_START, + ROUND_COMPLETE_EVENT, + WAITING_FOR_ROUND_SIGN_OFF)), + + // this can happen if there are no ballots for an audit board + O3(new ASMTransition(WAITING_FOR_ROUND_START, + ROUND_SIGN_OFF_EVENT, + WAITING_FOR_ROUND_START)), + + /* We probably want this transition eventually, but not for CDOS + EARLY(new ASMTransition(ROUND_IN_PROGRESS, + RISK_LIMIT_ACHIEVED_EVENT, + AUDIT_COMPLETE)), */ + P(new ASMTransition(ROUND_IN_PROGRESS_NO_AUDIT_BOARD, + SIGN_IN_AUDIT_BOARD_EVENT, + ROUND_IN_PROGRESS)), + Q(new ASMTransition(WAITING_FOR_ROUND_SIGN_OFF, + SIGN_OUT_AUDIT_BOARD_EVENT, + WAITING_FOR_ROUND_SIGN_OFF_NO_AUDIT_BOARD)), + R(new ASMTransition(WAITING_FOR_ROUND_SIGN_OFF, + ROUND_SIGN_OFF_EVENT, + WAITING_FOR_ROUND_START)), + S(new ASMTransition(WAITING_FOR_ROUND_SIGN_OFF, + SetCreator.setOf(RISK_LIMIT_ACHIEVED_EVENT, + BALLOTS_EXHAUSTED_EVENT), + AUDIT_COMPLETE)), + T(new ASMTransition(WAITING_FOR_ROUND_SIGN_OFF_NO_AUDIT_BOARD, + SIGN_IN_AUDIT_BOARD_EVENT, + WAITING_FOR_ROUND_SIGN_OFF)), + U(new ASMTransition(SetCreator.setOf(AUDIT_INITIAL_STATE, + WAITING_FOR_ROUND_START, + WAITING_FOR_ROUND_START_NO_AUDIT_BOARD, + ROUND_IN_PROGRESS, + ROUND_IN_PROGRESS_NO_AUDIT_BOARD, + WAITING_FOR_ROUND_SIGN_OFF, + WAITING_FOR_ROUND_SIGN_OFF_NO_AUDIT_BOARD), + ABORT_AUDIT_EVENT, + AUDIT_ABORTED)); + + /** + * A single transition. + */ + @SuppressWarnings("PMD.ConstantsInInterface") + private final transient ASMTransition my_transition; + + /** + * Create a transition. + * @param the_transition the transition. + */ + AuditBoardDashboardTransitionFunction(final ASMTransition the_transition) { + my_transition = the_transition; + } + + /** + * @return the transition. + */ + public ASMTransition value() { + return my_transition; + } + } + + /** + * @return the value of this transition function element. + */ + ASMTransition value(); +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/ASMUtilities.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/ASMUtilities.java new file mode 100644 index 00000000..14372999 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/ASMUtilities.java @@ -0,0 +1,136 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 17, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.asm; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; + +import javax.persistence.PersistenceException; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.query.PersistentASMStateQueries; + +/** + * Utility classes that are generally useful for working with ASMs. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public final class ASMUtilities { + /** + * Private constructor to prevent instantiation. + */ + private ASMUtilities() { + // do nothing + } + + /** + * Gets the ASM for the specified ASM class and identity, initialized to its + * state on the database. + * + * @param the_class The class. + * @param the_identity The identity. + * @return the ASM, or null if the ASM cannot be instantiated. + */ + public static T asmFor(final Class the_class, + final String the_identity) { + T result = null; + + try { + // first, check for a no-argument constructor + final Constructor[] constructors = the_class.getConstructors(); + for (final Constructor c : constructors) { + @SuppressWarnings("unchecked") // this cast is safe because of the call above + final Constructor constructor = (Constructor) c; + if (constructor.getParameterTypes().length == 0) { + // default constructor + result = constructor.newInstance(); + break; + } else if (constructor.getParameterTypes().length == 1 && + constructor.getParameterTypes()[0].equals(String.class)) { + // 1-argument constructor that takes a String + result = constructor.newInstance(the_identity); + break; + } + } + } catch (final IllegalAccessException | InstantiationException | + InvocationTargetException e) { + Main.LOGGER.error("Unable to construct ASM of class " + the_class + + " with identity " + the_identity); + } + + final PersistentASMState asm_state = + PersistentASMStateQueries.get(the_class, the_identity); + + if (asm_state == null) { + Main.LOGGER.error("Unable to retrieve ASM state for class " + the_class + + " with identity " + the_identity); + } else if (result != null) { + asm_state.applyTo(result); + } + + return result; + } + + /** + * Saves the state of the specified ASM to the database. + * + * @param the_asm The ASM. + * @return true if the save was successful, false otherwise + */ + public static boolean save(final AbstractStateMachine the_asm) { + boolean result = false; + + final PersistentASMState asm_state = + PersistentASMStateQueries.get(the_asm.getClass(), the_asm.identity()); + + if (asm_state == null) { + Main.LOGGER.error("Unable to retrieve ASM state for " + the_asm); + } else { + asm_state.updateFrom(the_asm); + try { + Persistence.saveOrUpdate(asm_state); + result = true; + } catch (final PersistenceException e) { + Main.LOGGER.error("Could not save state for ASM " + the_asm); + } + } + + return result; + } + + /** + * Attempts to step with the specified event on the ASM of the specified + * class and identity, and persist the resulting state. + * + * @param the_event The event. + * @param the_asm_class The class. + * @param the_asm_identity The identity. + * @return true if the state transition succeeds, false if the state machine + * could not be loaded or the resulting state could not be persisted. + * @exception IllegalStateException if the state transition is illegal. + */ + public static boolean step(final ASMEvent the_event, + final Class the_asm_class, + final String the_asm_identity) { + boolean result = false; + final AbstractStateMachine asm = asmFor(the_asm_class, the_asm_identity); + + if (asm != null) { + asm.stepEvent(the_event); + result = save(asm); + } + + return result; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/AbstractStateMachine.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/AbstractStateMachine.java new file mode 100644 index 00000000..ce6b25bb --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/AbstractStateMachine.java @@ -0,0 +1,303 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 8, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.asm; + +import java.io.Serializable; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +/** + * @description A generic Abstract State Machine (ASM). + * @trace asm.asm + * @author Joseph R. Kiniry + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +// we'll need to investigate the complexity of this class later +@SuppressWarnings("PMD.CyclomaticComplexity") +public abstract class AbstractStateMachine implements Serializable { + /** + * Class-wide logger + */ + public static final Logger LOGGER = + LogManager.getLogger(AbstractStateMachine.class); + + /** + * The serialVersionUID. + */ + private static final long serialVersionUID = 1; + + /** + * This ASM's set of states. + */ + protected final Set my_states; + + /** + * This ASM's initial state. + */ + protected final ASMState my_initial_state; + + /** + * This AMS's final states. + */ + protected final Set my_final_states; + + /** + * This ASM's set of events. + */ + protected final Set my_events; + + /** + * A map from (state, event) pairs to state. + */ + protected final Set my_transition_function; + + /** + * The relation between UI events and ASM transitions. + */ + protected final UIToASMEventRelation my_ui_to_asm_relation = + new UIToASMEventRelation(); + + /** + * The current state of this ASM. Initialized to the initial state + * provided in the constructor. + */ + protected ASMState my_current_state; + + /** + * The identity of this ASM. The structure of this string is child + * class implementation dependent. + */ + protected String my_identity; + + /** + * Constructs an ASM. This constructor takes ownership of all the + * Collections passed to it. + * + * @param the_states the states of the new ASM. + * @param the_events the events of the new ASM. + * @param the_transition_function the transition function of the new + * ASM. This function, represented as a set of ASMTransitionFunction + * elements, need only specify legal transitions; all unspecified + * transitions are considered illegal. + * @param the_initial_state The initial state of the new ASM. + * @param the_final_states The final states of the new ASM. + * @param the_identity The identity of the new ASM. + */ + public AbstractStateMachine(final Set the_states, + final Set the_events, + final Set the_transition_function, + final ASMState the_initial_state, + final Set the_final_states, + final String the_identity) { + my_states = the_states; + my_events = the_events; + my_transition_function = the_transition_function; + my_initial_state = the_initial_state; + my_current_state = the_initial_state; + my_final_states = the_final_states; + my_identity = the_identity; + } + + /** + * @return are we in the initial state? + */ + public boolean isInInitialState() { + return my_current_state.equals(my_initial_state); + } + + /** + * @return are we in a final state? + */ + public boolean isInFinalState() { + return my_final_states.contains(my_current_state); + } + + /** + * @return the current state of this ASM. + * @trace asm.current_state + */ + public ASMState currentState() { + return my_current_state; + } + + /** + * Sets the current state. This method ignores any constraints + * imposed by the current state, and should only be used as part of + * reconstructing an ASM at a particular state. + * + * @param the_state The new state. + */ + protected void setCurrentState(final ASMState the_state) { + my_current_state = the_state; + } + + /** This sets the state machine back to the beginning. It should be used + * carefully. It is a shortcut. **/ + public void reinitialize() { + my_current_state = my_initial_state; + } + + /** + * @return the ASM's identity, or null if this ASM is a singleton. + */ + public String identity() { + return my_identity; + } + + /** + * Sets the ASM's identity. This method should only be used as part + * of reconstructing an ASM at a particular state. + * + * @param the_identity The new identity. + */ + protected void setIdentity(final String the_identity) { + my_identity = the_identity; + } + + /** + * @return the UI events enabled in this ASM. I.e., which UI events + * correspond to those states reachable from the current state? + */ + public Set enabledUIEvents() { + final Set asm_events_enabled = enabledASMEvents(); + final Set result = new HashSet(); + // For each enabled ASM event, look up which UI events it corresponds to. + for (final ASMEvent e : asm_events_enabled) { + result.addAll(my_ui_to_asm_relation.leftArrow(e)); + } + return result; + } + + /** + * @return the transitions of this ASM that are enabled. I.e., which + * states are reachable from the current state, given any possible + * event? + * @trace asm.enabled_events + */ + public Set enabledASMEvents() { + final Set result = new HashSet<>(); + for (final ASMTransition t : my_transition_function) { + if (t.startStates().contains(my_current_state)) { + result.addAll(t.events()); + } + } + return result; + } + + /** + * Transition to the next state of this ASM given the provided + * transition and its current state. + * @param the_transition the transition that is triggered. + * @return the new current state of the ASM after the transition. + * @throws IllegalStateException if this ASM cannot take a step + * given the provided transition. + */ + public ASMState stepTransition(final ASMTransition the_transition) + throws IllegalStateException { + // If we are in the right state then transition to the new state. + if (the_transition.startStates().contains(my_current_state)) { + my_current_state = the_transition.endState(); + LOGGER.debug("ASM transition " + the_transition + " succeeded from state " + + my_current_state + " for " + getClass().getSimpleName() + "/" + + my_identity); + } else { + LOGGER.error("ASM transition " + the_transition + + " failed from state " + my_current_state); + throw new IllegalStateException("Attempted to transition ASM " + + getClass().getName() + "/" + my_identity + + " from " + my_current_state + + " using transition " + + the_transition); + } + return my_current_state; + } + + public boolean checkEvent(final ASMEvent the_event) + throws IllegalStateException { + ASMState result = null; + for (final ASMTransition t : my_transition_function) { + if (t.startStates().contains(my_current_state) && + t.events().contains(the_event)) { + result = t.endState(); + break; + } + } + if (result == null) { + return false; + } + return true; + } + + /** + * Transition to the next state of this ASM given the provided event + * and its current state. + * @return the next state given the specified event and input. + * @throws IllegalStateException is this ASM cannot transition given + * the provided event. + */ + @SuppressWarnings("PMD.CyclomaticComplexity") + public ASMState stepEvent(final ASMEvent the_event) + throws IllegalStateException { + ASMState result = null; + for (final ASMTransition t : my_transition_function) { + if (t.startStates().contains(my_current_state) && + t.events().contains(the_event)) { + result = t.endState(); + break; + } + } + if (result == null) { + LOGGER.error("ASM event " + the_event + + " failed from state " + my_current_state); + throw new IllegalStateException("Illegal transition on ASM " + + getClass().getSimpleName() + "/" + my_identity + + ": (" + my_current_state + ", " + + the_event + ")"); + } else { + my_current_state = result; + LOGGER.debug("ASM event " + the_event + " caused transition to " + + my_current_state + " for " + getClass().getSimpleName() + + "/" + my_identity); + return result; + } + } + + /** + * Converts a list of ASMTransitionFunctions to a set of + * ASMTransitions. + * + * @param the set of ASMTransitionFunctions. + * @return the set of ASMTransitions for the specified list of + * ASMTransitionFunctions. + */ + public static Set + transitionsFor(final List the_list) { + final Set result = new HashSet(); + for (final ASMTransitionFunction atf : the_list) { + result.add(atf.value()); + } + return result; + } + + /** + * @return a String representation of this ASM. + */ + public String toString() { + return getClass().getSimpleName() + ", identity=" + my_identity + + ", current_state=" + my_current_state; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/AuditBoardDashboardASM.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/AuditBoardDashboardASM.java new file mode 100644 index 00000000..d3f3281a --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/AuditBoardDashboardASM.java @@ -0,0 +1,63 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 8, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.asm; + +import java.util.Arrays; +import java.util.HashSet; + +import javax.persistence.DiscriminatorValue; +import javax.persistence.Entity; + +import us.freeandfair.corla.asm.ASMEvent.AuditBoardDashboardEvent; +import us.freeandfair.corla.asm.ASMState.AuditBoardDashboardState; +import us.freeandfair.corla.asm.ASMTransitionFunction.AuditBoardDashboardTransitionFunction; +import us.freeandfair.corla.util.SetCreator; + +/** + * The ASM for the Audit Board Dashboard. + * @trace asm.dos_dashboard_next_state + * @author Joseph R. Kiniry + * @version 1.0.0 + */ +@Entity +@DiscriminatorValue(value = "AuditBoardDashboardASM") +public class AuditBoardDashboardASM extends AbstractStateMachine { + /** + * The serialVersionUID. + */ + private static final long serialVersionUID = 1; + + /** + * The final states of this ASM. + */ + private static final ASMState[] FINAL_STATES = + {AuditBoardDashboardState.AUDIT_COMPLETE, + AuditBoardDashboardState.UNABLE_TO_AUDIT, + AuditBoardDashboardState.AUDIT_ABORTED}; + + /** + * Create the Audit Board Dashboard ASM for the specified county. + * + * @param the_county_id The county identifier. + * @trace asm.county_dashboard_asm + */ + //@ requires the_county_id != null; + public AuditBoardDashboardASM(final String the_county_id) { + super(new HashSet(Arrays.asList(AuditBoardDashboardState.values())), + new HashSet(Arrays.asList(AuditBoardDashboardEvent.values())), + transitionsFor(Arrays. + asList(AuditBoardDashboardTransitionFunction.values())), + AuditBoardDashboardState.AUDIT_INITIAL_STATE, + SetCreator.setOf(FINAL_STATES), + the_county_id); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/CountyDashboardASM.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/CountyDashboardASM.java new file mode 100644 index 00000000..b4075cf3 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/CountyDashboardASM.java @@ -0,0 +1,62 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 8, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.asm; + +import java.util.Arrays; +import java.util.HashSet; + +import javax.persistence.DiscriminatorValue; +import javax.persistence.Entity; + +import us.freeandfair.corla.asm.ASMEvent.CountyDashboardEvent; +import us.freeandfair.corla.asm.ASMState.CountyDashboardState; +import us.freeandfair.corla.asm.ASMTransitionFunction.CountyDashboardTransitionFunction; +import us.freeandfair.corla.util.SetCreator; + +/** + * The ASM for the County Dashboard. + * @trace asm.dos_dashboard_next_state + * @author Joseph R. Kiniry + * @version 1.0.0 + */ +@Entity +@DiscriminatorValue(value = "CountyDashboardASM") +public class CountyDashboardASM extends AbstractStateMachine { + /** + * The serialVersionUID. + */ + private static final long serialVersionUID = 1; + + /** + * The final states of this ASM. + */ + private static final ASMState[] FINAL_STATES = + {CountyDashboardState.DEADLINE_MISSED, + CountyDashboardState.COUNTY_AUDIT_COMPLETE}; + + /** + * Create the County Dashboard ASM. + * + * @param the_county_id The county identifier. + * @trace asm.county_dashboard_asm + */ + //@ requires the_county_id != null + public CountyDashboardASM(final String the_county_id) { + super(new HashSet(Arrays.asList(CountyDashboardState.values())), + new HashSet(Arrays.asList(CountyDashboardEvent.values())), + transitionsFor(Arrays. + asList(CountyDashboardTransitionFunction.values())), + CountyDashboardState.COUNTY_INITIAL_STATE, + SetCreator.setOf(FINAL_STATES), + the_county_id); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/DoSDashboardASM.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/DoSDashboardASM.java new file mode 100644 index 00000000..5bd450e8 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/DoSDashboardASM.java @@ -0,0 +1,63 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 8, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.asm; + +import java.util.Arrays; +import java.util.HashSet; + +import javax.persistence.DiscriminatorValue; +import javax.persistence.Entity; + +import us.freeandfair.corla.asm.ASMEvent.DoSDashboardEvent; +import us.freeandfair.corla.asm.ASMState.DoSDashboardState; +import us.freeandfair.corla.asm.ASMTransitionFunction.DoSDashboardTransitionFunction; +import us.freeandfair.corla.util.SetCreator; + +/** + * The ASM for the Department of State Dashboard. + * @trace asm.dos_dashboard_next_state + * @author Joseph R. Kiniry + * @version 1.0.0 + */ +@Entity +@DiscriminatorValue(value = "DoSDashboardASM") +public class DoSDashboardASM extends AbstractStateMachine { + /** + * The identity of the singleton DoS dashboard. + */ + public static final String IDENTITY = "DoS"; + + /** + * The serialVersionUID. + */ + private static final long serialVersionUID = 1; + + /** + * The final states of this ASM. + */ + private static final ASMState[] FINAL_STATES = + {DoSDashboardState.AUDIT_RESULTS_PUBLISHED}; + + /** + * Create the Department of State Dashboard ASM. + * @trace asm.dos_asm + */ + public DoSDashboardASM() { + super(new HashSet(Arrays.asList(DoSDashboardState.values())), + new HashSet(Arrays.asList(DoSDashboardEvent.values())), + transitionsFor(Arrays. + asList(DoSDashboardTransitionFunction.values())), + DoSDashboardState.DOS_INITIAL_STATE, + SetCreator.setOf(FINAL_STATES), + IDENTITY); // there is only one DoS dashboard + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/Event.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/Event.java new file mode 100644 index 00000000..9445de11 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/Event.java @@ -0,0 +1,22 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 11, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.asm; + +/** + * An event in the system. This is effectively a marker interface. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public interface Event { + // no methods or fields +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/PersistentASMState.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/PersistentASMState.java new file mode 100644 index 00000000..6343464d --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/PersistentASMState.java @@ -0,0 +1,313 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 11, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.asm; + +import static us.freeandfair.corla.util.EqualsHashcodeHelper.nullableEquals; + +import java.io.Serializable; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Version; + +import us.freeandfair.corla.persistence.PersistentEntity; + +/** + * An abstract state machine state, and sufficient information to + * reconstruct the state machine it belongs to. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@Entity +@Table(name = "asm_state") +// this class has many fields that would normally be declared final, but +// cannot be for compatibility with Hibernate and JPA. +@SuppressWarnings("PMD.ImmutableField") +public class PersistentASMState implements PersistentEntity, Serializable { + /** + * The serialVersionUID. + */ + private static final long serialVersionUID = 1L; + + /** + * The ID number. + */ + @Id + @Column(updatable = false, nullable = false) + @GeneratedValue(strategy = GenerationType.SEQUENCE) + private Long my_id; + + /** + * The version (for optimistic locking). + */ + @Version + private Long my_version; + + /** + * The class of AbstractStateMachine to which this state belongs, as + * a String. + */ + @Column(updatable = false, nullable = false) + private String my_asm_class; + + /** + * The identifying information for the state machine, if any, as a + * String. + */ + @Column(updatable = false) + private String my_asm_identity; + + /** + * The ASMState class (enum) containing this state, as a String. + */ + private String my_state_class; + + /** + * The state value, as a String. + */ + private String my_state_value; + + /** + * Constructs an empty PersistentASMState, solely for persistence. + */ + protected PersistentASMState() { + super(); + } + + /** + * Constructs a PersistentASMState with the specified parameters. + */ + protected PersistentASMState(final String the_asm_class, + final String the_asm_identity, + final String the_state_class, + final String the_state_value) { + super(); + my_asm_class = the_asm_class; + my_asm_identity = the_asm_identity; + my_state_class = the_state_class; + my_state_value = the_state_value; + } + + /** + * Obtains a PersistentASMState from an abstract state machine. + * + * @param the_asm The ASM from which to obtain the state. + * @return The state. + */ + //@ requires the_asm != null; + public static PersistentASMState stateFor(final AbstractStateMachine the_asm) { + final String asm_class = the_asm.getClass().getName(); + // identifying info to be dealt with later + final ASMState state = the_asm.currentState(); + final String state_class = state.getClass().getName(); + String state_value = null; + if (state instanceof Enum) { + final Enum state_enum = (Enum) state; + state_value = state_enum.name(); + } + return new PersistentASMState(asm_class, the_asm.identity(), state_class, state_value); + } + + /** + * Obtains an abstract state machine from a PersistentASMState. + * + * @param the_state The state. + * @return the state machine. + * @exception IllegalArgumentException if the state machine cannot + * be constructed because the persistent state contains invalid + * information. + */ + //@ requires the_state != null; + public static AbstractStateMachine asmFor(final PersistentASMState the_state) { + try { + // first, construct an ASM of the correct class + final AbstractStateMachine result = + (AbstractStateMachine) Class.forName(the_state.asmClass()).newInstance(); + result.setIdentity(the_state.asmIdentity()); + the_state.applyTo(result); + return result; + } catch (final ClassNotFoundException | IllegalAccessException | + InstantiationException e) { + throw new IllegalArgumentException(e); + } + } + + /** + * Constructs a state value for the specified PersistentASMState. + * + * @param the_state The state, or null if no matching state could + * be constructed. + */ + public static ASMState asmStateFor(final PersistentASMState the_state) { + ASMState result = null; + try { + // construct the class for the ASM state + final Class state_class = Class.forName(the_state.stateClass()); + if (state_class.isEnum() && ASMState.class.isAssignableFrom(state_class)) { + // see if it has the right enum value + for (final Object o : state_class.getEnumConstants()) { + final Enum enum_constant = (Enum) o; + if (enum_constant.name().equals(the_state.stateValue())) { + result = (ASMState) enum_constant; + break; + } + } + } + } catch (final ClassNotFoundException e) { + // result is already null + } + return result; + } + + /** + * {@inheritDoc} + */ + @Override + public Long id() { + return my_id; + } + + /** + * {@inheritDoc} + */ + @Override + public void setID(final Long the_id) { + my_id = the_id; + } + + /** + * {@inheritDoc} + */ + @Override + public Long version() { + return my_version; + } + + /** + * Applies the state in this PersistentASMState to an existing state + * machine. + * + * @param the_asm The ASM. + * @exception IllegalArgumentException if the ASM is not the one + * described in this persistent state, or if this persistent state + * contains invalid information. + */ + //@ requires the_asm != null + public void applyTo(final AbstractStateMachine the_asm) { + if (the_asm.getClass().getName().equals(asmClass()) && + nullableEquals(the_asm.identity(), asmIdentity())) { + final ASMState state = asmStateFor(this); + if (state == null) { + throw new IllegalArgumentException("no ASM state found for state " + this); + } else { + the_asm.setCurrentState(state); + } + } else { + throw new IllegalArgumentException("invalid ASM class " + + the_asm.getClass().getName() + + " for state " + this); + } + } + + /** + * Updates this PersistentASMState from an existing state machine. + * + * @param the_asm The ASM + * @exception IllegalArgumentException if the specified state + * machine is not the one described in this persistent state. + */ + //@ requires the_asm != null + public void updateFrom(final AbstractStateMachine the_asm) { + final PersistentASMState new_state = stateFor(the_asm); + if (new_state.asmClass().equals(asmClass()) && + nullableEquals(new_state.asmIdentity(), asmIdentity())) { + my_state_class = new_state.stateClass(); + my_state_value = new_state.stateValue(); + } else { + throw new IllegalArgumentException("invalid ASM " + the_asm + + " for updating state " + this); + } + } + + /** + * @return the ASM class. + */ + public String asmClass() { + return my_asm_class; + } + + /** + * @return the ASM identity. + */ + public String asmIdentity() { + return my_asm_identity; + } + + /** + * @return the state class. + */ + public String stateClass() { + return my_state_class; + } + + /** + * @return the state value. + */ + public String stateValue() { + return my_state_value; + } + + /** + * @return a String representation of this ASM state. + */ + @Override + public String toString() { + return "PersistentASMState [asm_class=" + my_asm_class + + ", asm_identity=" + my_asm_identity + + ", state_class=" + my_state_class + + ", state_value=" + my_state_value + "]"; + } + + /** + * Compare this object with another for equivalence. + * + * @param the_other The other object. + * @return true if the objects are equivalent, false otherwise. + */ + @Override + public boolean equals(final Object the_other) { + boolean result = true; + if (the_other instanceof PersistentASMState) { + final PersistentASMState other_state = (PersistentASMState) the_other; + result &= nullableEquals(other_state.asmClass(), asmClass()); + result &= nullableEquals(other_state.asmIdentity(), asmIdentity()); + result &= nullableEquals(other_state.stateClass(), stateClass()); + result &= nullableEquals(other_state.stateValue(), stateValue()); + } else { + result = false; + } + return result; + } + + /** + * @return a hash code for this object. + */ + @Override + public int hashCode() { + return toString().hashCode(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/UIEvent.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/UIEvent.java new file mode 100644 index 00000000..d4efe825 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/UIEvent.java @@ -0,0 +1,30 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 11, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.asm; + +/** + * An enumeration of all user-triggered external inbound events in the + * client UI. + * + * @author Joseph R. Kiniry + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public enum UIEvent implements Event { + LOGIN, + FETCH_INITIAL_STATE_SEND, + FETCH_INITIAL_STATE_RECEIVE, + SELECT_NEXT_BALLOT, + UPDATE_BOARD_MEMBER, + UPDATE_BALLOT_MARKS, + UNDEFINED +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/UIToASMEventRelation.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/UIToASMEventRelation.java new file mode 100644 index 00000000..138fec06 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/asm/UIToASMEventRelation.java @@ -0,0 +1,126 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 8, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.asm; + +import static us.freeandfair.corla.asm.ASMEvent.AuditBoardDashboardEvent.*; +import static us.freeandfair.corla.asm.ASMEvent.CountyDashboardEvent.*; +import static us.freeandfair.corla.asm.ASMEvent.DoSDashboardEvent.*; +import static us.freeandfair.corla.asm.UIEvent.*; + +import java.util.HashSet; +import java.util.Set; + +import us.freeandfair.corla.util.Pair; + +/** + * @description The mapping between UI events and ASM events. + * @trace asm.ui_to_asm_event_relation + * @author Joseph R. Kiniry + * @version 1.0.0 + * @todo dmz/kiniry use an entity instead of Pair<> to enable persistence + */ +public class UIToASMEventRelation { + /** + * The relation encoded via a set of pairs. + */ + private final Set> my_relation = + new HashSet>(); + + /** + * Create an instance of this relation, which contains the full set + * of public inbound UI and ASM events. + * @design kiniry This should probably be refactored as a singleton. + */ + public UIToASMEventRelation() { + addDoSDashboardPairs(); + addCountyDashboardPairs(); + addAuditBoardDashboardPairs(); + } + + private void addDoSDashboardPairs() { + // All Department of State Dashboard pairs. + my_relation.add(new Pair(UNDEFINED, + PARTIAL_AUDIT_INFO_EVENT)); + my_relation.add(new Pair(UNDEFINED, + DOS_START_ROUND_EVENT)); + my_relation.add(new Pair(UNDEFINED, + PUBLISH_AUDIT_REPORT_EVENT)); + } + + private void addCountyDashboardPairs() { + // All County Dashboard pairs. + my_relation.add(new Pair(UNDEFINED, + IMPORT_BALLOT_MANIFEST_EVENT)); + my_relation.add(new Pair(UNDEFINED, + IMPORT_CVRS_EVENT)); + my_relation.add(new Pair(UNDEFINED, + COUNTY_START_AUDIT_EVENT)); + } + + private void addAuditBoardDashboardPairs() { + // All Audit Board Dashboard pairs. + my_relation.add(new Pair(UNDEFINED, + REPORT_MARKINGS_EVENT)); + my_relation.add(new Pair(UNDEFINED, + REPORT_BALLOT_NOT_FOUND_EVENT)); + my_relation.add(new Pair(UNDEFINED, + SUBMIT_AUDIT_INVESTIGATION_REPORT_EVENT)); + my_relation.add(new Pair(UNDEFINED, + SUBMIT_INTERMEDIATE_AUDIT_REPORT_EVENT)); + } + + /** + * Is a_pair a member of this relation? + * @param a_pair the UIEvent/ASMEvent pair to check. + */ + public boolean member(final UIEvent a_ue, final ASMEvent an_ae) { + return my_relation.contains(new Pair(a_ue, an_ae)); + } + + /** + * Follow the relation from left to right. + * @param a_ue the UI event to lookup. + * @return the ASM events corresponding to 'a_ue', or null if no such + * events exists. + */ + public Set rightArrow(final UIEvent a_ue) { + // iterate over all elements in the map and, for each one whose + // left element matches a_ue, include the right element in the + // resulting set. + final Set result = new HashSet(); + for (final Pair p : my_relation) { + if (p.first().equals(a_ue)) { + result.add(p.second()); + } + } + return result; + } + + /** + * Follow the relation from right to left. + * @param a_ae the ASM event to lookup. + * @return the UI event corresponding to 'an_ae', or null if no such + * event exists. + */ + public Set leftArrow(final ASMEvent an_ae) { + // iterate over all elements in the map and, for each one whose + // right element matches an_ae, include the left element in the + // resulting set. + final Set result = new HashSet(); + for (final Pair p : my_relation) { + if (p.second().equals(an_ae)) { + result.add(p.first()); + } + } + return result; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/auth/AbstractAuthentication.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/auth/AbstractAuthentication.java new file mode 100644 index 00000000..204173b1 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/auth/AbstractAuthentication.java @@ -0,0 +1,439 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 29, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.auth; + +import static us.freeandfair.corla.auth.AuthenticationStage.*; + +import java.util.Locale; + +import javax.persistence.PersistenceException; + +import org.apache.log4j.Logger; + +import com.google.gson.Gson; +import com.google.gson.JsonParseException; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.json.SubmittedCredentials; +import us.freeandfair.corla.model.Administrator; +import us.freeandfair.corla.model.Administrator.AdministratorType; +import us.freeandfair.corla.model.County; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.query.AdministratorQueries; + +/** + * An abstract base class that enforces the two-stage state machine for two-factor + * authentication. + * + * @author Joseph R. Kiniry + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings({"PMD.AtLeastOneConstructor", "PMD.CyclomaticComplexity", + "PMD.ModifiedCyclomaticComplexity", "PMD.StdCyclomaticComplexity", + "PMD.EmptyMethodInAbstractClassShouldBeAbstract", "PMD.GodClass"}) +public abstract class AbstractAuthentication implements AuthenticationInterface { + /** + * Authenticate the administrator `the_username` with credentials + * `the_password` (for traditional authentication) or `the_second_factor` + * (for two-factor authentication). This method should be called twice in + * succession, first for traditional authentication and second for two-factor + * authentication. + * + * @trace authentication.authenticate_county_administrator + * @trace authentication.authenticate_state_administrator + * @return true iff authentication succeeds. + * @param the_request The request. + * @param the_username the username of the person to attempt to authenticate. + * @param the_password the password for `username`. + * @param the_second_factor the second factor for `username`. + * + * @todo kiniry Refactor method into helper methods for each phase. + */ + @Override + @SuppressWarnings({"PMD.NPathComplexity", "PMD.ExcessiveMethodLength", "PMD.SwitchDensity", + "checkstyle:methodlength"}) + public boolean authenticateAdministrator(final Request the_request, + final Response the_response, + final String the_username, + final String the_password, + final String the_second_factor) { + boolean result = true; + AuthenticationStage auth_stage = null; + final Object auth_stage_attribute = the_request.session().attribute(AUTH_STAGE); + if (auth_stage_attribute instanceof AuthenticationStage) { + auth_stage = (AuthenticationStage) auth_stage_attribute; + } + if (auth_stage == null) { + auth_stage = NOT_AUTHENTICATED; + } else if (auth_stage != NOT_AUTHENTICATED) { + // if the existing authenticated admin is not this user, deauthenticate + // the session + final Object admin_attribute = the_request.session().attribute(ADMIN); + if (admin_attribute instanceof Administrator && + !((Administrator) admin_attribute).username().equals(the_username)) { + deauthenticate(the_request); + auth_stage = NOT_AUTHENTICATED; + } + } + try { + // If we didn't get a well-formed request in the first place, fail. + if (the_username == null || the_username.isEmpty()) { + result = false; + } else { + final String lowercase_username = the_username.toLowerCase(Locale.US); + switch (auth_stage) { + case NOT_AUTHENTICATED: + final AuthenticationResult auth_result = + traditionalAuthenticate(the_request, the_response, + lowercase_username, the_password); + if (auth_result.success()) { + // We have traditionally authenticated. + final Administrator admin = + AdministratorQueries.byUsername(lowercase_username); + if (admin == null) { + Main.LOGGER.info("User " + lowercase_username + " not found in database, " + + "aborting authentication."); + result = false; + } else { + admin.updateLastLoginTime(); + Persistence.saveOrUpdate(admin); + the_request.session().attribute(AUTH_STAGE, TRADITIONALLY_AUTHENTICATED); + the_request.session().attribute(ADMIN, admin); + the_request.session().attribute(CHALLENGE, auth_result.challenge()); + Main.LOGGER.info("Traditional authentication succeeded for administrator " + + lowercase_username); + } + } else { + Main.LOGGER.info("Traditional authentication failed for administrator " + + lowercase_username); + result = false; + } + break; + + case TRADITIONALLY_AUTHENTICATED: + if (secondFactorAuthenticate(the_request, lowercase_username, the_second_factor)) { + // We have both traditionally and second-factor authenticated. + final Administrator admin = + AdministratorQueries.byUsername(lowercase_username); + admin.updateLastLoginTime(); + Persistence.saveOrUpdate(admin); + the_request.session().attribute(AUTH_STAGE, SECOND_FACTOR_AUTHENTICATED); + the_request.session().attribute(ADMIN, admin); + the_request.session().removeAttribute(CHALLENGE); + Main.LOGGER.info("Second factor authentication succeeded for administrator " + + lowercase_username + " in role " + admin.type()); + if (admin.type() == AdministratorType.COUNTY) { + Main.LOGGER.info(lowercase_username + " is an administrator for county " + + admin.county()); + } + if (admin.type() == AdministratorType.STATE) { + Main.LOGGER.info(lowercase_username + " is a state administrator"); + } + } else { + // Send the authentication state machine back to its initial state. + the_request.session().attribute(AUTH_STAGE, NOT_AUTHENTICATED); + the_request.session().removeAttribute(CHALLENGE); + Main.LOGGER.info("Second factor authentication failed for administrator" + + lowercase_username); + result = false; + } + break; + + case SECOND_FACTOR_AUTHENTICATED: + // we are already second-factor authenticated as this user + break; + + default: + // this should never happen + deauthenticate(the_request); + break; + } + } + } catch (final PersistenceException e) { + // there's nothing we can really do here other than saying that the + // authentication failed + deauthenticate(the_request); + } + + if (!result) { + // a failed authentication attempt removes any existing session authentication + deauthenticate(the_request); + Main.LOGGER.info("Authentication failed for user " + the_username); + } + return result; + } + + /** + * {@inheritDoc} + */ + @Override + public County authenticatedCounty(final Request the_request) { + County result = null; + if (secondFactorAuthenticated(the_request)) { + final Administrator admin = + (Administrator) the_request.session().attribute(ADMIN); + if (admin != null) { + result = admin.county(); + } + } + return result; + } + + /** + * {@inheritDoc} + */ + @Override + public Administrator authenticatedAdministrator(final Request the_request) { + Administrator result = null; + final Object admin_attribute = the_request.session().attribute(ADMIN); + if (admin_attribute instanceof Administrator) { + result = (Administrator) admin_attribute; + } + return result; + } + + /** + * {@inheritDoc} + */ + @Override + public AuthenticationStatus authenticationStatus(final Request the_request) { + final Object admin_attribute = the_request.session().attribute(ADMIN); + final Object stage_attribute = the_request.session().attribute(AUTH_STAGE); + final AdministratorType type; + if (admin_attribute instanceof Administrator) { + type = ((Administrator) admin_attribute).type(); + } else { + type = null; + } + final AuthenticationStage stage; + if (stage_attribute instanceof AuthenticationStage) { + stage = (AuthenticationStage) stage_attribute; + } else { + stage = null; + } + return new AuthenticationStatus(type, stage, + the_request.session().attribute(CHALLENGE)); + } + + /** + * {@inheritDoc} + */ + @Override + public void traditionalDeauthenticate(final Request the_request, + final String the_username) { + the_request.session().removeAttribute(ADMIN); + the_request.session().removeAttribute(CHALLENGE); + Main.LOGGER.info("session is now traditionally deauthenticated"); + } + + /** + * {@inheritDoc} + */ + @Override + public void twoFactorDeauthenticate(final Request the_request, + final String the_username) { + the_request.session().removeAttribute(ADMIN); + the_request.session().removeAttribute(CHALLENGE); + Main.LOGGER.info("session is now second factor deauthenticated"); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean traditionalAuthenticated(final Request the_request) { + final Object auth_stage_attribute = + the_request.session().attribute(AuthenticationInterface.AUTH_STAGE); + AuthenticationStage auth_stage = null; + if (auth_stage_attribute instanceof AuthenticationStage) { + auth_stage = (AuthenticationStage) auth_stage_attribute; + } + return auth_stage != null && + auth_stage == TRADITIONALLY_AUTHENTICATED; + } + + /** + * {@inheritDoc} + */ + @Override + public boolean secondFactorAuthenticated(final Request the_request) { + final Object auth_stage_attribute = + the_request.session().attribute(AuthenticationInterface.AUTH_STAGE); + AuthenticationStage auth_stage = null; + if (auth_stage_attribute instanceof AuthenticationStage) { + auth_stage = (AuthenticationStage) auth_stage_attribute; + } + return auth_stage != null && + auth_stage == SECOND_FACTOR_AUTHENTICATED; + } + + /** + * {@inheritDoc} + */ + @Override + public boolean authenticated(final Request the_request) { + final Object auth_stage_attribute = + the_request.session().attribute(AuthenticationInterface.AUTH_STAGE); + AuthenticationStage auth_stage = null; + if (auth_stage_attribute instanceof AuthenticationStage) { + auth_stage = (AuthenticationStage) auth_stage_attribute; + } + return auth_stage != null && + auth_stage != NOT_AUTHENTICATED; + } + + /** + * {@inheritDoc} + */ + @Override + public boolean authenticatedAs(final Request the_request, + final AdministratorType the_type, + final String the_username) { + boolean result = false; + final Object auth_stage_attribute = + the_request.session().attribute(AuthenticationInterface.AUTH_STAGE); + final Object admin_attribute = the_request.session().attribute(ADMIN); + + if (auth_stage_attribute instanceof AuthenticationStage && + admin_attribute instanceof Administrator) { + final AuthenticationStage stage = (AuthenticationStage) auth_stage_attribute; + final Administrator admin = (Administrator) admin_attribute; + result = stage != NOT_AUTHENTICATED && + admin.type() == the_type && + admin.username().equals(the_username); + } else if (auth_stage_attribute != null || admin_attribute != null) { + // this should never happen since we control what's in the session object, + // but if it does, we'll clear out that attribute and thereby force another + // authentication + Main.LOGGER.error("Invalid admin or auth stage type detected in session."); + deauthenticate(the_request); + } + return result; + } + + /** + * {@inheritDoc} + */ + @Override + public boolean secondFactorAuthenticatedAs(final Request the_request, + final AdministratorType the_type, + final String the_username) { + boolean result = false; + final Object auth_stage_attribute = + the_request.session().attribute(AuthenticationInterface.AUTH_STAGE); + final Object admin_attribute = the_request.session().attribute(ADMIN); + + if (auth_stage_attribute instanceof AuthenticationStage && + admin_attribute instanceof Administrator) { + final AuthenticationStage stage = (AuthenticationStage) auth_stage_attribute; + final Administrator admin = (Administrator) admin_attribute; + result = stage == SECOND_FACTOR_AUTHENTICATED && + admin.type() == the_type && + the_username.equals(admin.username()); + } else if (auth_stage_attribute != null || admin_attribute != null) { + // this should never happen since we control what's in the session object, + // but if it does, we'll clear out that attribute and thereby force another + // authentication + Main.LOGGER.error("Invalid admin or auth stage type detected in session."); + deauthenticate(the_request); + } + return result; + } + + /** + * {@inheritDoc} + */ + @Override + public void deauthenticate(final Request the_request) { + // If we are authenticated in any fashion + final Object admin_attribute = + the_request.session().attribute(ADMIN); + final Administrator admin; + + if (admin_attribute instanceof Administrator) { + admin = (Administrator) admin_attribute; + } else { + admin = null; + } + + if (admin == null) { + Main.LOGGER.warn("Deauthenticated an unauthenticated session."); + } else { + // update the last logout time for the logged in administrator + admin.updateLastLogoutTime(); + Persistence.saveOrUpdate(admin); + Main.LOGGER.info("Deauthenticated user '" + admin.username() + "'"); + // Take care of any specific back-end deauthentication logic. + traditionalDeauthenticate(the_request, admin.username()); + twoFactorDeauthenticate(the_request, admin.username()); + } + + the_request.session().removeAttribute(ADMIN); + the_request.session().removeAttribute(AUTH_STAGE); + } + + /** + * {@inheritDoc} + */ + @Override + public void setLogger(final Logger the_logger) { + // skip, as we have access to Main.LOGGER + } + + /** + * {@inheritDoc} + */ + @Override + public void setGSON(final Gson the_gson) { + // skip, as we have access to Main.GSON + } + + /** + * {@inheritDoc} + */ + @Override + public void setAuthenticationServerName(final String the_name) { + // skip, as there is no server necessary for the built-in test service + } + + /** + * {@inheritDoc} + */ + @Override + public final SubmittedCredentials authenticationCredentials(final Request the_request) { + SubmittedCredentials result = null; + // Check for JSON credentials in the request. + try { + result = Main.GSON.fromJson(the_request.body(), SubmittedCredentials.class); + } catch (final JsonParseException jse) { + // There wasn't JSON there! + } + // If there wasn't a JSON request, is there an HTTP params one? + if (result == null && the_request.queryParams(USERNAME) != null) { + result = + new SubmittedCredentials( + the_request.queryParams(USERNAME), + the_request.queryParams(PASSWORD), + the_request.queryParams(SECOND_FACTOR)); + } + // If there wasn't an HTTP params one, is the session already authenticated? + if (result == null && authenticated(the_request)) { + final Administrator admin = (Administrator) the_request.session().attribute(ADMIN); + result = new SubmittedCredentials(admin.username(), null, null); + } + return result; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/auth/AuthenticationInterface.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/auth/AuthenticationInterface.java new file mode 100644 index 00000000..5aee95cb --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/auth/AuthenticationInterface.java @@ -0,0 +1,281 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 27, 2017 + * @copyright 2017 Colorado Department of State + * @license GNU Affero General Public License v3 with + * Grant of Additional Permission + * @creator Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +/* +This code is licensed under the GNU Affero General Public License Version 3 +(AGPLv3) with the following additional permission. + +1. Grant of Additional Permission. + +The copyright holders of this software give you permission to link the +ColoradoRLA application server with an alternative implementation of +us.freeandfair.corla.auth.AuthenticationInterface.java to produce an +executable, regardless of the license terms of the implementation, and to copy +and distribute the resulting executable under terms of your choice, provided +that you also meet the terms and conditions of the license of that +implementation. + +If you modify the application server, you may extend this exception to your +version of the software, but you are not obliged to do so. If you do not wish +to do so, delete this exception statement from your version. + +This Grant of Additional Permission is only for the purpose of allowing the +application server to be linked with a proprietary two-factor authentication +method. +*/ + +package us.freeandfair.corla.auth; + +import org.apache.log4j.Logger; + +import com.google.gson.Gson; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.json.SubmittedCredentials; +import us.freeandfair.corla.model.Administrator; +import us.freeandfair.corla.model.Administrator.AdministratorType; +import us.freeandfair.corla.model.County; + +/** + * The interface to authentication providers. + * + * @author Joseph R. Kiniry + * @author Daniel M. Zimmerman + * @version 1.0.0 + * @todo Add a model for authentication to make the specifications herein much + * more clear. + */ +public interface AuthenticationInterface { + /** + * The constant for the "admin" attribute of the current session. + */ + String ADMIN = "admin"; + + /** + * The constant for the "challenge" attribute of the current session. + */ + String CHALLENGE = "challenge"; + + /** + * The constant for the "username" request parameter. + */ + String USERNAME = "username"; + + /** + * The constant for the "password" request parameter. + */ + String PASSWORD = "password"; + + /** + * The constant for the "second factor" request parameter. + */ + String SECOND_FACTOR = "second_factor"; + + /** + * The constant used to denote which authentication stage the session is in. + * The value of this attribute in a value of the `AuthenticationStage` + * enumeration. + */ + String AUTH_STAGE = "authentication_stage"; + + /** + * Set the logger for the authentication subsystem. This method should + * be called immediately after construction and before the subsystem is used. + * @param the_logger the logger to use. + */ + void setLogger(Logger the_logger); + + /** + * Set the GSON serialization/deserialization subsystem to use. This method + * should be called immediately after construction and before the subsystem + * is used. + * @param the_gson the GSON subsystem to use. + */ + void setGSON(Gson the_gson); + + /** + * Set the DNS name of the authentication server to use for this particular + * authentication service. + * @param the_name the full DNS name of the authentication server. + */ + void setAuthenticationServerName(String the_name); + + /** + * Authenticate the administrator `the_username` with credentials + * `the_password` (for traditional authentication) or `the_second_factor` + * (for two-factor authentication). + * @trace authentication.authenticate_county_administrator + * @trace authentication.authenticate_state_administrator + * @return true iff authentication succeeds. + * @param the_request The request. + * @param the_response The response, which is used in the case that a second + * factor challenge must be sent to the client. + * @param the_username the username of the person to attempt to authenticate. + * @param the_password the password for `username`. + * @param the_second_factor the second factor for `username`. + */ + //@ requires 0 < the_username.length(); + //@ requires the_password != null || the_second_factor != null; + boolean authenticateAdministrator(Request the_request, + Response the_response, + String the_username, + String the_password, + String the_second_factor); + + /** + * Attempt to authenticate `the_username` using `the_second_factor`. + * @trace authentication.second_factor_authenticate + * @return true iff two-factor authentication with credential pair + * (username, password) succeeds. + * @param the_request The request. + * @param the_username the username of the person to attempt to authenticate. + * @param the_second_factor the second factor for `username`. + */ + //@ requires 0 < the_username.length(); + //@ requires the_second_factor != null; + //@ ensures (* If the_second_factor is a correct response to the challenge + //@ returned in the AuthenticateResult from the immediately preceding + //@ successful traditionalAuthenticate call for the_username, then + //@ secondFactorAuthenticate(the_request) holds and a true is returned; + //@ otherwise, a false is returned and + //@ secondFactorAuthenticated(the_request) will be false. *); + boolean secondFactorAuthenticate(Request the_request, + String the_username, + String the_second_factor); + + /** + * Is the session authenticated with a second factor? + * @trace authenticated.second_factor_authenticated? + * @param the_request The request. + * @return true iff the session is second-factor authenticated + */ + boolean secondFactorAuthenticated(Request the_request); + + /** + * @trace authentication.traditional_authenticate + * @return an AuthenticationResult with a positive success() and a second factor + * challenge() if traditional authentication with credential pair + * (username, password) succeeds; a negative success() otherwise. + * @param the_request The request. + * @param the_response The response. + * @param the_username the username of the person to attempt to authenticate. + * @param the_password the password for `username`. + */ + //@ requires 0 < the_username.length(); + //@ requires the_password != null; + //@ ensures (* If the provided credentials are correct, then + //@ traditionalAuthenticate(the_request) holds and the returned + //@ AuthenticationResult will contain a second factor challenge and + //@ will claim success. *); + AuthenticationResult traditionalAuthenticate(Request the_request, + Response the_response, + String the_username, + String the_password); + + /** + * @trace authentication.traditional_authenticated? + * @return true iff the session is traditionally authenticated. + * @param the_request The request. + */ + boolean traditionalAuthenticated(Request the_request); + + /** + * @return true iff the session is authenticated either traditionally + * or with a second factor. + */ + boolean authenticated(Request the_request); + + /** + * @return true iff the session is authenticated in any way + * as the specified administrator type and username. + * @param the_request The request. + * @param the_username the username of the person to check. + * @param the_type the type of the administrator. + */ + boolean authenticatedAs(Request the_request, + AdministratorType the_type, + String the_username); + + /** + * @return true iff the session is authenticated with a second factor + * as the specified administrator type and username. + * @param the_request The request. + * @param the_username the username of the person to check. + * @param the_type the type of the administrator. + */ + boolean secondFactorAuthenticatedAs(Request the_request, + AdministratorType the_type, + String the_username); + + /** + * Deauthenticate the currently authenticated user. + * @param the_request The request. + */ + void deauthenticate(Request the_request); + + /** + * @trace authentication.traditional_deauthenticate + * @param the_request The request. + * @param the_username the user to deauthenticate. + */ + //@ ensures (* If `the_username` was logged in via traditional authentication, + //@ now they are not. *); + void traditionalDeauthenticate(Request the_request, + String the_username); + + /** + * @trace authentication.two_factor_deauthenticate + * @param the_request The request. + * @param the_username the user to deauthenticate. + */ + //@ ensures (* If `the_username` was logged in via two-factor authentication, + //@ now they are not. *); + void twoFactorDeauthenticate(Request the_request, + String the_username); + + /** + * Gets the authenticated county for a request. + * + * @param the_request The request. + * @return the authenticated county, or null if this session is not authenticated + * as a county administrator. + */ + County authenticatedCounty(Request the_request); + + /** + * Gets the authenticated administrator for a request. + * + * @param the_request The request. + * @return the authenticated administrator, or null if this session is not + * authenticated. + */ + Administrator authenticatedAdministrator(Request the_request); + + /** + * Gets an authentication response based on the current status of a request. + * + * @param the_request The request. + * @return the authentication response. + */ + AuthenticationStatus authenticationStatus(Request the_request); + + /** + * Gets the authenticated username. + */ + /** + * @return the submitted credentials associated with any request. + * @param the_request The request. + */ + SubmittedCredentials authenticationCredentials(Request the_request); +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/auth/AuthenticationResult.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/auth/AuthenticationResult.java new file mode 100644 index 00000000..9331a3b2 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/auth/AuthenticationResult.java @@ -0,0 +1,57 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 31, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.auth; + +import java.io.Serializable; + +/** + * An authentication result from traditional authentication. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public class AuthenticationResult { + /** + * A flag indicating whether authentication was successful. + */ + private final boolean my_success; + + /** + * A challenge object that should be sent to the client. + */ + private final Serializable my_challenge; + + /** + * Constructs a new AuthenticationResult. + * + * @param the_success true if authentication was successful, false otherwise + * @param the_challenge The challenge to send for second factor authentication. + */ + public AuthenticationResult(final boolean the_success, final Serializable the_challenge) { + my_success = the_success; + my_challenge = the_challenge; + } + + /** + * @return true if authentication was successful, false otherwise. + */ + public boolean success() { + return my_success; + } + + /** + * @return the second factor challenge, if any. + */ + public Serializable challenge() { + return my_challenge; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/auth/AuthenticationStage.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/auth/AuthenticationStage.java new file mode 100644 index 00000000..bef1234c --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/auth/AuthenticationStage.java @@ -0,0 +1,30 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 31, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.auth; + +/** + * The stages through which authentication passes. Basically a small state machine + * for two-factor authentication. The authentication subsystem begins in the + * NO_AUTHENTICATION state, if traditional authenticate succeeds it moves to + * the TRADITIONALLY_AUTHENTICATED state, and then if second factor authentication + * succeeds it moves to the SECOND_FACTOR_AUTHENTICATED state. The current + * state of the state machine is encoded in the HTTP session's + * AuthenticationInterface.AUTH_STAGE attribute. + * + * @author Joseph R. Kiniry + * @version 1.0.0 + */ +public enum AuthenticationStage { + NOT_AUTHENTICATED, + TRADITIONALLY_AUTHENTICATED, + SECOND_FACTOR_AUTHENTICATED +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/auth/AuthenticationStatus.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/auth/AuthenticationStatus.java new file mode 100644 index 00000000..9a771adc --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/auth/AuthenticationStatus.java @@ -0,0 +1,76 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 30, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.auth; + +import java.io.Serializable; + +import us.freeandfair.corla.model.Administrator.AdministratorType; + +/** + * The response provided by the server after a successful second factor + * authentication which indicates what kind of user just authenticated. This + * particular response class is general purpose, encoding generic role used by the + * CORLA system. + * + * @author Joseph R. Kiniry + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public class AuthenticationStatus { + /** + * The role of the user that just authenticated. + */ + private final AdministratorType my_role; + + /** + * The authentication stage. + */ + private final AuthenticationStage my_stage; + + /** + * The authentication challenge. + */ + private final Serializable my_challenge; + + /** + * Create a new response object. + * @param the_role the role that was just successfully authenticated. + */ + public AuthenticationStatus(final AdministratorType the_role, + final AuthenticationStage the_stage, + final Serializable the_challenge) { + my_role = the_role; + my_stage = the_stage; + my_challenge = the_challenge; + } + + /** + * @return the role. + */ + public AdministratorType role() { + return my_role; + } + + /** + * @return the stage. + */ + public AuthenticationStage stage() { + return my_stage; + } + + /** + * @return the challenge. + */ + public Serializable challenge() { + return my_challenge; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/auth/DatabaseAuthentication.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/auth/DatabaseAuthentication.java new file mode 100644 index 00000000..7d2bf4ea --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/auth/DatabaseAuthentication.java @@ -0,0 +1,93 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 27, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.auth; + +import java.security.SecureRandom; +import java.util.Random; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.model.Administrator; +import us.freeandfair.corla.query.AdministratorQueries; + +/** + * A demonstration implementation of AuthenticationInterface used during + * development to mock an actual back-end authentication system. + * + * @author Joseph R. Kiniry + * @author Daniel M. Zimmerman + * @version 1.0.0 + * @trace authentication + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public final class DatabaseAuthentication extends AbstractAuthentication { + /** + * {@inheritDoc} + */ + @Override + public boolean secondFactorAuthenticate(final Request the_request, + final String the_username, + final String the_second_factor) { + // skip, as we do not implement a second factor in test mode + return true; + } + + /** + * {@inheritDoc} + * The returned AuthenticationResult's second factor challenge is a random + * triple of three uppercase characters early in the alphabet, similar to + * that which simple code cards use. + */ + @Override + @SuppressWarnings({"PMD.ConsecutiveLiteralAppends", "checkstyle:magicnumber"}) + public AuthenticationResult traditionalAuthenticate(final Request the_request, + final Response the_response, + final String the_username, + final String the_password) { + final Administrator admin = + AdministratorQueries.byUsername(the_username); + final Random random = new SecureRandom(); + final StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 3; i++) { + sb.append('['); + sb.append(Character.toString((char) ('A' + random.nextInt(9)))); + sb.append(','); + sb.append(random.nextInt(8) + 1); + sb.append("] "); + } + return new AuthenticationResult(admin != null, sb.toString().trim()); + } + + /** + * {@inheritDoc} + */ + @Override + public void traditionalDeauthenticate(final Request the_request, + final String the_username) { + the_request.session().removeAttribute(ADMIN); + the_request.session().removeAttribute(CHALLENGE); + Main.LOGGER.info("session is now traditionally deauthenticated"); + } + + /** + * {@inheritDoc} + */ + @Override + public void twoFactorDeauthenticate(final Request the_request, + final String the_username) { + the_request.session().removeAttribute(ADMIN); + the_request.session().removeAttribute(CHALLENGE); + Main.LOGGER.info("session is now second factor deauthenticated"); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/controller/AuditReport.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/controller/AuditReport.java new file mode 100644 index 00000000..acf1a28e --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/controller/AuditReport.java @@ -0,0 +1,223 @@ + +package us.freeandfair.corla.controller; + +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.io.IOException; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import java.io.Reader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.zip.ZipOutputStream; +import java.util.zip.ZipEntry; + +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import us.freeandfair.corla.report.WorkbookWriter; +import us.freeandfair.corla.report.ReportRows; +import us.freeandfair.corla.report.StateReport; +import us.freeandfair.corla.model.DoSDashboard; +import us.freeandfair.corla.persistence.Persistence; +//import us.freeandfair.corla.query.CSVParser; +import us.freeandfair.corla.query.ExportQueries; +//import us.freeandfair.corla.query.Reader; + +/** + * Find the data for a report and format it to be rendered into a presentation + * format elsewhere + **/ +public final class AuditReport { + + /** + * Class-wide logger + */ + public static final Logger LOGGER = LogManager.getLogger(AuditReport.class); + private static Pattern p = Pattern.compile(".* (\\w+) County$"); + + /** no instantiation **/ + private AuditReport() { + } + + /** + * Generate a report file and return the bytes activity: a log of acvr + * submissions for a particular Contest (includes all participating counties) + * activity-all: same as above for all targeted contests results: the acvr + * submissions for each random number that was generated (to audit this + * program's calculations) results-all: same as above for all targeted + * contests + * + * Here are the specific differences: - the Activity report is sorted by + * timestamp, the Audit Report by random number sequence - the Activity report + * shows previous revisions, the Audit Report does not - the Result Report + * shows the random number that was generated for the CVR (and the position), + * the Activity Report does not - the Result Report shows + * duplicates(multiplicity), the Activity Report does not + * + * contestName is optional if reportType is *-all + **/ + public static byte[] generate(final String contentType, final String reportType, + final String contestName) + throws IOException { + // xlsx + final WorkbookWriter writer = new WorkbookWriter(); + List> rows; + DoSDashboard dosdb; + dosdb = Persistence.getByID(DoSDashboard.ID, DoSDashboard.class); + Map contestMap = createUniqueSheetNames(dosdb.targetedContestNames()); + + switch (reportType) { + case "activity": + rows = ReportRows.getContestActivity(contestName); + writer.addSheet(contestMap.get(contestName), rows); + break; + case "activity-all": + writer.addSheet("Contests to Sheets", getMappingSheet(contestMap)); + for (final Entry entry : contestMap.entrySet()) { + writer.addSheet(entry.getValue(), ReportRows.getContestActivity(entry.getKey())); + } + break; + case "results": + rows = ReportRows.getResultsReport(contestName); + writer.addSheet(contestMap.get(contestName), rows); + break; + case "results-all": + writer.addSheet("Contests to Sheets", getMappingSheet(contestMap)); + writer.addSheet("Summary", ReportRows.genSumResultsReport()); + for (final Entry entry : contestMap.entrySet()) { + writer.addSheet(entry.getValue(), ReportRows.getResultsReport(entry.getKey())); + } + break; + default: + LOGGER.error("invalid reportType: " + reportType); + break; + + } + + return writer.write(); + } + + public static List> getMappingSheet(Map contestMap) { + final List> rows = new ArrayList<>(); + rows.add(Arrays.asList("Contest Name","Sheet Name")); + for (Entry pair : contestMap.entrySet()) { + rows.add(Arrays.asList(pair.getKey(), pair.getValue())); + } + return rows ; + + } + + public static String getFirst31Characters(String inName) { + inName = inName.replace("the",""); + inName = inName.replace("of",""); + inName = inName.replace("and",""); + inName = inName.replace("to",""); + inName = inName.replace("Judicial District",""); + inName = inName.replace("Congressional District",""); + inName = inName.replace("Congress-District",""); + inName = inName.replace(" "," "); + inName = inName.replace(" "," "); + String first31 = inName.length() > 31 ? inName.substring(0, 30) : inName ; + String newUniqueName = getUniqueNameUsingCounty(inName, first31); + return newUniqueName.replaceAll("\\s+$", ""); + } + + private static String getUniqueName(String inUniqueName) { + char lastChar = inUniqueName.charAt(inUniqueName.length()-1) ; + if (lastChar >= 'a' && lastChar <= 'z') { + lastChar++; + return inUniqueName.substring(0, inUniqueName.length()-1) + lastChar; + } + return inUniqueName.substring(0, inUniqueName.length()-1) + 'a'; + } + + private static String getUniqueNameUsingCounty(String originalName, String inUniqueName) { + Matcher m = p.matcher(originalName) ; + if ( m.matches()) { + String countyName = m.group(1); + return inUniqueName.substring(0, inUniqueName.length()-countyName.length()-2) + " " + countyName; + } + return inUniqueName ; + } + + private static Map createUniqueSheetNames(Set contestNames) { + Map uniqueNames = new HashMap<>(contestNames.size()); + for (String contestName : contestNames) { + String newUniqueName = getFirst31Characters(contestName); + + //TRK 279 2021 practice period, dowload audit report hangs +// for (Map.Entry me : uniqueNames.entrySet()) { +// LOGGER.info("Key: "+me.getKey() + " & Value: " + me.getValue()); +// } + while (uniqueNames.containsKey(newUniqueName)) { + newUniqueName = getUniqueName(newUniqueName); + } + uniqueNames.put(newUniqueName, contestName); + } + return + uniqueNames.entrySet() + .stream() + .sorted(Map.Entry.comparingByValue()) + .collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey,(oldValue, newValue) -> oldValue, LinkedHashMap::new)) + ; + } + + /** all the reports in one "package" **/ + public static void generateZip(final OutputStream os) { + final ZipOutputStream zos = new ZipOutputStream(os); + + try { + final Map files = ExportQueries.sqlFiles(); + + for (final Map.Entry entry : files.entrySet()) { + final String filename = entry.getKey() + ".csv"; + final ZipEntry zipEntry = new ZipEntry(filename); + zos.putNextEntry(zipEntry); + ExportQueries.csvOut(entry.getValue(), zos); + zos.closeEntry(); + } + + for (final Map.Entry entry : files.entrySet()) { + final String filename = entry.getKey() + ".json"; + final ZipEntry zipEntry = new ZipEntry(filename); + zos.putNextEntry(zipEntry); + ExportQueries.jsonOut(entry.getValue(), zos); + zos.closeEntry(); + } + + zos.putNextEntry(new ZipEntry("ActivityReport.xlsx")); + zos.write(generate("xlsx", "activity-all", null)); + zos.closeEntry(); + + zos.putNextEntry(new ZipEntry("ResultsReport.xlsx")); + zos.write(generate("xlsx", "results-all", null)); + zos.closeEntry(); + + final StateReport sr = new StateReport(); + zos.putNextEntry(new ZipEntry(sr.filenameExcel())); + zos.write(sr.generateExcel()); + zos.closeEntry(); + } catch (IOException e) { + LOGGER.error(e.getMessage()); + } finally { + try { + zos.close(); + } catch (IOException e) { + LOGGER.warn(String.format("Cannot close stream: %s", e.getMessage())); + } + } + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/controller/BallotSelection.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/controller/BallotSelection.java new file mode 100644 index 00000000..0773b80c --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/controller/BallotSelection.java @@ -0,0 +1,583 @@ +/** + * Prepare a list of ballots from a list of random numbers + **/ +package us.freeandfair.corla.controller; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import us.freeandfair.corla.crypto.PseudoRandomNumberGenerator; +import us.freeandfair.corla.json.CVRToAuditResponse; +import us.freeandfair.corla.model.BallotManifestInfo; +import us.freeandfair.corla.model.CastVoteRecord; +import us.freeandfair.corla.model.ContestResult; +import us.freeandfair.corla.model.CVRAuditInfo; +import us.freeandfair.corla.model.Tribute; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.query.BallotManifestInfoQueries; +import us.freeandfair.corla.query.CastVoteRecordQueries; +import us.freeandfair.corla.util.BallotSequencer; +import us.freeandfair.corla.util.PhantomBallots; + +// TODO remove suppression and refactor +@SuppressWarnings({"PMD.ExcessivePublicCount"}) +public final class BallotSelection { + /** + * Class-wide logger + */ + public static final Logger LOGGER = + LogManager.getLogger(BallotSelection.class); + + /** + * Prevent construction + */ + private BallotSelection() { + } + + /** + * A Segment is the unit of work for a single contest during one round + * of a comparison audit. It might contain work for one or more + * counties. + */ + public static class Segment { + /** + * The set of CVRs to audit, unordered but without duplicates. + */ + public Set cvrs = new HashSet<>(); + + /** + * CVR IDs in audit sequence order, possible duplicates. + */ + public List cvrIds = new ArrayList<>(); + + /** + * Let's rethink the name 'Tribute'. It's enough metadata to find a + * CVR from a manifest given a sample position. + */ + public List tributes = new ArrayList<>(); + + /** + * Add the nth ballot from a manifest to our segment + * @param bmi the manifest entry containing the sample + * @param ballotPosition the position within a batch to sample + */ + public void addTribute(final BallotManifestInfo bmi, + final Integer ballotPosition, + final Integer rand, + final Integer randSequencePosition, + final String contestName) { + final Tribute t = new Tribute(); + t.countyId = bmi.countyID(); + t.scannerId = bmi.scannerID(); + t.batchId = bmi.batchID(); + t.ballotPosition = ballotPosition; + t.rand = rand; + t.randSequencePosition = randSequencePosition; + t.contestName = contestName; + t.setUri(); + tributes.add(t); + } + + /** + * Add some CVRs to this thing + */ + public void addCvrs(final Collection cvrs) { + this.cvrs.addAll(cvrs); + } + + /** + * add CVR IDs. CVR flavored. + */ + public void addCvrIds(final Collection cvrs) { + this.cvrIds.addAll(cvrs.stream().map(cvr -> cvr.id()).collect(Collectors.toList())); + } + + /** + * add CVR IDs. Long flavored. + */ + public void addCvrIds(final List cvrIds) { + this.cvrIds.addAll(cvrIds); + } + + /** + * in the order of the random selection, not deduped and not sorted + */ + public List auditSequence() { + return cvrIds; + } + + /** + * Return a list of CVRs, de-duplicated and sorted. + * + * The IDs of these CVRs are expected to be given to an audit board. + * + * PERF: This is not a fast operation, avoid calling it more than once. + */ + public List cvrsInBallotSequence() { + return BallotSequencer.sortAndDeduplicateCVRs( + cvrs.stream().collect(Collectors.toList())); + } + + /** + * a good idea + */ + public String toString() { + return String.format("[Segment auditSequence=%s ballotPositions=%s]", + auditSequence(), + tributes.stream().map(t -> t.ballotPosition).collect(Collectors.toList())); + } + } + + /** + * An ADT for thinking about selecting a sample from a contest + */ + public static class Selection { + /** + * the audit segments for some collection of counties + */ + public Map segments = new HashMap<>(); + + /** + * How large is our collection of ballots? + */ + public Integer domainSize = Integer.MIN_VALUE; + + /** + * the PRNG output + */ + public List generatedNumbers = new ArrayList(); + + /** + * what's the contest called? + */ + public String contestName; + + /** + * The contest result associated with this selection + */ + public ContestResult contestResult; + + /** + * Combines a collection of segments (selections from different + * contests) into a single segment that can be given to a county. + */ + public static Segment combineSegments(final Collection segments) { + return segments.stream() + .filter(s -> null != s) + .reduce(new Segment(), + (acc,s) -> { + // can't ask segment.cvrs for raw data because it is a set + // so we get the cvrIds + acc.addCvrIds(s.cvrIds); + acc.addCvrs(s.cvrs); + return acc;}); + } + + /** + * Initializer for a county segment + */ + public void initCounty(final Long countyId) { + if (null == forCounty(countyId)) { + final Segment segment = new Segment(); + this.segments.put(countyId, segment); + } + } + + /** + * record ballot position metadata + */ + public void addBallotPosition(final BallotManifestInfo bmi, + final Integer ballotPosition, + final Integer rand, + final Integer randSequencePosition, + final String ContestName) { + this.forCounty(bmi.countyID()).addTribute(bmi, + ballotPosition, + rand, + randSequencePosition, + contestName); + } + + /** + * return the segment for a county + */ + public Segment forCounty(final Long countyId) { + return this.segments.get(countyId); + } + + /** + * getter of all segments + */ + public Collection allSegments() { + return segments.values(); + } + + /** + * CVR IDs for a contest, from a segment + */ + public List contestCVRIds() { + return contestResult.countyIDs().stream() + .map(id -> forCounty(id)) + .filter(s -> s != null) + .map(segment -> segment.cvrIds) + .flatMap(List::stream) + .collect(Collectors.toList()); + } + + /** + * a good idea + */ + public String toString() { + return String.format("[Selection contestName=%s generatedNumbers=%s domainSize=%s]", + contestName, generatedNumbers, domainSize); + } + } + + /** + * create a random list of numbers and divide them into the appropriate + * counties + * FIXME: setSegments on contestResult for now + **/ + public static Selection randomSelection(final ContestResult contestResult, + final String seed, + final Integer minIndex, + final Integer maxIndex) { + + if (minIndex > maxIndex) { + // you are done, silly + final Selection selection = new Selection(); + selection.contestResult = contestResult; + return selection; + } + + final int domainSize = ballotsCast(contestResult.countyIDs()).intValue(); + final PseudoRandomNumberGenerator gen = + new PseudoRandomNumberGenerator(seed, true, 1, domainSize); + + final List generatedNumbers = gen.getRandomNumbers(minIndex, maxIndex); + + final Selection selection = new Selection(); + selection.contestResult = contestResult; + selection.contestName = contestResult.getContestName();//posterity + selection.domainSize = domainSize; //posterity + selection.generatedNumbers = generatedNumbers; //posterity + + // make the theoretical selections (avoiding cvrs) + selectTributes(selection, contestResult.countyIDs()); + + LOGGER.info(String.format("[randomSelection] selected %s samples for %s ", + selection.generatedNumbers.size(), + contestResult.getContestName())); + LOGGER.debug("randomSelection: selection= " + selection); + // get the CVRs from the theoretical + resolveSelection(selection); + return selection; + } + + /** + * Divide a list of random numbers into segments by county + **/ + public static void selectTributes(final Selection selection, + final Set countyIds) { + selectTributes(selection, countyIds, BallotManifestInfoQueries::getMatching); + } + + /** + * Divide a list of random numbers into segments + * transitional refactor step (3 arities is too many) + **/ + public static void selectTributes(final Selection selection, + final Set countyIds, + final MATCHINGQ queryMatching) { + + final Set contestBmis = queryMatching.apply(countyIds); + selectTributes(selection, countyIds, contestBmis); + } + + /** + * Divide a list of random numbers into segments by county + **/ + public static void selectTributes(final Selection selection, + final Set countyIds, + final Set contestBmis) { + countyIds.forEach(id -> selection.initCounty(id)); + int i = 0; + for (final Integer rand: selection.generatedNumbers) { + final BallotManifestInfo bmi = selectCountyId(Long.valueOf(rand), contestBmis); + selection.addBallotPosition(bmi, + // translate rand from Contest scope to bmi/batch scope + bmi.translateRand(rand), + // keep rand around to store on cvr for reporting + rand, + // preserve the order of random selections, 0-based + i++, + selection.contestName); + } + } + + /** + * When we draw more than one phantom ballot, we need to make sure + * that the persistence context knows about only one instance of each. + * (Phantom ballots are POJOs, so every phantom ballot looks identical + * to the persistence context.) + * + * @param county The county. + * @param cvrs A list of CastVoteRecord objects that might contain phantom ballots + */ + public static List dedupePhantomBallots(final List cvrs) { + // A map of a CVR to a CVR so we can get a unique persisted entity from the + // database. + final Map phantomCvrs = + cvrs.stream() + .filter(cvr -> PhantomBallots.isPhantomRecord(cvr)) + .collect(Collectors.toMap( + Function.identity(), + Function.identity(), + (a, b) -> b)); + + // Assign database identifiers to newly-created phantom CVRs. + phantomCvrs.entrySet().stream() + .forEach(e -> Persistence.saveOrUpdate(e.getValue())); + + // Use the database-mapped CVR if it exists. + return cvrs.stream() + .map(cvr -> phantomCvrs.getOrDefault(cvr, cvr)) + .collect(Collectors.toList()); + } + + /** look for the cvrs, some may be phantom records **/ + public static Selection resolveSelection(final Selection selection) { + + selection.allSegments().forEach(segment -> { + final List cvrs = + dedupePhantomBallots(CastVoteRecordQueries.atPosition(segment.tributes)); + + segment.addCvrs(cvrs); + segment.addCvrIds(cvrs); // keep raw data separate + }); + LOGGER.debug(String.format("[resolveSelection: selection=%s, combinedSegments=%s]", + selection.segments, + Selection.combineSegments(selection.allSegments()).cvrIds)); + return selection; + } + + /** + * project a sequence across counties + * + * Uses special fields on BallotManifestInfo to hold temorary values. + * These values are only valid in this set of BallotManifestInfos + **/ + public static Set projectUltimateSequence(final Set bmis) { + Long last = 0L; + for (final BallotManifestInfo bmi: bmis) { + // plus one to make the sequence start and end inclusive in bmi.isHolding + bmi.setUltimate(last + 1L); + last = bmi.ultimateSequenceEnd; + } + return bmis; + } + + /** + * How much of an audit sequence have we checked? + * + * @param cvrIds A list of IDs to check. Presumably the unsorted + * original sampling. + * @return the number of ballot cards that have been audited + */ + public static Integer auditedPrefixLength(final List cvrIds) { + // FIXME extract-fn, then use + // Map isAuditedById = checkAudited(cvrIds); + + if (cvrIds.isEmpty()) { return 0; } + + final Map isAuditedById = new HashMap<>(); + + for (final Long cvrId: cvrIds) { + final CVRAuditInfo cvrai = Persistence.getByID(cvrId, CVRAuditInfo.class); + // has an acvr + final boolean isAudited = cvrai != null && cvrai.acvr() != null; + isAuditedById.put(cvrId, isAudited); + } + + Integer idx = 0; + for (int i=0; i < cvrIds.size(); i++) { + final boolean audited = isAuditedById.get(cvrIds.get(i)); + if (audited) { + idx = i + 1; + } else { break; } + } + LOGGER.debug(String.format("[auditedPrefixLength: isAuditedById=%s, apl=%d]", + isAuditedById, idx)); + return idx; + } + + /** + * Find the manifest entry holding a random selection + */ + public static BallotManifestInfo selectCountyId(final Long rand, + final Set bmis) { + final Optional holding = projectUltimateSequence(bmis).stream() + .filter(bmi -> bmi.isHolding(rand)) + .findFirst(); + if (holding.isPresent()) { + return holding.get(); + } else { + final String msg = "Could not find BallotManifestInfo holding random number: " + rand; + throw new MissingBallotManifestException(msg); + } + } + + /** + * The total number of ballots across a set of counties + * @param countyIds a set of counties to count + * @return the number of ballots in the ballot manifests belonging to + * countyIds + **/ + public static Long ballotsCast(final Set countyIds) { + // could use voteTotals but that would be impure; using cvr data + // + // If a county has only one ballot for a contest, all the ballots from that + // county are used to get a total number of ballots + return BallotManifestInfoQueries.totalBallots(countyIds); + } + + /** + * Joins provided CVRs to the ballot manifest. + * + * Produces a list of CVRToAuditResponse elements which represent the CVRs + * augmented with ballot manifest data. + * + * @return CVRs joind with ballot manifest data + */ + public static List + toResponseList(final List cvrs) { + return toResponseList(cvrs, BallotManifestInfoQueries::locationFor); + } + + + /** + * Joins provided CVRs to the ballot manifest. + * + * Produces a list of CVRToAuditResponse elements which represent the CVRs + * augmented with ballot manifest data. + * + * Uses a passed-in BallotManifestInfo query. + * + * @return CVRs joind with ballot manifest data + */ + public static List + toResponseList(final List cvrs, final BMILOCQ bmiq) { + + final List responses = new LinkedList(); + + final Set uris = cvrs.stream() + .map(cvr -> cvr.bmiUri()) + .collect(Collectors.toSet()); + final List bmis = BallotManifestInfoQueries.locationFor(uris); + final Map uriToLoc = bmis.stream().collect(Collectors.toMap(bmi -> bmi.getUri(), + bmi -> bmi.storageLocation())); + int i = 0; + for (final CastVoteRecord cvr: cvrs) { + + final String storageLocation = uriToLoc.get(cvr.bmiUri()); + if (null == storageLocation) { + LOGGER.error("could not find a ballot manifest for cvr: "+ cvr.getUri()); + continue; + } + responses.add(toResponse(i, storageLocation, cvr)); + i++; + } + return responses; + } + + /** + * get ready to render the data + **/ + public static CVRToAuditResponse toResponse(final int i, + final String storageLocation, + final CastVoteRecord cvr) { + + // the only field that the cvr doesn't have is storageLocation. Also, it would be + // ideal to render data from bmis but the cvrs have already been selected + // from info from bmis so that integrity concern can be checked off as met + return new CVRToAuditResponse(i, + cvr.scannerID(), + cvr.batchID(), + cvr.recordID(), + cvr.imprintedID(), + cvr.cvrNumber(), + cvr.id(), + cvr.ballotType(), + storageLocation, + cvr.auditFlag(), + cvr.previouslyAudited()); + } + + /** + * this is bad, it could be one of two things: + * - a random number was generated outside of the number of (theoretical) ballots + * - there is a gap in the sequence_start and sequence_end values of the + * ballot_manifest_infos + **/ + public static class MissingBallotManifestException extends RuntimeException { + /** constructor **/ + public MissingBallotManifestException(final String msg) { + super(msg); + } + } + + /** + * a functional interface to pass a function as an argument that takes two + * arguments + **/ + public interface CVRQ { + + /** how to query the database **/ + CastVoteRecord apply(Long county_id, + Integer scanner_id, + String batch_id, + Long position); + } + + /** + * a functional interface to pass a function as an argument that takes two + * arguments + **/ + public interface BMIQ { + + /** how to query the database **/ + Optional apply(Long rand, + Long countyId); + } + + + /** + * a functional interface to pass a function as an argument + **/ + public interface BMILOCQ { + + /** how to query the database **/ + Optional apply(CastVoteRecord cvr); + } + + /** + * a functional interface to pass a function as an argument + **/ + public interface MATCHINGQ { + + /** how to query the database **/ + Set apply(final Set county_ids); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/controller/ComparisonAuditController.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/controller/ComparisonAuditController.java new file mode 100644 index 00000000..e783189b --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/controller/ComparisonAuditController.java @@ -0,0 +1,691 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 23, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.controller; + +import java.math.BigDecimal; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.OptionalInt; +import java.util.Set; + +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import us.freeandfair.corla.math.Audit; +import us.freeandfair.corla.model.AuditReason; +import us.freeandfair.corla.model.CVRAuditInfo; +import us.freeandfair.corla.model.CVRContestInfo; +import us.freeandfair.corla.model.CVRContestInfo.ConsensusValue; +import us.freeandfair.corla.model.CastVoteRecord; +import us.freeandfair.corla.model.ContestResult; +import us.freeandfair.corla.model.ComparisonAudit; +import us.freeandfair.corla.model.CountyDashboard; +import us.freeandfair.corla.model.Round; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.query.CastVoteRecordQueries; + +/** + * Controller methods relevant to comparison audits. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings({"PMD.GodClass", "PMD.CyclomaticComplexity", "PMD.ExcessiveImports", + "PMD.ModifiedCyclomaticComplexity", "PMD.StdCyclomaticComplexity"}) +public final class ComparisonAuditController { + /** + * Class-wide logger + */ + public static final Logger LOGGER = + LogManager.getLogger(ComparisonAuditController.class); + + /** + * Private constructor to prevent instantiation. + */ + private ComparisonAuditController() { + // empty + } + + /** + * Gets all CVRs to audit in the specified round for the specified county + * dashboard. This returns a list in audit random sequence order. + * + * @param the_dashboard The dashboard. + * @param the_round_number The round number (indexed from 1). + * @return the CVRs to audit in the specified round. + * @exception IllegalArgumentException if the specified round doesn't exist. + */ + public static List cvrsToAuditInRound(final CountyDashboard the_cdb, + final int the_round_number) { + if (the_round_number < 1 || the_cdb.rounds().size() < the_round_number) { + throw new IllegalArgumentException("invalid round specified"); + } + final Round round = the_cdb.rounds().get(the_round_number - 1); + final Set id_set = new HashSet<>(); + final List result = new ArrayList<>(); + + for (final Long cvr_id : round.auditSubsequence()) { + if (!id_set.contains(cvr_id)) { + id_set.add(cvr_id); + result.add(Persistence.getByID(cvr_id, CVRAuditInfo.class)); + } + } + + return result; + } + + /** + * @return the CVR IDs remaining to audit in the current round, or an empty + * list if there are no CVRs remaining to audit or if no round is in progress. + */ + public static List cvrIDsRemainingInCurrentRound(final CountyDashboard the_cdb) { + final List result = new ArrayList(); + final Round round = the_cdb.currentRound(); + if (round != null) { + for (int i = 0; + i + round.actualAuditedPrefixLength() < round.expectedAuditedPrefixLength(); + i++) { + result.add(round.auditSubsequence().get(i + round.actualAuditedPrefixLength())); + } + } + return result; + } + + /** + * Return the ballot cards to audit for a particular county and round. + * + * The returned list will not have duplicates and is in an undefined order. + * + * @param countyDashboard county dashboard owning the rounds + * @param roundNumber 1-based round number + * + * @return the list of ballot cards for audit. If the query does not result in + * any ballot cards, for instance when the round number is invalid, + * the returned list is empty. + */ + public static List + ballotsToAudit(final CountyDashboard countyDashboard, + final int roundNumber) { + final List rounds = countyDashboard.rounds(); + Round round; + + try { + // roundNumber is 1-based + round = rounds.get(roundNumber - 1); + } catch (IndexOutOfBoundsException e) { + return new ArrayList(); + } + + LOGGER.debug( + String.format( + "Ballot cards to audit: " + + "[round=%s, round.ballotSequence.size()=%d, round.ballotSequence()=%s]", + round, + round.ballotSequence().size(), + round.ballotSequence() + ) + ); + + // Get all ballot cards for the target round + final List cvrs = CastVoteRecordQueries.get(round.ballotSequence()); + + // Fetch the CVRs from previous rounds in order to set a flag determining + // whether they had been audited previously. + final Set previousCvrs = new HashSet(); + for (int i = 1; i < roundNumber; i++) { + // i is 1-based + final Round r = rounds.get(i - 1); + previousCvrs.addAll(CastVoteRecordQueries.get(r.ballotSequence())); + } + + // PERF TODO: We may be able to replace calls to `audited` with a query that + // determines the audit status of all the CVRs when they are fetched. + for (final CastVoteRecord cvr : cvrs) { + cvr.setAuditFlag(audited(countyDashboard, cvr)); + cvr.setPreviouslyAudited(previousCvrs.contains(cvr)); + } + + return cvrs; + } + + /** + * @param the_cdb The dashboard. + * @return true if an audit round is started for the dashboard, false otherwise; + * an audit round might not be started if there are no driving contests in the + * county, or if the county needs to audit 0 ballots to meet the risk limit. + */ + public static ComparisonAudit createAudit(final ContestResult contestResult, + final BigDecimal riskLimit) { + final ComparisonAudit ca = + new ComparisonAudit(contestResult, riskLimit, contestResult.getDilutedMargin(), + Audit.GAMMA, contestResult.getAuditReason()); + Persistence.save(ca); + LOGGER.debug(String.format("[createAudit: contestResult=%s, ComparisonAudit=%s]", + contestResult, ca)); + return ca; + } + + /** + * Do the part of setup for a county dashboard to start their round. + * - updateRound + * - updateCVRUnderAudit + */ + public static boolean startRound(final CountyDashboard cdb, + final Set audits, + final List auditSequence, + final List ballotSequence) { + LOGGER.info(String.format("Starting a round for %s, drivingContests=%s", + cdb.county(), cdb.drivingContestNames())); + cdb.startRound(ballotSequence.size(), auditSequence.size(), + 0, ballotSequence, auditSequence); + // FIXME it appears these two must happen in this order. + updateRound(cdb, cdb.currentRound()); + updateCVRUnderAudit(cdb); + + // if the round was started there will be ballots to count + return cdb.ballotsRemainingInCurrentRound() > 0; + } + + + /** unaudit and audit a submitted ACVR **/ + public static boolean reaudit(final CountyDashboard cdb, + final CastVoteRecord cvr, + final CastVoteRecord newAcvr, + final String comment) { + + LOGGER.info("[reaudit] cvr: " + cvr.toString()); + final CVRAuditInfo cai = + Persistence.getByID(cvr.id(), CVRAuditInfo.class); + final CastVoteRecord oldAcvr = cai.acvr(); + if (null == oldAcvr) { + LOGGER.error("can't reaudit a cvr that hasn't been audited"); + return false; + } + + final Integer former_count = unaudit(cdb, cai); + LOGGER.debug("[reaudit] former_count: " + former_count.toString()); + + + Long revision = CastVoteRecordQueries.maxRevision(cvr); + // sets revision to 1 if this is the original(revision is zero) + if (0L == revision) { + revision = 1L; + oldAcvr.setRevision(revision); + } + oldAcvr.setToReaudited(); + CastVoteRecordQueries.forceUpdate(oldAcvr); + + // the original will not have a re-audit comment + newAcvr.setComment(comment); + + // sets revision to 2 if this is the first revision(revision is zero) + newAcvr.setRevision(revision + 1L); + cai.setACVR(newAcvr); + Persistence.save(newAcvr); + Persistence.save(cai); + + final Integer new_count = audit(cdb, cai, true); + LOGGER.debug("[reaudit] new_count: " + new_count.toString()); + cdb.updateAuditStatus(); + + return true; + } + + + /** + * Submit an audit CVR for a CVR under audit to the specified county dashboard. + * + * @param cdb The dashboard. + * @param the_cvr_under_audit The CVR under audit. + * @param the_audit_cvr The corresponding audit CVR. + * @return true if the audit CVR is submitted successfully, false if it doesn't + * correspond to the CVR under audit, or the specified CVR under audit was + * not in fact under audit. + */ + //@ require the_cvr_under_audit != null; + //@ require the_acvr != null; + @SuppressWarnings({"PMD.CyclomaticComplexity", "PMD.AvoidDeeplyNestedIfStmts"}) + public static boolean submitAuditCVR(final CountyDashboard cdb, + final CastVoteRecord the_cvr_under_audit, + final CastVoteRecord the_audit_cvr) { + // performs a sanity check to make sure the CVR under audit and the ACVR + // are the same card + boolean result = false; + + final CVRAuditInfo info = + Persistence.getByID(the_cvr_under_audit.id(), CVRAuditInfo.class); + + if (info == null) { + LOGGER.warn("attempt to submit ACVR for county " + + cdb.id() + ", cvr " + + the_cvr_under_audit.id() + " not under audit"); + } else if (checkACVRSanity(the_cvr_under_audit, the_audit_cvr)) { + LOGGER.trace("[submitAuditCVR: ACVR seems sane]"); + // if the record is the current CVR under audit, or if it hasn't been + // audited yet, we can just process it + if (info.acvr() == null) { + // this audits all instances of the ballot in our current sequence; + // they might be out of order, but that's OK because we have strong + // requirements about finishing rounds before looking at results as + // final and valid + LOGGER.trace("[submitAuditCVR: ACVR is null, creating]"); + info.setACVR(the_audit_cvr); + final int new_count = audit(cdb, info, true); + cdb.addAuditedBallot(); + // there could be a problem here, maybe the cdb counts for all contests + // and that is good enough?? + cdb.setAuditedSampleCount(cdb.auditedSampleCount() + new_count); + } else { + // the record has been audited before, so we need to "unaudit" it + LOGGER.trace("[submitAuditCVR: ACVR is seen, un/reauditing]"); + final int former_count = unaudit(cdb, info); + info.setACVR(the_audit_cvr); + final int new_count = audit(cdb, info, true); + cdb.setAuditedSampleCount(cdb.auditedSampleCount() - former_count + new_count); + } + result = true; + } else { + LOGGER.warn("attempt to submit non-corresponding ACVR " + + the_audit_cvr.id() + " for county " + cdb.id() + + ", cvr " + the_cvr_under_audit.id()); + } + Persistence.flush(); + + LOGGER.trace(String.format("[Before recalc: auditedSampleCount=%d, estimatedSamples=%d, optimisticSamples=%d", + cdb.auditedSampleCount(), + cdb.estimatedSamplesToAudit(), + cdb.optimisticSamplesToAudit())); + updateCVRUnderAudit(cdb); + LOGGER.trace(String.format("[After recalc: auditedSampleCount=%d, estimatedSamples=%d, optimisticSamples=%d", + cdb.auditedSampleCount(), + cdb.estimatedSamplesToAudit(), + cdb.optimisticSamplesToAudit())); + cdb.updateAuditStatus(); + return result; + } + + /** + * Computes the estimated total number of samples to audit on the specified + * county dashboard. This uses the minimum samples to audit calculation, + * increased by the percentage of discrepancies seen in the audited ballots + * so far. + * + * @param cdb The dashboard. + */ + public static int estimatedSamplesToAudit(final CountyDashboard cdb) { + int to_audit = Integer.MIN_VALUE; + final Set drivingContests = cdb.drivingContestNames(); + + // FIXME might look better as a stream().filter(). + for (final ComparisonAudit ca : cdb.comparisonAudits()) { // to_audit = cdb.comparisonAudits.stream() + final String contestName = ca.contestResult().getContestName(); // strike + if (drivingContests.contains(contestName)) { // .filter(ca -> drivingContests.contains(ca.contestResult().getContestName())) + final int bta = ca.estimatedSamplesToAudit(); // .map(ComparisonAudit::estimatedSamplesToAudit) + to_audit = Math.max(to_audit, bta); // .max() gets the biggest of all driving contest estimated samples + LOGGER.debug(String.format("[estimatedSamplesToAudit: " + + "driving contest=%s, bta=%d, to_audit=%d]", + ca.contestResult().getContestName(), bta, to_audit)); + } + } + return Math.max(0, to_audit); + } + + /** + * Checks to see if the specified CVR has been audited on the specified county + * dashboard. This check sets the audit flag on the CVR record in memory, + * so its result can be accessed later without an expensive database hit. + * + * @param the_cdb The county dashboard. + * @param the_cvr The CVR. + * @return true if the specified CVR has been audited, false otherwise. + */ + public static boolean audited(final CountyDashboard the_cdb, + final CastVoteRecord the_cvr) { + final CVRAuditInfo info = Persistence.getByID(the_cvr.id(), CVRAuditInfo.class); + final boolean result; + if (info == null || info.acvr() == null) { + result = false; + } else { + result = true; + } + return result; + } + + /** + * Updates a round object with the disagreements and discrepancies + * that already exist for CVRs in its audit subsequence, creates + * any CVRAuditInfo objects that don't exist but need to, and + * increases the multiplicity of any CVRAuditInfo objects that already + * exist and are duplicated in this round. + * + * @param cdb The county dashboard to update. + * @param round The round to update. + */ + private static void updateRound(final CountyDashboard cdb, + final Round round) { + for (final Long cvrID : new HashSet<>(round.auditSubsequence())) { + final Map auditReasons = new HashMap<>(); + final Set discrepancies = new HashSet<>(); + final Set disagreements = new HashSet<>(); + + CVRAuditInfo cvrai = Persistence.getByID(cvrID, CVRAuditInfo.class); + if (cvrai == null) { + cvrai = new CVRAuditInfo(Persistence.getByID(cvrID, CastVoteRecord.class)); + } + + if (cvrai.acvr() != null) { + // do the thing + // update the round statistics as necessary + for (final ComparisonAudit ca : cdb.comparisonAudits()) { + final String contestName = ca.contestResult().getContestName(); + AuditReason auditReason = ca.auditReason(); + + if (ca.isCovering(cvrID) && auditReason.isTargeted()) { + // If this CVR is interesting to this audit, the discrepancy + // should be in the audited contests part of the dashboard. + LOGGER.debug(String.format("[updateRound: CVR %d is covered in a targeted audit." + + " contestName=%s, auditReason=%s]", + cvrID, contestName, auditReason)); + auditReasons.put(contestName, auditReason); + } else { + // Otherwise, let's put it in the unaudited contest bucket. + auditReason = AuditReason.OPPORTUNISTIC_BENEFITS; + LOGGER.debug(String.format("[updateRound: CVR %d has a discrepancy; not covered by" + + " contestName=%s, auditReason=%s]", + cvrID, contestName, auditReason)); + auditReasons.put(contestName, auditReason); + } + + final OptionalInt discrepancy = ca.computeDiscrepancy(cvrai.cvr(), cvrai.acvr()); + if (!discrepancies.contains(auditReason) && discrepancy.isPresent()) { + discrepancies.add(auditReason); + } + + + final int multiplicity = ca.multiplicity(cvrID); + for (int i = 0; i < multiplicity; i++) { + round.addDiscrepancy(discrepancies); + round.addDisagreement(disagreements); + } + + cvrai.setMultiplicityByContest(ca.id(), multiplicity); + } + + for (final CVRContestInfo ci : cvrai.acvr().contestInfo()) { + final AuditReason reason = auditReasons.get(ci.contest().name()); + if (ci.consensus() == ConsensusValue.NO) { + // TODO check to see if we have disagreement problems. this + // is being added in the other loop. + disagreements.add(reason); + } + } + } + + Persistence.saveOrUpdate(cvrai); + } + } + + /** + * Audits a CVR/ACVR pair by adding it to all the audits in progress. + * This also updates the local audit counters, as appropriate. + * + * @param cdb The dashboard. + * @param auditInfo The CVRAuditInfo to audit. + * @param updateCounters true to update the county dashboard + * counters, false otherwise; false is used when this ballot + * has already been audited once. + * @return the number of times the record was audited. + */ + @SuppressWarnings({"PMD.ModifiedCyclomaticComplexity", "PMD.StdCyclomaticComplexity", + "PMD.NPathComplexity"}) + private static int audit(final CountyDashboard cdb, + final CVRAuditInfo auditInfo, + final boolean updateCounters) { + final Set contestDisagreements = new HashSet<>(); + final Set discrepancies = new HashSet<>(); + final Set disagreements = new HashSet<>(); + final CastVoteRecord cvrUnderAudit = auditInfo.cvr(); + final Long cvrID = cvrUnderAudit.id(); + final CastVoteRecord auditCvr = auditInfo.acvr(); + int totalCount = 0; + + // discrepancies + for (final ComparisonAudit ca : cdb.comparisonAudits()) { + AuditReason auditReason = ca.auditReason(); + final String contestName = ca.contestResult().getContestName(); + + // how many times does this cvr appear in the audit samples; how many dups? + final int multiplicity = ca.multiplicity(cvrID); + + // how many times does a discrepancy need to be recorded, while counting + // each sample(or occurance) only once - across rounds + final int auditCount = multiplicity - auditInfo.getCountByContest(ca.id()); + + // to report something to the caller + totalCount += auditCount; + + auditInfo.setMultiplicityByContest(ca.id(), multiplicity); + auditInfo.setCountByContest(ca.id(), multiplicity); + + final OptionalInt discrepancy = ca.computeDiscrepancy(cvrUnderAudit, auditCvr); + if (discrepancy.isPresent()) { + for (int i = 0; i < auditCount; i++) { + ca.recordDiscrepancy(auditInfo, discrepancy.getAsInt()); + } + + if (ca.isCovering(cvrID) && auditReason.isTargeted()) { + LOGGER.debug(String.format("[audit: CVR %d is covered in a targeted audit." + + " contestName=%s, auditReason=%s]", + cvrID, contestName, auditReason)); + discrepancies.add(auditReason); + } else { + auditReason = AuditReason.OPPORTUNISTIC_BENEFITS; + LOGGER.debug(String.format("[audit: CVR %d has a discrepancy, but isn't covered by" + + " contestName=%s, auditReason=%s]", + cvrID, contestName, auditReason)); + discrepancies.add(auditReason); + } + } + + // disagreements + for (final CVRContestInfo ci : auditCvr.contestInfo()) { + if (ci.consensus() == ConsensusValue.NO) { + contestDisagreements.add(ci.contest().name()); + } + } + + // NOTE: this may or may not be correct, we're not sure + if (contestDisagreements.contains(contestName)) { + for (int i = 0; i < auditCount; i++) { + ca.recordDisagreement(auditInfo); + } + if (ca.isCovering(cvrID) && auditReason.isTargeted()) { + LOGGER.debug(String.format("[audit: CVR %d is covered in a targeted audit." + + " contestName=%s, auditReason=%s]", + cvrID, contestName, auditReason)); + disagreements.add(auditReason); + } else { + auditReason = AuditReason.OPPORTUNISTIC_BENEFITS; + LOGGER.debug(String.format("[audit: CVR %d has a disagreement, but isn't covered by" + + " contestName=%s, auditReason=%s]", + cvrID, contestName, auditReason)); + disagreements.add(auditReason); + } + } + + ca.signalSampleAudited(auditCount, cvrID); + Persistence.saveOrUpdate(ca); + } + + // todo does this need to be in the loop? + auditInfo.setDiscrepancy(discrepancies); + auditInfo.setDisagreement(disagreements); + Persistence.saveOrUpdate(auditInfo); + + if (updateCounters) { + cdb.addDiscrepancy(discrepancies); + cdb.addDisagreement(disagreements); + LOGGER.debug(String.format("[audit: %s County discrepancies=%s, disagreements=%s]", + cdb.county().name(), discrepancies, disagreements)); + } + + return totalCount; + } + + /** + * "Unaudits" a CVR/ACVR pair by removing it from all the audits in + * progress in the specified county dashboard. This also updates the + * dashboard's counters as appropriate. + * + * @param the_cdb The county dashboard. + * @param the_info The CVRAuditInfo to unaudit. + */ + @SuppressWarnings("PMD.NPathComplexity") + private static int unaudit(final CountyDashboard the_cdb, final CVRAuditInfo the_info) { + final Set contest_disagreements = new HashSet<>(); + final Set discrepancies = new HashSet<>(); + final Set disagreements = new HashSet<>(); + final CastVoteRecord cvr_under_audit = the_info.cvr(); + final Long cvrID = cvr_under_audit.id(); + final CastVoteRecord audit_cvr = the_info.acvr(); + int totalCount = 0; + + for (final CVRContestInfo ci : audit_cvr.contestInfo()) { + if (ci.consensus() == ConsensusValue.NO) { + contest_disagreements.add(ci.contest().name()); + } + } + + for (final ComparisonAudit ca : the_cdb.comparisonAudits()) { + AuditReason auditReason = ca.auditReason(); + final String contestName = ca.contestResult().getContestName(); + + // how many times does this cvr appear in the audit samples; how many dups? + final int multiplicity = ca.multiplicity(cvr_under_audit.id()); + + // if the cvr has been audited, which is must have been to be here, then + final int auditCount = multiplicity; + + // to report something to the caller + totalCount += auditCount; + + final OptionalInt discrepancy = + ca.computeDiscrepancy(cvr_under_audit, audit_cvr); + if (discrepancy.isPresent()) { + for (int i = 0; i < auditCount; i++) { + ca.removeDiscrepancy(the_info, discrepancy.getAsInt()); + } + + if (ca.isCovering(cvrID) && auditReason.isTargeted()) { + LOGGER.debug(String.format("[audit: CVR %d is covered in a targeted audit." + + " contestName=%s, auditReason=%s]", + cvrID, contestName, auditReason)); + discrepancies.add(auditReason); + } else { + auditReason = AuditReason.OPPORTUNISTIC_BENEFITS; + LOGGER.debug(String.format("[audit: CVR %d has a discrepancy, but isn't covered by" + + " contestName=%s, auditReason=%s]", + cvrID, contestName, auditReason)); + discrepancies.add(auditReason); + } + } + if (contest_disagreements.contains(ca.contestResult().getContestName())) { + for (int i = 0; i < auditCount; i++) { + ca.removeDisagreement(the_info); + } + if (ca.isCovering(cvrID) && auditReason.isTargeted()) { + LOGGER.debug(String.format("[audit: CVR %d is covered in a targeted audit." + + " contestName=%s, auditReason=%s]", + cvrID, contestName, auditReason)); + disagreements.add(auditReason); + } else { + auditReason = AuditReason.OPPORTUNISTIC_BENEFITS; + LOGGER.debug(String.format("[audit: CVR %d has a disagreement, but isn't covered by" + + " contestName=%s, auditReason=%s]", + cvrID, contestName, auditReason)); + disagreements.add(auditReason); + } + } + ca.signalSampleUnaudited(auditCount, cvr_under_audit.id()); + Persistence.saveOrUpdate(ca); + } + + the_info.setDisagreement(null); + the_info.setDiscrepancy(null); + the_info.resetCounted(); + Persistence.saveOrUpdate(the_info); + + the_cdb.removeDiscrepancy(discrepancies); + the_cdb.removeDisagreement(disagreements); + + return totalCount; + } + + /** + * Updates the current CVR to audit index of the specified county + * dashboard to the first CVR after the current CVR under audit that + * lacks an ACVR. This "audits" all the CVR/ACVR pairs it finds + * in between, and extends the sequence of ballots to audit if it + * reaches the end and the audit is not concluded. + * + * @param cdb The dashboard. + */ + public static void updateCVRUnderAudit(final CountyDashboard cdb) { + // start from where we are in the current round + final Round round = cdb.currentRound(); + + if (round != null) { + final Set checked_ids = new HashSet<>(); + int index = round.actualAuditedPrefixLength() - round.startAuditedPrefixLength(); + + while (index < round.auditSubsequence().size()) { + final Long cvr_id = round.auditSubsequence().get(index); + if (!checked_ids.contains(cvr_id)) { + checked_ids.add(cvr_id); + + final CVRAuditInfo cai = Persistence.getByID(cvr_id, CVRAuditInfo.class); + + if (cai == null || cai.acvr() == null) { + break; // ok, so this hasn't been audited yet. + } else { + final int audit_count = audit(cdb, cai, false); + cdb.setAuditedSampleCount(cdb.auditedSampleCount() + audit_count); + } + } + index = index + 1; + } + // FIXME audited prefix length might not mean the same things that + // it once meant. + cdb.setAuditedPrefixLength(index + round.startAuditedPrefixLength()); + cdb.updateAuditStatus(); + } + } + + /** + * Checks that the specified CVR and ACVR are an audit pair, and that + * the specified ACVR is auditor generated. + * + * @param the_cvr The CVR. + * @param the_acvr The ACVR. + */ + private static boolean checkACVRSanity(final CastVoteRecord the_cvr, + final CastVoteRecord the_acvr) { + return the_cvr.isAuditPairWith(the_acvr) && + (the_acvr.recordType().isAuditorGenerated() + || the_acvr.recordType().isSystemGenerated()) + ; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/controller/ContestCounter.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/controller/ContestCounter.java new file mode 100644 index 00000000..6494e792 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/controller/ContestCounter.java @@ -0,0 +1,214 @@ +/** Copyright (C) 2018 the Colorado Department of State **/ +package us.freeandfair.corla.controller; + + +import java.math.BigDecimal; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map.Entry; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import us.freeandfair.corla.math.Audit; +import us.freeandfair.corla.model.ContestResult; +import us.freeandfair.corla.model.CountyContestResult; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.query.BallotManifestInfoQueries; +import us.freeandfair.corla.query.ContestResultQueries; + +public final class ContestCounter { + /** + * Class-wide logger + */ + public static final Logger LOGGER = + LogManager.getLogger(ContestCounter.class); + + /** prevent contruction **/ + private ContestCounter() { + } + + /** + * Group all CountyContestResults by contest name and tally the votes + * across all counties that have reported results. + * + * @return List A high level view of contests and their + * participants. + */ + public static List countAllContests() { + return + Persistence.getAll(CountyContestResult.class) + .stream() + .collect(Collectors.groupingBy(x -> x.contest().name())) + .entrySet() + .stream() + .map(ContestCounter::countContest) + .collect(Collectors.toList()); + } + + /** + * Calculates all the pairwise margins - like a cross product - using + * the vote totals. When there are no losers, all margins are zero. + * + * @param winners those who won the contest + * @param losers those who did not win the contest + * @param voteTotals a map of choice name to number of votes received + * @return a Set of Integers representing all margins between winners + * and losers + */ + public static Set pairwiseMargins(final Set winners, + final Set losers, + final Map voteTotals) { + final Set margins = new HashSet<>(); + + if (losers.isEmpty()) { + margins.add(0); + } else { + for (final String w : winners) { + for (final String l : losers) { + margins.add(voteTotals.get(w) - voteTotals.get(l)); + } + } + } + + return margins; + } + + /** + * Set voteTotals on CONTEST based on all counties that have that + * Contest name in their uploaded CVRs + **/ + public static ContestResult + countContest(final Map.Entry> countyContestResults) { + final String contestName = countyContestResults.getKey(); + final ContestResult contestResult = ContestResultQueries.findOrCreate(contestName); + + final Map voteTotals = + accumulateVoteTotals(countyContestResults.getValue().stream() + .map((cr) -> cr.voteTotals()) + .collect(Collectors.toList())); + contestResult.setVoteTotals(voteTotals); + + int numWinners; + final Set winnersAllowed = countyContestResults.getValue().stream() + .map(x -> x.winnersAllowed()) + .collect(Collectors.toSet()); + + if (winnersAllowed.isEmpty()) { + LOGGER.error(String.format("[countContest: %s doesn't have any winners allowed." + + " Assuming 1 allowed! Check the CVRS!", contestName)); + numWinners = 1; + } else { + if (winnersAllowed.size() > 1) { + LOGGER.error(String.format("[countContest: County results for %s contain different" + + " numbers of winners allowed: %s. Check the CVRS!", + contestName, winnersAllowed)); + } + numWinners = Collections.max(winnersAllowed); + } + + contestResult.setWinnersAllowed(numWinners); + contestResult.setWinners(winners(voteTotals, numWinners)); + contestResult.setLosers(losers(voteTotals, contestResult.getWinners())); + + contestResult.addContests(countyContestResults.getValue().stream() + .map(cr -> cr.contest()) + .collect(Collectors.toSet())); + contestResult.addCounties(countyContestResults.getValue().stream() + .map(cr -> cr.county()) + .collect(Collectors.toSet())); + + final Long ballotCount = BallotManifestInfoQueries.totalBallots(contestResult.countyIDs()); + final Set margins = pairwiseMargins(contestResult.getWinners(), + contestResult.getLosers(), + voteTotals); + final Integer minMargin = Collections.min(margins); + final Integer maxMargin = Collections.max(margins); + final BigDecimal dilutedMargin = Audit.dilutedMargin(minMargin, ballotCount); + // dilutedMargin of zero is ok here, it means the contest is uncontested + // and the contest will not be auditable, so samples should not be selected for it + contestResult.setBallotCount(ballotCount); + contestResult.setMinMargin(minMargin); + contestResult.setMaxMargin(maxMargin); + contestResult.setDilutedMargin(dilutedMargin); + + if (ballotCount == 0L) { + LOGGER.error(String.format("[countContest: %s has no ballot manifests for" + + " countyIDs: %s", contestName, contestResult.countyIDs())); + } + + return contestResult; + } + + /** add em up **/ + public static Map + accumulateVoteTotals(final List> voteTotals) { + final Map acc = new HashMap(); + return voteTotals.stream().reduce(acc, + (a, vt) -> addVoteTotal(a, vt)); + } + + /** add one vote total to another **/ + public static Map addVoteTotal(final Map acc, + final Map vt) { + // we iterate over vt because it may have a key that the accumulator has not + // seen yet + vt.forEach((k,v) -> acc.merge(k, v, + (v1,v2) -> { return (null == v1) ? v2 : v1 + v2; })); + return acc; + } + + /** + * Ranks a list of the choices in descending order by number of votes + * received. + **/ + public static List> rankTotals(final Map voteTotals) { + return voteTotals.entrySet().stream() + .sorted(Collections.reverseOrder(Entry.comparingByValue())) + .collect(Collectors.toList()); + } + + /** + * Find the set of winners for the ranking of voteTotals. Assumes only + * one winner allowed. + * + * @param voteTotals a map of choice name to number of votes + */ + public static Set winners(final Map voteTotals) { + return winners(voteTotals, 1); + } + + /** + * Find the set of winners for the ranking of voteTotals + * + * @param voteTotals a map of choice name to number of votes + * @param winnersAllowed how many can win this contest? + */ + public static Set winners(final Map voteTotals, + final Integer winnersAllowed) { + return rankTotals(voteTotals).stream() + .limit(winnersAllowed) + .map(Entry::getKey) + .collect(Collectors.toSet()); + } + + /** + * Find the set of losers given a ranking of voteTotals and some set + * of contest winners. + * + * @param voteTotals a map of choice name to number of votes + * @param winners the choices that aren't losers + */ + public static Set losers(final Map voteTotals, + final Set winners) { + final Set l = new HashSet(); + l.addAll((Set)voteTotals.keySet()); + l.removeAll(winners); + return l; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/controller/DeleteFileController.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/controller/DeleteFileController.java new file mode 100644 index 00000000..408978d4 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/controller/DeleteFileController.java @@ -0,0 +1,162 @@ +package us.freeandfair.corla.controller; + + +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import us.freeandfair.corla.asm.ASMEvent.CountyDashboardEvent; +import us.freeandfair.corla.asm.ASMUtilities; +import us.freeandfair.corla.asm.CountyDashboardASM; +import us.freeandfair.corla.model.CountyDashboard; +import us.freeandfair.corla.model.DoSDashboard; +import us.freeandfair.corla.model.ImportStatus; +import us.freeandfair.corla.model.ImportStatus.ImportState; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.query.BallotManifestInfoQueries; +import us.freeandfair.corla.query.CastVoteRecordQueries; +import us.freeandfair.corla.query.CountyContestResultQueries; + +/** + * + */ +public abstract class DeleteFileController { + + /** + * Class-wide logger + */ + public static final Logger LOGGER = + LogManager.getLogger(DeleteFileController.class); + + /** + * Perform all the steps to undo a file upload. + * fileType can be either "bmi" or "cvr" + * returns true if all the steps succeeded, false if one or more failed + * if any steps don't succeed they will throw a DeleteFileFail exception + */ + public static Boolean deleteFile(final Long countyId, final String fileType) + throws DeleteFileFail { + if ("cvr".equals(fileType)) { + LOGGER.info("deleting a CVR file for countyId: " + countyId); + + // deleteCastVoteRecords will also delete cvr_contest_infos due to + // constraints, also, deleteCastVoteRecords needs to be called before + // contests are deleted due to constraints + deleteCastVoteRecords(countyId); + deleteResultsAndContests(countyId); + } else if ("bmi".equals(fileType)) { + LOGGER.info("deleting a BMI file for countyId: " + countyId); + deleteBallotManifestInfos(countyId); + } else { + throw new DeleteFileFail("Did not recognize fileType: " + fileType); + } + + resetDashboards(countyId, fileType); + return true; + } + + /** reset cvr file info or bmi info on the county dashboard **/ + public static Boolean resetDashboards(final Long countyId, final String fileType) + throws DeleteFileFail { + final CountyDashboard cdb = Persistence.getByID(countyId, CountyDashboard.class); + if ("cvr".equals(fileType)) { + resetDashboardCVR(cdb); + final DoSDashboard dosdb = Persistence.getByID(DoSDashboard.ID, DoSDashboard.class); + dosdb.removeContestsToAuditForCounty(cdb.county()); + LOGGER.debug("Removed contests to audit for county"); + } else if ("bmi".equals(fileType)) { + resetDashboardBMI(cdb); + } else { + throw new DeleteFileFail("Did not recognize fileType: " + fileType); + } + + // this must come after the other resetDashboard*s + reinitializeCDB(cdb); + return true; + } + + /** reset cvr file info on the county dashboard **/ + public static Boolean resetDashboardCVR(final CountyDashboard cdb) { + Persistence.delete(cdb.cvrFile()); + cdb.setCVRFile(null); + cdb.setCVRsImported(0); + cdb.setCVRImportStatus(new ImportStatus(ImportState.NOT_ATTEMPTED)); + LOGGER.debug("Updated the county dashboard to remove CVR stuff"); + return true; + } + + /** + * Re-initialize county dashboard ASM state based on newly-deleted files + * + * Uses an ASM shortcut if both are deleted, otherwises assumes that before + * the deletion, both the CVR and ballot manifests were uploaded successfully, + * and transitions the ASM backward to indicate removal of the respective + * files. + * + * @param cdb the county dashboard to modify + */ + public static void reinitializeCDB(final CountyDashboard cdb) { + final CountyDashboardASM cdbASM = + ASMUtilities.asmFor(CountyDashboardASM.class, String.valueOf(cdb.id())); + + // no CVR, no manifest + if (null == cdb.cvrFile() && null == cdb.manifestFile()) { + cdbASM.reinitialize(); + ASMUtilities.save(cdbASM); + // no CVR, yes manifest + } else if (null == cdb.cvrFile() && null != cdb.manifestFile()) { + cdbASM.stepEvent(CountyDashboardEvent.DELETE_CVRS_EVENT); + ASMUtilities.save(cdbASM); + // yes CVR, no manifest + } else if (null != cdb.cvrFile() && null == cdb.manifestFile()) { + cdbASM.stepEvent(CountyDashboardEvent.DELETE_BALLOT_MANIFEST_EVENT); + ASMUtilities.save(cdbASM); + } + } + + /** reset bmi info on the county dashboard **/ + public static Boolean resetDashboardBMI(final CountyDashboard cdb) { + Persistence.delete(cdb.manifestFile()); + cdb.setManifestFile(null); + cdb.setBallotsInManifest(0); + LOGGER.debug("Updated the county dashboard to remove BMI stuff"); + return true; + } + + /** + * Remove all CountyContestResults and Contests for a county + */ + public static void deleteResultsAndContests(final Long countyId) + throws DeleteFileFail { + // this will also delete the contests - surprise! + final Integer rowsDeleted = CountyContestResultQueries.deleteForCounty(countyId); + LOGGER.info(String.format("%d ContestResults and Contests deleted!", rowsDeleted)); + } + + /** + * Remove all CastVoteRecords for a county + */ + public static void deleteCastVoteRecords(final Long countyId) + throws DeleteFileFail { + final Integer rowsDeleted = CastVoteRecordQueries.deleteAll(countyId); + LOGGER.info(String.format("%d cvrs deleted!", rowsDeleted)); + } + + /** + * Remove all BallotManifestInfo for a county + * @param countyId + */ + public static void deleteBallotManifestInfos(final Long countyId) + throws DeleteFileFail { + final Integer rowsDeleted = BallotManifestInfoQueries.deleteMatching(countyId); + LOGGER.info(String.format("%d bmis deleted!", rowsDeleted)); + } + + /** used to abort the set of operations (transaction) **/ + public static class DeleteFileFail extends Exception { + + /** used to abort the set of operations (transaction) **/ + public DeleteFileFail(final String message) { + super(message); + } + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/controller/ImportFileController.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/controller/ImportFileController.java new file mode 100644 index 00000000..b0135632 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/controller/ImportFileController.java @@ -0,0 +1,235 @@ +package us.freeandfair.corla.controller; + +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import java.io.InputStream; +import java.io.InputStreamReader; + +import javax.persistence.PersistenceException; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.asm.ASMEvent.CountyDashboardEvent; +import us.freeandfair.corla.asm.CountyDashboardASM; +import us.freeandfair.corla.asm.ASMUtilities; +import us.freeandfair.corla.csv.DominionCVRExportParser; +import us.freeandfair.corla.csv.Result; +import us.freeandfair.corla.json.UploadedFileDTO; +import us.freeandfair.corla.model.County; +import us.freeandfair.corla.model.CountyDashboard; +import us.freeandfair.corla.model.ImportStatus; +import us.freeandfair.corla.model.ImportStatus.ImportState; +import us.freeandfair.corla.model.UploadedFile.FileStatus; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.query.CastVoteRecordQueries; +import us.freeandfair.corla.query.CountyContestResultQueries; +import us.freeandfair.corla.query.UploadedFileQueries; +import us.freeandfair.corla.util.UploadedFileStreamer; + +public class ImportFileController implements Runnable { + + /** + * Class-wide logger + */ + public static final Logger LOGGER = + LogManager.getLogger(ImportFileController.class); + + + private UploadedFileDTO uploadedFileDTO; + private Long countyId; + + /** + * Constructs a new ImportFileController for the given file info which can be run + * in a separate, independent, thread. + * + */ + public ImportFileController(final UploadedFileDTO upF) { + this.uploadedFileDTO = upF; + this.countyId = upF.getCountyId(); + } + + public void run() { + LOGGER.debug("run()"); + try { + // We need the endpoint transaction, which sets the cdb state to + // "importing", to finish first. This is an easy way to dodge updates to + // the same hibernate object across threads. I have no idea why this + // didn't work for the UploadedFile object. + Thread.sleep(1000); + + // There is lots of transaction management here because we want to be sure + // that an error gets written to the database and not rolled back. We also + // want to make sure hibernate does not overwrite any non-hibernate + // updates. We have non-hibernate updates because we are in a thread that + // shares a hibernate session with other threads which all want to use the + // same object: the UploadedFile. Instead of that UploadedFile, we have + // have a UploadedFileDTO(Data Transfer Object) which is not a hibernate + // object and does not get updates across threads. + Persistence.beginTransaction(); + cleanSlate(); + commit(); + + setCVRFile(); + commit(); + + runOnThread(); + Persistence.flush(); + Persistence.commitTransaction(); + } catch (final RuntimeException | java.lang.InterruptedException e) { + final Result result = new Result(); + result.success = false; + result.errorMessage = e.getClass() +" "+ e.getMessage(); + error(result); + Persistence.flush(); + Persistence.commitTransaction(); + } + } + + public void runOnThread() { + LOGGER.debug("runOnThread()"); + Result result = parse(); + if (result.success) { + success(result); + } else { + error(result); + } + } + + + // Note: setCVRFile must come after the cdb is saved so it isn't overwritten. + // We want CVRFile set, whether the import succeeds or fails, for CDOS to be able + // to examine it + public void setCVRFile() { + UploadedFileQueries.setCVRFileOnCounty(this.uploadedFileDTO); + } + + + /** + * Aborts the import with the specified error description. + * + * This does everything that deleteFile does, except for: + * cdb.setCVRFile(null) + * because the state may want to examine the file + * + * @param the_description The error description. + */ + public void error(final Result result) { + LOGGER.debug("error("+ result.errorMessage + ")"); + + Persistence.rollbackTransaction(); + Persistence.beginTransaction(); + + // if we are here because of a persistence exception.. + // commit will throw another so rollback what's been done + // record the result + commit(); + this.uploadedFileDTO.setStatus(FileStatus.FAILED.toString()); + this.uploadedFileDTO.setResult(result); + UploadedFileQueries.updateStatusAndResult(this.uploadedFileDTO); + commit(); + + + // update status and state machine + final CountyDashboard cdb = + Persistence.getByID(this.countyId, CountyDashboard.class); + final CountyDashboardASM cdb_asm = + ASMUtilities.asmFor(CountyDashboardASM.class, this.countyId.toString()); + + cdb.setCVRImportStatus(new ImportStatus(ImportState.FAILED, result.errorMessage)); + cdb.setCVRsImported(0); + cdb_asm.stepEvent(CountyDashboardEvent.CVR_IMPORT_FAILURE_EVENT); + ASMUtilities.save(cdb_asm); + Persistence.saveOrUpdate(cdb); + + // then delete any imported cvrs + cleanSlate(); + + LOGGER.error(result.errorMessage + this.uploadedFileDTO.toString()); + } + + + public void success(final Result result) { + + // update status and state machine + final CountyDashboard cdb = + Persistence.getByID(this.countyId, CountyDashboard.class); + final CountyDashboardASM cdb_asm = + ASMUtilities.asmFor(CountyDashboardASM.class, this.countyId.toString()); + + cdb.setCVRImportStatus(new ImportStatus(ImportState.SUCCESSFUL)); + cdb.setCVRsImported(result.importedCount); + cdb_asm.stepEvent(CountyDashboardEvent.CVR_IMPORT_SUCCESS_EVENT); + ASMUtilities.save(cdb_asm); + Persistence.saveOrUpdate(cdb); + + // record the result + commit(); + this.uploadedFileDTO.setStatus(FileStatus.IMPORTED.toString()); + this.uploadedFileDTO.setResult(result); + UploadedFileQueries.updateStatusAndResult(this.uploadedFileDTO); + commit(); + + LOGGER.info(result.importedCount + " CVRs parsed from file " + this.uploadedFileDTO.toString()); + } + + public void cleanSlate() { + LOGGER.debug("cleanSlate()"); + + CastVoteRecordQueries.deleteAll(this.countyId); + //seems like an extra saftey gaurantee is needed here to protect + //against foreign key violations, not sure why + commit(); + CountyContestResultQueries.deleteForCounty(this.countyId); + commit(); + } + + public void commit() { + Persistence.flush(); + Persistence.commitTransaction(); + Persistence.beginTransaction(); + } + + /** + * Parses an uploaded CVR export and attempts to persist it to the database. + * with the default impl UploadedFileStreamer + * + */ + public Result parse() { + UploadedFileStreamer ufs = new UploadedFileStreamer(this.uploadedFileDTO); + try { + (new Thread(ufs)).start(); + return parse(ufs.inputStream()); + } finally { + ufs.stop(); + } + } + + /** + * Parses an uploaded CVR export and attempts to persist it to the database. + * with the default given InputStreamReader + * + */ + public Result parse(final InputStream inputStream) { + LOGGER.debug("parse()"); + try { + InputStreamReader inputStreamReader = new InputStreamReader(inputStream, + "UTF-8"); + final DominionCVRExportParser parser = + new DominionCVRExportParser(inputStreamReader, + Persistence.getByID(this.countyId, + County.class), + Main.properties(), + true); + return parser.parse(); + } catch (final RuntimeException | java.io.IOException e) { + // we could make parse() catch all possible exceptions because it already + // catches some, but we'll keep this here for now as a short cut. + LOGGER.error(e.getMessage()); + LOGGER.error(e.getClass()); + Result parseResult = new Result(); + parseResult.success = false; + parseResult.errorMessage = "System Error"; + return parseResult; + } + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/crypto/HashChecker.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/crypto/HashChecker.java new file mode 100644 index 00000000..779bfd38 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/crypto/HashChecker.java @@ -0,0 +1,101 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 13, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.crypto; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.math.BigInteger; +import java.security.DigestInputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +import us.freeandfair.corla.Main; + +/** + * Generate a SHA-256 hash of a given file. + * + * @author Joseph R. Kiniry + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public final class HashChecker { + /** + * The size of the buffer that we repeatedly read into to compute the SHA-256 + * of a file. + */ + public static final int BUFFER_SIZE = 8192; + + /** + * Private constructor to prevent instantiation. + */ + private HashChecker() { + // empty + } + + /** + * @trace cryptography.sha256 + * @param a_filename The name of the file to read. + * @return the SHA-256 hash of `a_filename`, encoded as a hexadecimal string, + * or null if the file cannot be hashed. + */ + public static String hashFile(final String a_filename) + throws Exception { + return hashFile(new File(a_filename)); + } + + /** + * @trace cryptography.sha256 + * @param a_file the file to read. + * @return the SHA-256 hash of `a_file`, encoded as a hexadecimal string, + * or null if the file cannot be hashed. + */ + public static String hashFile(final File a_file) + throws FileNotFoundException,IOException,NoSuchAlgorithmException { + String result = null; + final byte[] buffer = new byte[BUFFER_SIZE]; + + try { + final MessageDigest md = MessageDigest.getInstance("SHA-256"); + + try ( + final InputStream is = new FileInputStream(a_file); + final DigestInputStream dis = new DigestInputStream(is, md); + ) { + int bytes; + do { + bytes = dis.read(buffer); + } while (bytes != -1); + final BigInteger bi = new BigInteger(1, md.digest()); + result = String.format("%0" + (md.digest().length << 1) + "X", bi); + } catch (final FileNotFoundException e) { + Main.LOGGER.error("File to hash '" + a_file + + "' disappeared before it could be hashed."); + + throw e; //hash or die + } catch (final IOException e) { + Main.LOGGER.error("Unable to close file '" + a_file + + "' after hashing it."); + throw e; //hash or die + } + + } catch (final NoSuchAlgorithmException e) { + Main.LOGGER.error("No Java security framework installed."); + Main.LOGGER.error("Unable to compute SHA-256 hashes."); + throw e; //hash or die + } + + return result; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/crypto/PseudoRandomNumberGenerator.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/crypto/PseudoRandomNumberGenerator.java new file mode 100644 index 00000000..2ddaa305 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/crypto/PseudoRandomNumberGenerator.java @@ -0,0 +1,194 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 25, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joey Dodds + * @description A system to assist in conducting state-wide risk-limiting audits. + */ + +package us.freeandfair.corla.crypto; + +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.LinkedList; +import java.util.List; + +/** + * A pseudo-random number generator based on Philip Stark's pseudo-random number + * generator found at + * + * https://www.stat.berkeley.edu/~stark/Java/Html/sha256Rand.htm. + * + * @author Joey Dodds + * @author Joseph R. Kiniry + * @version 1.0.0 + * @review kiniry Why is this not just a static class? + */ +public class PseudoRandomNumberGenerator { + /** + * The minimum seed length specified in CRLS, and hence our formal specification, + * is 20 characters. + * @trace corla.randomness.seed + */ + public static final int MINIMUM_SEED_LENGTH = 20; + + /** + * The message digest we will use for generating hashes. + */ + private MessageDigest my_sha256_digest; + + /** + * The random numbers generated so far. + */ + private final List my_random_numbers; + + /** + * The current number to use for generation. + */ + //@ private invariant 0 <= my_count; + private int my_count; + + /** + * True if we should "replace" drawn numbers once they are drawn. True allows + * repeats. + */ + private final boolean my_with_replacement; + + /** + * The seed given, which must be at least of length MININUM_SEED_LENGTH and + * whose contents must only be digits. + */ + //@ private invariant MINIMUM_SEED_LENGTH <= my_seed.length(); + //@ private invariant seedOnlyContainsDigits(my_seed); + private final String my_seed; + + /** + * The minimum value to generate. + */ + private final int my_minimum; + + /** + * The maximum value to generate. + */ + private final int my_maximum; + + //@ private invariant my_minimum <= my_maximum; + + /** + * The maximum index that can be generated without replacement. + */ + private final int my_maximum_index; + + /** + * Create a pseudo-random number generator with functionality identical to + * Rivest's sampler.py example implementation in Python of an + * RLA sampler. + * + * @param the_seed The seed to generate random numbers from + * @param the_with_replacement True if duplicates can be generated + * @param the_minimum The minimum value to generate + * @param the_maximum The maximum value to generate + */ + //@ requires 20 <= the_seed.length(); + //@ requires seedOnlyContainsDigits(the_seed); + //@ requires the_minimum <= the_maximum; + public PseudoRandomNumberGenerator(final String the_seed, + final boolean the_with_replacement, + final int the_minimum, + final int the_maximum) { + // @trace randomness.seed side condition + assert MINIMUM_SEED_LENGTH <= the_seed.length(); + try { + my_sha256_digest = MessageDigest.getInstance("SHA-256"); + } catch (final NoSuchAlgorithmException e) { + assert false; + } + my_random_numbers = new LinkedList(); + my_with_replacement = the_with_replacement; + my_seed = the_seed; + assert the_minimum < the_maximum; + my_minimum = the_minimum; + my_maximum = the_maximum; + my_maximum_index = my_maximum - my_minimum + 1; + } + + /** + * Generate the specified list of random numbers. + * + * @param the_from the "index" of the first random number to give + * @param the_to the "index" of the final random number to give + * + * @return A list containing the_to - the_from + 1 random numbers + */ + //@ requires the_from <= the_to; + // @todo kiniry Refine this specification to include public model fields. + // requires my_with_replacement || the_to <= my_maximum_index; + public List getRandomNumbers(final int the_from, final int the_to) { + assert the_from <= the_to; + assert my_with_replacement || the_to <= my_maximum_index; + if (the_to + 1 > my_random_numbers.size()) { + extendList(the_to + 1); + } + // subList has an exclusive upper bound, but we have an inclusive one + return my_random_numbers.subList(the_from, the_to + 1); + } + + /** + * A helper function to extend the list of generated random numbers. + * @param the_length the number of random numbers to generate. + */ + //@ private behavior + //@ requires 0 <= the_length; + //@ ensures my_random_numbers.size() == the_length; + private void extendList(final int the_length) { + while (my_random_numbers.size() < the_length) { + generateNext(); + } + } + + /** + * Attempt to generate the next random number. This will either extend the + * list of random numbers in length or leave it the same. It will always + * advance the count. + */ + public void generateNext() { + my_count++; + assert my_with_replacement || my_count <= my_maximum_index; + + final String hash_input = my_seed + "," + my_count; + + final byte[] hash_output = + my_sha256_digest.digest(hash_input.getBytes(StandardCharsets.UTF_8)); + final BigInteger int_output = new BigInteger(1, hash_output); + + final BigInteger in_range = + int_output.mod(BigInteger.valueOf(my_maximum - my_minimum + 1)); + final int pick = my_minimum + in_range.intValueExact(); + + if (my_with_replacement || !my_random_numbers.contains(pick)) { + my_random_numbers.add(pick); + } + } + + /** + * Checks to see if the passed potential seed only contains digits. + * @param the_seed is the seed to check. + */ + /*@ behavior + @ ensures (\forall int i; 0 <= i && i < the_seed.length(); + @ Character.isDigit(the_seed.charAt(i))); + @*/ + public /*@ pure @*/ static boolean seedOnlyContainsDigits(final String the_seed) { + for (int i = 0; i < the_seed.length(); i++) { + if (!Character.isDigit(the_seed.charAt(i))) { + return false; + } + } + return true; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/csv/BallotManifestParser.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/csv/BallotManifestParser.java new file mode 100644 index 00000000..d3d7eb54 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/csv/BallotManifestParser.java @@ -0,0 +1,45 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 25, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joey Dodds + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.csv; + +import java.util.OptionalInt; + +/** + * A common interface to parsers for ballot manifest info in various formats. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public interface BallotManifestParser { + /** + * Parse the CVRs. The export data and other supplementary information is provided + * to the CVR parser in some way outside this interface (such as a constructor + * call). + * + * @return true if the parse was successful, false otherwise. + */ + boolean parse(); + + /** + * The number of records parsed from the ballot manifest file. + * + * @return the number of records; empty if parsing has not yet occurred. + */ + OptionalInt recordCount(); + + /** + * The number of ballots represented by the parsed ballot manifest records. + * + * @return the number of ballots; empty if parsing has not yet occurred. + */ + OptionalInt ballotCount(); +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/csv/CVRExportParser.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/csv/CVRExportParser.java new file mode 100644 index 00000000..500bca55 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/csv/CVRExportParser.java @@ -0,0 +1,45 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 25, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joey Dodds + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.csv; + +import java.util.OptionalInt; + +/** + * A common interface to parsers for CVR info in various formats. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public interface CVRExportParser { + /** + * Parse the CVRs. The export data and other supplementary information is provided + * to the CVR parser in some way outside this interface (such as a constructor + * call). + * + * @return true if the parse was successful, false otherwise. + */ + boolean parse(); + + /** + * The number of records parsed from the CVR file. + * + * @return the number of records; empty if parsing has not yet occurred. + */ + OptionalInt recordCount(); + + /** + * The error message, if any, generated by the parser. + * + * @return the error message, or null if there was no error. + */ + String errorMessage(); +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/csv/ColoradoBallotManifestParser.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/csv/ColoradoBallotManifestParser.java new file mode 100644 index 00000000..4349b6db --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/csv/ColoradoBallotManifestParser.java @@ -0,0 +1,233 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 25, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joey Dodds + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.csv; + +import java.io.IOException; +import java.io.Reader; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.OptionalInt; +import java.util.Set; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import us.freeandfair.corla.model.BallotManifestInfo; +import us.freeandfair.corla.persistence.Persistence; + +/** + * The parser for Colorado ballot manifests. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public class ColoradoBallotManifestParser { + /** + * Class-wide logger + */ + public static final Logger LOGGER = + LogManager.getLogger(ColoradoBallotManifestParser.class); + + /** + * The size of a batch of ballot manifests to be flushed to the database. + */ + private static final int BATCH_SIZE = 50; + + /** + * The column containing the scanner ID. + */ + private static final int SCANNER_ID_COLUMN = 1; + + /** + * The column containing the batch number. + */ + private static final int BATCH_NUMBER_COLUMN = 2; + + /** + * The column containing the number of ballots in the batch. + */ + private static final int NUM_BALLOTS_COLUMN = 3; + + /** + * The column containing the storage location. + */ + private static final int BATCH_LOCATION_COLUMN = 4; + + /** + * The parser to be used. + */ + private final CSVParser my_parser; + + /** + * The county ID to apply to the parsed manifest lines. + */ + private final Long my_county_id; + + /** + * The number of ballots represented by the parsed records. + */ + private int my_ballot_count = -1; + + /** + * The set of parsed ballot manifests that haven't yet been flushed to the + * database. + */ + private final Set my_parsed_manifests = new HashSet<>(); + + /** + * Construct a new Colorado ballot manifest parser using the specified Reader. + * + * @param the_reader The reader from which to read the CSV to parse. + * @param the_timestamp The timestamp to apply to the parsed records. + * @param the_county_id The county ID for the parsed records. + * @exception IOException if an error occurs while constructing the parser. + */ + public ColoradoBallotManifestParser(final Reader the_reader, + final Long the_county_id) + throws IOException { + my_parser = new CSVParser(the_reader, CSVFormat.DEFAULT); + my_county_id = the_county_id; + } + + /** + * Construct a new Colorado ballot manifest parser using the specified String. + * + * @param the_string The CSV string to parse. + * @param the_timestamp The timestamp to apply to the parsed records. + * @param the_county_id The county ID for the parsed records. + * @exception IOException if an error occurs while constructing the parser. + */ + public ColoradoBallotManifestParser(final String the_string, + final Long the_county_id) + throws IOException { + my_parser = CSVParser.parse(the_string, CSVFormat.DEFAULT); + my_county_id = the_county_id; + } + + /** + * Checks to see if the set of parsed manifests needs flushing, and does so + * if necessary. + */ + private void checkForFlush() { + if (my_parsed_manifests.size() % BATCH_SIZE == 0) { + Persistence.flush(); + for (final BallotManifestInfo bmi : my_parsed_manifests) { + Persistence.evict(bmi); + } + my_parsed_manifests.clear(); + } + } + + /** + * Extracts ballot manifest information from a single CSV line. + * + * @param the_line The CSV line. + * @param the_timestamp The timestamp to apply to the result. + * @return the extracted information. + */ + private BallotManifestInfo extractBMI(final CSVRecord the_line) { + BallotManifestInfo bmi = null; + + final int batch_size = Integer.parseInt(the_line.get(NUM_BALLOTS_COLUMN)); + final Long sequence_start; + if (my_ballot_count == 0) { + // this is the first row. also, sequence is not zero based + sequence_start = 1L; + } else { + // rest of the rows. also, batch sequences don't overlap or touch + sequence_start = Long.valueOf(my_ballot_count) + 1L; + } + // this is used to set my_ballot_count below + final Long sequence_end = sequence_start + Long.valueOf(batch_size) - 1L; + // TODO: should we check for mismatched county IDs between the + // one we were passed at construction and the county name string + // in the file? + bmi = new BallotManifestInfo(my_county_id, + Integer.parseInt(the_line.get(SCANNER_ID_COLUMN)), + the_line.get(BATCH_NUMBER_COLUMN), + batch_size, + the_line.get(BATCH_LOCATION_COLUMN), + sequence_start, + sequence_end); + Persistence.saveOrUpdate(bmi); + my_parsed_manifests.add(bmi); + checkForFlush(); + LOGGER.debug("parsed ballot manifest: " + bmi); + + return bmi; + } + + /** + * Parse the supplied data export. If it has already been parsed, this + * method returns immediately. + * + * @return true if the parse was successful, false otherwise + */ + public synchronized Result parse() { + final Result result = new Result(); + final Iterator records = my_parser.iterator(); + + int my_record_count = 0; + my_ballot_count = 0; + // bmi line may not have been initialized + CSVRecord bmi_line = null; + BallotManifestInfo bmi; + + try { + // we expect the first line to be the headers, which we currently discard + records.next(); + // subsequent lines contain ballot manifest info + while (records.hasNext()) { + bmi_line = records.next(); + bmi = extractBMI(bmi_line); + my_record_count = my_record_count + 1; + my_ballot_count = Math.toIntExact(bmi.sequenceEnd()); + } + + result.success = true; + result.importedCount = my_record_count; + } catch (final IllegalStateException | NoSuchElementException e) { + result.success = false; + result.errorMessage = e.getClass().toString() + " " + e.getMessage(); + result.errorRowNum = my_record_count; + if (null != bmi_line) { + final List values = new ArrayList<>(); + bmi_line.iterator().forEachRemaining(values::add); + result.errorRowContent = String.join(",", values); + } + // this log message is partially here to make findbugs happy. For some + // reason URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD would not be suppressed. + LOGGER.error(e.getClass().toString() + " " + e.getMessage() + + "\n line number: " + result.errorRowNum + + "\n content:" + result.errorRowContent); + } + + return result; + } + + /** + * {@inheritDoc} + */ + public synchronized OptionalInt ballotCount() { + if (my_ballot_count < 0) { + return OptionalInt.empty(); + } else { + return OptionalInt.of(my_ballot_count); + } + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/csv/ContestNameParser.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/csv/ContestNameParser.java new file mode 100644 index 00000000..72503fbd --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/csv/ContestNameParser.java @@ -0,0 +1,319 @@ +package us.freeandfair.corla.csv; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.apache.commons.io.IOUtils; +import org.apache.commons.io.input.BOMInputStream; +import org.apache.commons.io.input.ReaderInputStream; + + +/** + * A simple CSV parser built atop commons.csv + * + */ + +public class ContestNameParser { + /** + * A CSVParser + */ + private final CSVParser parser; + + /** + * The format of our CSV + */ + private final CSVFormat csvFormat = + CSVFormat + .DEFAULT + .withHeader(); + + /** + * A mapping of county to contests + */ + private final Map> contests = new TreeMap>(); + + /** + * A mapping of contest name to choices + */ + private final Map> choices = new TreeMap>(); + + /** + * A mapping of county to those duplicate contests + */ + private final Map> duplicates = new TreeMap>(); + + /** + * A set to hold our ParseErrors in line order + */ + private final SortedSet errors = new TreeSet(); + + /** + * A ContestNameParser can be built for a Reader + */ + public ContestNameParser(final Reader r) + throws IOException { + final Reader reader = + new InputStreamReader(new BOMInputStream(new ReaderInputStream(r)), "UTF-8"); + parser = new CSVParser(reader, csvFormat); + } + + /** + * A ContestNameParser can be built for a String + */ + public ContestNameParser(final String string) + throws IOException { + final Reader reader = + new InputStreamReader(new BOMInputStream(IOUtils.toInputStream(string)), "UTF-8"); + parser = new CSVParser(reader, csvFormat); + } + + /** + * a good practice + */ + @Override + public String toString() { + return String.format("[contests=%s; duplicates=%s; errors=%s]", + contests(), + duplicates(), + errors()); + } + /** + * @return OptionalInt Maybe the number of contests we found across + * all counties, maybe not. + */ + public OptionalInt contestCount() { + return contests + .values() + .stream() + .mapToInt((x) -> { + return x.size(); + }) + .reduce((a, b) -> { + return a + b; + }); + } + /** + * @return Map> A map of county to + * contests-within-county, sans duplicates. + */ + public Map> contests() { + return contests; + } + + /** add to the map **/ + public void addContest(final String countyName, final String contestName) { + final Set v = this.contests.getOrDefault(countyName, new TreeSet()); + final boolean newElement = v.add(contestName); + if (!newElement) { + addDuplicateContest(countyName, contestName); + } + this.contests.put(countyName, v); + } + + /** + * @return Map> A map of contest name to set of choices + **/ + public Map> getChoices() { + return this.choices; + } + + /** add to the map **/ + public void addChoices(final String contestName, final String... splitResult) { + final Set choiceNames = new HashSet(); + Collections.addAll(choiceNames, splitResult); + + this.choices.merge(contestName, choiceNames, + (s1,s2) -> { + s1.addAll(s2); + return s1; + }); + } + + /** add to the map **/ + public void addDuplicateContest(final String countyName, final String contestName) { + final Set v = this.duplicates.getOrDefault(countyName, new TreeSet()); + v.add(contestName); + this.duplicates.put(countyName, v); + } + + /** + * @return Map> A map of county to + * duplicate-contests-within-county. + */ + public Map> duplicates() { + return duplicates; + } + + /** + * @return SortedSet Any ParserErrors that may have + * occured, in line order. + */ + + public SortedSet errors() { + return errors; + } + + /** + * @return boolean Did the parser complete successfully? + */ + public boolean isSuccess() { + return errors.isEmpty() && duplicates.isEmpty(); + } + + /** + * Execute the parser, returning whether or not it was successful. + * TODO pure function? + * FIXME cyclomatic complexity + */ + public synchronized boolean parse() { + final Iterable records = parser; + + try { + for (final CSVRecord r : records) { + final Map record = r.toMap(); + final String countyName = record.getOrDefault("CountyName", ""); + final String contestName = record.getOrDefault("ContestName", ""); + final String choiceNames = record.getOrDefault("ContestChoices", ""); + + if (!choiceNames.isEmpty()){ + addChoices(contestName, choiceNames.split("\\s*,\\s*")); + } + + if (countyName.isEmpty() || contestName.isEmpty()) { + errors.add(new ParseError("malformed record: (" + record + ")", + parser.getCurrentLineNumber())); + break; + } else { + addContest(countyName, contestName); + } + } + } catch (final NoSuchElementException e) { + errors.add(new ParseError("Could not parse contests file", + parser.getCurrentLineNumber(), e)); + } + + return this.isSuccess(); + } + + /** + * A ParseError is an object that we can put in a sorted set + */ + public static class ParseError implements Comparable { + /** + * a message about the ParseError + */ + private final String msg; + + /** + * The line on which the error occured. In some cases - where a + * CSV field contains linebreaks - the line number may be + * nonsensical. For single line records, you're OK. + */ + private final long line; + + /** + * The exception related to a ParseError, if any. + */ + private final Optional e; + + /** + * A ParseError can be just a message and a line number. + */ + ParseError(final String msg, final long n) { + this.msg = msg; + this.line = n; + this.e = Optional.empty(); + } + + /** + * A ParseError can be a message, line number, and an Exception. + */ + ParseError(final String msg, final long n, final Exception e) { + this.msg = msg; + this.line = n; + this.e = Optional.of(e); + } + + /** + * a good practice + */ + @Override + public boolean equals(final Object o) { + if (o == this) { + return true; + } + + if (!(o instanceof ParseError)) { + return false; + } + + final ParseError pe = (ParseError) o; + return pe.line == this.line && pe.msg == this.msg; + } + + /** + * a good practice + */ + @Override + public int hashCode() { + return Long.hashCode(line) + 31 + msg.hashCode(); + } + + /** + * ParseErrors are comparable by the tuple [line number, message]. + */ + public int compareTo(final ParseError pe) { + if (this == pe) { + return 0; + } + + int result = Long.compare(this.line, pe.line); + if (result == 0) { + result = String.CASE_INSENSITIVE_ORDER.compare(this.msg, pe.msg); + } + + return result; + } + + /** + * @return Optional e Maybe the exception thrown, + * maybe not. + */ + public Optional getException() { + return e; + } + + /** + * @return long The line number of the ParseError + */ + public long getLine() { + return line; + } + + /** + * @return String A printable representation + */ + @Override + public String toString() { + if (e.isPresent()) { + return msg + " on line " + line + ". Exception: " + e; + } else { + return msg + " on line " + line; + } + } + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/csv/DominionCVRExportParser.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/csv/DominionCVRExportParser.java new file mode 100644 index 00000000..7da9172c --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/csv/DominionCVRExportParser.java @@ -0,0 +1,769 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 25, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joey Dodds + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.csv; + +import java.io.IOException; +import java.io.Reader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +import javax.persistence.PersistenceException; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.text.WordUtils; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import us.freeandfair.corla.model.CVRContestInfo; +import us.freeandfair.corla.model.CastVoteRecord; +import us.freeandfair.corla.model.CastVoteRecord.RecordType; +import us.freeandfair.corla.model.Choice; +import us.freeandfair.corla.model.Contest; +import us.freeandfair.corla.model.County; +import us.freeandfair.corla.model.CountyContestResult; +import us.freeandfair.corla.model.CountyDashboard; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.query.CountyContestResultQueries; +import us.freeandfair.corla.util.DBExceptionUtil; +import us.freeandfair.corla.util.ExponentialBackoffHelper; + +/** + * Parser for Dominion CVR export files. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings({"PMD.GodClass", "PMD.CyclomaticComplexity", "PMD.ExcessiveImports", + "PMD.ModifiedCyclomaticComplexity", "PMD.StdCyclomaticComplexity"}) +public class DominionCVRExportParser { + /** + * Class-wide logger + */ + public static final Logger LOGGER = + LogManager.getLogger(DominionCVRExportParser.class); + + /** + * The name of the transaction size property. + */ + public static final String TRANSACTION_SIZE_PROPERTY = "cvr_import_transaction_size"; + + /** + * The name of the batch size property. + */ + public static final String BATCH_SIZE_PROPERTY = "cvr_import_batch_size"; + + /** + * The number of times to retry a county dashboard update operation. + */ + private static final int UPDATE_RETRIES = 15; + + /** + * The number of milliseconds to sleep between transaction retries. + */ + private static final long TRANSACTION_SLEEP_MSEC = 10; + + /** + * The interval at which to log progress. + */ + private static final int PROGRESS_INTERVAL = 500; + + /** + * The default size of a batch of CVRs to be flushed to the database. + */ + private static final int DEFAULT_BATCH_SIZE = 80; + + /** + * The default size of a batch of CVRs to be committed as a transaction. + */ + private static final int DEFAULT_TRANSACTION_SIZE = 400; + + /** + * The column containing the CVR number in a Dominion export file. + */ + private static final String CVR_NUMBER_HEADER = "CvrNumber"; + + /** + * The column containing the tabulator number in a Dominion export file. + */ + private static final String TABULATOR_NUMBER_HEADER = "TabulatorNum"; + + /** + * The column containing the batch ID in a Dominion export file. + */ + private static final String BATCH_ID_HEADER = "BatchId"; + + /** + * The column containing the record ID in a Dominion export file. + */ + private static final String RECORD_ID_HEADER = "RecordId"; + + /** + * The column containing the imprinted ID in a Dominion export file. + */ + private static final String IMPRINTED_ID_HEADER = "ImprintedId"; + + /** + * The column containing the counting group in a Dominion export file. + */ + private static final String COUNTING_GROUP_HEADER = "CountingGroup"; + + /** + * The column containing the precinct portion in a Dominion export file. + */ + @SuppressWarnings({"PMD.UnusedPrivateField", "unused"}) + private static final String PRECINCT_PORTION_HEADER = "PrecinctPortion"; + + /** + * The column containing the ballot type in a Dominion export file. + */ + private static final String BALLOT_TYPE_HEADER = "BallotType"; + + /** + * The prohibited headers. + */ + private static final String[] PROHIBITED_HEADERS = {COUNTING_GROUP_HEADER}; + + /** + * The required headers. + */ + private static final String[] REQUIRED_HEADERS = { + CVR_NUMBER_HEADER, TABULATOR_NUMBER_HEADER, BATCH_ID_HEADER, + RECORD_ID_HEADER, IMPRINTED_ID_HEADER, BALLOT_TYPE_HEADER + }; + + /** + * The parser to be used. + */ + private final CSVParser my_parser; + + /** + * The map from column names to column numbers. + */ + private final Map my_columns = new HashMap(); + + /** + * The index of the first choice/contest column. + */ + private int my_first_contest_column; + + /** + * The list of contests parsed from the supplied data export. + */ + private final List my_contests = new ArrayList(); + + /** + * The list of county contest results we build from the supplied + * data export. + */ + private final List my_results = + new ArrayList(); + + /** + * The county whose CVRs we are parsing. + */ + private final County my_county; + + /** + * The number of parsed CVRs. + */ + private int my_record_count = -1; + + /** + * The set of parsed CVRs that haven't yet been flushed to the database. + */ + private final Set my_parsed_cvrs = new HashSet<>(); + + /** + * The size of a batch of CVRs to be flushed to the database. + */ + private final int my_batch_size; + + /** + * The size of a batch of CVRs to be committed as a transaction. + */ + private final int my_transaction_size; + + /** + * A flag that indicates whether the parse is processed as multiple + * transactions. + */ + private final boolean my_multi_transaction; + + /** + * Construct a new Dominion CVR export parser using the specified Reader, + * for CVRs provided by the specified county. + * + * @param the_reader The reader from which to read the CSV to parse. + * @param the_county The county whose CVRs are to be parsed. + * @param the_properties The properties from which to read any overrides to the + * default transaction and batch sizes. + * @param the_multi_transaction true to commit the CVRs in multiple transactions, + * false otherwise. If this is true, the parser assumes that a transaction is + * in progress when invoked, and periodically commits that transaction and + * starts a new one to continue parsing, leaving a transaction open at completion. + * @exception IOException if an error occurs while constructing the parser. + */ + public DominionCVRExportParser(final Reader the_reader, final County the_county, + final Properties the_properties, + final boolean the_multi_transaction) + throws IOException { + my_parser = new CSVParser(the_reader, CSVFormat.DEFAULT); + my_county = the_county; + my_multi_transaction = the_multi_transaction; + my_batch_size = parseProperty(the_properties, BATCH_SIZE_PROPERTY, + DEFAULT_BATCH_SIZE); + my_transaction_size = parseProperty(the_properties, TRANSACTION_SIZE_PROPERTY, + DEFAULT_TRANSACTION_SIZE); + } + + /** + * Construct a new Dominion CVR export parser to parse the specified + * CSV string, for CVRs provided by the specified county. + * + * @param the_string The CSV string to parse. + * @param the_county The county whose CVRs are to be parsed. + * @exception IOException if an error occurs while constructing the parser. + */ + public DominionCVRExportParser(final String the_string, final County the_county) + throws IOException { + my_parser = CSVParser.parse(the_string, CSVFormat.DEFAULT); + my_county = the_county; + my_multi_transaction = false; + my_batch_size = DEFAULT_BATCH_SIZE; + my_transaction_size = DEFAULT_TRANSACTION_SIZE; + } + + /** + * Parse an integer value from the specified property, returning the specified + * default if the property doesn't exist or is not an integer. + * + * @param the_properties The properties to use. + * @param the_property_name The name of the property to parse. + * @param the_default_value The default value. + */ + private int parseProperty(final Properties the_properties, + final String the_property_name, + final int the_default_value) { + int result; + + try { + result = Integer.parseInt(the_properties.getProperty(the_property_name, + String.valueOf(the_default_value))); + } catch (final NumberFormatException e) { + result = the_default_value; + } + + return result; + } + + /** + * Strip the '="..."' from a column. + * + * @param the_value The value to strip. + * @return the stripped value, as a String, or the original String if it + * does not have the '="..."' form. + */ + private String stripEqualQuotes(final String the_value) { + String result = the_value; + if (the_value.startsWith("=\"") && the_value.endsWith("\"")) { + result = the_value.substring(0, the_value.length() - 1).replaceFirst("=\"", ""); + } + return result; + } + + /** + * Updates the contest names, max selections, and choice counts structures. + * + * @param the_line The CSV line containing the contest information. + * @param the_names The contest names. + * @param the_votes_allowed The votes allowed table. + * @param the_choice_counts The choice counts table. + */ + private void updateContestStructures(final CSVRecord the_line, + final List the_names, + final Map the_votes_allowed, + final Map the_choice_counts) { + int index = my_first_contest_column; + do { + final String c = the_line.get(index); + int count = 0; + while (index < the_line.size() && + c.equals(the_line.get(index))) { + index = index + 1; + count = count + 1; + } + // get the contest name and "(Vote For=" number + String cn = c.substring(0, c.indexOf("(Vote For=")); + final String vf = c.replace(cn, "").replace("(Vote For=", "").replace(")", ""); + // clean up the contest name (we used it to get the number, before cleaning) + cn = cn.trim(); + int ms = 1; // this is our default maximum selections + try { + ms = Integer.parseInt(vf); + } catch (final NumberFormatException e) { + // ignored + } + the_names.add(cn); + the_choice_counts.put(cn, count); + the_votes_allowed.put(cn, ms); + } while (index < the_line.size()); + } + + /** + * Create contest and result objects for use later in parsing. + * + * @param choiceLine The CSV line containing the choice information. + * @param explanationLine The CSV line containing the choice explanations. + * @param contestNames The list of contest names. + * @param votesAllowed The table of votes allowed values. + * @param choiceCounts The table of contest choice counts. + */ + private Result addContests(final CSVRecord choiceLine, + final CSVRecord explanationLine, + final List contestNames, + final Map votesAllowed, + final Map choiceCounts) { + final Result result = new Result(); + int index = my_first_contest_column; + int contest_count = 0; + + for (final String contestName : contestNames) { + final List choices = new ArrayList(); + final int end = index + choiceCounts.get(contestName); + boolean isWriteIn = false; + + while (index < end) { + String choice =choiceLine.get(index).trim(); + final String explanation = explanationLine.get(index).trim(); + // "Write-in" is a fictitious candidate that denotes the beginning of + // the list of qualified write-in candidates + final boolean isFictitious = "Write-in".equalsIgnoreCase(choice); + choices.add(new Choice(choice, explanation, isWriteIn, isFictitious)); + if (isFictitious) { + // consider all subsequent choices in this contest to be qualified + // write-in candidates + isWriteIn = true; + } + index = index + 1; + } + // now that we have all the choices, we can create a Contest object for + // this contest (note the empty contest description at the moment, below, + // as that's not in the CVR files and may not actually be used) + // note that we're using the "Vote For" number as the number of winners + // allowed as well, because the Dominion format doesn't give us that + // separately + final Contest c = new Contest(contestName, my_county, "", choices, + votesAllowed.get(contestName), votesAllowed.get(contestName), + contest_count); + LOGGER.debug(String.format("[addContests: county=%s, contest=%s", my_county.name(), c)); + + contest_count = contest_count + 1; + try { + Persistence.saveOrUpdate(c); + final CountyContestResult r = CountyContestResultQueries.matching(my_county, c); + my_contests.add(c); + my_results.add(r); + } catch (PersistenceException pe) { + result.success = false; + result.errorMessage = StringUtils.abbreviate(DBExceptionUtil.getConstraintFailureReason(pe),250); + result.errorRowContent = StringUtils.abbreviate("Error adding " + c.shortToString(),250) ; + return result ; + } + } + result.success = true ; + return result ; + } + + /** + * Checks to see if the set of parsed CVRs needs flushing, and does so + * if necessary. + */ + private void checkForFlush() { + if (my_multi_transaction && my_record_count % my_transaction_size == 0) { + commitCVRsAndUpdateCountyDashboard(); + } + + if (my_record_count % my_batch_size == 0) { + Persistence.flush(); + for (final CastVoteRecord cvr : my_parsed_cvrs) { + Persistence.evict(cvr); + } + my_parsed_cvrs.clear(); + } + } + + /** + * Commits the currently outstanding CVRs and updates the county dashboard + * accordingly. + */ + private void commitCVRsAndUpdateCountyDashboard() { + // commit all the CVR records and contest tracking data + Persistence.commitTransaction(); + + boolean success = false; + int retries = 0; + while (!success && retries < UPDATE_RETRIES) { + try { + retries = retries + 1; + LOGGER.debug("updating county " + my_county.id() + " dashboard, attempt " + + retries); + Persistence.beginTransaction(); + final CountyDashboard cdb = + Persistence.getByID(my_county.id(), CountyDashboard.class); + // if we can't get a reference to the county dashboard, we've got problems - + // but we'll deal with them elsewhere + if (cdb == null) { + Persistence.rollbackTransaction(); + } else { + cdb.setCVRsImported(my_record_count); + Persistence.saveOrUpdate(cdb); + Persistence.commitTransaction(); + success = true; + } + } catch (final PersistenceException e) { + // something went wrong, let's try again + if (Persistence.canTransactionRollback()) { + try { + Persistence.rollbackTransaction(); + } catch (final PersistenceException ex) { + // not much we can do about it + } + } + // let's give other transactions time to breathe + try { + final long delay = + ExponentialBackoffHelper.exponentialBackoff(retries, TRANSACTION_SLEEP_MSEC); + LOGGER.info("retrying county " + my_county.id() + + " dashboard update in " + delay + "ms"); + Thread.sleep(delay); + } catch (final InterruptedException ex) { + // it's OK to be interrupted + } + } + } + // we always need a running transaction + Persistence.beginTransaction(); + if (success && retries > 1) { + LOGGER.info("updated state machine for county " + my_county.id() + + " in " + retries + " tries"); + } else if (!success) { + throw new PersistenceException("could not update state machine for county " + + my_county.id() + " after " + retries + " tries"); + } + } + + /** + * Extract a CVR from a line of the file. + * + * @param the_line The line representing the CVR. + * @return the resulting CVR. + */ + @SuppressWarnings("PMD.CyclomaticComplexity") + private CastVoteRecord extractCVR(final CSVRecord the_line) { + final int cvr_id = + Integer.parseInt( + stripEqualQuotes(the_line.get(my_columns.get(CVR_NUMBER_HEADER)))); + final int tabulator_id = + Integer.parseInt( + stripEqualQuotes( + the_line.get(my_columns.get(TABULATOR_NUMBER_HEADER)))); + final String batch_id = + stripEqualQuotes(the_line.get(my_columns.get(BATCH_ID_HEADER))); + final int record_id = + Integer.parseInt( + stripEqualQuotes(the_line.get(my_columns.get(RECORD_ID_HEADER)))); + final String imprinted_id = + stripEqualQuotes(the_line.get(my_columns.get(IMPRINTED_ID_HEADER))); + final String ballot_type = + stripEqualQuotes(the_line.get(my_columns.get(BALLOT_TYPE_HEADER))); + final List contest_info = new ArrayList(); + + // for each contest, see if choices exist on the CVR; "0" or "1" are + // votes or absences of votes; "" means that the contest is not in this style + int index = my_first_contest_column; + for (final Contest co : my_contests) { + boolean present = false; + final List votes = new ArrayList(); + for (final Choice ch : co.choices()) { + final String mark_string = the_line.get(index); + final boolean p = !mark_string.isEmpty(); + final boolean mark = "1".equals(mark_string); + present |= p; + if (!ch.fictitious() && p && mark) { + votes.add(ch.name()); + } + index = index + 1; + } + // if this contest was on the ballot, add it to the votes + if (present) { + contest_info.add(new CVRContestInfo(co, null, null, votes)); + } + } + + // we don't need to look for an existing CVR with this data because, + // by definition, there cannot be one unless the same line appears + // twice in the CVR export file... and if it does, we need it to + // appear twice here too. + final CastVoteRecord new_cvr = + new CastVoteRecord(RecordType.UPLOADED, null, my_county.id(), + cvr_id, my_record_count, tabulator_id, + batch_id, record_id, imprinted_id, + ballot_type, contest_info); + Persistence.saveOrUpdate(new_cvr); + my_parsed_cvrs.add(new_cvr); + + // add the CVR to all of our results + for (final CountyContestResult r : my_results) { + r.addCVR(new_cvr); + } + LOGGER.debug("parsed CVR: " + new_cvr); + return new_cvr; + } + + /** + * Processes the headers from the specified CSV record. This includes checking + * for the use of forbidden headers, and that all required headers are + * present. + * + * @return true if the headers are OK, false otherwise; this method also + * sets the error message if necessary. + */ + @SuppressWarnings({"PMD.AvoidLiteralsInIfCondition", "PMD.AvoidDeeplyNestedIfStmts", + "PMD.ModifiedCyclomaticComplexity", "PMD.CyclomaticComplexity", + "PMD.StdCyclomaticComplexity", "PMD.NPathComplexity"}) + private Result processHeaders(final CSVRecord the_line) { + final Result result = new Result(); + + // the explanations line includes the column names for the non-contest/choice + // columns, so let's get those + for (int i = 0; i < my_first_contest_column; i++) { + my_columns.put(the_line.get(i), i); + } + + // let's make sure none of our prohibited headers are present + final List prohibited_headers = new ArrayList<>(); + for (final String h : PROHIBITED_HEADERS) { + if (my_columns.get(h) != null) { + result.success = false; + prohibited_headers.add(h); + } + } + + // let's make sure no required headers are missing + final Set required_headers = + new HashSet<>(Arrays.asList(REQUIRED_HEADERS)); + for (final String header : REQUIRED_HEADERS) { + if (my_columns.get(header) != null) { + required_headers.remove(header); + } + } + + result.success = prohibited_headers.isEmpty() && required_headers.isEmpty(); + + if (!result.success) { + final StringBuilder sb = new StringBuilder(); + sb.append("malformed CVR file: "); + + if (!prohibited_headers.isEmpty()) { + sb.append("prohibited header"); + if (prohibited_headers.size() > 1) { + sb.append('s'); + } + sb.append(' '); + sb.append(stringList(prohibited_headers)); + sb.append(" present"); + if (!required_headers.isEmpty()) { + sb.append(", "); + } + } + + if (!required_headers.isEmpty()) { + sb.append("required header"); + if (required_headers.size() > 1) { + sb.append('s'); + } + sb.append(' '); + sb.append(stringList(required_headers)); + sb.append(" missing"); + } + + result.errorMessage = sb.toString(); + result.errorRowNum = Long.valueOf( the_line.getRecordNumber()).intValue(); + List values = new ArrayList<>(); + the_line.iterator().forEachRemaining(values::add); + result.errorRowContent = String.join(",", values); + } + + return result; + } + + /** + * Makes a comma-separated string of the specified collection of + * strings. + * + * @param the_list The list. + * @return the comma-separated string. + */ + private String stringList(final Collection the_strings) { + final List strings = new ArrayList<>(the_strings); + final StringBuilder sb = new StringBuilder(); + + Collections.sort(strings); + sb.append(strings.get(0)); + for (int i = 1; i < strings.size(); i++) { + sb.append(", "); + sb.append(strings.get(i)); + } + + return sb.toString(); + } + + /** + * Parse the supplied data export. If it has already been parsed, this + * method returns immediately. + * + * @return true if the parse was successful, false otherwise + */ + public Result parse() { + final Result result = new Result(); + + LOGGER.info("parsing CVR export for county " + my_county.id() + + ", batch_size=" + my_batch_size + + ", transaction_size=" + my_transaction_size); + + final Iterator records = my_parser.iterator(); + + my_record_count = 0; + + + // 1) we expect the first line to be the election name, which we currently discard + final CSVRecord electionName; + + // 2) for the second line, we count the number of empty strings to find the first + // contest/choice column + final CSVRecord contest_line; + + // 3) we expect the third line to be a list of contest choices + final CSVRecord choice_line; + + // 4) a list of explanations of those choices (such as party affiliations) + final CSVRecord expl_line; + + // the combination of line 2-5 + final Result headerResult; + + // tracker for errors + int lineNum = 1; + + try { + // electionName + records.next(); + lineNum++; + contest_line = records.next(); + lineNum++; + choice_line = records.next(); + lineNum++; + expl_line = records.next(); + } catch (final Exception e) { + LOGGER.error(e.getClass()); + LOGGER.error(e.getMessage()); + result.success = false; + result.errorMessage = "Not a valid CSV"; + result.errorRowNum = lineNum; + result.errorRowContent = "? (could not parse)"; + return result; + } + + my_first_contest_column = 0; + while ("".equals(contest_line.get(my_first_contest_column))) { + my_first_contest_column = my_first_contest_column + 1; + } + // find all the contest names, how many choices each has, + // and how many choices can be made in each + final List contest_names = new ArrayList(); + final Map contest_votes_allowed = new HashMap(); + final Map contest_choice_counts = new HashMap(); + + // we expect the second line to be a list of contest names, each appearing once + // for each choice in the contest + + updateContestStructures(contest_line, contest_names, contest_votes_allowed, + contest_choice_counts); + + + headerResult = processHeaders(expl_line); + + if (headerResult.success == false) { + return headerResult; + } else { + Result addContestResult =addContests(choice_line, expl_line, contest_names, + contest_votes_allowed, contest_choice_counts); + if (!addContestResult.success) { + return addContestResult ; + } + + + // subsequent lines contain cast vote records + while (records.hasNext()) { + final CSVRecord cvr_line = records.next(); + try { + extractCVR(cvr_line); + } catch (final Exception e) { + LOGGER.error(e.getClass()); + LOGGER.error(e.getMessage()); + result.success = false; + // we don't know what went wrong + result.errorMessage = e.getClass().toString() + " - "+ e.getMessage(); + result.errorRowNum = Long.valueOf( cvr_line.getRecordNumber()).intValue(); + List values = new ArrayList<>(); + cvr_line.iterator().forEachRemaining(values::add); + result.errorRowContent = String.join(",", values); + // get out of here now! break and return + return result; + } + + my_record_count = my_record_count + 1; + if (my_record_count % PROGRESS_INTERVAL == 0) { + LOGGER.info("parsed " + my_record_count + + " CVRs for county " + my_county.id()); + } + checkForFlush(); + } + + for (final CountyContestResult r : my_results) { + r.updateResults(); + Persistence.saveOrUpdate(r); + } + + // commit any uncommitted records + + commitCVRsAndUpdateCountyDashboard(); + } + + result.success = true; // we made it through, yay! + result.importedCount = my_record_count; + + return result; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/csv/Result.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/csv/Result.java new file mode 100644 index 00000000..b293ae7d --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/csv/Result.java @@ -0,0 +1,49 @@ +package us.freeandfair.corla.csv; + +import static us.freeandfair.corla.util.EqualsHashcodeHelper.nullableEquals; + +import java.util.Objects; + + +/** The result of parsing/importing a csv file **/ +public class Result { + public boolean success; + public Integer importedCount; + public String errorMessage; + public Integer errorRowNum; + public String errorRowContent; + + /** + * The attributes determine whether a Hibernate update happens, not the + * table-object (which has been the prevailing assumption in this project). + * Since this class can be used as a Hibernate attribute or field, it is + * important for it to be able to be checked for equality by + * org.hibernate.internal.util.compare.EqualsHelper#equals. Otherwise + * Hibernate will always think that an update has happened and keep + * incrementing the version which can cause errors such as: + * "ERROR: could not serialize access due to concurrent update". + **/ + public boolean equals(final Object other) { + Result a = this; + boolean result = true; + if (other instanceof Result) { + final Result b = (Result) other; + result &= nullableEquals(a.success, b.success); + result &= nullableEquals(a.importedCount, b.importedCount); + result &= nullableEquals(a.errorMessage, b.errorMessage); + result &= nullableEquals(a.errorRowNum, b.errorRowNum); + result &= nullableEquals(a.errorRowContent, b.errorRowContent); + } else { + result = false; + } + return result; + } + + public int hashCode() { + return Objects.hash(success, + importedCount, + errorMessage, + errorRowNum, + errorRowContent); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ACVRDownload.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ACVRDownload.java new file mode 100644 index 00000000..1c58f770 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ACVRDownload.java @@ -0,0 +1,98 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 27, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.UncheckedIOException; +import java.util.stream.Stream; + +import javax.persistence.PersistenceException; + +import com.google.gson.stream.JsonWriter; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.model.CastVoteRecord; +import us.freeandfair.corla.model.CastVoteRecord.RecordType; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.query.CastVoteRecordQueries; +import us.freeandfair.corla.util.SparkHelper; + +/** + * The ballot manifest download endpoint. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class ACVRDownload extends AbstractEndpoint { + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.GET; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/acvr"; + } + + /** + * This endpoint requires any kind of authentication. + */ + @Override + public AuthorizationType requiredAuthorization() { + return AuthorizationType.EITHER; + } + + /** + * {@inheritDoc} + */ + @Override + // necessary to break out of the lambda expression in case of IOException + @SuppressWarnings("PMD.ExceptionAsFlowControl") + public String endpointBody(final Request the_request, final Response the_response) { + try (OutputStream os = SparkHelper.getRaw(the_response).getOutputStream(); + BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(os, "UTF-8")); + JsonWriter jw = new JsonWriter(bw)) { + jw.beginArray(); + final Stream matches = + Stream.concat(CastVoteRecordQueries.getMatching(RecordType.AUDITOR_ENTERED), + CastVoteRecordQueries.getMatching(RecordType.PHANTOM_BALLOT)); + matches.forEach((the_cvr) -> { + try { + jw.jsonValue(Main.GSON.toJson(Persistence.unproxy(the_cvr))); + Persistence.evict(the_cvr); + } catch (final IOException e) { + throw new UncheckedIOException(e); + } + }); + jw.endArray(); + jw.flush(); + jw.close(); + ok(the_response); + } catch (final UncheckedIOException | IOException | PersistenceException e) { + serverError(the_response, "Unable to stream response"); + } + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ACVRDownloadByCounty.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ACVRDownloadByCounty.java new file mode 100644 index 00000000..e55fc6b4 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ACVRDownloadByCounty.java @@ -0,0 +1,129 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 27, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.UncheckedIOException; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Stream; + +import javax.persistence.PersistenceException; + +import com.google.gson.stream.JsonWriter; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.model.CastVoteRecord; +import us.freeandfair.corla.model.CastVoteRecord.RecordType; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.query.CastVoteRecordQueries; +import us.freeandfair.corla.util.SparkHelper; + +/** + * The ballot manifest download endpoint. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class ACVRDownloadByCounty extends AbstractEndpoint { + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.GET; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/acvr/county"; + } + + /** + * This endpoint requires any kind of authentication. + */ + @Override + public AuthorizationType requiredAuthorization() { + return AuthorizationType.EITHER; + } + + /** + * {@inheritDoc} + */ + @Override + // necessary to break out of the lambda expression in case of IOException + @SuppressWarnings("PMD.ExceptionAsFlowControl") + public String endpointBody(final Request the_request, final Response the_response) { + final Set county_set = new HashSet(); + for (final String s : the_request.queryParams()) { + county_set.add(Long.valueOf(s)); + } + try (OutputStream os = SparkHelper.getRaw(the_response).getOutputStream(); + BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(os, "UTF-8")); + JsonWriter jw = new JsonWriter(bw)) { + jw.beginArray(); + for (final Long county : county_set) { + final Stream matches = + Stream.concat(CastVoteRecordQueries.getMatching(county, + RecordType.AUDITOR_ENTERED), + CastVoteRecordQueries.getMatching(county, + RecordType.PHANTOM_BALLOT)); + matches.forEach((the_cvr) -> { + try { + jw.jsonValue(Main.GSON.toJson(Persistence.unproxy(the_cvr))); + Persistence.evict(the_cvr); + } catch (final IOException e) { + throw new UncheckedIOException(e); + } + }); + } + jw.endArray(); + jw.flush(); + jw.close(); + ok(the_response); + } catch (final UncheckedIOException | IOException | PersistenceException e) { + serverError(the_response, "Unable to stream response"); + } + return my_endpoint_result.get(); + } + + /** + * For this endpoint, the parameter names must all be integers. + * + * @param the_request The request. + * @return true if the parameters are valid, false otherwise. + */ + protected boolean validateParameters(final Request the_request) { + boolean result = true; + + for (final String s : the_request.queryParams()) { + try { + Integer.parseInt(s); + } catch (final NumberFormatException e) { + result = false; + break; + } + } + + return result; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ACVRUpload.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ACVRUpload.java new file mode 100644 index 00000000..7cf85704 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ACVRUpload.java @@ -0,0 +1,203 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 27, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import static us.freeandfair.corla.asm.ASMEvent.AuditBoardDashboardEvent.*; + +import java.time.Instant; + +import javax.persistence.PersistenceException; + +import com.google.gson.JsonParseException; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.asm.ASMEvent; +import us.freeandfair.corla.controller.ComparisonAuditController; +import us.freeandfair.corla.json.SubmittedAuditCVR; +import us.freeandfair.corla.model.CastVoteRecord; +import us.freeandfair.corla.model.CastVoteRecord.RecordType; +import us.freeandfair.corla.model.CountyDashboard; +import us.freeandfair.corla.persistence.Persistence; + +/** + * The "audit CVR upload" endpoint. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings({"PMD.AtLeastOneConstructor", "PMD.CyclomaticComplexity"}) +// TODO: consider rewriting along the same lines as CVRExportUpload +public class ACVRUpload extends AbstractAuditBoardDashboardEndpoint { + /** + * Class-wide logger + */ + public static final Logger LOGGER = + LogManager.getLogger(ACVRUpload.class); + /** + * The event we will return for the ASM. + */ + private final ThreadLocal my_event = new ThreadLocal(); + + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.POST; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/upload-audit-cvr"; + } + + /** + * {@inheritDoc} + */ + @Override + protected ASMEvent endpointEvent() { + return my_event.get(); + } + + /** + * {@inheritDoc} + */ + @Override + protected void reset() { + my_event.set(null); + } + + /** build new acvr **/ + public CastVoteRecord buildNewAcvr(final SubmittedAuditCVR submission, + final CastVoteRecord cvr, + final CountyDashboard cdb) { + final CastVoteRecord s = submission.auditCVR(); + final CastVoteRecord newAcvr = + new CastVoteRecord(RecordType.AUDITOR_ENTERED, + Instant.now(), + s.countyID(), s.cvrNumber(), null, s.scannerID(), + s.batchID(), s.recordID(), s.imprintedID(), + s.ballotType(), s.contestInfo()); + newAcvr.setAuditBoardIndex(submission.getAuditBoardIndex()); + newAcvr.setCvrId(submission.cvrID()); + newAcvr.setRoundNumber(cdb.currentRound().number()); + newAcvr.setRand(cvr.getRand()); + + return newAcvr; + } + + /** + * {@inheritDoc} + */ + @SuppressWarnings({"PMD.ModifiedCyclomaticComplexity", "PMD.StdCyclomaticComplexity"}) + @Override + public String endpointBody(final Request the_request, final Response the_response) { + final CastVoteRecord newAcvr; + + try { + final SubmittedAuditCVR submission = + Main.GSON.fromJson(the_request.body(), SubmittedAuditCVR.class); + if (submission.auditCVR() == null || submission.cvrID() == null) { + LOGGER.error("empty audit CVR upload"); + badDataContents(the_response, "empty audit CVR upload"); + } else { + // FIXME extract-fn: handleACVR + final CountyDashboard cdb = + Persistence.getByID(Main.authentication().authenticatedCounty(the_request).id(), + CountyDashboard.class); + if (cdb == null) { + LOGGER.error("could not get audit board dashboard"); + serverError(the_response, "Could not save ACVR to dashboard"); + } else if (submission.isReaudit()) { + + final CastVoteRecord cvr = Persistence.getByID(submission.cvrID(), + CastVoteRecord.class); + newAcvr = buildNewAcvr(submission, cvr, cdb); + + if (ComparisonAuditController.reaudit(cdb,cvr,newAcvr, submission.getComment())) { + ok(the_response, "ACVR reaudited"); + } else { + LOGGER.error("CVR has not previously been audited"); + invariantViolation(the_response, "CVR has not previously been audited"); + } + + } else if (cdb.ballotsRemainingInCurrentRound() > 0) { + + // Now we have a thing we can give our controller, maybe. + final CastVoteRecord cvr = Persistence.getByID(submission.cvrID(), + CastVoteRecord.class); + + // FIXME extract-fn: setupACVR + final CastVoteRecord acvr = submission.auditCVR(); + acvr.setID(null); + + newAcvr = buildNewAcvr(submission, cvr, cdb); + + Persistence.saveOrUpdate(newAcvr); + LOGGER.info("Audit CVR for CVR id " + submission.cvrID() + + " parsed and stored as id " + newAcvr.id()); + + if (cvr == null) { + LOGGER.error("could not find original CVR"); + // FIXME throw and push HTTP response up. + this.badDataContents(the_response, "could not find original CVR"); + } else { + + // The positive outcome is a little hard to notice in all the noise + // FIXME return an appropriate value and push HTTP response up + if (ComparisonAuditController.submitAuditCVR(cdb, cvr, newAcvr)) { + LOGGER.debug("ACVR OK"); + Persistence.saveOrUpdate(cdb); + ok(the_response, "ACVR submitted"); + } else { + // FIXME throw and push HTTP response up + LOGGER.error("invalid audit CVR uploaded"); + badDataContents(the_response, "invalid audit CVR uploaded"); + } + } + } else { + // FIXME throw and push HTTP response up + LOGGER.error("ballot submission with no remaining ballots in round"); + invariantViolation(the_response, + "ballot submission with no remaining ballots in round"); + } + + // don't advance state machine if reaudit + if (!submission.isReaudit()) { + if (cdb.ballotsRemainingInCurrentRound() == 0) { + // TODO this has to happen before we can say RISK_LIMIT_ACHIEVED! + LOGGER.debug("The round is over and set ROUND_COMPLETE_EVENT"); + my_event.set(ROUND_COMPLETE_EVENT); + } else { + LOGGER.debug("Some ballots remaining according to the CDB: REPORT_MARKING_EVENT"); + my_event.set(REPORT_MARKINGS_EVENT); + } + } + } // extract-fn: handleACVR will have returned some value or thrown + } catch (final JsonParseException e) { + LOGGER.error("malformed audit CVR upload"); + badDataContents(the_response, "malformed audit CVR upload"); + } catch (final PersistenceException e) { + LOGGER.error("could not save audit CVR"); + serverError(the_response, "Unable to save audit CVR"); + } + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AbstractAuditBoardDashboardEndpoint.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AbstractAuditBoardDashboardEndpoint.java new file mode 100644 index 00000000..87368460 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AbstractAuditBoardDashboardEndpoint.java @@ -0,0 +1,54 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 12, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import spark.Request; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.asm.AuditBoardDashboardASM; + +/** + * Functionality that spans endpoints on the Audit Board Dashboard. + * + * @author Joseph R. Kiniry + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public abstract class AbstractAuditBoardDashboardEndpoint extends AbstractEndpoint { + /** + * @return County authorization is required for these endpoints. + */ + @Override + public AuthorizationType requiredAuthorization() { + return AuthorizationType.COUNTY; + } + + /** + * @return subclasses of this endpoint use the Department of State ASM. + */ + @Override + protected Class asmClass() { + return AuditBoardDashboardASM.class; + } + + /** + * Gets the ASM identity for the specified request. + * + * @param the_request The request. + * @return the county ID of the authenticated county. + */ + @Override + protected String asmIdentity(final Request the_request) { + return String.valueOf(Main.authentication().authenticatedCounty(the_request).id()); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AbstractCountyDashboardEndpoint.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AbstractCountyDashboardEndpoint.java new file mode 100644 index 00000000..db93e643 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AbstractCountyDashboardEndpoint.java @@ -0,0 +1,54 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 12, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import spark.Request; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.asm.CountyDashboardASM; + +/** + * Functionality that spans endpoints on the Department of State Dashboard. + * + * @author Joseph R. Kiniry + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public abstract class AbstractCountyDashboardEndpoint extends AbstractEndpoint { + /** + * @return State authorization is required for these endpoints. + */ + @Override + public AuthorizationType requiredAuthorization() { + return AuthorizationType.COUNTY; + } + + /** + * @return subclasses of this endpoint use the Department of State ASM. + */ + @Override + protected Class asmClass() { + return CountyDashboardASM.class; + } + + /** + * Gets the ASM identity for the specified request. + * + * @param the_request The request. + * @return the county ID of the authenticated county. + */ + @Override + protected String asmIdentity(final Request the_request) { + return String.valueOf(Main.authentication().authenticatedCounty(the_request).id()); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AbstractDoSDashboardEndpoint.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AbstractDoSDashboardEndpoint.java new file mode 100644 index 00000000..2899cfda --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AbstractDoSDashboardEndpoint.java @@ -0,0 +1,53 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 12, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import spark.Request; + +import us.freeandfair.corla.asm.DoSDashboardASM; + +/** + * Functionality that spans endpoints on the Department of State Dashboard. + * + * @author Joseph R. Kiniry + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public abstract class AbstractDoSDashboardEndpoint extends AbstractEndpoint { + /** + * @return State authorization is required for these endpoints. + */ + @Override + public AuthorizationType requiredAuthorization() { + return AuthorizationType.STATE; + } + + /** + * @return subclasses of this endpoint use the Department of State ASM. + */ + @Override + protected Class asmClass() { + return DoSDashboardASM.class; + } + + /** + * Gets the ASM identity for the specified request. + * + * @param the_request The request. + * @return DoSDashboardASM.IDENTITY + */ + @Override + protected String asmIdentity(final Request the_request) { + return DoSDashboardASM.IDENTITY; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AbstractEndpoint.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AbstractEndpoint.java new file mode 100644 index 00000000..3df7a7f3 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AbstractEndpoint.java @@ -0,0 +1,714 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 11, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import static us.freeandfair.corla.model.Administrator.AdministratorType.*; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +import javax.persistence.PersistenceException; + +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.log4j.Level; +import org.eclipse.jetty.http.HttpStatus; +import org.hibernate.HibernateException; + +import spark.HaltException; +import spark.Request; +import spark.Response; +import spark.Spark; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.asm.ASMEvent; +import us.freeandfair.corla.asm.ASMUtilities; +import us.freeandfair.corla.asm.AbstractStateMachine; +import us.freeandfair.corla.auth.AuthenticationInterface; +import us.freeandfair.corla.json.Result; +import us.freeandfair.corla.model.Administrator; +import us.freeandfair.corla.model.LogEntry; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.query.LogEntryQueries; +import us.freeandfair.corla.util.DBExceptionUtil; +import us.freeandfair.corla.util.SuppressFBWarnings; + +/** + * Basic behaviors that span all endpoints. In particular, standard exceptional + * behavior with regards to cross-cutting concerns like authentication or + * erroneous or bogus requests from clients. + * + * @trace endpoint + * @author Joseph R. Kiniry + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressFBWarnings({"UWF_FIELD_NOT_INITIALIZED_IN_CONSTRUCTOR", +// Justification: False positive because we are weaving in behavior +// in before() to initialize my_persistent_asm_state. + "SF_SWITCH_NO_DEFAULT"}) +@SuppressWarnings({"PMD.AtLeastOneConstructor", "PMD.TooManyMethods", + "PMD.EmptyMethodInAbstractClassShouldBeAbstract", "PMD.GodClass"}) +public abstract class AbstractEndpoint implements Endpoint { + /** + * A flag that disables ASM checks, when true. + */ + public static final boolean DISABLE_ASM = false; + + /** + * The number of times to retry committing logs for failed transactions. + */ + public static final int LOG_COMMIT_RETRIES = 5; + + /** + * The "Retry-After" value for a transaction failure response, in seconds. + */ + public static final String RETRY_AFTER_DELAY = "10"; + + /** + * The ASM for this endpoint. + */ + protected ThreadLocal my_asm = + new ThreadLocal(); + + /** + * The endpoint result for the ongoing transaction. + */ + protected ThreadLocal my_endpoint_result = new ThreadLocal(); + + /** + * The HTTP status for the ongoing transaction. + */ + protected ThreadLocal my_status = new ThreadLocal(); + + /** + * The log entries to be logged by this endpoint after execution. + */ + protected ThreadLocal> my_log_entries = + new ThreadLocal>(); + + /** + * Halts the endpoint execution by ending the request and returning the + * most recently set response code and endpoint result. + */ + protected final void halt(final Response the_response) { + Spark.halt(the_response.status(), my_endpoint_result.get()); + } + + /** + * @return the abstract state machine class for this endpoint. By default, + * the endpoint does not have an abstract state machine. + */ + // this method is not empty! + protected Class asmClass() { + return null; + } + + /** + * Gets the identity of the ASM used by this endpoint for this + * request (as necessary). By default, the endpoint does not have + * an abstract state machine. + * + * @param the_request The request. + * @return The identity of the ASM used by this endpoint for this + * request. + */ + // this method is not empty! + protected String asmIdentity(final Request the_request) { + return null; + } + + /** + * Which event does this endpoint wish to take? By default, it + * does not execute an event. + * + * @return the event. + */ + // this method is not empty! + protected ASMEvent endpointEvent() { + return null; + } + + /** + * The endpoint method. Delegates immediately to the child class + */ + /** + * Load the appropriate ASM from the database and make sure that + * the transition we wish to take is legal. + * + * @param the_request The request. + * @param the_response The response. + */ + protected void loadAndCheckASM(final Request the_request, + final Response the_response) { + // get the state of the ASM + if (DISABLE_ASM || asmClass() == null) { + // there is no ASM for this endpoint + my_asm.set(null); + return; + } + my_asm.set(ASMUtilities.asmFor(asmClass(), asmIdentity(the_request))); + // check that we are in the right ASM state + if (endpointEvent() != null && + !my_asm.get().enabledASMEvents().contains(endpointEvent())) { + illegalTransition(the_response, + endpointName() + + " attempted to apply illegal event " + endpointEvent() + + " from state " + my_asm.get().currentState()); + } + } + + /** + * Save the ASM back to the database. + * + * @param the_response The response. + * @return true if the ASM transitioned successfully + */ + protected boolean transitionAndSaveASM(final Response the_response) { + if (DISABLE_ASM || my_asm.get() == null || endpointEvent() == null) { + // there is no ASM event for this endpoint + return true; + } + try { + // this asmFor() will be a no-op in nearly all cases, but in multi-transaction + // endpoint hits like uploading large CVR imports, it is possible for the + // state to change out from underneath us + my_asm.set(ASMUtilities.asmFor(asmClass(), my_asm.get().identity())); + my_asm.get().stepEvent(endpointEvent()); + } catch (final IllegalStateException e) { + illegalTransition(the_response, e.getMessage(), false); + return false; + } + return ASMUtilities.save(my_asm.get()); + } + + /** + * Indicate that the operation completed successfully. This method is only + * to be used when no response should be sent in the body beyond any + * streaming that the endpoint has already done; to provide an OK response + * outside of a streaming context, use ok(Response, String) or + * okJSON(Response, String). + * + * @param the_response the HTTP response. + */ + public void ok(final Response the_response) { + my_log_entries.get().add(new LogEntry(HttpStatus.OK_200, endpointName(), Instant.now())); + my_status.set(HttpStatus.OK_200); + my_endpoint_result.set(""); + } + + /** + * Indicate and log that the operation completed successfully. + * @param the_response the HTTP response. + * @param the_body the body of the HTTP response. + */ + public void ok(final Response the_response, + final String the_body) { + okJSON(the_response, Main.GSON.toJson(new Result(the_body))); + } + + /** + * Indicate and log that the operation completed successfully, and + * send the specified JSON-formatted string. + * + * @param the_response The HTTP response. + * @param the_json The JSON string to send as the body of the response. + */ + public void okJSON(final Response the_response, final String the_json) { + my_log_entries.get().add(new LogEntry(HttpStatus.OK_200, endpointName(), Instant.now())); + my_status.set(HttpStatus.OK_200); + my_endpoint_result.set(the_json); + } + + /** + * Indicate the client has violated an invariant or precondition relating data + * to the endpoint in question. E.g., a digest is incorrect with regards to + * the file that it summarizes. + * @param the_response the HTTP response. + * @param the_body the body of the HTTP response. + */ + public void invariantViolation(final Response the_response, + final String the_body) { + my_log_entries.get().add(new LogEntry(HttpStatus.BAD_REQUEST_400, + "invariant violation on " + endpointName() + ": " + + the_body, + Instant.now())); + my_status.set(HttpStatus.BAD_REQUEST_400); + my_endpoint_result.set(Main.GSON.toJson(new Result(the_body))); + halt(the_response); + } + + /** + * Indicate that the client is not authorized to perform the requested action. + * @param the_response the HTTP response. + * @param the_body the body of the HTTP response. + */ + public void unauthorized(final Response the_response, + final String the_body) { + my_log_entries.get().add(new LogEntry(HttpStatus.UNAUTHORIZED_401, + "unauthorized access on " + endpointName() + ": " + + the_body, + Instant.now())); + my_status.set(HttpStatus.UNAUTHORIZED_401); + my_endpoint_result.set(Main.GSON.toJson(new Result(the_body))); + halt(the_response); + } + + /** + * Indicate the client cannot perform the requested action because it violates + * the server's state machine. + * @param the_response the HTTP response. + * @param the_body the body of the HTTP response. + * @param the_halt true to halt, false otherwise. + */ + public void illegalTransition(final Response the_response, + final String the_body, + final boolean the_halt) { + my_log_entries.get().add(new LogEntry(HttpStatus.FORBIDDEN_403, + "illegal transition attempt on " + endpointName() + ": " + + the_body, + Instant.now())); + my_status.set(HttpStatus.FORBIDDEN_403); + my_endpoint_result.set(Main.GSON.toJson(new Result(the_body))); + if (the_halt) { + halt(the_response); + } + } + + /** + * Indicate the client cannot perform the requested action because it violates + * the server's state machine. + * + * @param the_response The HTTP response. + * @param the_body The HTTP response body. + */ + public void illegalTransition(final Response the_response, final String the_body) { + illegalTransition(the_response, the_body, true); + } + + /** + * Indicate that some data was not found. + * @param the_response the HTTP response. + * @param the_body the body of the HTTP response. + */ + public void dataNotFound(final Response the_response, + final String the_body) { + my_log_entries.get().add(new LogEntry(HttpStatus.NOT_FOUND_404, + "data not found on " + endpointName() + ": " + + the_body, + Instant.now())); + my_status.set(HttpStatus.NOT_FOUND_404); + my_endpoint_result.set(Main.GSON.toJson(new Result(the_body))); + halt(the_response); + } + + /** + * Indicate that the type/shape of data the client provided is ill-formed. + * @param the_response the HTTP response. + * @param the_body the body of the HTTP response. + */ + public void badDataType(final Response the_response, + final String the_body) { + my_log_entries.get().add(new LogEntry(HttpStatus.UNSUPPORTED_MEDIA_TYPE_415, + "bad data type from client on " + endpointName() + ": " + + the_body, + Instant.now())); + my_status.set(HttpStatus.UNSUPPORTED_MEDIA_TYPE_415); + my_endpoint_result.set(Main.GSON.toJson(new Result(the_body))); + halt(the_response); + } + + /** + * Indicate that data that the client provided is ill-formed. + * @param the_response the HTTP response. + * @param the_body the body of the HTTP response. + */ + public void badDataContents(final Response the_response, + final String the_body) { + my_log_entries.get().add(new LogEntry(HttpStatus.UNPROCESSABLE_ENTITY_422, + "bad data from client on " + endpointName() + ": " + + the_body, + Instant.now())); + my_status.set(HttpStatus.UNPROCESSABLE_ENTITY_422); + my_endpoint_result.set(Main.GSON.toJson(new Result(the_body))); + halt(the_response); + } + + /** + * Indicate that an internal server error has taken place. + * @param the_response the HTTP response. + * @param the_body the body of the HTTP response. + */ + public void serverError(final Response the_response, + final String the_body) { + my_log_entries.get().add(new LogEntry(HttpStatus.INTERNAL_SERVER_ERROR_500, + "server error on " + endpointName() + ": " + + the_body, + Instant.now())); + my_status.set(HttpStatus.INTERNAL_SERVER_ERROR_500); + my_endpoint_result.set(Main.GSON.toJson(new Result(the_body))); + halt(the_response); + } + + /** + * Indicate that the server is temporarily unavailable. This is typically due + * to server maintenance. + * @param the_response the HTTP response. + * @param the_body the body of the HTTP response. + */ + public void serverUnavailable(final Response the_response, + final String the_body) { + my_log_entries.get().add(new LogEntry(HttpStatus.SERVICE_UNAVAILABLE_503, + "service temporarily unavailable on " + + endpointName() + ": " + the_body, + Instant.now())); + my_status.set(HttpStatus.SERVICE_UNAVAILABLE_503); + my_endpoint_result.set(Main.GSON.toJson(new Result(the_body))); + halt(the_response); + } + + /** + * Indicate that the endpoint action failed due to a transaction failure. + * Unlike other error responses, this one does _not_ halt the connection. + * + * @param the_response The HTTP response. + * @param the_body The body of the HTTP response. + */ + public void transactionFailure(final Response the_response, final String the_body) { + my_log_entries.get().add(new LogEntry(HttpStatus.SERVICE_UNAVAILABLE_503, + "transaction failure on " + endpointName() + ": " + + the_body, + Instant.now())); + my_status.set(HttpStatus.SERVICE_UNAVAILABLE_503); + the_response.header("Retry-After", RETRY_AFTER_DELAY); + my_endpoint_result.set(Main.GSON.toJson(new Result(the_body))); + } + + /** + * This before-filter is evaluated before each request, and can read the + * request and read/modify the response. Our before-filter performs + * authentication checking. + */ + @SuppressWarnings("PMD.ConfusingTernary") + // this warning is caused by the call to queryParams(), + // but we really don't need the result + @SuppressFBWarnings("RV_RETURN_VALUE_IGNORED_NO_SIDE_EFFECT") + @Override + public void before(final Request the_request, final Response the_response) { + reset(); + my_log_entries.set(new ArrayList()); + Main.LOGGER.log(logLevel(), + "endpoint " + endpointName() + " hit by " + the_request.host()); + // make sure we get all the HTTP post parameters, if there are any, before + // anything has a chance to read the request body before Spark + the_request.queryParams(); + + // Start a transaction, if the database is functioning; otherwise abort + if (Persistence.hasDB()) { + Persistence.beginTransaction(); + } else { + serverError(the_response, "no database"); + halt(the_response); + } + + // Check that the user is authorized for this endpoint + if (!checkAuthorization(the_request, requiredAuthorization())) { + unauthorized(the_response, + "client not authorized to perform this action"); + halt(the_response); + } + + // Validate the parameters of the request. + if (!validateParameters(the_request)) { + dataNotFound(the_response, "parameter validation failed"); + halt(the_response); + } + + // Load and check the ASM + loadAndCheckASM(the_request, the_response); + } + + /** + * The main body of the endpoint. This method wraps an execution of the + * endpointBody() method of a child class in a way such that unexpected + * exceptions will be properly logged rather than causing thread deaths. + * + * @param the_request The request. + * @param the_response The response. + * @return the result of the endpoint execution. + */ + // these exception warnings are suppressed because this method is, exactly, + // attempting to suppress and log all non-Error exceptions other than + // HaltException + @SuppressWarnings({"PMD.AvoidCatchingGenericException", "PMD.AvoidRethrowingException"}) + public final String endpoint(final Request the_request, final Response the_response) { + String result = null; + + try { + result = endpointBody(the_request, the_response); + } catch (final HaltException e) { + // a HaltException should just be propagated, as that is an expected exception + // that is properly dealt with by Spark + throw e; + } catch (final HibernateException e) { + // a Hibernate exception is treated like a transaction failure + Main.LOGGER.error("JDBC error in endpoint " + endpointName() + ":\n" + + ExceptionUtils.getStackTrace(e)); + transactionFailure(the_response, e.toString()); + // must manually halt after a transaction failure + halt(the_response); + } catch (final Exception e) { + // some exception occurred that was not handled within the endpoint, so + // handle it as a generic server error and log the stack trace + Main.LOGGER.error("uncaught exception in endpoint " + endpointName() + ":\n" + + ExceptionUtils.getStackTrace(e)); + serverError(the_response, e.toString()); + // the server error halts processing + } + + return result; + } + + /** + * The main body of the endpoint to be executed in child classes. + * + * @param the_request The request + * @param the_response The response. + * @return the result of the endpoint execution. + */ + protected abstract String endpointBody(Request the_request, Response the_response); + + /** + * The after filter for this endpoint. Currently, the implementation is empty. + */ + public void after(final Request the_request, final Response the_response) { + // skip + } + + /** + * Persists, and logs to the system logger, all accumulated log entries for + * this endpoint. + * + * @param the_request The request (used to get the hostname of the client + * and the authentication data for the log). + */ + private void sendToLogger(final LogEntry the_log_entry) { + if (the_log_entry.resultCode() == null) { + Main.LOGGER.log(logLevel(), + the_log_entry.information() + " by " + + the_log_entry.authenticationData() + " from " + + the_log_entry.clientHost()); + } else if (HttpStatus.isSuccess(the_log_entry.resultCode())) { + Main.LOGGER.log(logLevel(), + "successful " + the_log_entry.information() + " by " + + the_log_entry.authenticationData() + " from " + + the_log_entry.clientHost()); + } else { + Main.LOGGER.error("error " + the_log_entry.resultCode() + " " + + the_log_entry.information() + " by " + + the_log_entry.authenticationData() + " from " + + the_log_entry.clientHost()); + } + } + + private void persistLogEntries(final Request the_request) { + LogEntry previous_entry = LogEntryQueries.last(); + final Object admin_attribute = + the_request.session().attribute(AuthenticationInterface.ADMIN); + final String admin_data; + if (admin_attribute instanceof Administrator) { + admin_data = ((Administrator) admin_attribute).username(); + } else { + admin_data = "(unauthenticated)"; + } + + for (final LogEntry entry : my_log_entries.get()) { + // create and persist a new hash-chained log entry for each log entry + final LogEntry real_entry = + new LogEntry(entry.resultCode(), entry.information(), + admin_data, the_request.host(), + entry.timestamp(), previous_entry); + Persistence.save(real_entry); + sendToLogger(real_entry); + previous_entry = real_entry; + } + } + + /** + * @returns true if the current set of log entries indicates a successful + * request, false otherwise. + */ + private boolean successful() { + return !my_log_entries.get().isEmpty() && + HttpStatus.isSuccess(my_log_entries. + get().get(my_log_entries.get().size() - 1).resultCode()); + } + + /** + * Attempts to commit, in a separate transaction, any straggling log entries that + * could not be committed due to error. + * + * @param the_request The request (used for log data). + */ + private void finalizeLogs(final Request the_request) { + int log_commit_retries = 0; + if (!my_log_entries.get().isEmpty() && log_commit_retries < LOG_COMMIT_RETRIES) { + try { + log_commit_retries = log_commit_retries + 1; + Persistence.beginTransaction(); + persistLogEntries(the_request); + Persistence.commitTransaction(); + my_log_entries.get().clear(); + } catch (final PersistenceException e) { + Main.LOGGER.error("could not persist log entries for error response after " + + log_commit_retries + " attempt(s)"); + } + } else if (!my_log_entries.get().isEmpty()) { + Main.LOGGER.error("maximum number of log entry commit attempts reached, aborting"); + } + } + + /** + * The afterAfter filter for this endpoint. By default, it attempts to commit + * any open transaction (this makes writing endpoint code more straightforward, + * as the vast majority of endpoints will never have to deal with transactions + * themselves). + */ + public void afterAfter(final Request the_request, final Response the_response) { + // try to take the transition for this endpoint in the ASM and save it to the DB + // note that we do not try to commit when we have an error code in the response + if (successful() && + transitionAndSaveASM(the_response) && + Persistence.isTransactionActive()) { + try { + // since the transition finished, let's log all the log entries and commit + persistLogEntries(the_request); + Persistence.commitTransaction(); + my_log_entries.get().clear(); + } catch (final PersistenceException e) { + // this is an internal server error because we don't know what didn't + // get committed + transactionFailure(the_response, + DBExceptionUtil.getConstraintFailureReason(e)); + } + } else { + if (Persistence.canTransactionRollback()) { + try { + Persistence.rollbackTransaction(); + } catch (final PersistenceException ex) { + Main.LOGGER.error("could not roll back transaction for error response: " + + ex.getMessage()); + } + } else { + Main.LOGGER.error("could not roll back transaction for error response"); + } + } + // if there are still log entries left, we need to persist them and print them + finalizeLogs(the_request); + Integer status = my_status.get(); + String endpoint_result = my_endpoint_result.get(); + if (status == null) { + status = HttpStatus.INTERNAL_SERVER_ERROR_500; + endpoint_result = + Main.GSON.toJson(new Result("server error, no response from endpoint")); + } + the_response.body(endpoint_result); + the_response.status(status); + } + + /** + * @return the type of authorization required to use this endpoint. + * The default is NONE. + */ + @Override + public AuthorizationType requiredAuthorization() { + return AuthorizationType.NONE; + } + + /** + * @return the priority level at which the endpoint's activity will be + * logged. The default is Priority.INFO. + */ + @Override + public Level logLevel() { + return Level.INFO; + } + + /** + * Validates the parameters of a request. The default behavior is to + * return 'true'. + * + * @param the_request The request. + * @return true if the parameters are valid, false otherwise. + */ + protected boolean validateParameters(final Request the_request) { + return true; + } + + /** + * Resets any thread-local state of an endpoint. The default behavior is + * to do nothing. + */ + protected void reset() { + // do nothing + } + + /** + * Checks to see that the specified request satisfies the specified + * authorization type. + * + * @param the_request The request. + * @param the_type The authorization type. + * @return true if the request is appropriately authorized, false otherwise. + */ + public static boolean checkAuthorization(final Request the_request, + final AuthorizationType the_type) { + boolean result = the_type == AuthorizationType.NONE; + if (!result) { + final boolean state; + final boolean county; + + if (Main.authentication().secondFactorAuthenticated(the_request)) { + final Administrator admin = + Main.authentication().authenticatedAdministrator(the_request); + if (admin == null) { + state = false; + county = false; + } else { + state = + Main.authentication().authenticatedAs(the_request, STATE, admin.username()); + county = + Main.authentication().authenticatedAs(the_request, COUNTY, admin.username()); + } + + switch (the_type) { + case STATE: + result = state; + break; + + case COUNTY: + result = county; + break; + + case EITHER: + result = county || state; + break; + + case NONE: + default: + } + } + } + + return result; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AppInfoEndpoint.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AppInfoEndpoint.java new file mode 100644 index 00000000..a21b4592 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AppInfoEndpoint.java @@ -0,0 +1,100 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * + * @created Aug 12, 2017 + * + * @copyright 2017 Colorado Department of State + * + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * + * @creator Joseph R. Kiniry + * + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.asm.AbstractStateMachine; +import us.freeandfair.corla.asm.AuditBoardDashboardASM; +import us.freeandfair.corla.asm.CountyDashboardASM; +import us.freeandfair.corla.asm.DoSDashboardASM; +import us.freeandfair.corla.asm.PersistentASMState; +import us.freeandfair.corla.json.AppInfoJSON; +import us.freeandfair.corla.json.Result; +import us.freeandfair.corla.model.County; +import us.freeandfair.corla.model.CountyDashboard; +import us.freeandfair.corla.model.DoSDashboard; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.query.DatabaseResetQueries; +import us.freeandfair.corla.query.PersistentASMStateQueries; + +/** + * Reset the database, except for authentication information and uploaded + * artifact data (the latter is cleaned up at the database level, not by this + * code). + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +// the endpoint method here is long and has lots of loops, but is not +// at all difficult to understand +@SuppressWarnings({"PMD.AtLeastOneConstructor", "PMD.ModifiedCyclomaticComplexity", + "PMD.CyclomaticComplexity", "PMD.StdCyclomaticComplexity", "PMD.NPathComplexity"}) +public class AppInfoEndpoint extends AbstractEndpoint { + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.GET; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/app-info"; + } + + /** + * {@inheritDoc} + */ + @Override + public String asmIdentity(final Request the_request) { + return null; + } + + /** + * {@inheritDoc} + */ + @Override + public Class asmClass() { + return null; + } + + /** + * @return STATE authorization is necessary for this endpoint. + */ + @Override + public AuthorizationType requiredAuthorization() { + return AuthorizationType.EITHER; + } + + /** + * @return app info that contains the version info + */ + @Override + public String endpointBody(final Request the_request, final Response the_response) { + + String implementationVersion = Main.VERSION; + okJSON(the_response, Main.GSON.toJson(new AppInfoJSON(implementationVersion))); + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AuditBoardDashboardASMState.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AuditBoardDashboardASMState.java new file mode 100644 index 00000000..21fed592 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AuditBoardDashboardASMState.java @@ -0,0 +1,67 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 12, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import org.apache.log4j.Level; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.json.ServerASMResponse; + +/** + * An endpoint to provide the state of an audit board dashboard ASM to the client. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class AuditBoardDashboardASMState extends AbstractAuditBoardDashboardEndpoint { + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.GET; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/audit-board-asm-state"; + } + + /** + * {@inheritDoc} + */ + @Override + public Level logLevel() { + return Level.DEBUG; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointBody(final Request the_request, final Response the_response) { + // there's really nothing to do here other than get the ASM state, which we + // conveniently have locally already + + okJSON(the_response, + Main.GSON.toJson(new ServerASMResponse(my_asm.get().currentState(), + my_asm.get().enabledUIEvents()))); + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AuditBoardSignIn.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AuditBoardSignIn.java new file mode 100644 index 00000000..b6ec3ca2 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AuditBoardSignIn.java @@ -0,0 +1,168 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 12, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import static us.freeandfair.corla.asm.ASMEvent.AuditBoardDashboardEvent.SIGN_IN_AUDIT_BOARD_EVENT; +import static us.freeandfair.corla.asm.ASMState.AuditBoardDashboardState.*; + +import java.lang.reflect.Type; + +import java.util.List; + +import javax.persistence.PersistenceException; + +import com.google.gson.JsonParser; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.reflect.TypeToken; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.asm.ASMEvent; +import us.freeandfair.corla.asm.ASMState; +import us.freeandfair.corla.asm.ASMUtilities; +import us.freeandfair.corla.asm.AuditBoardDashboardASM; +import us.freeandfair.corla.model.County; +import us.freeandfair.corla.model.CountyDashboard; +import us.freeandfair.corla.model.Elector; +import us.freeandfair.corla.persistence.Persistence; + +/** + * Signs in the audit board for a county. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings({"PMD.AtLeastOneConstructor"}) +public class AuditBoardSignIn extends AbstractAuditBoardDashboardEndpoint { + /** + * The event to return for this endpoint. + */ + private final ThreadLocal asmEvent = new ThreadLocal(); + + /** + * Type of a list of electors for easier unmarshaling with GSON + */ + private static final Type ELECTOR_LIST = + new TypeToken>() { }.getType(); + + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.POST; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/audit-board-sign-in"; + } + + /** + * @return COUNTY authorization is required for this endpoint. + */ + public AuthorizationType requiredAuthorization() { + return AuthorizationType.COUNTY; + } + + /** + * {@inheritDoc} + */ + @Override + protected ASMEvent endpointEvent() { + return this.asmEvent.get(); + } + + /** + * Establish the audit board for a county. + */ + @Override + public String endpointBody(final Request the_request, + final Response the_response) { + final JsonParser parser = new JsonParser(); + final JsonObject object; + + try { + object = parser.parse(the_request.body()).getAsJsonObject(); + + final int index = object.get("index").getAsInt(); + final List parsed_audit_board = + Main.GSON.fromJson( + object.get("audit_board"), + ELECTOR_LIST); + + if (parsed_audit_board.size() >= CountyDashboard.MIN_AUDIT_BOARD_MEMBERS) { + final County county = Main.authentication().authenticatedCounty(the_request); + if (county == null) { + Main.LOGGER.error("could not get authenticated county"); + unauthorized(the_response, "not authorized to sign in audit board"); + } else { + final CountyDashboard cdb = Persistence.getByID(county.id(), CountyDashboard.class); + if (cdb == null) { + Main.LOGGER.error("could not get county dashboard"); + serverError(the_response, "could not sign in audit board"); + } else { + this.asmEvent.set(this.nextEvent(cdb)); + cdb.signInAuditBoard(index, parsed_audit_board); + Persistence.saveOrUpdate(cdb); + ok(the_response, + String.format("audit board #%d for county %d signed in: %s", + index, county.id(), parsed_audit_board)); + } + } + } else { + invariantViolation(the_response, "invalid audit board membership"); + } + } catch (final PersistenceException e) { + serverError(the_response, "unable to sign in audit board: " + e); + + } catch (final JsonParseException e) { + badDataContents(the_response, "invalid audit board data"); + } + + return my_endpoint_result.get(); + } + + /** + * Computes an ASM event to emit when the audit board is signed in. + * + * Currently only returns SIGN_IN_AUDIT_BOARD_EVENT when all audit boards are + * marked signed out. + * + * @param cdb the county dashboard + */ + private ASMEvent nextEvent(final CountyDashboard cdb) { + final AuditBoardDashboardASM asm = ASMUtilities.asmFor( + AuditBoardDashboardASM.class, + String.valueOf(cdb.id())); + + ASMState currentState = null; + if (null != asm) { + currentState = asm.currentState(); + } + + if (AUDIT_INITIAL_STATE == currentState + || WAITING_FOR_ROUND_START_NO_AUDIT_BOARD == currentState + || ROUND_IN_PROGRESS_NO_AUDIT_BOARD == currentState + || WAITING_FOR_ROUND_SIGN_OFF_NO_AUDIT_BOARD == currentState) { + return SIGN_IN_AUDIT_BOARD_EVENT; + } + + return null; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AuditBoardSignOut.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AuditBoardSignOut.java new file mode 100644 index 00000000..9ae4b57f --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AuditBoardSignOut.java @@ -0,0 +1,93 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 12, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import javax.persistence.PersistenceException; + +import spark.Request; +import spark.Response; + +import com.google.gson.JsonParser; +import com.google.gson.JsonParseException; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.model.County; +import us.freeandfair.corla.model.CountyDashboard; +import us.freeandfair.corla.persistence.Persistence; + +/** + * Signs out the audit board for a county. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings({"PMD.AtLeastOneConstructor"}) +public class AuditBoardSignOut extends AbstractAuditBoardDashboardEndpoint { + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.POST; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/audit-board-sign-out"; + } + + /** + * @return COUNTY authorization is required for this endpoint. + */ + public AuthorizationType requiredAuthorization() { + return AuthorizationType.COUNTY; + } + + /** + * Signs the audit board out for the logged in county at the specified index. + */ + @Override + public String endpointBody(final Request the_request, + final Response the_response) { + final JsonParser parser = new JsonParser(); + + try { + final int index = parser.parse(the_request.body()).getAsInt(); + final County county = Main.authentication().authenticatedCounty(the_request); + if (county == null) { + Main.LOGGER.error("could not get authenticated county"); + unauthorized(the_response, "not authorized to perform audit board login"); + } else { + final CountyDashboard cdb = Persistence.getByID(county.id(), CountyDashboard.class); + if (cdb == null) { + Main.LOGGER.error("could not get county dashboard"); + serverError(the_response, "could not log in audit board"); + } else { + cdb.signOutAuditBoard(index); + Persistence.saveOrUpdate(cdb); + ok(the_response, + String.format("audit board #%d for county %d signed out", + index, county.id())); + } + } + } catch (final PersistenceException e) { + serverError(the_response, "unable to sign out audit board: " + e); + } catch (final JsonParseException e) { + badDataContents(the_response, "unable to sign out audit board"); + } + + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AuditInvestigationReport.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AuditInvestigationReport.java new file mode 100644 index 00000000..89887f90 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AuditInvestigationReport.java @@ -0,0 +1,98 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 12, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import static us.freeandfair.corla.asm.ASMEvent.AuditBoardDashboardEvent.SUBMIT_AUDIT_INVESTIGATION_REPORT_EVENT; + +import javax.persistence.PersistenceException; + +import com.google.gson.JsonParseException; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.asm.ASMEvent; +import us.freeandfair.corla.model.AuditInvestigationReportInfo; +import us.freeandfair.corla.model.CountyDashboard; +import us.freeandfair.corla.persistence.Persistence; + +/** + * Submit an audit investigation report. + * + * @author Joseph R. Kiniry + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +// TODO: consider rewriting along the same lines as CVRExportUpload +public class AuditInvestigationReport extends AbstractAuditBoardDashboardEndpoint { + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.POST; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/audit-investigation-report"; + } + + /** + * @return COUNTY authorization is required for this endpoint. + */ + public AuthorizationType requiredAuthorization() { + return AuthorizationType.COUNTY; + } + + /** + * {@inheritDoc} + */ + @Override + protected ASMEvent endpointEvent() { + return SUBMIT_AUDIT_INVESTIGATION_REPORT_EVENT; + } + + /** + * Submit an audit investigation report. + */ + @Override + public String endpointBody(final Request the_request, + final Response the_response) { + try { + final AuditInvestigationReportInfo report = + Main.GSON.fromJson(the_request.body(), AuditInvestigationReportInfo.class); + final CountyDashboard cdb = + Persistence.getByID(Main.authentication().authenticatedCounty(the_request).id(), + CountyDashboard.class); + if (cdb == null) { + Main.LOGGER.error("could not get audit board dashboard"); + serverError(the_response, "Could not save audit investigation report"); + } else { + cdb.submitInvestigationReport(report); + } + Persistence.saveOrUpdate(cdb); + } catch (final JsonParseException e) { + Main.LOGGER.error("malformed audit investigation report"); + badDataContents(the_response, "Invalid audit investigation report"); + } catch (final PersistenceException e) { + Main.LOGGER.error("could not save audit investigation report"); + serverError(the_response, "Unable to save audit investigation report"); + } + ok(the_response, "Report submitted"); + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AuthenticateAdministrator.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AuthenticateAdministrator.java new file mode 100644 index 00000000..0b6e8c4c --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AuthenticateAdministrator.java @@ -0,0 +1,115 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 9, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.asm.ASMEvent; +import us.freeandfair.corla.asm.AbstractStateMachine; +import us.freeandfair.corla.json.SubmittedCredentials; + +/** + * The endpoint for authenticating an administrator. + * + * @author Daniel M Zimmerman + * @author Joseph R. Kiniry + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class AuthenticateAdministrator extends AbstractEndpoint { + /** + * @return no authorization is required for this endpoint. + */ + @Override + public AuthorizationType requiredAuthorization() { + return AuthorizationType.NONE; + } + + /** + * @return this endpoint does not use an ASM. + */ + @Override + protected Class asmClass() { + return null; + } + + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.POST; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/auth-admin"; + } + + /** + * {@inheritDoc} + */ + @Override + protected ASMEvent endpointEvent() { + return null; + } + + /** + * Gets the ASM identity for the specified request. + * + * @param the_request The request. + * @return the county ID of the authenticated county. + */ + @Override + protected String asmIdentity(final Request the_request) { + return null; + } + + /** + * Attempts to authenticate an administrator; if the authentication is + * successful, authentication data is added to the session. + * + * Session query parameters: username, password, + * second_factor + * + * @param the_request The request. + * @param the_response The response. + */ + @Override + public String endpointBody(final Request the_request, final Response the_response) { + final SubmittedCredentials credentials = + Main.authentication().authenticationCredentials(the_request); + if (Main.authentication().secondFactorAuthenticated(the_request) && + Main.authentication().authenticatedAdministrator(the_request).username(). + equals(credentials.username())) { + okJSON(the_response, + Main.GSON.toJson(Main.authentication().authenticationStatus(the_request))); + } else { + if (Main.authentication(). + authenticateAdministrator(the_request, the_response, + credentials.username(), + credentials.password(), + credentials.secondFactor())) { + okJSON(the_response, + Main.GSON.toJson(Main.authentication().authenticationStatus(the_request))); + } else { + unauthorized(the_response, "Authentication failed"); + } + } + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AuthenticateCountyAdministrator.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AuthenticateCountyAdministrator.java new file mode 100644 index 00000000..39b0cbe0 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AuthenticateCountyAdministrator.java @@ -0,0 +1,85 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 9, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import static us.freeandfair.corla.model.Administrator.AdministratorType.COUNTY; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.json.SubmittedCredentials; +import us.freeandfair.corla.model.Administrator; + +/** + * The endpoint for authenticating a county administrator. + * + * @author Daniel M Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class AuthenticateCountyAdministrator extends AbstractEndpoint { + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.POST; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/auth-county-admin"; + } + + /** + * Attempts to authenticate a county administrator; if the authentication is + * successful, authentication data is added to the session. + * + * Session query parameters: username, password, + * second_factor + * + * @param the_request The request. + * @param the_response The response. + */ + @Override + public String endpointBody(final Request the_request, final Response the_response) { + final SubmittedCredentials credentials = + Main.authentication().authenticationCredentials(the_request); + if (Main.authentication(). + secondFactorAuthenticatedAs(the_request, COUNTY, credentials.username())) { + okJSON(the_response, + Main.GSON.toJson(Main.authentication().authenticationStatus(the_request))); + } else { + if (Main.authentication(). + authenticateAdministrator(the_request, the_response, + credentials.username(), + credentials.password(), + credentials.secondFactor())) { + final Administrator admin = + Main.authentication().authenticatedAdministrator(the_request); + if (admin.type() == COUNTY) { + okJSON(the_response, + Main.GSON.toJson(Main.authentication().authenticationStatus(the_request))); + } else { + unauthorized(the_response, "Authentication failed"); + } + } else { + unauthorized(the_response, "Authentication failed"); + } + } + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AuthenticateStateAdministrator.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AuthenticateStateAdministrator.java new file mode 100644 index 00000000..06a886ab --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/AuthenticateStateAdministrator.java @@ -0,0 +1,87 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 9, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import static us.freeandfair.corla.model.Administrator.AdministratorType.STATE; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.auth.AuthenticationInterface; +import us.freeandfair.corla.json.SubmittedCredentials; +import us.freeandfair.corla.model.Administrator; + +/** + * The endpoint for authenticating a state administrator. + * + * @author Daniel M Zimmerman + * @author Joseph R. Kiniry + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class AuthenticateStateAdministrator extends AbstractEndpoint { + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.POST; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/auth-state-admin"; + } + + /** + * Attempts to authenticate a state administrator; if the authentication is + * successful, authentication data is added to the session. + * + * Session query parameters: username, password, + * second_factor + * + * @param the_request The request. + * @param the_response The response. + */ + @Override + public String endpointBody(final Request the_request, final Response the_response) { + final SubmittedCredentials credentials = + Main.authentication().authenticationCredentials(the_request); + if (Main.authentication(). + secondFactorAuthenticatedAs(the_request, STATE, credentials.username())) { + okJSON(the_response, + Main.GSON.toJson(Main.authentication().authenticationStatus(the_request))); + } else { + if (Main.authentication(). + authenticateAdministrator(the_request, the_response, + credentials.username(), + credentials.password(), + credentials.secondFactor())) { + final Administrator admin = + (Administrator) the_request.session().attribute(AuthenticationInterface.ADMIN); + if (admin.type() == STATE) { + okJSON(the_response, + Main.GSON.toJson(Main.authentication().authenticationStatus(the_request))); + } else { + unauthorized(the_response, "Authentication failed"); + } + } else { + unauthorized(the_response, "Authentication failed"); + } + } + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/BallotManifestDownload.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/BallotManifestDownload.java new file mode 100644 index 00000000..cffe4995 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/BallotManifestDownload.java @@ -0,0 +1,83 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 27, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.UncheckedIOException; +import java.util.stream.Stream; + +import com.google.gson.stream.JsonWriter; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.model.BallotManifestInfo; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.util.SparkHelper; + +/** + * The ballot manifest download endpoint. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class BallotManifestDownload extends AbstractCountyDashboardEndpoint { + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.GET; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/ballot-manifest"; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointBody(final Request the_request, final Response the_response) { + try (OutputStream os = SparkHelper.getRaw(the_response).getOutputStream(); + BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(os, "UTF-8")); + JsonWriter jw = new JsonWriter(bw)) { + jw.beginArray(); + final Stream bmi_stream = + Persistence.getAllAsStream(BallotManifestInfo.class); + bmi_stream.forEach((the_bmi) -> { + try { + jw.jsonValue(Main.GSON.toJson(Persistence.unproxy(the_bmi))); + Persistence.evict(the_bmi); + } catch (final IOException e) { + throw new UncheckedIOException(e); + } + }); + jw.endArray(); + jw.flush(); + jw.close(); + ok(the_response); + } catch (final IOException e) { + serverError(the_response, "Unable to stream response"); + } + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/BallotManifestDownloadByCounty.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/BallotManifestDownloadByCounty.java new file mode 100644 index 00000000..12510acc --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/BallotManifestDownloadByCounty.java @@ -0,0 +1,114 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 27, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.util.HashSet; +import java.util.Set; + +import com.google.gson.stream.JsonWriter; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.model.BallotManifestInfo; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.query.BallotManifestInfoQueries; +import us.freeandfair.corla.util.SparkHelper; + +/** + * The ballot manifest by county download endpoint. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class BallotManifestDownloadByCounty extends AbstractCountyDashboardEndpoint { + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.GET; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/ballot-manifest/county"; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointBody(final Request the_request, final Response the_response) { + if (validateParameters(the_request)) { + final Set county_set = new HashSet(); + for (final String s : the_request.queryParams()) { + county_set.add(Long.valueOf(s)); + } + final Set matches = + BallotManifestInfoQueries.getMatching(county_set); + if (matches.isEmpty()) { + serverError(the_response, "Error retrieving records from database"); + } else { + try (OutputStream os = SparkHelper.getRaw(the_response).getOutputStream(); + BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(os, "UTF-8")); + JsonWriter jw = new JsonWriter(bw)) { + jw.beginArray(); + for (final BallotManifestInfo bmi : matches) { + jw.jsonValue(Main.GSON.toJson(Persistence.unproxy(bmi))); + Persistence.evict(bmi); + } + jw.endArray(); + jw.flush(); + jw.close(); + } catch (final IOException e) { + serverError(the_response, "Unable to stream response"); + } + } + ok(the_response); + } else { + dataNotFound(the_response, "Invalid county ID specified"); + } + return my_endpoint_result.get(); + } + + /** + * Validates the parameters of a request. For this endpoint, + * the parameter names must all be integers. + * + * @param the_request The request. + * @return true if the parameters are valid, false otherwise. + */ + protected boolean validateParameters(final Request the_request) { + boolean result = true; + + for (final String s : the_request.queryParams()) { + try { + Integer.parseInt(s); + } catch (final NumberFormatException e) { + result = false; + break; + } + } + + return result; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/BallotManifestImport.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/BallotManifestImport.java new file mode 100644 index 00000000..59ed4cdc --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/BallotManifestImport.java @@ -0,0 +1,211 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 27, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import static us.freeandfair.corla.asm.ASMEvent.CountyDashboardEvent.IMPORT_BALLOT_MANIFEST_EVENT; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.Map; + +import javax.persistence.PersistenceException; + +import com.google.gson.JsonParseException; + +import spark.Request; +import spark.Response; + +import org.apache.log4j.Logger; +import org.apache.log4j.LogManager; + + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.asm.ASMEvent; +import us.freeandfair.corla.csv.ColoradoBallotManifestParser; +import us.freeandfair.corla.csv.Result; +import us.freeandfair.corla.json.UploadedFileDTO; +import us.freeandfair.corla.model.County; +import us.freeandfair.corla.model.CountyDashboard; +import us.freeandfair.corla.model.UploadedFile; +import us.freeandfair.corla.model.UploadedFile.FileStatus; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.query.BallotManifestInfoQueries; + +/** + * The "ballot manifest import" endpoint. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings({"PMD.AtLeastOneConstructor", "PMD.ExcessiveImports"}) +public class BallotManifestImport extends AbstractCountyDashboardEndpoint { + + /** + * Class-wide logger + */ + public static final Logger LOGGER = + LogManager.getLogger(BallotManifestImport.class); + + /** + * The " (id " string. + */ + private static final String PAREN_ID = " (id "; + + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.POST; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/import-ballot-manifest"; + } + + /** + * {@inheritDoc} + */ + @Override + protected ASMEvent endpointEvent() { + return IMPORT_BALLOT_MANIFEST_EVENT; + } + + /** + * Updates the appropriate county dashboard to reflect a new + * ballot manifest upload. + * @param the_response The response object (for error reporting). + * @param the_file The uploaded file. + * @param the_ballot_count The ballot count from the manifest. + */ + private void updateCountyDashboard(final Response the_response, + final UploadedFile the_file, + final int the_ballot_count) { + final CountyDashboard cdb = + Persistence.getByID(the_file.county().id(), CountyDashboard.class); + if (cdb == null) { + serverError(the_response, "could not locate county dashboard"); + } else { + // now set the new manifest info + cdb.setManifestFile(the_file); + cdb.setBallotsInManifest(the_ballot_count); + try { + Persistence.saveOrUpdate(cdb); + } catch (final PersistenceException e) { + serverError(the_response, "could not update county dashboard"); + } + } + } + + /** + * Parses an uploaded ballot manifest and attempts to persist it to the database. + * + * @param the_response The response (for error reporting). + * @param the_file The uploaded file. + */ + // the CSV parser can throw arbitrary runtime exceptions, which we must catch + @SuppressWarnings({"PMD.AvoidCatchingGenericException"}) + private void parseFile(final Response the_response, final UploadedFile the_file) { + try (InputStream bmi_is = the_file.file().getBinaryStream()) { + final InputStreamReader bmi_isr = new InputStreamReader(bmi_is, "UTF-8"); + final ColoradoBallotManifestParser parser = + new ColoradoBallotManifestParser(bmi_isr, + the_file.county().id()); + final int deleted = BallotManifestInfoQueries.deleteMatching(the_file.county().id()); + Result result = parser.parse(); + if (result.success) { + final int imported = result.importedCount; + result = new Result(); + result.success = true; + result.importedCount = imported; + the_file.setResult(result); + LOGGER.info(imported + " ballot manifest records parsed from file " + + the_file.filename() + PAREN_ID + the_file.id() + ") for county " + + the_file.county().id()); + updateCountyDashboard(the_response, the_file, + parser.ballotCount().getAsInt()); + the_file.setStatus(FileStatus.IMPORTED); + Persistence.saveOrUpdate(the_file); + final Map response = new HashMap(); + response.put("records_imported", imported); + if (deleted > 0) { + response.put("records_deleted", deleted); + } + okJSON(the_response, Main.GSON.toJson(response)); + } else { + the_file.setResult(result); + Persistence.saveOrUpdate(the_file); + LOGGER.info("could not parse malformed ballot manifest file " + + the_file.filename() + PAREN_ID + the_file.id() + ") for county " + + the_file.county().id()); + badDataContents(the_response, "malformed ballot manifest file " + + the_file.filename() + PAREN_ID + the_file.id() + ")"); + } + } catch (final RuntimeException | IOException e) { + LOGGER.info("could not parse malformed ballot manifest file " + + the_file.filename() + PAREN_ID + the_file.id() + ") for county " + + the_file.county().id() + ": " + e); + badDataContents(the_response, "malformed ballot manifest file " + + the_file.filename() + PAREN_ID + the_file.id() + ")"); + } catch (final SQLException e) { + LOGGER.info("could not read file " + the_file.filename() + + PAREN_ID + the_file.id() + ") from persistent storage"); + } + } + + /** + * {@inheritDoc} + */ + @Override + @SuppressWarnings({"PMD.ConfusingTernary"}) + public String endpointBody(final Request the_request, final Response the_response) { + // we know we have county authorization, so let's find out which county + final County county = Main.authentication().authenticatedCounty(the_request); + + if (county == null) { + unauthorized(the_response, "unauthorized administrator for ballot manifest upload"); + return my_endpoint_result.get(); + } + + try { + UploadedFileDTO upF = Main.GSON.fromJson(the_request.body(), UploadedFileDTO.class); + if (null == upF.getFileId()) { + LOGGER.error(the_request.body() + " did not have a fileId attribute"); + badDataContents(the_response, "missing fileId attribute"); + return my_endpoint_result.get(); + } + final UploadedFile file = Persistence.getByID(upF.getFileId(), UploadedFile.class); + if (file == null) { + badDataContents(the_response, "nonexistent file"); + } else if (!file.county().equals(county)) { + unauthorized(the_response, "county " + county.id() + " attempted to import " + + "file " + file.filename() + " uploaded by county " + + file.county().id()); + } else if (file.getStatus() == FileStatus.HASH_VERIFIED) { + parseFile(the_response, file); + } else { + badDataContents(the_response, "attempt to import a file without a verified hash"); + } + } catch (final JsonParseException e) { + badDataContents(the_response, "malformed request: " + e.getMessage()); + } + + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/BallotNotFound.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/BallotNotFound.java new file mode 100644 index 00000000..85038691 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/BallotNotFound.java @@ -0,0 +1,193 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 12, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import static us.freeandfair.corla.asm.ASMEvent.AuditBoardDashboardEvent.*; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +import javax.persistence.PersistenceException; + +import com.google.gson.JsonParseException; +import com.google.gson.JsonSyntaxException; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.asm.ASMEvent; +import us.freeandfair.corla.controller.ComparisonAuditController; +import us.freeandfair.corla.json.SubmittedBallotNotFound; +import us.freeandfair.corla.model.CVRAuditInfo; +import us.freeandfair.corla.model.CVRContestInfo; +import us.freeandfair.corla.model.CastVoteRecord; +import us.freeandfair.corla.model.CastVoteRecord.RecordType; +import us.freeandfair.corla.model.County; +import us.freeandfair.corla.model.CountyDashboard; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.util.SuppressFBWarnings; + +/** + * The endpoint for reporting ballots that could not be found by auditors. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings({"PMD.AtLeastOneConstructor", "PMD.CyclomaticComplexity", + "PMD.ModifiedCyclomaticComplexity", "PMD.StdCyclomaticComplexity"}) +public class BallotNotFound extends AbstractAuditBoardDashboardEndpoint { + /** + * The event we will return for the ASM. + */ + private final ThreadLocal my_event = new ThreadLocal(); + + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.POST; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/ballot-not-found"; + } + + /** + * {@inheritDoc} + */ + @Override + public ASMEvent endpointEvent() { + return my_event.get(); + } + + /** + * {@inheritDoc} + */ + @Override + protected void reset() { + my_event.set(null); + } + + /** build new acvr, similar to ACVRUpload#buildNewAcvr, + * but details come from the cvr + **/ + public CastVoteRecord buildNewAcvr(final SubmittedBallotNotFound submission, + final CastVoteRecord cvr, + final CountyDashboard cdb, + final RecordType recordType, + final List contestInfo) { + final CastVoteRecord newAcvr = + new CastVoteRecord(recordType, + Instant.now(), cvr.countyID(), cvr.cvrNumber(), null, + cvr.scannerID(), cvr.batchID(), cvr.recordID(), + cvr.imprintedID(), cvr.ballotType(), + contestInfo); + newAcvr.setAuditBoardIndex(submission.getAuditBoardIndex()); + newAcvr.setCvrId(submission.id()); + newAcvr.setRoundNumber(cdb.currentRound().number()); + newAcvr.setRand(cvr.getRand()); + + return newAcvr; + } + + /** + * Marks the specified ballot as "not found" by the audit board. + * The ballot to so mark is indicated by the ID of its corresponding + * CVR, which must match a ballot under audit by the authenticated + * county. + * + * @param the_request The request. + * @param the_response The response. + */ + @Override + // FindBugs thinks we can deference a null CVR, but we can't because + // badDataContents() (which ends the method's execution) would get called first + @SuppressFBWarnings("NP_NULL_ON_SOME_PATH") + @SuppressWarnings("PMD.NPathComplexity") + public String endpointBody(final Request the_request, final Response the_response) { + // we must be authenticated as a county + final County county = Main.authentication().authenticatedCounty(the_request); + if (county == null) { + unauthorized(the_response, "not authorized for audit board operations"); + return my_endpoint_result.get(); + } + + final CountyDashboard cdb = Persistence.getByID(county.id(), CountyDashboard.class); + if (cdb == null) { + serverError(the_response, "could not load audit board information"); + return my_endpoint_result.get(); + } + + // attempt to find the CVR under audit specified in the request + try { + final SubmittedBallotNotFound sbnf = + Main.GSON.fromJson(the_request.body(), SubmittedBallotNotFound.class); + if (sbnf.id() == null) { + throw new JsonSyntaxException("invalid ballot ID"); + } + final CastVoteRecord cvr = Persistence.getByID(sbnf.id(), CastVoteRecord.class); + if (cvr == null) { + badDataContents(the_response, "nonexistent CVR ID"); + } + final CVRAuditInfo matching = Persistence.getByID(cvr.id(), CVRAuditInfo.class); + if (matching == null) { + badDataContents(the_response, "specified CVR not under audit"); + } + + // construct a phantom ballot ACVR + final List contest_info = new ArrayList<>(); + for (final CVRContestInfo ci : cvr.contestInfo()) { + contest_info.add(new CVRContestInfo(ci.contest(), "ballot not found", + null, new ArrayList())); + } + final CastVoteRecord newAcvr = buildNewAcvr(sbnf, + cvr, + cdb, + RecordType.PHANTOM_BALLOT, + contest_info); + + boolean result; + if (sbnf.isReaudit()) { + final String comment = sbnf.getComment(); + result = ComparisonAuditController.reaudit(cdb, cvr, newAcvr, comment); + } else { + Persistence.saveOrUpdate(newAcvr); + result = ComparisonAuditController.submitAuditCVR(cdb, cvr, newAcvr); + } + + if (result) { + ok(the_response, "audit CVR submitted"); + } + if (cdb.ballotsRemainingInCurrentRound() == 0) { + // the round is over + my_event.set(ROUND_COMPLETE_EVENT); + } else { + my_event.set(REPORT_BALLOT_NOT_FOUND_EVENT); + } + } catch (final JsonParseException e) { + badDataType(the_response, "invalid request format"); + } catch (final NumberFormatException e) { + badDataContents(the_response, "malformed CVR id"); + } catch (final PersistenceException e) { + serverError(the_response, "unable to save audit CVR"); + } + return my_endpoint_result.get(); + } + +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CORSFilter.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CORSFilter.java new file mode 100644 index 00000000..ab342351 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CORSFilter.java @@ -0,0 +1,126 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 15, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import spark.Filter; +import spark.Request; +import spark.Response; + +/** + * A filter to enable CORS in our Spark endpoints. This is used as a Spark + * "afterAfterFilter". + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public class CORSFilter implements Filter { + /** + * The methods property name. + */ + public static final String METHODS_PROPERTY = "cors.methods"; + + /** + * The default methods. + */ + public static final String DEFAULT_METHODS = "GET,PUT,POST,DELETE,OPTIONS"; + + /** + * The origin property name. + */ + public static final String ORIGIN_PROPERTY = "cors.origin"; + + /** + * The default origin. + */ + public static final String DEFAULT_ORIGIN = "http://localhost:3000"; + + /** + * The headers property name. + */ + public static final String HEADERS_PROPERTY = "cors.headers"; + + /** + * The default headers. + */ + public static final String DEFAULT_HEADERS = + "Content-Type,Authorization,X-Requested-With,Content-Length,Accept,Origin,"; + + /** + * The credentials property name. + */ + public static final String CREDENTIALS_PROPERTY = "cors.credentials"; + + /** + * The default credentials. + */ + public static final String DEFAULT_CREDENTIALS = "true"; + + /** + * The CORS headers for this filter. + */ + private final Map my_cors_headers; + + /** + * The filter to run after this one, if any. + */ + private final Filter my_filter; + + /** + * Constructs a new CORSFilter with the specified properties. + * + * @param the_properties The properties. + * @param the_filter The filter to run after this one, if any. + */ + public CORSFilter(final Properties the_properties, final Filter the_filter) { + my_cors_headers = corsHeaders(the_properties); + my_filter = the_filter; + } + + /** + * Gets CORS headers from a set of properties. + * + * @param the_properties The properties. + */ + private Map corsHeaders(final Properties the_properties) { + final Map result = new HashMap<>(); + + result.put("Access-Control-Allow-Methods", + the_properties.getProperty(METHODS_PROPERTY, DEFAULT_METHODS)); + result.put("Access-Control-Allow-Origin", + the_properties.getProperty(ORIGIN_PROPERTY, DEFAULT_ORIGIN)); + result.put("Access-Control-Allow-Headers", + the_properties.getProperty(HEADERS_PROPERTY, DEFAULT_HEADERS)); + result.put("Access-Control-Allow-Credentials", + the_properties.getProperty(CREDENTIALS_PROPERTY, DEFAULT_CREDENTIALS)); + + return result; + } + + /** + * Handles a request. + * + * @param the_request The request. + * @param the_response The response. + */ + @SuppressWarnings("PMD.SignatureDeclareThrowsException") + public void handle(final Request the_request, final Response the_response) + throws Exception { + my_cors_headers.forEach((the_key, the_value) -> { + the_response.header(the_key, the_value); + }); + my_filter.handle(the_request, the_response); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CVRDownload.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CVRDownload.java new file mode 100644 index 00000000..a09924d7 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CVRDownload.java @@ -0,0 +1,97 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 27, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.UncheckedIOException; +import java.util.stream.Stream; + +import javax.persistence.PersistenceException; + +import com.google.gson.stream.JsonWriter; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.model.CastVoteRecord; +import us.freeandfair.corla.model.CastVoteRecord.RecordType; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.query.CastVoteRecordQueries; +import us.freeandfair.corla.util.SparkHelper; + +/** + * The ballot manifest download endpoint. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class CVRDownload extends AbstractEndpoint { + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.GET; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/cvr"; + } + + /** + * This endpoint requires any kind of authentication. + */ + @Override + public AuthorizationType requiredAuthorization() { + return AuthorizationType.EITHER; + } + + /** + * {@inheritDoc} + */ + @Override + // necessary to break out of the lambda expression in case of IOException + @SuppressWarnings("PMD.ExceptionAsFlowControl") + public String endpointBody(final Request the_request, final Response the_response) { + try (OutputStream os = SparkHelper.getRaw(the_response).getOutputStream(); + BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(os, "UTF-8")); + JsonWriter jw = new JsonWriter(bw)) { + jw.beginArray(); + final Stream matches = + CastVoteRecordQueries.getMatching(RecordType.UPLOADED); + matches.forEach((the_cvr) -> { + try { + jw.jsonValue(Main.GSON.toJson(Persistence.unproxy(the_cvr))); + Persistence.evict(the_cvr); + } catch (final IOException e) { + throw new UncheckedIOException(e); + } + }); + jw.endArray(); + jw.flush(); + jw.close(); + ok(the_response); + } catch (final UncheckedIOException | IOException | PersistenceException e) { + serverError(the_response, "Unable to stream response"); + } + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CVRDownloadByCounty.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CVRDownloadByCounty.java new file mode 100644 index 00000000..b7bdc47a --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CVRDownloadByCounty.java @@ -0,0 +1,127 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 27, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.UncheckedIOException; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Stream; + +import javax.persistence.PersistenceException; + +import com.google.gson.stream.JsonWriter; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.model.CastVoteRecord; +import us.freeandfair.corla.model.CastVoteRecord.RecordType; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.query.CastVoteRecordQueries; +import us.freeandfair.corla.util.SparkHelper; + +/** + * The ballot manifest download endpoint. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class CVRDownloadByCounty extends AbstractEndpoint { + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.GET; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/cvr/county"; + } + + /** + * This endpoint requires any kind of authentication. + */ + @Override + public AuthorizationType requiredAuthorization() { + return AuthorizationType.EITHER; + } + + /** + * {@inheritDoc} + */ + @Override + // necessary to break out of the lambda expression in case of IOException + @SuppressWarnings("PMD.ExceptionAsFlowControl") + public String endpointBody(final Request the_request, final Response the_response) { + final Set county_set = new HashSet(); + for (final String s : the_request.queryParams()) { + county_set.add(Long.valueOf(s)); + } + try (OutputStream os = SparkHelper.getRaw(the_response).getOutputStream(); + BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(os, "UTF-8")); + JsonWriter jw = new JsonWriter(bw)) { + jw.beginArray(); + for (final Long county : county_set) { + final Stream matches = + CastVoteRecordQueries.getMatching(county, RecordType.UPLOADED); + matches.forEach((the_cvr) -> { + try { + jw.jsonValue(Main.GSON.toJson(Persistence.unproxy(the_cvr))); + Persistence.evict(the_cvr); + } catch (final IOException e) { + throw new UncheckedIOException(e); + } + }); + } + jw.endArray(); + jw.flush(); + jw.close(); + ok(the_response); + } catch (final UncheckedIOException | IOException | PersistenceException e) { + serverError(the_response, "Unable to stream response"); + } + return my_endpoint_result.get(); + } + + /** + * Validates the parameters of a request. For this endpoint, + * the paramter names must all be integers. + * + * @param the_request The request. + * @return true if the parameters are valid, false otherwise. + */ + protected boolean validateParameters(final Request the_request) { + boolean result = true; + + for (final String s : the_request.queryParams()) { + try { + Integer.parseInt(s); + } catch (final NumberFormatException e) { + result = false; + break; + } + } + + return result; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CVRDownloadByID.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CVRDownloadByID.java new file mode 100644 index 00000000..a016c784 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CVRDownloadByID.java @@ -0,0 +1,83 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 27, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import org.apache.log4j.Level; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.model.CastVoteRecord; +import us.freeandfair.corla.persistence.Persistence; + +/** + * The CVR by ID download endpoint. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class CVRDownloadByID extends AbstractEndpoint { + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.GET; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/cvr/id/:id"; + } + + /** + * This endpoint requires any kind of authentication. + */ + @Override + public AuthorizationType requiredAuthorization() { + return AuthorizationType.EITHER; + } + + /** + * {@inheritDoc} + */ + @Override + public Level logLevel() { + return Level.DEBUG; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointBody(final Request the_request, final Response the_response) { + try { + final CastVoteRecord c = + Persistence.getByID(Long.parseLong(the_request.params(":id")), + CastVoteRecord.class); + if (c == null) { + dataNotFound(the_response, "CVR not found"); + } else { + okJSON(the_response, Main.GSON.toJson(Persistence.unproxy(c))); + } + } catch (final NumberFormatException e) { + invariantViolation(the_response, "Bad CVR ID"); + } + + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CVRExportImport.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CVRExportImport.java new file mode 100644 index 00000000..279dd16b --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CVRExportImport.java @@ -0,0 +1,147 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 27, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import static us.freeandfair.corla.asm.ASMEvent.CountyDashboardEvent.IMPORT_CVRS_EVENT; + +import java.io.InputStreamReader; +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; + +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import com.google.gson.JsonParseException; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.asm.ASMEvent; +import us.freeandfair.corla.controller.ImportFileController; +import us.freeandfair.corla.json.UploadedFileDTO; +import us.freeandfair.corla.model.County; +import us.freeandfair.corla.model.CountyDashboard; +import us.freeandfair.corla.model.ImportStatus; +import us.freeandfair.corla.model.ImportStatus.ImportState; +import us.freeandfair.corla.model.UploadedFile.FileStatus; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.query.UploadedFileQueries; + +/** + * The "CVR export import" endpoint. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings({"PMD.DoNotUseThreads"}) +public class CVRExportImport extends AbstractCountyDashboardEndpoint { + + /** + * Class-wide logger + */ + public static final Logger LOGGER = + LogManager.getLogger(CVRExportImport.class); + + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.POST; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/import-cvr-export"; + } + + /** + * {@inheritDoc} + */ + @Override + protected ASMEvent endpointEvent() { + return IMPORT_CVRS_EVENT; + } + + /** + * {@inheritDoc} + */ + @Override + @SuppressWarnings({"PMD.ConfusingTernary"}) + public String endpointBody(final Request the_request, final Response the_response) { + final County county = Main.authentication().authenticatedCounty(the_request); + // check logged in as county admin + if (county == null) { + unauthorized(the_response, "unauthorized administrator for CVR import"); + return my_endpoint_result.get(); + } + + + final CountyDashboard cdb = Persistence.getByID(county.id(), CountyDashboard.class); + UploadedFileDTO upF = null; + final Map responseBody = new HashMap<>(); + + + // check valid json + try { + upF = Main.GSON.fromJson(the_request.body(), UploadedFileDTO.class); + } catch (final JsonParseException e) { + badDataContents(the_response, "malformed request: " + e.getMessage()); + return my_endpoint_result.get(); + } + + // check parameter presence + if (null == upF.getFileId()) { + badDataContents(the_response, "missing file_id attribute"); + return my_endpoint_result.get(); + } + + UploadedFileDTO uploadedFileAttrs = UploadedFileQueries.getAttrs(upF); + + // check presence + if (null == uploadedFileAttrs) { + badDataContents(the_response, "nonexistent file"); + + // check county + } else if (!county.id().equals(uploadedFileAttrs.getCountyId())) { + badDataContents(the_response, "wrong file id, not for current county"); + + // check status + } else if (!"HASH_VERIFIED".equals(uploadedFileAttrs.getStatus())) { + if (!"IMPORTING".equals(uploadedFileAttrs.getStatus())) { + badDataContents(the_response, "currently importing a cvr file"); + } else { + badDataContents(the_response, "submitted hash failed verification"); + } + + // ok, proceed with correct county, and status + } else { + upF.setStatus(FileStatus.IMPORTING.toString()); + upF.setCountyId(county.id()); + UploadedFileQueries.updateStatus(upF); + cdb.setCVRImportStatus(new ImportStatus(ImportState.IN_PROGRESS)); + // spawn a thread to do the import; this endpoint always immediately + // returns a successful result if we get to this point + (new Thread(new ImportFileController(upF))).start(); + + responseBody.put("import_start_time", Instant.now()); + okJSON(the_response, Main.GSON.toJson(responseBody)); + } + + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CVRToAuditDownload.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CVRToAuditDownload.java new file mode 100644 index 00000000..93620cbd --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CVRToAuditDownload.java @@ -0,0 +1,317 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 27, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import static us.freeandfair.corla.util.PrettyPrinter.booleanYesNo; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.UnsupportedEncodingException; +import java.io.Writer; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.OptionalInt; + +import javax.persistence.PersistenceException; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; +import org.apache.commons.csv.QuoteMode; +import org.apache.cxf.attachment.Rfc5987Util; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.controller.BallotSelection; +import us.freeandfair.corla.controller.ComparisonAuditController; +import us.freeandfair.corla.json.CVRToAuditResponse; +import us.freeandfair.corla.model.CastVoteRecord; +import us.freeandfair.corla.model.County; +import us.freeandfair.corla.model.CountyDashboard; +import us.freeandfair.corla.model.Round; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.util.SparkHelper; + +/** + * The CVR to audit download endpoint. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings({"PMD.AtLeastOneConstructor", "PMD.CyclomaticComplexity", + "PMD.ModifiedCyclomaticComplexity", "PMD.StdCyclomaticComplexity"}) +public class CVRToAuditDownload extends AbstractEndpoint { + /** + * The "start" parameter. + */ + public static final String START = "start"; + + /** + * The "ballot_count" parameter. + */ + public static final String BALLOT_COUNT = "ballot_count"; + + /** + * The "round" parameter. + */ + public static final String ROUND = "round"; + + /** + * The "county" parameter. + */ + public static final String COUNTY = "county"; + + /** + * The CSV headers for formatting the response. + */ + private static final String[] CSV_HEADERS = { + "storage_location", "scanner_id", "batch_id", "record_id", "imprinted_id", + "ballot_type", "cvr_number", "audited", "audit_board" + }; + + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.GET; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/cvr-to-audit-download"; + } + + /** + * This endpoint requires any kind of authentication. + */ + @Override + public AuthorizationType requiredAuthorization() { + return AuthorizationType.EITHER; + } + + /** + * Validate the request parameters. In this case, the two parameters + * must exist and both be non-negative integers. + * + * @param the_request The request. + */ + @Override + protected boolean validateParameters(final Request the_request) { + final String start = the_request.queryParams(START); + final String ballot_count = the_request.queryParams(BALLOT_COUNT); + final String round = the_request.queryParams(ROUND); + final String county = the_request.queryParams(COUNTY); + + boolean result = start != null && ballot_count != null || + round != null; + + if (result) { + try { + if (start != null) { + final int s = Integer.parseInt(start); + result &= s >= 0; + final int b = Integer.parseInt(ballot_count); + result &= b >= 0; + } + + if (round != null) { + final int r = Integer.parseInt(round); + result &= r > 0; + } + + if (county == null && Main.authentication().authenticatedCounty(the_request) == null) { + // it's a DoS user, but they didn't specify a county + result = false; + } else if (county != null) { + Long.parseLong(county); + } + } catch (final NumberFormatException e) { + result = false; + } + } + + return result; + } + + /** + * {@inheritDoc} + */ + @Override + @SuppressWarnings({"PMD.NPathComplexity", "PMD.ExcessiveMethodLength", + "checkstyle:methodlength", "checkstyle:executablestatementcount"}) + public String endpointBody(final Request the_request, final Response the_response) { + // we know we have either state or county authentication; this will be null + // for state authentication + County county = Main.authentication().authenticatedCounty(the_request); + + if (county == null) { + county = + Persistence.getByID(Long.parseLong(the_request.queryParams(COUNTY)), County.class); + if (county == null) { + badDataContents(the_response, "county " + the_request.queryParams(COUNTY) + + " does not exist"); + } + assert county != null; // makes FindBugs happy + } + + try { + // get the request parameters + final String start_param = the_request.queryParams(START); + final String ballot_count_param = the_request.queryParams(BALLOT_COUNT); + final String round_param = the_request.queryParams(ROUND); + + int ballot_count = 0; + if (ballot_count_param != null) { + ballot_count = Integer.parseInt(ballot_count_param); + } + + int index = 0; + if (start_param != null) { + index = Integer.parseInt(start_param); + } + + // get other things we need + final CountyDashboard cdb = Persistence.getByID(county.id(), CountyDashboard.class); + final List cvr_to_audit_list; + final List response_list = new ArrayList<>(); + + // compute the round, if any + OptionalInt round = OptionalInt.empty(); + if (round_param != null) { + final int round_number = Integer.parseInt(round_param); + if (0 < round_number && round_number <= cdb.rounds().size()) { + round = OptionalInt.of(round_number); + } else { + badDataContents(the_response, "cvr list requested for invalid round " + + round_param + " for county " + cdb.id()); + } + } + + if (round.isPresent()) { + cvr_to_audit_list = ComparisonAuditController.ballotsToAudit( + cdb, + round.getAsInt() + ); + response_list.addAll(BallotSelection.toResponseList(cvr_to_audit_list)); + response_list.sort(null); + + final Round roundObject = cdb.rounds().get(round.getAsInt() - 1); + + final List> bsa = + roundObject.ballotSequenceAssignment(); + + if (bsa != null) { + // Walk the sequence assignments getting the audit boards' index and + // count values. Use that information to set the audit board index for + // each response row. + // + // Note: the board assignment has already been created, so the list of + // ballots returned to the client cannot change after the round + // starts. + for (int i = 0; i < bsa.size(); i++) { + final Map m = bsa.get(i); + + final Integer boardIndex = m.get("index"); + final Integer boardCount = m.get("count"); + + for (int j = boardIndex; j < boardIndex + boardCount; j++) { + final CVRToAuditResponse row = response_list.get(j); + row.setAuditBoardIndex(i); + } + } + } + } + + // generate a CSV file from the response list + the_response.type("text/csv"); + + // the file name should be constructed from the county name and round + // or start/count + final StringBuilder sb = new StringBuilder(32); + sb.append("ballot-list-"); + sb.append(county.name().toLowerCase(Locale.getDefault()).replace(" ", "_")); + sb.append('-'); + if (round.isPresent()) { + sb.append("round-"); + sb.append(round.getAsInt()); + } else { + sb.append("start-"); + sb.append(index); + sb.append("-count-"); + sb.append(ballot_count); + } + sb.append(".csv"); + + try { + the_response.raw().setHeader("Content-Disposition", "attachment; filename=\"" + + Rfc5987Util.encode(sb.toString(), "UTF-8") + "\""); + } catch (final UnsupportedEncodingException e) { + serverError(the_response, "UTF-8 is unsupported (this should never happen)"); + } + + try (OutputStream os = SparkHelper.getRaw(the_response).getOutputStream(); + BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(os, "UTF-8"))) { + writeCSV(response_list, bw); + ok(the_response); + } catch (final IOException e) { + serverError(the_response, "Unable to stream response"); + } + } catch (final PersistenceException e) { + serverError(the_response, "could not generate cvr list"); + } + return my_endpoint_result.get(); + } + + /** + * Writes the specified list of CVRToAuditResponse objects as CSV. + * + * @param the_cvrs The list of objects. + * @param the_writer The writer to write to. + * @exception IOException if there is a problem writing the CSV file. + */ + private void writeCSV(final List the_cvrs, final Writer the_writer) + throws IOException { + try (CSVPrinter csvp = new CSVPrinter(the_writer, + CSVFormat.DEFAULT.withHeader(CSV_HEADERS). + withQuoteMode(QuoteMode.NON_NUMERIC))) { + for (final CVRToAuditResponse cvr : the_cvrs) { + csvp.printRecord(cvr.storageLocation(), cvr.scannerID(), cvr.batchID(), + cvr.recordID(), cvr.imprintedID(), cvr.ballotType(), + cvr.cvrNumber(), booleanYesNo(cvr.audited()), + this.boardIndexToName(cvr.auditBoardIndex())); + } + } + } + + /** + * Converts an audit board index to a human-readable board name. + * + * @param index audit board index + * @return String human-readable name + */ + private String boardIndexToName(final Integer index) { + if (index == null) { + return ""; + } + + return String.format("Audit board %d", index + 1); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CVRToAuditList.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CVRToAuditList.java new file mode 100644 index 00000000..506e1a09 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CVRToAuditList.java @@ -0,0 +1,214 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 27, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.OptionalInt; + +import javax.persistence.PersistenceException; + +import org.apache.log4j.Level; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.controller.BallotSelection; +import us.freeandfair.corla.controller.ComparisonAuditController; +import us.freeandfair.corla.json.CVRToAuditResponse; +import us.freeandfair.corla.model.CastVoteRecord; +import us.freeandfair.corla.model.County; +import us.freeandfair.corla.model.CountyDashboard; +import us.freeandfair.corla.model.Round; +import us.freeandfair.corla.persistence.Persistence; + +/** + * The CVR to audit list endpoint. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings({"PMD.AtLeastOneConstructor", "PMD.CyclomaticComplexity", + "PMD.ModifiedCyclomaticComplexity", "PMD.StdCyclomaticComplexity"}) +public class CVRToAuditList extends AbstractEndpoint { + /** + * The "start" parameter. + */ + public static final String START = "start"; + + /** + * The "round" parameter. + */ + public static final String ROUND = "round"; + + /** + * The "county" parameter. + */ + public static final String COUNTY = "county"; + + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.GET; + } + + /** + * {@inheritDoc} + */ + @Override + public Level logLevel() { + return Level.DEBUG; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/cvr-to-audit-list"; + } + + /** + * This endpoint requires any kind of authentication. + */ + @Override + public AuthorizationType requiredAuthorization() { + return AuthorizationType.EITHER; + } + + /** + * Validate the request parameters. In this case, the two parameters + * must exist and both be non-negative integers. + * + * @param the_request The request. + */ + @Override + protected boolean validateParameters(final Request the_request) { + final String start = the_request.queryParams(START); + final String round = the_request.queryParams(ROUND); + final String county = the_request.queryParams(COUNTY); + + boolean result = start != null || round != null; + + if (result) { + try { + if (start != null) { + final int s = Integer.parseInt(start); + result &= s >= 0; + } + + if (round != null) { + final int r = Integer.parseInt(round); + result &= r > 0; + } + + if (county == null && Main.authentication().authenticatedCounty(the_request) == null) { + // it's a DoS user, but they didn't specify a county + result = false; + } else if (county != null) { + Long.parseLong(county); + } + } catch (final NumberFormatException e) { + result = false; + } + } + + return result; + } + + /** + * {@inheritDoc} + */ + @Override + @SuppressWarnings({"PMD.ExcessiveMethodLength", "PMD.NPathComplexity"}) + public String endpointBody(final Request the_request, final Response the_response) { + // we know we have either state or county authentication; this will be null + // for state authentication + County county = Main.authentication().authenticatedCounty(the_request); + + if (county == null) { + county = + Persistence.getByID(Long.parseLong(the_request.queryParams(COUNTY)), County.class); + if (county == null) { + badDataContents(the_response, "county " + the_request.queryParams(COUNTY) + + " does not exist"); + } + assert county != null; // makes FindBugs happy + } + + try { + // get the request parameter + final String round_param = the_request.queryParams(ROUND); + + // get other things we need + final CountyDashboard cdb = Persistence.getByID(county.id(), CountyDashboard.class); + final List cvr_to_audit_list; + final List response_list = new ArrayList<>(); + + // compute the round, if any + OptionalInt round = OptionalInt.empty(); + if (round_param != null) { + final int round_number = Integer.parseInt(round_param); + if (0 < round_number && round_number <= cdb.rounds().size()) { + round = OptionalInt.of(round_number); + } else { + badDataContents(the_response, "cvr list requested for invalid round " + + round_param + " for county " + cdb.id()); + } + } + + if (round.isPresent()) { + cvr_to_audit_list = ComparisonAuditController.ballotsToAudit( + cdb, + round.getAsInt() + ); + response_list.addAll(BallotSelection.toResponseList(cvr_to_audit_list)); + response_list.sort(null); + + final Round roundObject = cdb.rounds().get(round.getAsInt() - 1); + + final List> bsa = + roundObject.ballotSequenceAssignment(); + + if (bsa != null) { + // Walk the sequence assignments getting the audit boards' index and + // count values. Use that information to set the audit board index for + // each response row. + // + // Note: the board assignment has already been created, so the list of + // ballots returned to the client cannot change after the round + // starts. + for (int i = 0; i < bsa.size(); i++) { + final Map m = bsa.get(i); + + final Integer boardIndex = m.get("index"); + final Integer boardCount = m.get("count"); + + for (int j = boardIndex; j < boardIndex + boardCount; j++) { + final CVRToAuditResponse row = response_list.get(j); + row.setAuditBoardIndex(i); + } + } + } + } + + okJSON(the_response, Main.GSON.toJson(response_list)); + } catch (final PersistenceException e) { + serverError(the_response, "could not generate cvr list"); + } + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ContestDownload.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ContestDownload.java new file mode 100644 index 00000000..a525da72 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ContestDownload.java @@ -0,0 +1,107 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 27, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.log4j.Level; + +import com.google.gson.stream.JsonWriter; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.model.Contest; +import us.freeandfair.corla.model.County; +import us.freeandfair.corla.model.CountyDashboard; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.query.ContestQueries; +import us.freeandfair.corla.util.SparkHelper; + +/** + * The contest download endpoint. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class ContestDownload extends AbstractEndpoint { + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.GET; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/contest"; + } + + /** + * This endpoint requires any kind of authentication. + */ + @Override + public AuthorizationType requiredAuthorization() { + return AuthorizationType.EITHER; + } + + /** + * {@inheritDoc} + */ + @Override + public Level logLevel() { + return Level.DEBUG; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointBody(final Request the_request, final Response the_response) { + // only return contests for counties that have finished their uploads + final Set county_set = new HashSet<>(); + for (final CountyDashboard cdb : Persistence.getAll(CountyDashboard.class)) { + if (cdb.manifestFile() != null && cdb.cvrFile() != null) { + county_set.add(cdb.county()); + } + } + final List contest_list = ContestQueries.forCounties(county_set); + try (OutputStream os = SparkHelper.getRaw(the_response).getOutputStream(); + BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(os, "UTF-8")); + JsonWriter jw = new JsonWriter(bw)) { + jw.beginArray(); + for (final Contest contest : contest_list) { + jw.jsonValue(Main.GSON.toJson(Persistence.unproxy(contest))); + Persistence.evict(contest); + } + jw.endArray(); + jw.flush(); + jw.close(); + ok(the_response); + } catch (final IOException e) { + serverError(the_response, "Unable to stream response"); + } + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ContestDownloadByCounty.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ContestDownloadByCounty.java new file mode 100644 index 00000000..56512cad --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ContestDownloadByCounty.java @@ -0,0 +1,138 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 27, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.log4j.Level; + +import com.google.gson.stream.JsonWriter; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.model.Contest; +import us.freeandfair.corla.model.County; +import us.freeandfair.corla.model.CountyDashboard; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.query.ContestQueries; +import us.freeandfair.corla.util.SparkHelper; + +/** + * The contest by county download endpoint. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class ContestDownloadByCounty extends AbstractEndpoint { + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.GET; + } + + /** + * {@inheritDoc} + */ + @Override + public Level logLevel() { + return Level.DEBUG; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/contest/county"; + } + + /** + * This endpoint requires any kind of authentication. + */ + @Override + public AuthorizationType requiredAuthorization() { + return AuthorizationType.EITHER; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointBody(final Request the_request, final Response the_response) { + if (validateParameters(the_request)) { + final Set county_set = new HashSet(); + for (final String s : the_request.queryParams()) { + final Long county_id = Long.valueOf(s); + final CountyDashboard cdb = Persistence.getByID(county_id, CountyDashboard.class); + // only get contests for counties that have finished their uploads + if (cdb == null) { + dataNotFound(the_response, "Nonexistent county ID specified"); + } else if (cdb.manifestFile() != null && + cdb.cvrFile() != null) { + county_set.add(Persistence.getByID(county_id, County.class)); + } + } + final List contest_list = ContestQueries.forCounties(county_set); + try (OutputStream os = SparkHelper.getRaw(the_response).getOutputStream(); + BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(os, "UTF-8")); + JsonWriter jw = new JsonWriter(bw)) { + jw.beginArray(); + for (final Contest contest : contest_list) { + jw.jsonValue(Main.GSON.toJson(Persistence.unproxy(contest))); + Persistence.evict(contest); + } + jw.endArray(); + jw.flush(); + jw.close(); + ok(the_response); + } catch (final IOException e) { + serverError(the_response, "Unable to stream response"); + } + } else { + dataNotFound(the_response, "Invalid county ID specified"); + } + return my_endpoint_result.get(); + } + + /** + * Validates the parameters of a request. For this endpoint, + * the parameter names must all be integers. + * + * @param the_request The request. + * @return true if the parameters are valid, false otherwise. + */ + protected boolean validateParameters(final Request the_request) { + boolean result = true; + + for (final String s : the_request.queryParams()) { + try { + Integer.parseInt(s); + } catch (final NumberFormatException e) { + result = false; + break; + } + } + + return result; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ContestDownloadByID.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ContestDownloadByID.java new file mode 100644 index 00000000..c9d802c4 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ContestDownloadByID.java @@ -0,0 +1,82 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 27, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import org.apache.log4j.Level; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.model.Contest; +import us.freeandfair.corla.persistence.Persistence; + +/** + * The contest by ID endpoint. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class ContestDownloadByID extends AbstractEndpoint { + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.GET; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/contest/id/:id"; + } + + /** + * This endpoint requires any kind of authentication. + */ + @Override + public AuthorizationType requiredAuthorization() { + return AuthorizationType.EITHER; + } + + /** + * {@inheritDoc} + */ + @Override + public Level logLevel() { + return Level.DEBUG; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointBody(final Request the_request, final Response the_response) { + try { + final Contest c = + Persistence.getByID(Long.parseLong(the_request.params(":id")), + Contest.class); + if (c == null) { + dataNotFound(the_response, "Contest not found"); + } else { + okJSON(the_response, Main.GSON.toJson(Persistence.unproxy(c))); + } + } catch (final NumberFormatException e) { + invariantViolation(the_response, "Bad contest ID"); + } + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CountyDashboardASMState.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CountyDashboardASMState.java new file mode 100644 index 00000000..8f64430c --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CountyDashboardASMState.java @@ -0,0 +1,67 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 12, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import org.apache.log4j.Level; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.json.ServerASMResponse; + +/** + * An endpoint to provide the state of a county dashboard ASM to the client. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class CountyDashboardASMState extends AbstractCountyDashboardEndpoint { + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.GET; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/county-asm-state"; + } + + /** + * {@inheritDoc} + */ + @Override + public Level logLevel() { + return Level.DEBUG; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointBody(final Request the_request, final Response the_response) { + // there's really nothing to do here other than get the ASM state, which we + // conveniently have locally already + + okJSON(the_response, + Main.GSON.toJson(new ServerASMResponse(my_asm.get().currentState(), + my_asm.get().enabledUIEvents()))); + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CountyDashboardRefresh.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CountyDashboardRefresh.java new file mode 100644 index 00000000..1f1f76f8 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CountyDashboardRefresh.java @@ -0,0 +1,87 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 12, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import javax.persistence.PersistenceException; + +import org.apache.log4j.Level; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.json.CountyDashboardRefreshResponse; +import us.freeandfair.corla.model.County; +import us.freeandfair.corla.model.CountyDashboard; +import us.freeandfair.corla.persistence.Persistence; + +/** + * The endpoint for refreshing the county dashboard status. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +// endpoints don't need constructors +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class CountyDashboardRefresh extends AbstractCountyDashboardEndpoint { + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.GET; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/county-dashboard"; + } + + /** + * {@inheritDoc} + */ + @Override + public Level logLevel() { + return Level.DEBUG; + } + + /** + * Provides information about the County and Audit Board dashboards. + * + * @param the_request The request. + * @param the_response The response. + */ + @Override + public String endpointBody(final Request the_request, final Response the_response) { + try { + final County county = Main.authentication().authenticatedCounty(the_request); + + okJSON(the_response, + Main.GSON.toJson(CountyDashboardRefreshResponse.createResponse + (Persistence.getByID(county.id(), CountyDashboard.class)))); + } catch (final PersistenceException e) { + serverError(the_response, "could not obtain dashboard state"); + } + return my_endpoint_result.get(); + } + + /** + * This endpoint requires COUNTY authorization. + */ + @Override + public AuthorizationType requiredAuthorization() { + return AuthorizationType.COUNTY; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CountyReportDownload.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CountyReportDownload.java new file mode 100644 index 00000000..92065cba --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/CountyReportDownload.java @@ -0,0 +1,175 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 27, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import java.util.Arrays; +import java.util.List; + +import javax.persistence.PersistenceException; + +import org.apache.cxf.attachment.Rfc5987Util; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.asm.ASMState; +import us.freeandfair.corla.asm.ASMState.DoSDashboardState; +import us.freeandfair.corla.asm.ASMUtilities; +import us.freeandfair.corla.asm.DoSDashboardASM; +import us.freeandfair.corla.model.County; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.report.CountyReport; +import us.freeandfair.corla.util.SparkHelper; + +/** + * The county report download endpoint. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class CountyReportDownload extends AbstractEndpoint { + /** + * The "county" parameter. + */ + public static final String COUNTY = "county"; + + /** + * The states in which this endpoint can provide a result. + */ + private static final List LEGAL_STATES = + Arrays.asList(DoSDashboardState.COMPLETE_AUDIT_INFO_SET, + DoSDashboardState.DOS_AUDIT_ONGOING, + DoSDashboardState.DOS_ROUND_COMPLETE, + DoSDashboardState.DOS_AUDIT_COMPLETE, + DoSDashboardState.AUDIT_RESULTS_PUBLISHED); + + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.GET; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/county-report"; + } + + /** + * This endpoint requires any kind of authentication. + */ + @Override + public AuthorizationType requiredAuthorization() { + return AuthorizationType.EITHER; + } + + /** + * Validate the request parameters. In this case, if the county + * parameter exists, it must be parseable as a long. + * + * @param the_request The request. + */ + @Override + protected boolean validateParameters(final Request the_request) { + final String county = the_request.queryParams(COUNTY); + boolean result = true; + + try { + if (county == null && Main.authentication().authenticatedCounty(the_request) == null) { + // it's a DoS user, but they didn't specify a county + result = false; + } else if (county != null) { + Long.parseLong(county); + } + } catch (final NumberFormatException e) { + result = false; + } + + return result; + } + + /** + * {@inheritDoc} + */ + @Override + // necessary to break out of the lambda expression in case of IOException + @SuppressWarnings("PMD.ExceptionAsFlowControl") + public String endpointBody(final Request the_request, final Response the_response) { + // if we haven't defined the election, this is "data not found" + final DoSDashboardASM dos_asm = ASMUtilities.asmFor(DoSDashboardASM.class, + DoSDashboardASM.IDENTITY); + if (!LEGAL_STATES.contains(dos_asm.currentState())) { + dataNotFound(the_response, "No state report available in this state."); + } + + // we know we have either state or county authentication; this will be null + // for state authentication + County county = Main.authentication().authenticatedCounty(the_request); + + if (county == null) { + county = + Persistence.getByID(Long.parseLong(the_request.queryParams(COUNTY)), County.class); + if (county == null) { + badDataContents(the_response, "county " + the_request.queryParams(COUNTY) + + " does not exist"); + } + assert county != null; // makes FindBugs happy + } + + final boolean pdf = "pdf".equalsIgnoreCase(the_request.queryParams("file_type")); + final CountyReport cr = new CountyReport(county); + byte[] file = new byte[0]; + String filename = ""; + + if (pdf) { + the_response.type("application/pdf"); + filename = cr.filenamePDF(); + file = cr.generatePDF(); + } else { + the_response.type("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); + filename = cr.filenameExcel(); + try { + file = cr.generateExcel(); + } catch (final IOException e) { + serverError(the_response, "Unable to generate Excel file"); + } + } + + try { + the_response.raw().setHeader("Content-Disposition", "attachment; filename=\"" + + Rfc5987Util.encode(filename, "UTF-8") + "\""); + } catch (final UnsupportedEncodingException e) { + serverError(the_response, "UTF-8 is unsupported (this should never happen)"); + } + + try (OutputStream os = SparkHelper.getRaw(the_response).getOutputStream(); + BufferedOutputStream bos = new BufferedOutputStream(os)) { + bos.write(file); + bos.flush(); + ok(the_response); + } catch (final IOException | PersistenceException e) { + serverError(the_response, "Unable to stream response"); + } + + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/DeleteFile.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/DeleteFile.java new file mode 100644 index 00000000..ea1e6968 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/DeleteFile.java @@ -0,0 +1,101 @@ +package us.freeandfair.corla.endpoint; + +import javax.persistence.PersistenceException; + +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.controller.DeleteFileController; +import us.freeandfair.corla.Main; +import us.freeandfair.corla.model.Administrator; +import us.freeandfair.corla.model.Administrator.AdministratorType; + +/** + * The endpoint for deleting a file or files for a county + * + * @author Democracy Works, Inc + * @version 1.0.0 + */ +// endpoints don't need constructors +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class DeleteFile extends AbstractEndpoint { + /** + * Class-wide logger + */ + public static final Logger LOGGER = + LogManager.getLogger(DeleteFile.class); + + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.POST; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/delete-file"; + } + + /** + * This endpoint requires COUNTY authorization. + */ + @Override + public AuthorizationType requiredAuthorization() { + return AuthorizationType.EITHER; + } + + /** + * + */ + @Override + public String endpointBody(final Request request, + final Response response) { + final JsonParser parser = new JsonParser(); + final JsonObject o; + Long countyId = null; + String fileType = null; + + try { + o = parser.parse(request.body()).getAsJsonObject(); + Administrator admin = request.session().attribute("admin"); + + if (admin.type() == AdministratorType.STATE) { + if (null != o.get("countyId")) { + countyId = o.get("countyId").getAsLong(); + } else { + // bad request + badDataContents(response, "Missing countyId in post body"); + } + } else if (admin.type() == AdministratorType.COUNTY) { + countyId = Main.authentication().authenticatedCounty(request).id(); + } + + if (null != o.get("fileType")) { + fileType = o.get("fileType").getAsString(); + } else { + // bad request + badDataContents(response, "Missing fileType in post body"); + } + + LOGGER.debug(String.format("[parsed request for deleting file: countyId=%d, fileType=%s", + countyId, fileType)); + DeleteFileController.deleteFile(countyId, fileType); + } catch (final PersistenceException | DeleteFileController.DeleteFileFail e) { + // this will roll back the transaction in afterAfter() + serverError(response, "could not delete file"); + } + + // for a full circle, we can return the data that was sent; the fileType + okJSON(response, request.body()); + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/DoSDashboardASMState.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/DoSDashboardASMState.java new file mode 100644 index 00000000..6ce2a058 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/DoSDashboardASMState.java @@ -0,0 +1,67 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 12, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import org.apache.log4j.Level; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.json.ServerASMResponse; + +/** + * An endpoint to provide the state of the DoS dashboard ASM to the client. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class DoSDashboardASMState extends AbstractDoSDashboardEndpoint { + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.GET; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/dos-asm-state"; + } + + /** + * {@inheritDoc} + */ + @Override + public Level logLevel() { + return Level.DEBUG; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointBody(final Request the_request, final Response the_response) { + // there's really nothing to do here other than get the ASM state, which we + // conveniently have locally already + + okJSON(the_response, + Main.GSON.toJson(new ServerASMResponse(my_asm.get().currentState(), + my_asm.get().enabledUIEvents()))); + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/DoSDashboardRefresh.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/DoSDashboardRefresh.java new file mode 100644 index 00000000..e802b1da --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/DoSDashboardRefresh.java @@ -0,0 +1,95 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 12, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import javax.persistence.PersistenceException; + +import org.apache.log4j.Level; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.json.DoSDashboardRefreshResponse; +import us.freeandfair.corla.model.DoSDashboard; +import us.freeandfair.corla.persistence.Persistence; + +/** + * The endpoint for refreshing the Department of State dashboard status. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +// endpoints don't need constructors +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class DoSDashboardRefresh extends AbstractDoSDashboardEndpoint { + /** + * Class-wide logger + */ + public static final Logger LOGGER = + LogManager.getLogger(DoSDashboardRefresh.class); + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.GET; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/dos-dashboard"; + } + + /** + * {@inheritDoc} + */ + @Override + public Level logLevel() { + return Level.DEBUG; + } + + /** + * Provides information about the DoS dashboard. + * + * @param the_request The request. + * @param the_response The response. + */ + @Override + public String endpointBody(final Request the_request, final Response the_response) { + try { + okJSON(the_response, + Main.GSON.toJson(DoSDashboardRefreshResponse.createResponse + (Persistence.getByID(DoSDashboard.ID, DoSDashboard.class)))); + + LOGGER.debug("dos-dashboard:\n " + + my_endpoint_result.get()); + + } catch (final PersistenceException e) { + serverError(the_response, "could not obtain dashboard state"); + } + return my_endpoint_result.get(); + } + + /** + * This endpoint requires STATE authorization. + */ + @Override + public AuthorizationType requiredAuthorization() { + return AuthorizationType.STATE; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/Endpoint.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/Endpoint.java new file mode 100644 index 00000000..44f9bb19 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/Endpoint.java @@ -0,0 +1,95 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 27, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import org.apache.log4j.Level; + +import spark.Request; +import spark.Response; + +/** + * An interface implemented by all our Spark endpoints. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public interface Endpoint { + /** + * The endpoint type for this endpoint. + */ + enum EndpointType { + GET, POST, PUT; + } + + /** + * @return the type of this endpoint. + */ + EndpointType endpointType(); + + /** + * The name of the endpoint implemented by this class. + * + * @return the endpoint name. + */ + String endpointName(); + + /** + * The main body of the endpoint. + * + * @param the_request The request object. + * @param the_response The response object. + * @return the String response. + */ + String endpoint(Request the_request, Response the_response); + + /** + * The before-filter for this endpoint. + * + * @param the_request The request object. + * @param the_response The response object. + */ + void before(Request the_request, Response the_response); + + /** + * The after-filter for this endpoint. + * + * @param the_request The request object. + * @param the_response The response object. + */ + void after(Request the_request, Response the_response); + + /** + * The after-after-filter for this endpoint. + * + * @param the_request The request object. + * @param the_response The response object. + */ + void afterAfter(Request the_request, Response the_response); + + /** + * @return the required authorization type for this endpoint. + */ + AuthorizationType requiredAuthorization(); + + /** + * @return the priority level at which the activity of this endpoint should + * be logged. + */ + Level logLevel(); + + /** + * The authorization types. + */ + enum AuthorizationType { + STATE, COUNTY, EITHER, NONE; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/FileDownload.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/FileDownload.java new file mode 100644 index 00000000..bf5e364d --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/FileDownload.java @@ -0,0 +1,135 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 27, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import java.sql.SQLException; + +import org.apache.cxf.attachment.Rfc5987Util; + +import com.google.gson.JsonParseException; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.model.County; +import us.freeandfair.corla.model.UploadedFile; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.util.FileHelper; +import us.freeandfair.corla.util.SparkHelper; + +/** + * The file download endpoint. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings({"PMD.AtLeastOneConstructor", "PMD.ExcessiveImports"}) +public class FileDownload extends AbstractEndpoint { + + /** + * The download buffer size, in bytes. + */ + private static final int BUFFER_SIZE = 1048576; // 1 MB + + /** + * The maximum download size, in bytes. + */ + private static final int MAX_DOWNLOAD_SIZE = 1073741824; // 1 GB + + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.GET; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/download-file"; + } + + /** + * This endpoint requires either authorization, but only allows downloads + * by the county that made the upload, or by the state. + * + * @return EITHER + */ + @Override + public AuthorizationType requiredAuthorization() { + return AuthorizationType.EITHER; + } + + /** + * Validates the parameters of this request. The only requirement is that there be + * a parameter with the name in QUERY_PARAMETER; its parsing happens later. + */ + @Override + public boolean validateParameters(final Request the_request) { + return the_request.queryParams().contains("fileId"); + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointBody(final Request the_request, final Response the_response) { + // we know we have either state or county authentication; this will be null + // for state authentication + final County county = Main.authentication().authenticatedCounty(the_request); + + UploadedFile uploadedFile = null; + + try { + final String fileId = the_request.queryParams("fileId"); + if (null != fileId) { + uploadedFile = Persistence.getByID(Long.valueOf(fileId), UploadedFile.class); + } + if (uploadedFile == null) { + badDataContents(the_response, "nonexistent file requested"); + } else if (county == null || county.id().equals(uploadedFile.county().id())) { + the_response.type("text/csv"); + try { + the_response.raw().setHeader("Content-Disposition", "attachment; filename=\"" + + Rfc5987Util.encode(uploadedFile.filename(), "UTF-8") + "\""); + } catch (final UnsupportedEncodingException e) { + serverError(the_response, "UTF-8 is unsupported (this should never happen)"); + } + + try (OutputStream os = SparkHelper.getRaw(the_response).getOutputStream()) { + final int total = + FileHelper.bufferedCopy(uploadedFile.file().getBinaryStream(), os, + BUFFER_SIZE, MAX_DOWNLOAD_SIZE); + Main.LOGGER.debug("sent file " + uploadedFile.filename() + " of size " + total); + ok(the_response); + } catch (final SQLException | IOException e) { + serverError(the_response, "Unable to stream response"); + } + } else { + unauthorized(the_response, "county " + county.id() + " attempted to download " + + "file " + uploadedFile.filename() + " uploaded by county " + + uploadedFile.county().id()); + } + } catch (final JsonParseException e) { + badDataContents(the_response, "malformed request: " + e.getMessage()); + } + + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/FileUpload.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/FileUpload.java new file mode 100644 index 00000000..0b88bd81 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/FileUpload.java @@ -0,0 +1,489 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 27, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.LineNumberReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Blob; +import java.text.SimpleDateFormat; +import java.time.Instant; +import java.util.Date; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; + +import javax.persistence.PersistenceException; +import javax.servlet.http.HttpServletRequest; + +import org.apache.commons.fileupload.FileItemIterator; +import org.apache.commons.fileupload.FileItemStream; +import org.apache.commons.fileupload.FileUploadException; +import org.apache.commons.fileupload.servlet.ServletFileUpload; +import org.apache.commons.fileupload.util.Streams; +import org.apache.commons.lang3.StringUtils; + +import org.apache.log4j.Logger; +import org.apache.log4j.LogManager; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.crypto.HashChecker; +import us.freeandfair.corla.csv.Result; +import us.freeandfair.corla.json.UploadedFileDTO; +import us.freeandfair.corla.model.County; +import us.freeandfair.corla.model.UploadedFile; +import us.freeandfair.corla.model.UploadedFile.FileStatus; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.util.FileHelper; +import us.freeandfair.corla.util.SparkHelper; +import us.freeandfair.corla.util.SuppressFBWarnings; + +/** + * The file upload endpoint. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings({"PMD.AtLeastOneConstructor", "PMD.ExcessiveImports"}) +public class FileUpload extends AbstractEndpoint { + + /** + * Class-wide logger + */ + public static final Logger LOGGER = + LogManager.getLogger(FileUpload.class); + + /** + * The "hash" form data field name. + */ + public static final String HASH = "hash"; + + /** + * The "file" form data field name. + */ + public static final String FILE = "file"; + + /** + * The upload buffer size, in bytes. + */ + private static final int BUFFER_SIZE = 1048576; // 1 MB + + /** + * The maximum upload size, in bytes. + */ + private static final int MAX_UPLOAD_SIZE = 1073741824; // 1 GB + + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.POST; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/upload-file"; + } + + /** + * This endpoint requires county authorization. + * + * @return COUNTY + */ + @Override + public AuthorizationType requiredAuthorization() { + return AuthorizationType.COUNTY; + } + + /** + * Attempts to save the specified file in the database. + * + * @param the_response The response object (for error reporting). + * @param the_info The upload info about the file and hash. + * @param the_county The county that uploaded the file. + * @return the resulting entity if successful, null otherwise + */ + // we are deliberately ignoring the return value of lnr.skip() + @SuppressFBWarnings("SR_NOT_CHECKED") + private UploadedFile attemptFilePersistence(final Response the_response, + final UploadInformation the_info, + final County the_county) { + UploadedFile uploadedFile = null; + FileStatus file_status = null; + Result result = new Result(); + + try (FileInputStream is = new FileInputStream(the_info.my_file); + LineNumberReader lnr = + new LineNumberReader(new InputStreamReader(new FileInputStream(the_info.my_file), + "UTF-8"))) { + final Blob blob = Persistence.blobFor(is, the_info.my_file.length()); + + // first, compute the approximate number of records in the file + lnr.skip(Integer.MAX_VALUE); + final int approx_records = lnr.getLineNumber(); + + if (the_info.my_computed_hash.equals(the_info.my_uploaded_hash)) { + file_status = FileStatus.HASH_VERIFIED; + } else { + file_status = FileStatus.HASH_MISMATCH; + result.success = false; + result.errorMessage = "Submitted hash does not equal computed hash"; + } + uploadedFile = new UploadedFile(the_info.my_timestamp, + the_county, + the_info.my_filename, + file_status, + the_info.my_computed_hash, + the_info.my_uploaded_hash, + blob, + the_info.my_file.length(), + approx_records); + uploadedFile.setResult(result); + Persistence.save(uploadedFile); + Persistence.flush(); + } catch (final PersistenceException | IOException e) { + LOGGER.error("could not persist file of size " + e.getMessage()); + badDataType(the_response, "could not persist file of size " + + the_info.my_file.length()); + the_info.my_ok = false; + } + return uploadedFile; + } + + /** + * Handles the upload of the file, updating the provided UploadInformation. + * sets the_info.my_file to a tempfile and writes to it + * + * @param the_request The request to use. + * @param the_info The upload information to update. + */ + // I don't see any other way to implement the buffered reading + // than a deeply nested if statement + @SuppressWarnings("PMD.AvoidDeeplyNestedIfStmts") + private void handleUpload(final Request the_request, + final Response the_response, + final UploadInformation the_info) { + try { + final HttpServletRequest raw = SparkHelper.getRaw(the_request); + the_info.my_ok = ServletFileUpload.isMultipartContent(raw); + + LOGGER.info("handling file upload request from " + raw.getRemoteHost()); + if (the_info.my_ok) { + final ServletFileUpload upload = new ServletFileUpload(); + final FileItemIterator fii = upload.getItemIterator(raw); + while (fii.hasNext()) { + final FileItemStream item = fii.next(); + final String name = item.getFieldName(); + final InputStream stream = item.openStream(); + + if (item.isFormField()) { + the_info.my_form_fields.put(item.getFieldName(), Streams.asString(stream)); + } else if (FILE.equals(name)) { + // save the file + the_info.my_filename = item.getName(); + the_info.my_file = File.createTempFile("upload", ".csv"); + final OutputStream os = new FileOutputStream(the_info.my_file); + final int total = + FileHelper.bufferedCopy(stream, os, BUFFER_SIZE, MAX_UPLOAD_SIZE); + + if (total >= MAX_UPLOAD_SIZE) { + LOGGER.info("attempt to upload file greater than max size from " + + raw.getRemoteHost()); + badDataContents(the_response, "Upload Failed"); + the_info.my_ok = false; + } else { + LOGGER.info("successfully saved file of size " + total + " from " + + raw.getRemoteHost()); + } + os.close(); + } + } + } + + if (the_info.my_file == null) { + // no file was actually uploaded + the_info.my_ok = false; + badDataContents(the_response, "No file was uploaded"); + } else if (!the_info.my_form_fields.containsKey(HASH)) { + // no hash was provided + the_info.my_ok = false; + badDataContents(the_response, "No hash was provided with the uploaded file"); + } + } catch (final IOException | FileUploadException e) { + the_info.my_ok = false; + badDataContents(the_response, "Upload Failed"); + } + } + + /** + * Copies uploaded file which is in a temporary location and its hash into + * an archival location + * + * Steps: + * 1. Based on operating system, fetches archival file path from property file + * 2. Creates the file path if not existing + * 3. Appends timestamp to file and archives it by making a copy of the + * temporary file. + * 4. Appends the same timestamp to create a new file that would contain the + * hash value and archives it. + * + * @param the_upload_information contains all the specifices of the uploaded + * file that is in temporary location along with + * its hash value. + */ + // we are deliberately ignoring the return value of archive_file_dir.mkdirs() + @SuppressFBWarnings("RV_RETURN_VALUE_IGNORED_BAD_PRACTICE") + private void archive(final UploadInformation the_upload_information) { + // name of file that was uploaded + final String uploaded_file_name = the_upload_information.my_filename; + + // Prepend timestamp to file name. + // Not append timestamp as we can't assure if file names follow + // filename.ext format + // Example: 2018-05-17-T10-44-04.244-Arapahoe 2016 Primary Ballot Manifest.csv + // Cannot use ':' as in HH:mm:ss because, when running in Windows, it + // throws java.nio.file.InvalidPathException + final String archive_file_name = + new SimpleDateFormat("yyyy-MM-dd-'T'HH-mm-ss.SSS-", + Locale.US).format(new Date()) + + uploaded_file_name; + + // create corresponding hash file name to archive. Prepend archive_file_name + // so it is paired correctly + String archive_hash_file_name = null; + if (StringUtils.contains(uploaded_file_name, ".")) { // file name contains a "." + // get the file name till the "." and append "-Hash.text" to it + // Example: + // Archive File Name: 2018-05-17-T10-44-04.390-Arapahoe 2016 + // Primary Ballot Manifest.csv + // Archive Hash File Name: 2018-05-17-T10-44-04.390-Arapahoe 2016 + // Primary Ballot Manifest-Hash.txt + archive_hash_file_name = StringUtils.substringBeforeLast(archive_file_name, ".") + + "-Hash.txt"; + } else { + // file name has no "." just append "-Hash.text" to it + // Example: + // Archive File Name: 2018-05-17-T10-44-04.390-Arapahoe 2016 + // Primary Ballot Manifest + // Archive Hash File Name: 2018-05-17-T10-44-04.390-Arapahoe 2016 + // Primary Ballot Manifest-Hash.txt + archive_hash_file_name = archive_file_name + "-Hash.txt"; + } + + // fetch location where file needs to be uploaded to for archival + final String archive_file_path = fetchArchiveFilePath(); + // create directory if not existing + final File archive_file_dir = new File(archive_file_path); + archive_file_dir.mkdirs(); + + // archive file by copying it to destination + archiveFile(archive_file_path + archive_file_name, + the_upload_information.my_file.toPath()); + // create corresponding hash text file with hash value in it + archiveHashFile(archive_file_path + archive_hash_file_name, + the_upload_information.my_uploaded_hash); + } + + /** + * Copies passed in temporary file into archive destination, also renames it + * to its original name. + * + * @param the_file_path_and_name path and file name of the file to be archived + * @param the_source_path path and file name of the temporary file created by + * the server + */ + private void archiveFile(final String the_file_path_and_name, + final Path the_source_path) { + try { + // copy the temp file into archive destination + final Path path = Files.copy(the_source_path, + Paths.get(the_file_path_and_name)); + + if (path == null) { + LOGGER.info("Error archiving file (" + the_file_path_and_name + ")."); + } else { + LOGGER.info("Successfully archived file (" + the_file_path_and_name + ")."); + } + } catch (final IOException e) { + LOGGER.info("Encountered exception while archiving file (" + + the_file_path_and_name + + ")", + e); + } + } + + /** + * Creates a new hash file and copies passed in hash value and archives it. + * + * @param the_archive_hash_file_name path and file name of the hash file to be + * archived + * @param the_hash_value hash content that will be written into the file + */ + private void archiveHashFile(final String the_archive_hash_file_name, + final String the_hash_value) { + try (BufferedWriter bw = + new BufferedWriter( + new OutputStreamWriter( + new FileOutputStream(the_archive_hash_file_name), + StandardCharsets.UTF_8))) { + + bw.write(the_hash_value); + LOGGER.info("Successfully archived hash file (" + + the_archive_hash_file_name + + ")."); + } catch (final IOException e) { + LOGGER.info("Encountered exception while archiving hash file (" + + the_archive_hash_file_name + + ")", + e); + } + } + + /** + * Based on operating system, retrieves archive file location from property file. + * + * @return archive file location + */ + private String fetchArchiveFilePath() { + final Properties properties = Main.properties(); + final String os_name = System.getProperty("os.name").toLowerCase(Locale.US); + final boolean is_windows = os_name.startsWith("windows"); + final String archive_file_location; + if (is_windows) { + archive_file_location = properties.getProperty("windows_upload_file_location"); + } else { // it's UNIX + archive_file_location = properties.getProperty("unix_upload_file_location"); + } + return archive_file_location; + } + + /** + * {@inheritDoc} + * + */ + @Override + public String endpointBody(final Request the_request, final Response the_response) { + final UploadInformation info = new UploadInformation(); + info.my_timestamp = Instant.now(); + info.my_ok = true; + + // we know we have county authorization, so let's find out which county + final County county = Main.authentication().authenticatedCounty(the_request); + + if (county == null) { + unauthorized(the_response, "unauthorized administrator for CVR export upload"); + return my_endpoint_result.get(); + } + + // we can exit in several different ways, so let's make sure we delete + // the temp file even if we exit exceptionally + try { + handleUpload(the_request, the_response, info); + + // now process the temp file, putting it in the database + UploadedFile uploaded_file = null; + + if (info.my_ok) { + try { + + info.my_computed_hash = HashChecker.hashFile(info.my_file); + + info.my_uploaded_hash = + info.my_form_fields.get(HASH).toUpperCase(Locale.US).trim(); + uploaded_file = attemptFilePersistence(the_response, info, county); + if (uploaded_file != null) { + LOGGER.info("Upload File " + uploaded_file.toString()); + UploadedFileDTO upF = new UploadedFileDTO(uploaded_file); + okJSON(the_response, Main.GSON.toJson(upF)); + } // else another result code has already been set + } catch (final java.io.IOException | java.security.NoSuchAlgorithmException e) { + info.my_ok = false; + LOGGER.error("Upload Failed " + e.getMessage()); + badDataContents(the_response, "Upload Failed"); + } + } + } finally { + // delete the temp file, if it exists + if (info.my_file != null) { + try { + // archive file before deleting + archive(info); + if (!info.my_file.delete()) { + LOGGER.error("Unable to delete temp file " + info.my_file); + } + } catch (final SecurityException e) { + // ignored - should never happen + } + } + } + return my_endpoint_result.get(); + } + + /** + * A small class to encapsulate data dealt with during an upload. + */ + private static class UploadInformation { + /** + * The uploaded file. + */ + protected File my_file; + + /** + * The original name of the uploaded file. + */ + protected String my_filename; + + /** + * The timestamp of the upload. + */ + protected Instant my_timestamp; + + /** + * A flag indicating whether the upload is "ok". + */ + protected boolean my_ok = true; + + /** + * A map of form field names and values. + */ + protected Map my_form_fields = new HashMap(); + + /** + * The uploaded hash. + */ + protected String my_uploaded_hash; + + /** + * The computed hash. + */ + protected String my_computed_hash; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/IndicateHandCount.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/IndicateHandCount.java new file mode 100644 index 00000000..192060ef --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/IndicateHandCount.java @@ -0,0 +1,175 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 12, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + + +import com.google.gson.JsonParseException; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; +import spark.Request; +import spark.Response; +import us.freeandfair.corla.Main; +import us.freeandfair.corla.asm.ASMEvent; +import us.freeandfair.corla.model.*; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.query.ComparisonAuditQueries; + +import javax.persistence.PersistenceException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * The endpoint for indicating that a contest must be hand-counted. + * + * @author Joseph R. Kiniry + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class IndicateHandCount extends AbstractDoSDashboardEndpoint { + /** + * Class-wide logger + */ + public static final Logger LOGGER = LogManager.getLogger(IndicateHandCount.class); + + + /** + * The event to return for this endpoint. + */ + private final ThreadLocal my_event = new ThreadLocal(); + + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.POST; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/hand-count"; + } + + /** + * @return STATE authorization is necessary for this endpoint. + */ + public AuthorizationType requiredAuthorization() { + return AuthorizationType.STATE; + } + + /** + * {@inheritDoc} + */ + @Override + protected ASMEvent endpointEvent() { + return my_event.get(); + } + + /** + * {@inheritDoc} + */ + @Override + protected void reset() { + my_event.set(null); + } + + /** + * Indicate that a contest must be hand-counted. + * + * @param the_request The request. + * @param the_response The response. + */ + @Override + public synchronized String endpointBody(final Request the_request, + final Response the_response) { + try { + final ContestToAudit[] supplied_ctas = + Main.GSON.fromJson(the_request.body(), ContestToAudit[].class); + final DoSDashboard dosdb = Persistence.getByID(DoSDashboard.ID, DoSDashboard.class); + if (dosdb == null) { + serverError(the_response, "Could not select contests"); + } else { + boolean hand_count = false; + final Set hand_count_contests = new HashSet<>(); + for (final ContestToAudit c : fixReasons(dosdb, supplied_ctas)) { + if (c.audit() == AuditType.HAND_COUNT && + dosdb.updateContestToAudit(c)) { + hand_count = true; + hand_count_contests.add(c.contest().name()); + } + } + if (hand_count) { + unTargetContests(dosdb, hand_count_contests); + LOGGER.info("HAND_COUNT set for: " + String.join(",", hand_count_contests)); + } else { + // bad data was submitted for hand count selection + badDataContents(the_response, "Invalid contest selection data"); + } + } + Persistence.saveOrUpdate(dosdb); + ok(the_response, "Contest selected for hand count"); + } catch (final JsonParseException e) { + badDataContents(the_response, "Invalid contest selection data"); + } catch (final PersistenceException e) { + serverError(the_response, "Unable to save contest selection"); + } + return my_endpoint_result.get(); + } + + /** + * Updates the supplied CTAs with the reasons that were originally specified + * on the DoS dashboard. + * + * @param the_dosdb The DoS dashboard. + * @param the_supplied_ctas The supplied CTAs. + */ + @SuppressWarnings("PMD.UseVarargs") + private Set fixReasons(final DoSDashboard the_dosdb, + final ContestToAudit[] the_supplied_ctas) { + final Set result = new HashSet<>(); + final Set existing_ctas = the_dosdb.contestsToAudit(); + final Map contest_cta = new HashMap<>(); + + // let's iterate over these only once, instead of once for each array element in + // the_supplied_ctas + for (final ContestToAudit c : existing_ctas) { + contest_cta.put(c.contest(), c); + } + + // update the supplied CTAs with the dashboard reasons + for (final ContestToAudit c : the_supplied_ctas) { + ContestToAudit real_cta = c; + if (contest_cta.containsKey(c.contest())) { + real_cta = new ContestToAudit(c.contest(), + contest_cta.get(c.contest()).reason(), + c.audit()); + } + result.add(real_cta); + } + + return result; + } + + private void unTargetContests(final DoSDashboard dosdb, + final Set hand_count_contests) { + for (final String contestName: hand_count_contests) { + dosdb.removeContestToAuditByName(contestName); + ComparisonAuditQueries.updateStatus(contestName, AuditStatus.HAND_COUNT); + } + } + +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/IntermediateAuditReport.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/IntermediateAuditReport.java new file mode 100644 index 00000000..d0b7d4cf --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/IntermediateAuditReport.java @@ -0,0 +1,98 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 12, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import static us.freeandfair.corla.asm.ASMEvent.AuditBoardDashboardEvent.SUBMIT_INTERMEDIATE_AUDIT_REPORT_EVENT; + +import javax.persistence.PersistenceException; + +import com.google.gson.JsonParseException; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.asm.ASMEvent; +import us.freeandfair.corla.model.CountyDashboard; +import us.freeandfair.corla.model.IntermediateAuditReportInfo; +import us.freeandfair.corla.persistence.Persistence; + +/** + * Publish the intermediate audit report by the audit board. + * + * @author Joseph R. Kiniry + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class IntermediateAuditReport extends AbstractAuditBoardDashboardEndpoint { + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.POST; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/intermediate-audit-report"; + } + + /** + * @return COUNTY authorization is necessary for this endpoint. + */ + public AuthorizationType requiredAuthorization() { + return AuthorizationType.COUNTY; + } + + /** + * {@inheritDoc} + */ + @Override + protected ASMEvent endpointEvent() { + return SUBMIT_INTERMEDIATE_AUDIT_REPORT_EVENT; + } + + /** + * Publish the intermediate audit report by the audit board. + */ + @Override + public String endpointBody(final Request the_request, + final Response the_response) { + try { + final IntermediateAuditReportInfo report = + Main.GSON.fromJson(the_request.body(), IntermediateAuditReportInfo.class); + final CountyDashboard cdb = + Persistence.getByID(Main.authentication().authenticatedCounty(the_request).id(), + CountyDashboard.class); + if (cdb == null) { + Main.LOGGER.error("could not get audit board dashboard"); + serverError(the_response, "Could not save intermediate audit report"); + } else { + cdb.submitIntermediateReport(report); + } + Persistence.saveOrUpdate(cdb); + } catch (final JsonParseException e) { + Main.LOGGER.error("malformed intermediate audit report"); + badDataContents(the_response, "Invalid intermediate audit report"); + } catch (final PersistenceException e) { + Main.LOGGER.error("could not save intermediate audit report"); + serverError(the_response, "Unable to save intermediate audit report"); + } + ok(the_response, "Report submitted"); + // de-authenticate user? + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/PublishAuditReport.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/PublishAuditReport.java new file mode 100644 index 00000000..5ded3e83 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/PublishAuditReport.java @@ -0,0 +1,123 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 12, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import spark.Request; +import spark.Response; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; + +import java.util.Locale; + +import org.apache.cxf.attachment.Rfc5987Util; + +import us.freeandfair.corla.controller.AuditReport; +import us.freeandfair.corla.util.SparkHelper; + +/** + * Download all of the data relevant to public auditing of a RLA. + * + * @author Joseph R. Kiniry + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class PublishAuditReport extends AbstractDoSDashboardEndpoint { + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.GET; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/publish-audit-report"; + } + + /** + * @return STATE authorization is necessary for this endpoint. + */ + public AuthorizationType requiredAuthorization() { + return AuthorizationType.STATE; + } + + private String capitalize(final String string) { + return string.substring(0,1).toUpperCase(Locale.US) + + string.substring(1); + } + + private String fileName(final String reportType, final String extension) { + try { + return Rfc5987Util.encode(String.format("%s_Report.%s", + capitalize(reportType), + extension), + "UTF-8"); + } catch(final UnsupportedEncodingException e) { + return String.format("%s_Report.%s", + capitalize(reportType), + extension); + } + } + + /** + * Download all of the data relevant to public auditing of a RLA. + */ + @Override + public String endpointBody(final Request request, + final Response response) { + String contentType; + final String contestName = request.queryParams("contestName"); // optional when reportType is *-all + final String reportType = request.queryParams("reportType"); // activity/results + + contentType = request.queryParams("contentType"); + if (null == contentType) { + contentType = request.headers("Accept"); //header wins + } + // todo ensure reportType is present + + byte[] reportBytes; + try { + + final OutputStream os = SparkHelper.getRaw(response).getOutputStream(); + switch (contentType) { + case "xlsx": case "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": + reportBytes = AuditReport.generate("xlsx", reportType, contestName); + response.header("Content-Type", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); + response.header("Content-Disposition", "attachment; filename*=UTF-8''" + fileName(reportType, "xlsx")); + os.write(reportBytes); + os.close(); + break; + case "zip": case "application/zip": + response.header("Content-Type", "application/zip"); + response.header("Content-Disposition", "attachment; filename*=UTF-8''" + fileName(reportType, "zip")); + AuditReport.generateZip(os); + os.close(); + break; + default: + invariantViolation(response, "Accept header or query param contentType is missing or invalid"); + return my_endpoint_result.get(); + } + + ok(response); + } catch (final IOException e) { + serverError(response, "Unable to stream response"); + } + + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/PublishDataToAudit.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/PublishDataToAudit.java new file mode 100644 index 00000000..f90771d2 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/PublishDataToAudit.java @@ -0,0 +1,58 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 12, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import spark.Request; +import spark.Response; + +/** + * Download all of the data relevant to public auditing of a RLA. + * + * @author Joseph R. Kiniry + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class PublishDataToAudit extends AbstractDoSDashboardEndpoint { + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.GET; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/publish-data-to-audit"; + } + + /** + * @return STATE authorization is necessary for this endpoint. + */ + public AuthorizationType requiredAuthorization() { + return AuthorizationType.STATE; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointBody(final Request the_request, + final Response the_response) { + ok(the_response, "When defined, the full set of data relevant to permitting the" + + "public to audit an RLA will be downloaded here."); + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ReportBallotsToAudit.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ReportBallotsToAudit.java new file mode 100644 index 00000000..70f2416b --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ReportBallotsToAudit.java @@ -0,0 +1,58 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 12, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import spark.Request; +import spark.Response; + +/** + * Download all ballots to audit for the entire state. + * + * @author Joseph R. Kiniry + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class ReportBallotsToAudit extends AbstractDoSDashboardEndpoint { + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.GET; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/ballots-to-audit"; + } + + /** + * @return STATE authorization is necessary for this endpoint. + */ + public AuthorizationType requiredAuthorization() { + return AuthorizationType.STATE; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointBody(final Request the_request, + final Response the_response) { + ok(the_response, "the report of ballots to audit is not yet implemented, but " + + "the information can be obtained from county dashboard states"); + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ResetAudit.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ResetAudit.java new file mode 100644 index 00000000..532ed521 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ResetAudit.java @@ -0,0 +1,105 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 12, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import java.util.List; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.asm.AbstractStateMachine; +import us.freeandfair.corla.asm.AuditBoardDashboardASM; +import us.freeandfair.corla.asm.CountyDashboardASM; +import us.freeandfair.corla.asm.DoSDashboardASM; +import us.freeandfair.corla.asm.PersistentASMState; +import us.freeandfair.corla.model.County; +import us.freeandfair.corla.model.CountyContestResult; +import us.freeandfair.corla.model.CountyDashboard; +import us.freeandfair.corla.model.DoSDashboard; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.query.DatabaseResetQueries; +import us.freeandfair.corla.query.PersistentASMStateQueries; + +/** + * Reset the database, except for authentication information and uploaded + * artifact data (the latter is cleaned up at the database level, not by this + * code). + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +// the endpoint method here is long and has lots of loops, but is not +// at all difficult to understand +@SuppressWarnings({"PMD.AtLeastOneConstructor", "PMD.ModifiedCyclomaticComplexity", + "PMD.CyclomaticComplexity", "PMD.StdCyclomaticComplexity", "PMD.NPathComplexity"}) +public class ResetAudit extends AbstractEndpoint { + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.POST; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/reset-audit"; + } + + /** + * {@inheritDoc} + */ + @Override + public String asmIdentity(final Request the_request) { + return null; + } + + /** + * {@inheritDoc} + */ + @Override + public Class asmClass() { + return null; + } + + /** + * @return STATE authorization is necessary for this endpoint. + */ + public AuthorizationType requiredAuthorization() { + return AuthorizationType.STATE; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointBody(final Request the_request, + final Response the_response) { + + + // create new dashboards + List dosdbList = Persistence.getAll(DoSDashboard.class); + for(DoSDashboard dosdb : dosdbList) { + Persistence.delete(dosdb); + } + + // create new dashboards + final DoSDashboard dosdb = new DoSDashboard(); + Persistence.saveOrUpdate(dosdb); + + ok(the_response, "database audit"); + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ResetDatabase.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ResetDatabase.java new file mode 100644 index 00000000..0381581a --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/ResetDatabase.java @@ -0,0 +1,126 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 12, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.asm.AbstractStateMachine; +import us.freeandfair.corla.asm.AuditBoardDashboardASM; +import us.freeandfair.corla.asm.CountyDashboardASM; +import us.freeandfair.corla.asm.DoSDashboardASM; +import us.freeandfair.corla.asm.PersistentASMState; +import us.freeandfair.corla.model.County; +import us.freeandfair.corla.model.CountyDashboard; +import us.freeandfair.corla.model.DoSDashboard; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.query.DatabaseResetQueries; +import us.freeandfair.corla.query.PersistentASMStateQueries; + +/** + * Reset the database, except for authentication information and uploaded + * artifact data (the latter is cleaned up at the database level, not by this + * code). + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +// the endpoint method here is long and has lots of loops, but is not +// at all difficult to understand +@SuppressWarnings({"PMD.AtLeastOneConstructor", "PMD.ModifiedCyclomaticComplexity", + "PMD.CyclomaticComplexity", "PMD.StdCyclomaticComplexity", "PMD.NPathComplexity"}) +public class ResetDatabase extends AbstractEndpoint { + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.POST; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/reset-database"; + } + + /** + * {@inheritDoc} + */ + @Override + public String asmIdentity(final Request the_request) { + return null; + } + + /** + * {@inheritDoc} + */ + @Override + public Class asmClass() { + return null; + } + + /** + * @return STATE authorization is necessary for this endpoint. + */ + public AuthorizationType requiredAuthorization() { + return AuthorizationType.STATE; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointBody(final Request the_request, + final Response the_response) { + // delete everything + + DatabaseResetQueries.resetDatabase(); + + // create new dashboards + final DoSDashboard dosdb = new DoSDashboard(); + Persistence.saveOrUpdate(dosdb); + + for (final County c : Persistence.getAll(County.class)) { + final CountyDashboard cdb = new CountyDashboard(c); + Persistence.saveOrUpdate(cdb); + } + + // reset the DoS dashboard ASM state + final PersistentASMState dos_asm = + PersistentASMStateQueries.get(DoSDashboardASM.class, DoSDashboardASM.IDENTITY); + dos_asm.updateFrom(new DoSDashboardASM()); + Persistence.saveOrUpdate(dos_asm); + + // for each County, reset the states of its ASMs + for (final County c : Persistence.getAll(County.class)) { + final String id = String.valueOf(c.id()); + final PersistentASMState county_asm = + PersistentASMStateQueries.get(CountyDashboardASM.class, id); + if (county_asm != null) { + county_asm.updateFrom(new CountyDashboardASM(id)); + Persistence.saveOrUpdate(county_asm); + } + final PersistentASMState audit_asm = + PersistentASMStateQueries.get(AuditBoardDashboardASM.class, id); + if (audit_asm != null) { + audit_asm.updateFrom(new AuditBoardDashboardASM(id)); + Persistence.saveOrUpdate(audit_asm); + } + } + + ok(the_response, "database reset"); + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/RiskLimitForComparisonAudits.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/RiskLimitForComparisonAudits.java new file mode 100644 index 00000000..4256b593 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/RiskLimitForComparisonAudits.java @@ -0,0 +1,105 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 9, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import static us.freeandfair.corla.asm.ASMEvent.DoSDashboardEvent.PARTIAL_AUDIT_INFO_EVENT; + +import java.math.BigDecimal; + +import javax.persistence.PersistenceException; + +import com.google.gson.JsonParseException; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.asm.ASMEvent; +import us.freeandfair.corla.model.AuditInfo; +import us.freeandfair.corla.model.DoSDashboard; +import us.freeandfair.corla.persistence.Persistence; + +/** + * The endpoint for establishing the risk limit for comparison audits. + * + * @author Daniel M Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class RiskLimitForComparisonAudits extends AbstractDoSDashboardEndpoint { + /** + * The "risk limit" parameter. + */ + public static final String RISK_LIMIT = "risk_limit"; + + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.POST; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/risk-limit-comp-audits"; + } + + /** + * {@inheritDoc} + */ + @Override + protected ASMEvent endpointEvent() { + return PARTIAL_AUDIT_INFO_EVENT; + } + + /** + * Attempts to set the risk limit for comparison audits. The risk limit + * should be provided as a decimal number (i.e., 0.10 for 10%). + * + * Session query parameters: risk-limit + * + * @param the_request The request. + * @param the_response The response. + */ + @Override + public String endpointBody(final Request the_request, final Response the_response) { + try { + final AuditInfo risk_limit = + Main.GSON.fromJson(the_request.body(), AuditInfo.class); + if (risk_limit == null || risk_limit.riskLimit() == null || + 0 < BigDecimal.ZERO.compareTo(risk_limit.riskLimit()) || + 0 < risk_limit.riskLimit().compareTo(BigDecimal.ONE)) { + invariantViolation(the_response, "invalid risk limit specified"); + } else { + final DoSDashboard dosd = Persistence.getByID(DoSDashboard.ID, DoSDashboard.class); + if (dosd == null) { + Main.LOGGER.error("could not get department of state dashboard"); + serverError(the_response, "could not set risk limit"); + } else { + dosd.updateAuditInfo(risk_limit); + Persistence.saveOrUpdate(dosd); + ok(the_response, "risk limit set to " + risk_limit.riskLimit()); + } + } + } catch (final PersistenceException e) { + serverError(the_response, "unable to set risk limit: " + e); + + } catch (final JsonParseException e) { + invariantViolation(the_response, "Invalid risk limit specified"); + } + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/Root.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/Root.java new file mode 100644 index 00000000..d145f1bb --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/Root.java @@ -0,0 +1,52 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 27, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; + +/** + * The root endpoint. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class Root extends AbstractEndpoint { + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.GET; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/"; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointBody(final Request the_request, final Response the_response) { + ok(the_response, "ColoradoRLA Server, Version " + Main.VERSION + " - " + + "Please Use a Valid Endpoint!"); + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/SelectContestsForAudit.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/SelectContestsForAudit.java new file mode 100644 index 00000000..317aad8e --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/SelectContestsForAudit.java @@ -0,0 +1,133 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 9, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import static us.freeandfair.corla.asm.ASMEvent.DoSDashboardEvent.*; + +import javax.persistence.PersistenceException; + +import com.google.gson.JsonParseException; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.asm.ASMEvent; +import us.freeandfair.corla.model.AuditInfo; +import us.freeandfair.corla.model.ContestToAudit; +import us.freeandfair.corla.model.DoSDashboard; +import us.freeandfair.corla.persistence.Persistence; + +/** + * The endpoint for selecting the contests to audit. + * + * @author Daniel M Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class SelectContestsForAudit extends AbstractDoSDashboardEndpoint { + /** + * The event to return for this endpoint. + */ + private final ThreadLocal my_event = new ThreadLocal(); + + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.POST; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/select-contests"; + } + + /** + * {@inheritDoc} + */ + @Override + protected ASMEvent endpointEvent() { + return my_event.get(); + } + + /** + * {@inheritDoc} + */ + @Override + protected void reset() { + my_event.set(null); + } + + /** + * Attempts to select contests for audit. + * + * @param the_request The request. + * @param the_response The response. + */ + @Override + public synchronized String endpointBody(final Request the_request, + final Response the_response) { + try { + final ContestToAudit[] contests = + Main.GSON.fromJson(the_request.body(), ContestToAudit[].class); + final DoSDashboard dosdb = Persistence.getByID(DoSDashboard.ID, DoSDashboard.class); + if (dosdb == null) { + Main.LOGGER.error("could not get department of state dashboard"); + serverError(the_response, "Could not select contests"); + } else { + // unchecked contests are not posted so that which is not added, is removed. + dosdb.removeAuditableContestsToAudit(); + for (final ContestToAudit c : contests) { + Main.LOGGER.info("updating contest audit status: " + c); + dosdb.updateContestToAudit(c); + Persistence.saveOrUpdate(dosdb); + } + my_event.set(nextEvent(dosdb)); + ok(the_response, "Contests selected"); + } + } catch (final JsonParseException e) { + Main.LOGGER.error("malformed contest selection"); + badDataContents(the_response, "Invalid contest selection data"); + } catch (final PersistenceException e) { + Main.LOGGER.error("could not save contest selection"); + serverError(the_response, "Unable to save contest selection"); + } + return my_endpoint_result.get(); + } + + /** + * Computes the event of this endpoint based on audit info completeness. + * + * @param the_dosdb The DoS dashboard. + */ + private ASMEvent nextEvent(final DoSDashboard the_dosdb) { + final ASMEvent result; + final AuditInfo info = the_dosdb.auditInfo(); + + if (info.electionDate() == null || info.electionType() == null || + info.publicMeetingDate() == null || info.riskLimit() == null || + info.seed() == null || the_dosdb.contestsToAudit().isEmpty()) { + Main.LOGGER.debug("partial audit information submitted"); + result = PARTIAL_AUDIT_INFO_EVENT; + } else { + Main.LOGGER.debug("complete audit information submitted"); + result = COMPLETE_AUDIT_INFO_EVENT; + } + + return result; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/SetAuditBoardCount.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/SetAuditBoardCount.java new file mode 100644 index 00000000..394233b9 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/SetAuditBoardCount.java @@ -0,0 +1,174 @@ +/* + * Colorado RLA System + * + * @title ColoradoRLA + * @copyright 2018 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Democracy Works, Inc. + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.persistence.PersistenceException; + +import com.google.gson.JsonParseException; +import com.google.gson.reflect.TypeToken; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.model.County; +import us.freeandfair.corla.model.CountyDashboard; +import us.freeandfair.corla.model.Round; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.util.BallotAssignment; + +/** + * The endpoint for setting the number of audit boards for a given county. + * + * @author Democracy Works, Inc. + */ +// TODO: This rule and checkstyle conflict. We need to pick one or the other, +// but with both we need a suppression rule for one of them. +@SuppressWarnings({"PMD.AtLeastOneConstructor"}) +public class SetAuditBoardCount extends AbstractCountyDashboardEndpoint { + /** + * Type information for easy unmarshalling of the request body. + */ + private static final Type TYPE_TOKEN = + new TypeToken>() { }.getType(); + + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + // TODO: Easy to make this PUT? + return EndpointType.POST; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/set-audit-board-count"; + } + + /** + * Set the number of audit boards for a given county. + * + * @param request HTTP request + * @param response HTTP response + */ + @Override + public String endpointBody(final Request request, final Response response) { + try { + final Map input = + Main.GSON.fromJson(request.body(), TYPE_TOKEN); + + final County county = Main.authentication().authenticatedCounty(request); + + if (!validateRequestBody(input)) { + badDataContents(response, "malformed audit board count request"); + } + + final CountyDashboard countyDashboard = + Persistence.getByID(county.id(), CountyDashboard.class); + + if (countyDashboard == null) { + Main.LOGGER.error(String.format( + "could not get county dashboard [countyId=%d]", + county.id())); + serverError(response, "could not set audit board count"); + } + + final Round round = countyDashboard.currentRound(); + + if (round == null) { + Main.LOGGER.error(String.format( + "round not started [countyId=%d]", + county.id())); + badDataContents(response, "round not started"); + } + + final Integer ballotCount = round.expectedCount(); + + if (ballotCount == null) { + Main.LOGGER.error(String.format( + "ballot count not yet set for round [countyId=%d, roundNumber=%d]", + county.id(), + round.number())); + badDataContents(response, "ballot count not yet set for round"); + } + + final Integer auditBoardCount = input.get("count"); + + countyDashboard.setAuditBoardCount(auditBoardCount); + round.setBallotSequenceAssignment( + this.calculateBallotAssignment(auditBoardCount, ballotCount)); + + Persistence.saveOrUpdate(countyDashboard); + + // TODO: Make sure we don't have to persist the round separately. + + Main.LOGGER.info(String.format( + "set the number of audit boards to %d", + countyDashboard.auditBoardCount())); + + Main.LOGGER.info(String.format( + "set the audit board assignment: %s", + round.ballotSequenceAssignment())); + + ok(response, String.format("set the number of audit boards to %d", + countyDashboard.auditBoardCount())); + } catch (final PersistenceException e) { + Main.LOGGER.error("unable to set audit board count", e); + serverError(response, "unable to set audit board count"); + } catch (final JsonParseException e) { + badDataContents(response, "malformed audit board count request"); + } + + return my_endpoint_result.get(); + } + + /** + * Check that the unmarshalled input looks like we expect it to. + * + * @param body the unmarshalled input + */ + private static boolean validateRequestBody(final Map body) { + return body.containsKey("count"); + } + + /** + * Calculate the ballot sequence assignment from the given input request. + */ + private static List> + calculateBallotAssignment(final Integer auditBoardCount, + final Integer ballotCount) { + final List> result = new ArrayList<>(); + + final List boardAssignment = + BallotAssignment.assignToBoards(ballotCount, auditBoardCount); + + int index = 0; + for (final int count : boardAssignment) { + final Map m = new HashMap<>(); + m.put("index", index); + m.put("count", count); + result.add(m); + index += count; + } + + return result; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/SetContestNames.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/SetContestNames.java new file mode 100644 index 00000000..737a6306 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/SetContestNames.java @@ -0,0 +1,187 @@ +/* + * Colorado RLA System + * + * @title ColoradoRLA + * @copyright 2018 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-LGO3.0-or-later + * @creator Democracy Works, Inc. + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import com.google.gson.JsonParseException; +import com.google.gson.reflect.TypeToken; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; +import spark.Request; +import spark.Response; +import us.freeandfair.corla.Main; +import us.freeandfair.corla.asm.ASMEvent; +import us.freeandfair.corla.json.CanonicalUpdate; +import us.freeandfair.corla.json.CanonicalUpdate.ChoiceChange; +import us.freeandfair.corla.model.AuditInfo; +import us.freeandfair.corla.model.Contest; +import us.freeandfair.corla.model.CountyContestResult; +import us.freeandfair.corla.model.DoSDashboard; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.query.CastVoteRecordQueries; +import us.freeandfair.corla.query.CountyContestResultQueries; + +import javax.persistence.PersistenceException; +import java.lang.reflect.Type; +import java.util.List; + +import static us.freeandfair.corla.asm.ASMEvent.DoSDashboardEvent.COMPLETE_AUDIT_INFO_EVENT; +import static us.freeandfair.corla.asm.ASMEvent.DoSDashboardEvent.PARTIAL_AUDIT_INFO_EVENT; + +/** + * The endpoint for renaming contests. + * + * This allows allows the state to rename contests uploaded by counties that + * may not conform to the state specifications. + * + * @author Democracy Works, Inc. + */ +// TODO: This rule and checkstyle conflict. We need to pick one or the other, +// but with both we need a suppression rule for one of them. +@SuppressWarnings({"PMD.AtLeastOneConstructor"}) +public class SetContestNames extends AbstractDoSDashboardEndpoint { + + /** + * Class-wide logger + */ + public static final Logger LOGGER = + LogManager.getLogger(SetContestNames.class); + + /** + * The event to return for this endpoint. + */ + private final ThreadLocal asmEvent = new ThreadLocal(); + + /** + * Type information for the new contest names. + */ + private static final Type TYPE_TOKEN = + new TypeToken>(){}.getType(); + + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.POST; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/set-contest-names"; + } + + /** + * {@inheritDoc} + */ + @Override + protected ASMEvent endpointEvent() { + return asmEvent.get(); + } + + /** + * Updates contest names based on the DoS-preferred contest names. + * + * @param request HTTP request + * @param response HTTP response + */ + @Override + public String endpointBody(final Request request, final Response response) { + + try { + final List canons = Main.GSON.fromJson(request.body(), TYPE_TOKEN); + if (canons == null) { + badDataContents(response, "malformed contest mappings"); + } else { + final DoSDashboard dosdb = Persistence.getByID(DoSDashboard.ID, DoSDashboard.class); + if (dosdb == null) { + serverError(response, "could not set contest mappings"); + } + final int updateCount = changeNames(canons); + asmEvent.set(nextEvent(dosdb)); + ok(response, String.format("re-mapped %d contest names", updateCount)); + } + } catch (final PersistenceException e) { + serverError(response, "unable to re-map contest names"); + } catch (final JsonParseException e) { + LOGGER.error("JsonParseException causing malformed error", e); + badDataContents(response, "malformed contest mapping"); + } catch (final Exception e) { + badDataContents(response, "Exception"); + } + return my_endpoint_result.get(); + } + + private int changeNames(final List canons) { + + int updateCount = 0; + for (final CanonicalUpdate canon : canons) { + + + final Long id = Long.parseLong(canon.contestId); + final Contest contest = Persistence.getByID(id, Contest.class); + // change contest name + if (null != canon.name) { + contest.setName(canon.name); + } + // change choice names + if (null == canon.choices) { + LOGGER.info("canon.choices IS NULL"); + } + if (null != canon.choices) { + for (final ChoiceChange choiceChange: canon.choices) { + if (null != choiceChange.oldName + && null != choiceChange.newName + && !choiceChange.oldName.equals(choiceChange.newName)) { + + LOGGER.info("changing choice name as part of canonicalization:\n " + + choiceChange.oldName +" -> "+ choiceChange.newName + + " contest: " + contest.name() + " county: " + contest.county()); + contest.updateChoiceName(choiceChange.oldName, choiceChange.newName); + + CastVoteRecordQueries.updateCVRContestInfos(contest.county().id(), + contest.id(), + choiceChange.oldName, + choiceChange.newName); + final CountyContestResult ccr = CountyContestResultQueries.matching(contest.county(), contest); + ccr.updateChoiceName(choiceChange.oldName, choiceChange.newName); + Persistence.update(ccr); + } + } + } + + updateCount += 1; + } + return updateCount; + } + + /** + * Computes the event of this endpoint based on audit info completeness. + * + * @param dosDashboard The DoS dashboard. + */ + private ASMEvent nextEvent(final DoSDashboard dosDashboard) { + final ASMEvent result; + final AuditInfo info = dosDashboard.auditInfo(); + + if (info.electionDate() == null || info.electionType() == null || + info.publicMeetingDate() == null || info.riskLimit() == null || + info.seed() == null || dosDashboard.contestsToAudit().isEmpty()) { + result = PARTIAL_AUDIT_INFO_EVENT; + } else { + result = COMPLETE_AUDIT_INFO_EVENT; + } + + return result; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/SetRandomSeed.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/SetRandomSeed.java new file mode 100644 index 00000000..671118d1 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/SetRandomSeed.java @@ -0,0 +1,137 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 9, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import static us.freeandfair.corla.asm.ASMEvent.DoSDashboardEvent.COMPLETE_AUDIT_INFO_EVENT; +import static us.freeandfair.corla.asm.ASMEvent.DoSDashboardEvent.PARTIAL_AUDIT_INFO_EVENT; + +import javax.persistence.PersistenceException; + +import com.google.gson.JsonParseException; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.asm.ASMEvent; +import us.freeandfair.corla.model.AuditInfo; +import us.freeandfair.corla.model.DoSDashboard; +import us.freeandfair.corla.persistence.Persistence; + +/** + * The endpoint for setting the random seed. + * + * @author Daniel M Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class SetRandomSeed extends AbstractDoSDashboardEndpoint { + /** + * The "random seed" parameter. + */ + public static final String RANDOM_SEED = "random_seed"; + + /** + * The event to return for this endpoint. + */ + private final ThreadLocal my_event = new ThreadLocal(); + + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.POST; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/random-seed"; + } + + /** + * {@inheritDoc} + */ + @Override + protected ASMEvent endpointEvent() { + return my_event.get(); + } + + /** + * Attempts to set the random seed for comparison audits. The random seed + * should be provided as an integer in base 10, as Colorado rolls a + * 10-sided die to determine each digit. + * + * @param the_request The request. + * @param the_response The response. + */ + @Override + public String endpointBody(final Request the_request, final Response the_response) { + try { + final AuditInfo submitted = + Main.GSON.fromJson(the_request.body(), AuditInfo.class); + + if (submitted == null) { + badDataContents(the_response, "malformed random seed"); + } else if (DoSDashboard.isValidSeed(submitted.seed())) { + // if the rest of the audit info isn't set, we can't set the seed + final DoSDashboard dosdb = Persistence.getByID(DoSDashboard.ID, DoSDashboard.class); + if (dosdb == null) { + Main.LOGGER.error("could not get department of state dashboard"); + serverError(the_response, "could not set random seed"); + } + + // anything in the submitted audit info that isn't a random seed is ignored + final AuditInfo seed = + new AuditInfo(null, null, null, submitted.seed(), null); + dosdb.updateAuditInfo(seed); + Persistence.saveOrUpdate(dosdb); + my_event.set(nextEvent(dosdb)); + ok(the_response, "random seed set to " + seed.seed()); + } else { + invariantViolation(the_response, "invalid random seed specified: " + submitted.seed()); + } + } catch (final PersistenceException e) { + + serverError(the_response, "unable to set random seed: " + e); + + } catch (final JsonParseException e) { + badDataContents(the_response, "malformed random seed"); + } + return my_endpoint_result.get(); + } + + + /** + * Computes the event of this endpoint based on audit info completeness. + * + * @param the_dosdb The DoS dashboard. + */ + private ASMEvent nextEvent(final DoSDashboard the_dosdb) { + final ASMEvent result; + final AuditInfo info = the_dosdb.auditInfo(); + + if (info.electionDate() == null || info.electionType() == null || + info.publicMeetingDate() == null || info.riskLimit() == null) { + Main.LOGGER.debug("partial audit information submitted"); + result = PARTIAL_AUDIT_INFO_EVENT; + } else { + Main.LOGGER.debug("complete audit information submitted"); + result = COMPLETE_AUDIT_INFO_EVENT; + } + + return result; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/SignOffAuditRound.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/SignOffAuditRound.java new file mode 100644 index 00000000..21c9cc73 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/SignOffAuditRound.java @@ -0,0 +1,409 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * + * @created Aug 12, 2017 + * + * @copyright 2017 Colorado Department of State + * + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * + * @creator Joseph R. Kiniry + * + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import static us.freeandfair.corla.asm.ASMEvent.AuditBoardDashboardEvent.*; +import static us.freeandfair.corla.asm.ASMEvent.DoSDashboardEvent.*; +import static us.freeandfair.corla.asm.ASMState.AuditBoardDashboardState.*; + +import static us.freeandfair.corla.asm.ASMEvent.CountyDashboardEvent.COUNTY_AUDIT_COMPLETE_EVENT; +import static us.freeandfair.corla.asm.ASMEvent.CountyDashboardEvent.COUNTY_START_AUDIT_EVENT; + +import java.lang.reflect.Type; +import java.text.MessageFormat; +import java.util.HashSet; +import java.util.List; + +import javax.persistence.PersistenceException; + +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonParser; + +import com.google.gson.reflect.TypeToken; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; + +import us.freeandfair.corla.asm.ASMEvent; +import us.freeandfair.corla.asm.ASMState; +import us.freeandfair.corla.asm.ASMUtilities; +import us.freeandfair.corla.asm.AuditBoardDashboardASM; +import us.freeandfair.corla.asm.CountyDashboardASM; +import us.freeandfair.corla.asm.DoSDashboardASM; + +import us.freeandfair.corla.model.County; +import us.freeandfair.corla.model.CountyDashboard; +import us.freeandfair.corla.model.DoSDashboard; +import us.freeandfair.corla.model.AuditReason; +import us.freeandfair.corla.model.AuditStatus; +import us.freeandfair.corla.model.ComparisonAudit; +import us.freeandfair.corla.model.Elector; +import us.freeandfair.corla.model.Round; + +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.report.ReportRows; + +/** + * Signs off on the current audit round for a county. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings({"PMD.AtLeastOneConstructor", "PMD.CyclomaticComplexity", + "PMD.ModifiedCyclomaticComplexity", "PMD.NPathComplexity", "PMD.StdCyclomaticComplexity"}) +public class SignOffAuditRound extends AbstractAuditBoardDashboardEndpoint { + + boolean updateAll = true; + /** + * The type of the JSON request. + */ + private static final Type AUDIT_BOARD = new TypeToken>() { + }.getType(); + + /** + * Class-wide logger + */ + public static final Logger LOGGER = LogManager.getLogger(SignOffAuditRound.class); + + /** + * The event to return for this endpoint. + */ + private final ThreadLocal my_event = new ThreadLocal(); + + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.POST; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/sign-off-audit-round"; + } + + /** + * @return COUNTY authorization is required for this endpoint. + */ + public AuthorizationType requiredAuthorization() { + return AuthorizationType.COUNTY; + } + + /** + * {@inheritDoc} + */ + @Override + protected ASMEvent endpointEvent() { + return my_event.get(); + } + + /** + * {@inheritDoc} + */ + @Override + protected void reset() { + my_event.set(null); + } + + /** + * Signs off on the current audit round, regardless of its state of + * completion. + * + * @param request The request. + * @param response The response. + */ + @Override + @SuppressWarnings({"PMD.ExcessiveMethodLength"}) + public String endpointBody(final Request request, final Response response) { + final County county = Main.authentication().authenticatedCounty(request); + + if (county == null) { + LOGGER.error("could not get authenticated county"); + unauthorized(response, "not authorized to sign off on the round"); + } + + final JsonParser parser = new JsonParser(); + final JsonObject o; + + try { + o = parser.parse(request.body()).getAsJsonObject(); + final int auditBoardIndex = o.get("index").getAsInt(); + final List signatories = Main.GSON.fromJson(o.get("audit_board"), AUDIT_BOARD); + + if (signatories.size() < CountyDashboard.MIN_ROUND_SIGN_OFF_MEMBERS) { + LOGGER.error("[signoff: too few signatories for round sign-off]"); + invariantViolation(response, "too few signatories for round sign-off sent"); + } + + final CountyDashboard cdb = Persistence.getByID(county.id(), CountyDashboard.class); + + if (cdb == null) { + LOGGER.error(String + .format("[signoff: Could not get county dashboard for %s County id=%d]", + county.name(), county.id())); + serverError(response, "could not get county dashboard"); + } + + if (cdb.currentRound() == null) { + LOGGER.error(String.format("[signoff: No current round for %s County]", + cdb.county().name())); + invariantViolation(response, "no current round on which to sign off"); + } + + final Round currentRound = cdb.currentRound(); + + currentRound.setSignatories(auditBoardIndex, signatories); + + if (cdb.auditBoardCount() == null) { + LOGGER.error(String.format("[signoff: Audit board count unset for %s County]", + cdb.county().name())); + invariantViolation(response, "audit board count unset"); + } + + // If we have not seen all the boards sign off yet, we do not want to end + // the round. + if (currentRound.signatories().size() < cdb.auditBoardCount()) { + LOGGER.info(String.format("%d of %d audit boards have signed off for county %d", + currentRound.signatories().size(), cdb.auditBoardCount(), + cdb.id())); + } else { + // We're done! + cdb.endRound(); + + final AuditBoardDashboardASM asm = + ASMUtilities.asmFor(AuditBoardDashboardASM.class, String.valueOf(cdb.id())); + + if (null != asm && asm.currentState() == ROUND_IN_PROGRESS) { + ASMUtilities.step(ROUND_COMPLETE_EVENT, AuditBoardDashboardASM.class, + String.valueOf(cdb.id())); + } + + logAuditsForCountyDashboard(cdb); + + // update the ASM state for the county and maybe DoS + if (!DISABLE_ASM) { + final boolean auditComplete; + LOGGER.info(String + .format("[signoff for %s County: cdb.estimatedSamplesToAudit()=%d," + + " cdb.auditedSampleCount()=%d," + " cdb.ballotsAudited()=%d]", + cdb.county().name(), cdb.estimatedSamplesToAudit(), + cdb.auditedSampleCount(), cdb.ballotsAudited())); + + if (cdb.allAuditsComplete()) { + my_event.set(RISK_LIMIT_ACHIEVED_EVENT); + // In this case, we'd be terminating single county audits + // for opportunistic benefits only. + final List terminated = cdb.endSingleCountyAudits(); + LOGGER.debug(String.format("[signoff: all targeted audits finished in %s County." + + " Terminated these audits: %s]", cdb.county().name(), + terminated)); + my_event.set(ROUND_SIGN_OFF_EVENT); + if (participatesInStateAudit(cdb)) { + auditComplete = allCountyAuditBoardsSignedOff(); + } else { + auditComplete = true; + } + } else if (cdb.cvrsImported() <= cdb.ballotsAudited()) { + // In this case, we'd be terminating targeted and + // opportunistic single county audits. + final List terminated = cdb.endSingleCountyAudits(); + auditComplete = cdb.allAuditsComplete(); + LOGGER.debug(String + .format("[signoff: no more ballots; terminated single-county audits" + + " %s in %s County. All complete? (%b)]", terminated, + cdb.county().name(), auditComplete)); + my_event.set(ROUND_SIGN_OFF_EVENT); + } else { + LOGGER.debug("[signoff: the round ended normally]"); + auditComplete = false; + my_event.set(ROUND_SIGN_OFF_EVENT); + } + + if (auditComplete) { + LOGGER.info(String.format("[signoff: round complete in %s County]", + cdb.county().name())); + + LOGGER.info(String.format("[signoff: audit complete in %s County]", + cdb.county().name())); + notifyAuditCompleteForDoS(); + notifyRoundCompleteForDoS(cdb.id()); + } + } + } + } catch (final PersistenceException e) { + LOGGER.error("[signoff: unable to sign off round.]"); + serverError(response, "unable to sign off round: " + e); + } catch (final JsonParseException e) { + LOGGER.error("[signoff: bad data sent in an attempt to sign off on round]", e); + badDataContents(response, "invalid request body attempting to sign off on round"); + } + LOGGER.debug("[signoff: a-ok]"); + ok(response, "audit board signed off"); + + return my_endpoint_result.get(); + } + + /** + * Notifies the DoS dashboard that the round is over if all the counties + * _except_ for the one identified in the parameter have completed their audit + * round, or are not auditing (the excluded county is not counted because its + * transition will not happen until this endpoint returns). + * + * @param the_id The ID of the county to exclude. + */ + private void notifyRoundCompleteForDoS(final Long the_id) { + boolean finished = true; + for (final CountyDashboard cdb : Persistence.getAll(CountyDashboard.class)) { + if (cdb.id().equals(the_id)) { + continue; // <- sneaky filter for all but this county + // ROUND_COMPLETE_EVENT has already happened for this county above, and + // the notifyAuditComplete will handle COUNTY_AUDIT_COMPLETE_EVENT for + // this county + } + + if (!cdb.id().equals(the_id)) { + finished &= cdb.currentRound() == null; + } + } + + if (finished) { + for (final CountyDashboard cdb : Persistence.getAll(CountyDashboard.class)) { + if (cdb.id().equals(the_id)) { + continue; + } + markCountyAsDone(cdb); + } + + DoSDashboardASM dashboardASM = ASMUtilities.asmFor(DoSDashboardASM.class, DoSDashboardASM.IDENTITY); + + if (dashboardASM.currentState().equals(DoSDashboardState.DOS_AUDIT_ONGOING)) { + ASMUtilities.step(DOS_ROUND_COMPLETE_EVENT, DoSDashboardASM.class, + DoSDashboardASM.IDENTITY); + LOGGER.debug("[notifyRoundComplete stepped DOS_ROUND_COMPLETE_EVENT]"); + } + } + } + + /** + * Notifies the county and DoS dashboards that the audit is complete. + */ + private void notifyAuditCompleteForDoS() { + ASMUtilities.step(COUNTY_AUDIT_COMPLETE_EVENT, CountyDashboardASM.class, + my_asm.get().identity()); + // check to see if all counties are complete + boolean all_complete = true; + for (final County c : Persistence.getAll(County.class)) { + final CountyDashboardASM asm = + ASMUtilities.asmFor(CountyDashboardASM.class, String.valueOf(c.id())); + all_complete &= asm.isInFinalState(); + } + if (all_complete) { + ASMUtilities.step(DOS_AUDIT_COMPLETE_EVENT, DoSDashboardASM.class, + DoSDashboardASM.IDENTITY); + } + } + + /** + * + * Marks a county as done, marks the risk limit achieved event and + * county as audit complete. + * + * Technically the county should be done at this point. The If check + * is a bit redundant but it was in the code so kept for an additional + * check. + * + * @param cdb County dashboard who signed off + */ + private void markCountyAsDone(CountyDashboard cdb) { + final CountyDashboardASM countyDashboardASM = + ASMUtilities.asmFor(CountyDashboardASM.class, String.valueOf(cdb.id())); + final AuditBoardDashboardASM auditBoardASM = + ASMUtilities.asmFor(AuditBoardDashboardASM.class, String.valueOf(cdb.id())); + final Boolean inProgress = + auditBoardASM.currentState().equals(ROUND_IN_PROGRESS) || auditBoardASM.currentState() + .equals(ROUND_IN_PROGRESS_NO_AUDIT_BOARD); + + if (countyDashboardASM.currentState().equals(CountyDashboardState.COUNTY_AUDIT_UNDERWAY) && + !inProgress && cdb.allAuditsComplete()) { + final List terminated = cdb.endSingleCountyAudits(); + LOGGER.debug(String + .format("[markCountyAsDone: all audits finished in %s County." + + " Terminated these audits: %s]", cdb.county().name(), terminated)); + auditBoardASM.stepEvent(RISK_LIMIT_ACHIEVED_EVENT); + countyDashboardASM.stepEvent(COUNTY_AUDIT_COMPLETE_EVENT); + + ASMUtilities.save(auditBoardASM); + ASMUtilities.save(countyDashboardASM); + } + } + + private boolean allCountyAuditBoardsSignedOff() { + + for (CountyDashboard cdb : Persistence.getAll(CountyDashboard.class)) { + final CountyDashboardASM countyDashboardASM = + ASMUtilities.asmFor(CountyDashboardASM.class, String.valueOf(cdb.id())); + LOGGER.debug(MessageFormat + .format("County={0} dashboard state={1} auditBoardCount={2} currentRound={3}", + cdb.county().name(), countyDashboardASM.currentState(), + cdb.auditBoardCount(), cdb.currentRound())); + + if (!countyDashboardASM.currentState() + .equals(CountyDashboardState.COUNTY_AUDIT_UNDERWAY)) { + continue; + } // do not include counties where audit is not underway + + final boolean currentRoundNotSignedOff = + cdb.currentRound() != null && ((cdb.currentRound().signatories().size() == 0) || + (cdb.currentRound().signatories().size() < cdb.auditBoardCount())); + + if (currentRoundNotSignedOff) { + LOGGER.info("allCountyAuditBoardsSignedOff: false"); + return false; + } + } + LOGGER.info("allCountyAuditBoardsSignedOff: true"); + return true; + } + + private boolean participatesInStateAudit(CountyDashboard currentCountyDashboard) { + boolean returnVal = (currentCountyDashboard.getAudits().stream() + .filter(audit -> audit.getCounties().size() > 1) + .filter(ca -> ca.auditReason() != AuditReason.OPPORTUNISTIC_BENEFITS) + .filter(audit -> !audit.isHandCount()).count() > 0); + LOGGER.debug(MessageFormat.format("participatesInStateAudit: {0}", returnVal)); + return returnVal; + } + + private void logAuditsForCountyDashboard(CountyDashboard cd) { + LOGGER.debug(MessageFormat.format("{0} {1} {2} {3}", "Audit Name", "Audit Reason", + "Audit Status", "Targeted")); + for (ComparisonAudit ca : cd.getAudits()) { + LOGGER.debug(MessageFormat.format("{0} {1} {2} {3}", ca.getContestName(), + ca.auditReason(), ca.auditStatus(), ca.isTargeted())); + } + } + +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/StartAuditRound.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/StartAuditRound.java new file mode 100644 index 00000000..7e016321 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/StartAuditRound.java @@ -0,0 +1,534 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * + * @created Aug 12, 2017 + * + * @copyright 2017 Colorado Department of State + * + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * + * @creator Joseph R. Kiniry + * + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import static us.freeandfair.corla.asm.ASMEvent.AuditBoardDashboardEvent.*; +import static us.freeandfair.corla.asm.ASMEvent.CountyDashboardEvent.*; +import static us.freeandfair.corla.asm.ASMEvent.DoSDashboardEvent.DOS_START_ROUND_EVENT; +import static us.freeandfair.corla.asm.ASMState.DoSDashboardState.COMPLETE_AUDIT_INFO_SET; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import javax.persistence.PersistenceException; + +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.asm.ASMEvent; +import us.freeandfair.corla.asm.ASMState.CountyDashboardState; +import us.freeandfair.corla.asm.ASMUtilities; +import us.freeandfair.corla.asm.AuditBoardDashboardASM; +import us.freeandfair.corla.asm.CountyDashboardASM; +import us.freeandfair.corla.controller.BallotSelection; +import us.freeandfair.corla.controller.BallotSelection.Segment; +import us.freeandfair.corla.controller.BallotSelection.Selection; +import us.freeandfair.corla.controller.ComparisonAuditController; +import us.freeandfair.corla.controller.ContestCounter; +import us.freeandfair.corla.model.AuditReason; +import us.freeandfair.corla.model.AuditSelection; +import us.freeandfair.corla.model.CastVoteRecord; +import us.freeandfair.corla.model.ComparisonAudit; +import us.freeandfair.corla.model.ContestResult; +import us.freeandfair.corla.model.ContestToAudit; +import us.freeandfair.corla.model.CountyDashboard; +import us.freeandfair.corla.model.DoSDashboard; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.report.ReportRows; +import us.freeandfair.corla.util.PhantomBallots; + +/** + * Starts a new audit round for one or more counties. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings({"PMD.CyclomaticComplexity", "PMD.StdCyclomaticComplexity", + "PMD.AtLeastOneConstructor", "PMD.ModifiedCyclomaticComplexity", "PMD.NPathComplexity", + "PMD.ExcessiveImports", "PMD.TooManyStaticImports"}) +public class StartAuditRound extends AbstractDoSDashboardEndpoint { + /** + * Class-wide logger + */ + public static final Logger LOGGER = LogManager.getLogger(StartAuditRound.class); + + /** + * The event to return for this endpoint. + */ + private final ThreadLocal my_event = new ThreadLocal(); + + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.POST; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/start-audit-round"; + } + + /** + * @return STATE authorization is necessary for this endpoint. + */ + public AuthorizationType requiredAuthorization() { + return AuthorizationType.STATE; + } + + /** + * {@inheritDoc} + */ + @Override + protected ASMEvent endpointEvent() { + return my_event.get(); + } + + /** + * {@inheritDoc} + */ + @Override + protected void reset() { + my_event.set(null); + } + + /** + * Count ContestResults, create ComparisonAudits and assign them to + * CountyDashboards. + */ + public void initializeAuditData(final DoSDashboard dosdb) { + final List contestResults = initializeContests(dosdb.contestsToAudit()); + final List comparisonAudits = + initializeAudits(contestResults, dosdb.auditInfo().riskLimit()); + final List cdbs = Persistence.getAll(CountyDashboard.class); + for (final CountyDashboard cdb : cdbs) { + initializeCountyDashboard(cdb, comparisonAudits); + } + } + + /** + * Create a ContestResult for every contest to audit. + */ + public List initializeContests(final Set cta) { + final List countedCRs = countAndSaveContests(cta); + LOGGER + .debug(String.format("[initializeContests: cta=%s, countedCRs=%s]", cta, countedCRs)); + return countedCRs; + } + + /** + * Warning: Contains Side Effects + */ + public List initializeAudits(final List contestResults, + final BigDecimal riskLimit) { + final List comparisonAudits = + contestResults.stream().map(cr -> ComparisonAuditController.createAudit(cr, riskLimit)) + .collect(Collectors.toList()); + + LOGGER + .debug(String.format("[initializeAudits: contestResults=%s, " + "comparisonAudits=%s]", + contestResults, comparisonAudits)); + + return comparisonAudits; + } + + /** + * Setup a county dashboard. Puts the dashboard into the + * COUNTY_START_AUDIT_EVENT state. + * + * Puts the right set of comparison audits on the cdb. + * + * Builds comparison audits for the driving contests. + */ + public void initializeCountyDashboard(final CountyDashboard cdb, + final List comparisonAudits) { + // FIXME extract-fn + final Set drivingContestNames = comparisonAudits.stream() + .filter(ca -> ca.contestResult() + .getAuditReason() != AuditReason.OPPORTUNISTIC_BENEFITS) + .map(ca -> ca.contestResult().getContestName()).collect(Collectors.toSet()); + + // OK. + cdb.setAuditedSampleCount(0); + cdb.setAuditedPrefixLength(0); + cdb.setDrivingContestNames(drivingContestNames); + + // FIXME extract-fn + Set countyAudits = new HashSet<>(); + if (cdb.getAudits().isEmpty()) { + countyAudits = comparisonAudits.stream().filter(ca -> ca.isForCounty(cdb.county().id())) + .collect(Collectors.toSet()); + cdb.setAudits(countyAudits); + } + + // FIXME extract-fn + // The county missed its deadline, nothing to start, so let's mark it so + final CountyDashboardASM countyDashboardASM = + ASMUtilities.asmFor(CountyDashboardASM.class, String.valueOf(cdb.id())); + final AuditBoardDashboardASM auditDashboardASM = + ASMUtilities.asmFor(AuditBoardDashboardASM.class, String.valueOf(cdb.id())); + + if (countyDashboardASM + .currentState() != CountyDashboardState.BALLOT_MANIFEST_AND_CVRS_OK) { + LOGGER.info(String.format("[%s County missed the file upload deadline]", + cdb.county().name())); + auditDashboardASM.stepEvent(NO_CONTESTS_TO_AUDIT_EVENT); + } + countyDashboardASM.stepEvent(COUNTY_START_AUDIT_EVENT); + ASMUtilities.save(countyDashboardASM); + ASMUtilities.save(auditDashboardASM); + + if (!countyDashboardASM.isInInitialState() && !countyDashboardASM.isInFinalState()) { + LOGGER.debug(String + .format("[initializeCountyDashboard: " + " cdb=%s, comparisonAudits=%s, " + + " drivingContestNames=%s, countyAudits=%s]", cdb, comparisonAudits, + drivingContestNames, countyAudits)); + } + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointBody(final Request the_request, final Response the_response) { + + final DoSDashboard dosdb = Persistence.getByID(DoSDashboard.ID, DoSDashboard.class); + if (my_asm.get().currentState() == COMPLETE_AUDIT_INFO_SET) { + // this is the first round + // this needs to happen after uploading is done but before the audit is + // started + initializeAuditData(dosdb); + } + + my_event.set(DOS_START_ROUND_EVENT); + return startRound(the_request, the_response); + } + + /** + * Provide the reasons for auditing each targeted contest + * + * @return a map of contest name to audit reason + */ + public Map targetedContestReasons(final Set ctas) { + final Map> contestToAudits = ctas.stream() + .collect(Collectors.groupingBy((ContestToAudit cta) -> cta.contest().name())); + + return contestToAudits.entrySet().stream().collect(Collectors + .toMap((Map.Entry> e) -> e.getKey(), + // every getValue has at least one because of groupingBy + // every ContestToAudit has a reason + (Map.Entry> e) -> e.getValue().get(0).reason())); + } + + /** + * Update every - targeted and opportunistic both - contest's voteTotals from + * the counties. This needs to happen between all counties uploading their + * data and before the ballot selection happens + */ + public List countAndSaveContests(final Set cta) { + LOGGER.debug(String.format("[countAndSaveContests: cta=%s]", cta)); + final Map tcr = targetedContestReasons(cta); + + return ContestCounter.countAllContests().stream().map(cr -> { + cr.setAuditReason(tcr.getOrDefault(cr.getContestName(), + AuditReason.OPPORTUNISTIC_BENEFITS)); + return cr; + }).map(Persistence::persist).collect(Collectors.toList()); + } + + /** + * sets selection on each contestResult, the results of + * BallotSelection.randomSelection + */ + public List makeSelections(final List comparisonAudits, + final String seed, final BigDecimal riskLimit) { + + final List selections = new ArrayList<>(); + + for (final ComparisonAudit comparisonAudit : comparisonAudits) { + final ContestResult contestResult = comparisonAudit.contestResult(); + // only make selection for targeted contests + if (contestResult.getAuditReason().isTargeted()) { + final Integer startIndex = + BallotSelection.auditedPrefixLength(comparisonAudit.getContestCVRIds()); + final Integer endIndex = comparisonAudit.optimisticSamplesToAudit(); + + final Selection selection = + BallotSelection.randomSelection(contestResult, seed, startIndex, endIndex); + + LOGGER.debug(String.format("[makeSelections for ContestResult: contestName=%s, " + + "contestResult.contestCVRIds=%s, selection=%s, " + + "selection.contestCVRIds=%s, startIndex=%d, endIndex=%d]", + contestResult.getContestName(), + comparisonAudit.getContestCVRIds(), selection, + selection.contestCVRIds(), startIndex, endIndex)); + + LOGGER.info(String.format("[makeSelections for ContestResult: contestName=%s, " + + "contestResult.contestCVRIds=%s, selection=%s, " + + "selection.contestCVRIds=%s, startIndex=%d, endIndex=%d]", + contestResult.getContestName(), + comparisonAudit.getContestCVRIds(), selection, + selection.contestCVRIds(), startIndex, endIndex)); + + comparisonAudit.addContestCVRIds(selection.contestCVRIds()); + + selections.add(selection); + } + } + return selections; + } + + /** + * Starts the first audit round. + * + * @param the_request The HTTP request. + * @param the_response The HTTP response. + * @return the result for endpoint. + */ + // FIXME With some refactoring, we won't have excessive method length. + @SuppressWarnings({"PMD.ExcessiveMethodLength"}) + public String startRound(final Request the_request, final Response the_response) { + final DoSDashboard dosdb = Persistence.getByID(DoSDashboard.ID, DoSDashboard.class); + final BigDecimal riskLimit = dosdb.auditInfo().riskLimit(); + final String seed = dosdb.auditInfo().seed(); + + List comparisonAudits = Persistence.getAll(ComparisonAudit.class).stream() + .filter(ca -> ca.isTargeted() && !ca.isFinished()).collect(Collectors.toList()); + + // [[ComparisonAudit for Adams COUNTY COMMISSIONER DISTRICT 3: + // counties=[County [name=Adams, id=1]], auditedSampleCount=4, + // overstatements=0.000000, contestResult.contestCvrIds=[559, 422, 537, + // 411], status=IN_PROGRESS, reason=COUNTY_WIDE_CONTEST]] + + List selections = makeSelections(comparisonAudits, seed, riskLimit); + + // Nothing in this try-block should know about HTTP requests / responses + try { + // this flag starts off true if we're going to conjoin it with all + // the ASM states, and false otherwise as we just assume audit + // reasonableness in the absence of ASMs. We'll remind you about + // it at the end. + // FIXME map a function over a collection of dashboardsToStart + // FIXME extract-fn (for days): update every county dashboard with + // a list of ballots to audit + for (final CountyDashboard cdb : dashboardsToStart()) { + try { + final CountyDashboardASM countyDashboardASM = + ASMUtilities.asmFor(CountyDashboardASM.class, String.valueOf(cdb.id())); + List> resultReports = ReportRows.genSumResultsReport(); + Set audits = cdb.getAudits(); + Set contestStrs = new HashSet<>(); + for (ComparisonAudit audit : audits) { + contestStrs.add(audit.getContestName()); + } + boolean isRisk = false; + for (int i = 0; i < resultReports.size(); i++) { + List name = resultReports.get(i); + if (contestStrs.contains(name.get(0))) { + if (name.get(3).equalsIgnoreCase("No")) { + isRisk = true; + break; + } + } + } + // If a county still has an audit underway, check to see if + // they've achieved their risk limit before starting anything + // else. A county that has met the risk limit is done. + if (countyDashboardASM.currentState() + .equals(CountyDashboardState.COUNTY_AUDIT_UNDERWAY) && cdb.allAuditsComplete() && + (!isRisk)) { + LOGGER + .info(String.format("[startRound: allAuditsComplete! %s County is FINISHED.]", + cdb.county().name())); + ASMUtilities.step(RISK_LIMIT_ACHIEVED_EVENT, AuditBoardDashboardASM.class, + String.valueOf(cdb.id())); + countyDashboardASM.stepEvent(COUNTY_AUDIT_COMPLETE_EVENT); + + ASMUtilities.save(countyDashboardASM); + continue; + } + // Round 2 start + // No discrepencies + // Go to next round + // + int disagreements = 0; + if (cdb.discrepancies().size() > 0) { + if (cdb.discrepancies().get(AuditSelection.AUDITED_CONTEST) != null) { + disagreements = cdb.discrepancies().get(AuditSelection.AUDITED_CONTEST); + } + } + + if (countyDashboardASM.currentState() + .equals(CountyDashboardState.COUNTY_AUDIT_UNDERWAY) && + !cdb.allAuditsComplete() && + (cdb.rounds().size() > 0) && (disagreements == 0)) { + cdb.getAudits().stream().filter(ca -> !ca.isFinished()).forEach(ca-> { + ca.updateAuditStatus(); + Persistence.save(ca); + }); + } // sometimes audit status is left in air... and make sure we are good + + if (countyDashboardASM.currentState() + .equals(CountyDashboardState.COUNTY_AUDIT_UNDERWAY) && cdb.allAuditsComplete() && + (cdb.rounds().size() > 0) && (disagreements == 0)) { + LOGGER + .info(String.format("[startRound: allAuditsComplete! %s County is FINISHED. Audited Disagreements == 0]", + cdb.county().name())); + ASMUtilities.step(RISK_LIMIT_ACHIEVED_EVENT, AuditBoardDashboardASM.class, + String.valueOf(cdb.id())); + countyDashboardASM.stepEvent(COUNTY_AUDIT_COMPLETE_EVENT); + + ASMUtilities.save(countyDashboardASM); + continue; + } + + // Risk limit hasn't been achieved and we were never given any + // audits to work on. + if (cdb.comparisonAudits().isEmpty()) { + LOGGER + .info("[startRound: county made its deadline but was assigned no contests to audit]"); + ASMUtilities.step(NO_CONTESTS_TO_AUDIT_EVENT, AuditBoardDashboardASM.class, + String.valueOf(cdb.id())); + countyDashboardASM.stepEvent(COUNTY_AUDIT_COMPLETE_EVENT); + ASMUtilities.save(countyDashboardASM); + continue; // this county is completely finished. + } + + // Find the ballot selections for all contests that this + // county is participating in. + final Segment segment = Selection.combineSegments(selections.stream() + .map(s -> s.forCounty(cdb.county().id())).collect(Collectors.toList())); + + // Obtain all de-duplicated, ordered CVRs, then audit phantom ballots, + // removing them from the sequence to audit so the boards don’t have + // to. + final List ballotSequenceCVRs = + PhantomBallots.removePhantomRecords(PhantomBallots + .auditPhantomRecords(cdb, segment.cvrsInBallotSequence())); + + // ballotSequence is *just* the CVR IDs, as expected. + final List ballotSequence = + ballotSequenceCVRs.stream().map(cvr -> cvr.id()).collect(Collectors.toList()); + + // similar message also sent to info below, this could be a big line + LOGGER.trace(String + .format("[startRound:" + " county=%s, round=%s, segment.auditSequence()=%s," + + " segment.ballotSequence()=%s, cdb.comparisonAudits=%s,", cdb.county(), + cdb.currentRound(), segment.auditSequence(), ballotSequence, + cdb.comparisonAudits())); + LOGGER.info(String + .format("[startRound:" + " county=%s, round=%s, segment.auditSequence()=%s," + + " segment.ballotSequence()=%s, cdb.comparisonAudits=%s,", cdb.county(), + cdb.currentRound(), segment.auditSequence(), ballotSequence, + cdb.comparisonAudits())); + // Risk limit hasn't been achieved. We were given some audits + // to work on, but have nothing to do in this round. Please + // wait patiently. + if (ballotSequence.isEmpty()) { + LOGGER.info(String + .format("[startRound: no ballots to audit in %s County, skipping round]", + cdb.county())); + cdb.startRound(0, 0, 0, Collections.emptyList(), Collections.emptyList()); + Persistence.saveOrUpdate(cdb); + ASMUtilities.step(ROUND_COMPLETE_EVENT, AuditBoardDashboardASM.class, + String.valueOf(cdb.id())); + continue; + } + + // Risk limit hasn't been achieved and we finally have something to + // do in this round! + ComparisonAuditController.startRound(cdb, cdb.comparisonAudits(), + segment.auditSequence(), ballotSequence); + Persistence.saveOrUpdate(cdb); + + LOGGER.info(String.format("[startRound: Round %d for %s County started normally." + + " Estimated to audit %d ballots.]", + cdb.currentRound().number(), cdb.county().name(), + cdb.estimatedSamplesToAudit())); + + ASMUtilities.step(ROUND_START_EVENT, AuditBoardDashboardASM.class, + String.valueOf(cdb.id())); + ASMUtilities.save(countyDashboardASM); + + // FIXME hoist me; we don't need to know about HTTP requests or + // responses at this level. + } catch (final IllegalArgumentException e) { + e.printStackTrace(System.out); + final String msg = + String.format("could not start round for %s County", cdb.county().name()); + serverError(the_response, msg); + LOGGER.error(msg); + } catch (final IllegalStateException e) { + LOGGER.error("IllegalStateException " + e); + illegalTransition(the_response, e.getMessage()); + } + } // end of dashboard twiddling + + ok(the_response, "round started"); + // end of extraction. Now we can talk about HTTP requests / responses + // again! + } catch (final PersistenceException e) { + LOGGER.error("PersistenceException " + e); + serverError(the_response, "could not start round"); + } + + return my_endpoint_result.get(); + } + + /** + * + * @return true if a county should be started + */ + public Boolean isReadyToStartAudit(final CountyDashboard cdb) { + final CountyDashboardASM countyDashboardASM = + ASMUtilities.asmFor(CountyDashboardASM.class, String.valueOf(cdb.id())); + if (countyDashboardASM.isInInitialState() || countyDashboardASM.isInFinalState()) { + + return false; + } else { + return true; + } + } + + /** + * A dashboard is ready to start if it isn't in an initial or final state. + * + * @return a list of the dashboards to start. + */ + public List dashboardsToStart() { + final List cdbs = Persistence.getAll(CountyDashboard.class); + + final List result = + cdbs.stream().filter(cdb -> isReadyToStartAudit(cdb)).collect(Collectors.toList()); + + LOGGER.debug("[dashboardsToStart: " + result); + return result; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/StateReportDownload.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/StateReportDownload.java new file mode 100644 index 00000000..520cbd08 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/StateReportDownload.java @@ -0,0 +1,130 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 27, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import java.util.Arrays; +import java.util.List; + +import javax.persistence.PersistenceException; + +import org.apache.cxf.attachment.Rfc5987Util; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.asm.ASMState; +import us.freeandfair.corla.asm.ASMState.DoSDashboardState; +import us.freeandfair.corla.asm.ASMUtilities; +import us.freeandfair.corla.asm.DoSDashboardASM; +import us.freeandfair.corla.report.StateReport; +import us.freeandfair.corla.util.SparkHelper; + +/** + * The state report download endpoint. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class StateReportDownload extends AbstractEndpoint { + /** + * The states in which this endpoint can provide a result. + */ + private static final List LEGAL_STATES = + Arrays.asList(DoSDashboardState.COMPLETE_AUDIT_INFO_SET, + DoSDashboardState.DOS_AUDIT_ONGOING, + DoSDashboardState.DOS_ROUND_COMPLETE, + DoSDashboardState.DOS_AUDIT_COMPLETE, + DoSDashboardState.AUDIT_RESULTS_PUBLISHED); + + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.GET; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/state-report"; + } + + /** + * This endpoint requires any kind of authentication. + */ + @Override + public AuthorizationType requiredAuthorization() { + return AuthorizationType.STATE; + } + + /** + * {@inheritDoc} + */ + @Override + // necessary to break out of the lambda expression in case of IOException + @SuppressWarnings("PMD.ExceptionAsFlowControl") + public String endpointBody(final Request the_request, final Response the_response) { + // if we haven't defined the election, this is "data not found" + final DoSDashboardASM dos_asm = ASMUtilities.asmFor(DoSDashboardASM.class, + DoSDashboardASM.IDENTITY); + if (!LEGAL_STATES.contains(dos_asm.currentState())) { + dataNotFound(the_response, "No state report available in this state."); + } + + final boolean pdf = "pdf".equalsIgnoreCase(the_request.queryParams("file_type")); + final StateReport sr = new StateReport(); + byte[] file = new byte[0]; + String filename = ""; + + if (pdf) { + the_response.type("application/pdf"); + filename = sr.filenamePDF(); + file = sr.generatePDF(); + } else { + the_response.type("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); + // the file name should be constructed from the election type and date, and + // the county name and round + filename = sr.filenameExcel(); + try { + file = sr.generateExcel(); + } catch (final IOException e) { + serverError(the_response, "Unable to generate Excel file"); + } + } + + try { + the_response.raw().setHeader("Content-Disposition", "attachment; filename=\"" + + Rfc5987Util.encode(filename, "UTF-8") + "\""); + } catch (final UnsupportedEncodingException e) { + serverError(the_response, "UTF-8 is unsupported (this should never happen)"); + } + + try (OutputStream os = SparkHelper.getRaw(the_response).getOutputStream(); + BufferedOutputStream bos = new BufferedOutputStream(os)) { + bos.write(file); + bos.flush(); + ok(the_response); + } catch (final IOException | PersistenceException e) { + serverError(the_response, "Unable to stream response"); + } + + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/Unauthenticate.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/Unauthenticate.java new file mode 100644 index 00000000..2c0574de --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/Unauthenticate.java @@ -0,0 +1,92 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 9, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.asm.ASMEvent; +import us.freeandfair.corla.asm.AbstractStateMachine; + +/** + * The endpoint for unauthenticating an administrator. + * + * @author Daniel M Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class Unauthenticate extends AbstractEndpoint { + /** + * @return no authorization is required for this endpoints. + */ + @Override + public AuthorizationType requiredAuthorization() { + return AuthorizationType.NONE; + } + + /** + * @return this endpoint does not use an ASM. + */ + @Override + protected Class asmClass() { + return null; + } + + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.POST; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + return "/unauthenticate"; + } + + /** + * {@inheritDoc} + */ + @Override + protected ASMEvent endpointEvent() { + return null; + } + + /** + * Gets the ASM identity for the specified request. + * + * @param the_request The request. + * @return the county ID of the authenticated county. + */ + @Override + protected String asmIdentity(final Request the_request) { + return null; + } + + /** + * Unauthenticates the session. + * + * @param the_request The request. + * @param the_response The response. + */ + @Override + public String endpointBody(final Request the_request, final Response the_response) { + Main.authentication().deauthenticate(the_request); + ok(the_response, "Unauthenticated"); + return my_endpoint_result.get(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/UpdateAuditInfo.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/UpdateAuditInfo.java new file mode 100644 index 00000000..5e68c08e --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/endpoint/UpdateAuditInfo.java @@ -0,0 +1,246 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 9, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.endpoint; + +import static us.freeandfair.corla.asm.ASMEvent.DoSDashboardEvent.*; + +import java.io.IOException; +import java.math.BigDecimal; +import java.time.Instant; + +import javax.persistence.PersistenceException; + +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonParser; + +import spark.Request; +import spark.Response; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.asm.ASMEvent; +import us.freeandfair.corla.csv.ContestNameParser; +import us.freeandfair.corla.model.AuditInfo; +import us.freeandfair.corla.model.DoSDashboard; +import us.freeandfair.corla.persistence.Persistence; + +/** + * The endpoint for setting the election information. + * + * @author Daniel M Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings({"PMD.AtLeastOneConstructor", "PMD.CyclomaticComplexity"}) +public class UpdateAuditInfo extends AbstractDoSDashboardEndpoint { + /** + * Uploaded file attribute + * + * Factored out due to the PMD rule AvoidDuplicateLiterals + */ + private final static String UPLOAD_KEY = "upload_file"; + + /** + * The event to return for this endpoint. + */ + private final ThreadLocal my_event = new ThreadLocal(); + + /** + * {@inheritDoc} + */ + @Override + public EndpointType endpointType() { + return EndpointType.POST; + } + + /** + * {@inheritDoc} + */ + @Override + public String endpointName() { + + return "/update-audit-info"; + } + + /** + * {@inheritDoc} + */ + @Override + protected ASMEvent endpointEvent() { + return my_event.get(); + } + + /** + * {@inheritDoc} + */ + @Override + protected void reset() { + my_event.set(null); + } + + /** + * JSON extraction + */ + public String contestsFromJSON(final String s) { + final JsonParser parser = new JsonParser(); + final JsonObject object; + + try { + object = parser.parse(s).getAsJsonObject(); + if (hasFile(object)){ + return object + .getAsJsonArray(UPLOAD_KEY) + .get(0) + .getAsJsonObject() + .get("contents") + .getAsString(); + } else { + return ""; + } + + } catch (final IndexOutOfBoundsException | JsonParseException e) { + Main.LOGGER.error("Failed to parse malformed JSON", e); + throw e; + } + } + + private Boolean hasFile(final JsonObject o) { + return o != null + && o.getAsJsonArray(UPLOAD_KEY) != null + && o.getAsJsonArray(UPLOAD_KEY).size() > 0 + && o.getAsJsonArray(UPLOAD_KEY).get(0) != null; + } + + /** + * Attempts to set the audit information. + * + * @param request The request. + * @param response The response. + */ + @Override + @SuppressWarnings("PMD.UselessParentheses") + public String endpointBody(final Request request, final Response response) { + try { + final String json = request.body(); + final String contests = contestsFromJSON(json); + final AuditInfo info = Main.GSON.fromJson(json, AuditInfo.class); + + if (!contests.isEmpty()) { + final ContestNameParser p = new ContestNameParser(contests); + if (p.parse()) { + info.setCanonicalContests(p.contests()); + info.setCanonicalChoices(p.getChoices()); + } else { + badDataContents(response, + String.format("[duplicates=%s; errors=%s]", + p.duplicates(), + p.errors())); + } + } + + + final DoSDashboard dosdb = Persistence.getByID(DoSDashboard.ID, DoSDashboard.class); + if (dosdb == null) { + Main.LOGGER.error("could not get department of state dashboard"); + serverError(response, "could not update audit information"); + } + + if (validateElectionInfo(info, dosdb, response)) { + dosdb.updateAuditInfo(info); + my_event.set(nextEvent(dosdb)); + Main.LOGGER.info("AuditInfo updated: " + info); + ok(response, "audit information updated."); + } + } catch (final PersistenceException e) { + serverError(response, "unable to update audit information: " + e.getMessage()); + } catch (final IndexOutOfBoundsException | JsonParseException e) { + badDataContents(response, "malformed audit information: " + e.getMessage()); + } catch (final IOException e) { + Main.LOGGER.error("Failed to parse canonical contest file", e); + serverError(response, "Failed to parse canonical contest file"); + } + + return my_endpoint_result.get(); + } + + /** + * Validates the specified election info. + * + * @param the_info The info. + * @param the_dosdb The DoS dashboard. + * @param the_response The response (for reporting failures). + */ + @SuppressWarnings({"PMD.UselessParentheses", "PMD.NPathComplexity"}) + private boolean validateElectionInfo(final AuditInfo the_info, + final DoSDashboard the_dosdb, + final Response the_response) { + boolean result = true; + + // check for valid relationship between meeting date and election date + final Instant effective_public_meeting_date; + if (the_info.publicMeetingDate() == null) { + effective_public_meeting_date = the_dosdb.auditInfo().publicMeetingDate(); + } else { + effective_public_meeting_date = the_info.publicMeetingDate(); + } + + final Instant effective_election_date; + if (the_info.electionDate() == null) { + effective_election_date = the_dosdb.auditInfo().electionDate(); + } else { + effective_election_date = the_info.electionDate(); + } + + if (effective_public_meeting_date != null && effective_election_date != null && + !effective_public_meeting_date.isAfter(effective_election_date)) { + result = false; + invariantViolation(the_response, "public meeting must be after election"); + } + + // check that the 0 <= risk limit <= 1 + if (the_info.riskLimit() != null && + (0 < BigDecimal.ZERO.compareTo(the_info.riskLimit()) || + 0 < the_info.riskLimit().compareTo(BigDecimal.ONE))) { + result = false; + invariantViolation(the_response, "invalid risk limit specified"); + } + + // check that no seed is specified + if (the_info.seed() != null) { + result = false; + invariantViolation(the_response, "cannot specify random seed through this endpoint"); + } + + return result; + } + + /** + * Computes the event of this endpoint based on audit info completeness. + * + * @param the_dosdb The DoS dashboard. + */ + private ASMEvent nextEvent(final DoSDashboard the_dosdb) { + final ASMEvent result; + final AuditInfo info = the_dosdb.auditInfo(); + + if (info.electionDate() == null || info.electionType() == null || + info.publicMeetingDate() == null || info.riskLimit() == null || + info.seed() == null || the_dosdb.contestsToAudit().isEmpty()) { + Main.LOGGER.debug("partial audit information submitted"); + result = PARTIAL_AUDIT_INFO_EVENT; + } else { + Main.LOGGER.debug("complete audit information submitted"); + result = COMPLETE_AUDIT_INFO_EVENT; + } + + return result; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/AppInfoJSON.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/AppInfoJSON.java new file mode 100644 index 00000000..a1a615d6 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/AppInfoJSON.java @@ -0,0 +1,40 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 13, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.json; + +/** + * App Info returned by the result + * + * @author Dogan Cibiceli + * @version 1.0.0 + */ +public class AppInfoJSON { + + /* ~semantic versioning such as 2.3.33 */ + private final String versionInfo; + + /** + * Constructs a new AppInfo. + * + * @param versionInfo Only one field for now. + */ + public AppInfoJSON(final String versionInfo) { + this.versionInfo = versionInfo; + } + + /** + * @return the version info. + */ + public String getVersionInfo() { + return versionInfo; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/AuditInvestigationReportInfoJsonAdapter.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/AuditInvestigationReportInfoJsonAdapter.java new file mode 100644 index 00000000..1f9f697e --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/AuditInvestigationReportInfoJsonAdapter.java @@ -0,0 +1,110 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 28, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.json; + +import java.io.IOException; +import java.time.Instant; + +import com.google.gson.JsonSyntaxException; +import com.google.gson.TypeAdapter; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.model.AuditInvestigationReportInfo; + +/** + * JSON adapter for audit investigation reports. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +// the default constructor suffices for type adapters +@SuppressWarnings("PMD.AtLeastOneConstructor") +public final class AuditInvestigationReportInfoJsonAdapter + extends TypeAdapter { + /** + * The "contest" string (for JSON serialization). + */ + private static final String TIMESTAMP = "timestamp"; + + /** + * The "reason" string (for JSON serialization). + */ + private static final String NAME = "name"; + + /** + * The "audit type" string (for JSON serialization). + */ + private static final String REPORT = "report"; + + /** + * Writes an audit investigation report object. + * + * @param the_writer The JSON writer. + * @param the_info The object to write. + */ + @Override + public void write(final JsonWriter the_writer, + final AuditInvestigationReportInfo the_report) + throws IOException { + the_writer.beginObject(); + the_writer.name(TIMESTAMP).value(Main.GSON.toJson(the_report.timestamp())); + the_writer.name(NAME).value(the_report.name()); + the_writer.name(REPORT).value(the_report.report()); + the_writer.endObject(); + } + + /** + * Reads an audit investigation report object. + * + * @param the_reader The JSON reader. + * @return the object. + */ + @Override + public AuditInvestigationReportInfo read(final JsonReader the_reader) + throws IOException { + boolean error = false; + String report_name = null; + String report = null; + Instant timestamp = null; + + the_reader.beginObject(); + while (the_reader.hasNext()) { + final String name = the_reader.nextName(); + switch (name) { + case TIMESTAMP: + timestamp = Main.GSON.fromJson(the_reader.nextString(), Instant.class); + break; + + case NAME: + report_name = the_reader.nextString(); + break; + + case REPORT: + report = the_reader.nextString(); + break; + + default: + error = true; + break; + } + } + the_reader.endObject(); + + if (error) { + throw new JsonSyntaxException("invalid data detected in audit investigation report"); + } + + return new AuditInvestigationReportInfo(timestamp, report_name, report); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/CVRContestInfoJsonAdapter.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/CVRContestInfoJsonAdapter.java new file mode 100644 index 00000000..f76eb24d --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/CVRContestInfoJsonAdapter.java @@ -0,0 +1,187 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 28, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.json; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import com.google.gson.JsonSyntaxException; +import com.google.gson.TypeAdapter; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; + +import us.freeandfair.corla.model.CVRContestInfo; +import us.freeandfair.corla.model.CVRContestInfo.ConsensusValue; +import us.freeandfair.corla.model.Contest; +import us.freeandfair.corla.persistence.Persistence; + +/** + * JSON adapter for CVR contest information. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +// the default constructor suffices for type adapters +@SuppressWarnings("PMD.AtLeastOneConstructor") +public final class CVRContestInfoJsonAdapter + extends TypeAdapter { + /** + * The "contest" string (for JSON serialization). + */ + private static final String CONTEST = "contest"; + + /** + * The "choices" string (for JSON serialization). + */ + private static final String CHOICES = "choices"; + + /** + * The "comment" string (for JSON serialization). + */ + private static final String COMMENT = "comment"; + + /** + * THe "comments" string (for erroneous client JSON). + */ + private static final String COMMENTS = "comments"; + + /** + * The "consensus" string (for JSON serialization). + */ + private static final String CONSENSUS = "consensus"; + + /** + * Writes a CVR contest info object. + * + * @param the_writer The JSON writer. + * @param the_info The object to write. + */ + @Override + public void write(final JsonWriter the_writer, + final CVRContestInfo the_info) + throws IOException { + the_writer.beginObject(); + the_writer.name(CONTEST).value(the_info.contest().id()); + the_writer.name(COMMENT).value(the_info.comment()); + if (the_info.consensus() != null) { + the_writer.name(CONSENSUS).value(the_info.consensus().toString()); + } + the_writer.name(CHOICES); + the_writer.beginArray(); + for (final String c : the_info.choices()) { + the_writer.value(c); + } + the_writer.endArray(); + the_writer.endObject(); + } + + /** + * Reads a set of choices. + */ + private List readChoices(final JsonReader the_reader) + throws IOException { + final List result = new ArrayList(); + the_reader.beginArray(); + while (the_reader.hasNext()) { + result.add(the_reader.nextString()); + } + the_reader.endArray(); + return result; + } + + /** + * Checks the sanity of a contest against a set of choices. + * + * @param the_id The contest ID. + * @param the_choices The choices. + * @return the resulting contest, if the data is sane, or null if the + * data is invalid. + */ + private Contest contestSanityCheck(final Long the_id, + final List the_choices) { + final Contest result = Persistence.getByID(the_id, Contest.class); + boolean error = the_choices == null; + + if (!error && result != null) { + for (final String c : the_choices) { + if (!result.isValidChoice(c)) { + error = true; + } + } + } + + if (error) { + return null; + } else { + return result; + } + } + + /** + * Reads a CVR contest info object. + * + * @param the_reader The JSON reader. + * @return the object. + */ + @Override + public CVRContestInfo read(final JsonReader the_reader) + throws IOException { + boolean error = false; + List choices = null; + long contest_id = -1; + String comment = null; + ConsensusValue consensus = null; + + the_reader.beginObject(); + while (the_reader.hasNext()) { + final String name = the_reader.nextName(); + switch (name) { + case CONTEST: + contest_id = the_reader.nextLong(); + break; + + case COMMENT: + case COMMENTS: + comment = the_reader.nextString(); + break; + + case CONSENSUS: + try { + consensus = ConsensusValue.valueOf(the_reader.nextString()); + } catch (final IllegalArgumentException e) { + // assume undefined consensus, because enum value was invalid + } + break; + + case CHOICES: + choices = readChoices(the_reader); + break; + + default: + error = true; + break; + } + } + the_reader.endObject(); + + // check the sanity of the contest + + final Contest contest = contestSanityCheck(contest_id, choices); + + if (error || contest == null) { + throw new JsonSyntaxException("invalid data detected in CVR contest info"); + } + + return new CVRContestInfo(contest, comment, consensus, choices); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/CVRToAuditResponse.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/CVRToAuditResponse.java new file mode 100644 index 00000000..2c89ca8f --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/CVRToAuditResponse.java @@ -0,0 +1,280 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 10, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.json; + +import us.freeandfair.corla.util.NaturalOrderComparator; +import us.freeandfair.corla.util.SuppressFBWarnings; + +import static us.freeandfair.corla.util.EqualsHashcodeHelper.*; + +/** + * The standard response provided by the server in response to a request + * for CVR locations. + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings({"PMD.UnusedPrivateField", "PMD.SingularField"}) +@SuppressFBWarnings(value = {"URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD"}, + justification = "Field is read by Gson.") +public class CVRToAuditResponse implements Comparable { + /** + * The (first) audit sequence number. + */ + protected final int my_audit_sequence_number; + + /** + * The CVR scanner ID. + */ + protected final int my_scanner_id; + + /** + * The CVR batch ID. + */ + protected final String my_batch_id; + + /** + * The CVR record ID. + */ + protected final int my_record_id; + + /** + * The CVR imprinted ID. + */ + protected final String my_imprinted_id; + + /** + * The CVR number (from the CSV file). + */ + protected final int my_cvr_number; + + /** + * The CVR database ID. + */ + protected final long my_db_id; + + /** + * The CVR ballot type. + */ + protected final String my_ballot_type; + + /** + * The CVR storage location. + */ + protected final String my_storage_location; + + /** + * The optional audit board index + */ + protected Integer my_audit_board_index; + + /** + * A flag that indicates whether or not the CVR has been audited. + */ + protected final boolean my_audited; + + /** + * A flag that indicates whether or not the CVR was audited in a previous + * round + */ + protected final boolean my_previously_audited; + + /** + * Create a new response object. + * + * @param the_audit_sequence_number The audit sequence number. + * @param the_scanner_id The scanner ID. + * @param the_batch_id The batch ID. + * @param the_record_id The record ID. + * @param the_imprinted_id The imprinted ID. + * @param the_cvr_number The CVR number (from the CSV file). + * @param the_db_id The database ID. + * @param the_ballot_type The ballot type. + * @param the_storage_location The storage location. + * @param the_audited true if the ballot has been audited, false otherwise. + * @param the_previously_audited true if the ballot has been audited in a + * previous round, false otherwise. + */ + @SuppressWarnings("PMD.ExcessiveParameterList") + public CVRToAuditResponse(final int the_audit_sequence_number, + final int the_scanner_id, + final String the_batch_id, + final int the_record_id, + final String the_imprinted_id, + final int the_cvr_number, + final long the_db_id, + final String the_ballot_type, + final String the_storage_location, + final boolean the_audited, + final boolean the_previously_audited) { + my_audit_sequence_number = the_audit_sequence_number; + my_scanner_id = the_scanner_id; + my_batch_id = the_batch_id; + my_record_id = the_record_id; + my_imprinted_id = the_imprinted_id; + my_cvr_number = the_cvr_number; + my_db_id = the_db_id; + my_ballot_type = the_ballot_type; + my_storage_location = the_storage_location; + my_audited = the_audited; + my_previously_audited = the_previously_audited; + } + + /** + * @return the audit sequence number. + */ + public int auditSequenceNumber() { + return my_audit_sequence_number; + } + + /** + * @return the scanner ID. + */ + public int scannerID() { + return my_scanner_id; + } + + /** + * @return the batch ID. + */ + public String batchID() { + return my_batch_id; + } + + /** + * @return the record ID. + */ + public int recordID() { + return my_record_id; + } + + /** + * @return the imprinted ID. + */ + public String imprintedID() { + return my_imprinted_id; + } + + /** + * @return the CVR number. + */ + public int cvrNumber() { + return my_cvr_number; + } + + /** + * @return the database ID. + */ + public long dbID() { + return my_db_id; + } + + /** + * @return the ballot type. + */ + public String ballotType() { + return my_ballot_type; + } + + /** + * @return the storage location. + */ + public String storageLocation() { + return my_storage_location; + } + + /** + * @return the audit board index or null if not set + */ + public Integer auditBoardIndex() { + return my_audit_board_index; + } + + /** + * Optionally set the audit board index that will be auditing this ballot. + */ + public void setAuditBoardIndex(final Integer auditBoardIndex) { + my_audit_board_index = auditBoardIndex; + } + + /** + * @return the audited flag. + */ + public boolean audited() { + return my_audited; + } + + /** + * @return the "previously audited" flag. + */ + public boolean previouslyAudited() { + return my_previously_audited; + } + + /** + * {@inheritDoc} + */ + @Override + public boolean equals(final Object other) { + boolean result = true; + if (other instanceof CVRToAuditResponse) { + final CVRToAuditResponse otherCtar = (CVRToAuditResponse) other; + result &= nullableEquals(this.dbID(), otherCtar.dbID()); + } else { + result = false; + } + + return result; + } + + /** + * {@inheritDoc} + */ + @Override + public int hashCode() { + return nullableHashCode(this.dbID()); + } + + /** + * Compares this object to another. + * + * The sorting happens by the tuple + * (storageLocation(), scannerID(), batchID(), recordID()) and will return a + * negative, positive, or 0-valued result if this should come before, after, + * or at the same point as the other object, respectively. + * + * @return int + */ + @Override + public int compareTo(final CVRToAuditResponse other) { + final int storageLocation = NaturalOrderComparator.INSTANCE.compare( + this.storageLocation(), other.storageLocation()); + + if (storageLocation != 0) { + return storageLocation; + } + + final int scanner = this.scannerID() - other.scannerID(); + + if (scanner != 0) { + return scanner; + } + + final int batch = NaturalOrderComparator.INSTANCE.compare( + this.batchID(), other.batchID()); + + if (batch != 0) { + return batch; + } + + return this.recordID() - other.recordID(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/CanonicalUpdate.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/CanonicalUpdate.java new file mode 100644 index 00000000..497112f0 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/CanonicalUpdate.java @@ -0,0 +1,29 @@ +package us.freeandfair.corla.json; + +import java.util.List; + +/** json deserializer for SetContestNames **/ +public class CanonicalUpdate { + + /** the contest db id **/ + public String contestId; + + /** the new name **/ + public String name; + + /** not needed, may be removed **/ + public Long countyId; + + /** list of choice changes **/ + public List choices; + + /** json deserializer for SetContestNames **/ + public static class ChoiceChange { + + /** aka current name **/ + public String oldName; + + /** the new name to change to **/ + public String newName; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/ContestJsonAdapter.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/ContestJsonAdapter.java new file mode 100644 index 00000000..33e795a4 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/ContestJsonAdapter.java @@ -0,0 +1,192 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 28, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.json; + +import java.io.IOException; +import java.util.Arrays; + +import com.google.gson.JsonParseException; +import com.google.gson.TypeAdapter; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.model.Choice; +import us.freeandfair.corla.model.Contest; +import us.freeandfair.corla.model.County; +import us.freeandfair.corla.persistence.Persistence; + +/** + * JSON adapter for contest. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings({"PMD.AtLeastOneConstructor", "PMD.CyclomaticComplexity", + "PMD.StdCyclomaticComplexity", "PMD.NPathComplexity"}) +public final class ContestJsonAdapter extends TypeAdapter { + /** + * The "id" string (for JSON serialization). + */ + private static final String ID = "id"; + + /** + * The "name" string (for JSON serialization). + */ + private static final String NAME = "name"; + + /** + * The "county_id" string (for JSON serialization). + */ + private static final String COUNTY_ID = "county_id"; + + /** + * The "description" string (for JSON serialization). + */ + private static final String DESCRIPTION = "description"; + + /** + * The "choices" string (for JSON serialization). + */ + private static final String CHOICES = "choices"; + + /** + * The "votes allowed" string (for JSON serialization). + */ + private static final String VOTES_ALLOWED = "votes_allowed"; + + /** + * The "winners allowed" string (for JSON serialization). + */ + private static final String WINNERS_ALLOWED = "winners_allowed"; + + /** + * The "sequence number" string (for JSON serialization). + */ + private static final String SEQUENCE_NUMBER = "sequence_number"; + + /** + * Writes a contest object. + * + * @param the_writer The JSON writer. + * @param the_info The object to write. + */ + @Override + @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") + public void write(final JsonWriter the_writer, + final Contest the_contest) + throws IOException { + the_writer.beginObject(); + the_writer.name(ID).value(the_contest.id()); + the_writer.name(NAME).value(the_contest.name()); + the_writer.name(COUNTY_ID).value(the_contest.county().id()); + the_writer.name(DESCRIPTION).value(the_contest.description()); + the_writer.name(CHOICES); + the_writer.beginArray(); + for (final Choice c : the_contest.choices()) { + if (!c.fictitious()) { + the_writer.jsonValue(Main.GSON.toJson(Persistence.unproxy(c))); + } + } + the_writer.endArray(); + the_writer.name(VOTES_ALLOWED).value(the_contest.votesAllowed()); + the_writer.name(WINNERS_ALLOWED).value(the_contest.winnersAllowed()); + the_writer.name(SEQUENCE_NUMBER).value(the_contest.sequenceNumber()); + the_writer.endObject(); + } + + /** + * Reads a contest object. + * + * @param the_reader The JSON reader. + * @return the object. + */ + @Override + public Contest read(final JsonReader the_reader) + throws IOException { + boolean error = false; + Long contest_id = null; + String contest_name = null; + County county = null; + String description = null; + Choice[] choices = null; + Integer votes_allowed = null; + Integer winners_allowed = null; + Integer sequence_number = null; + + the_reader.beginObject(); + while (the_reader.hasNext()) { + final String name = the_reader.nextName(); + switch (name) { + case ID: + contest_id = the_reader.nextLong(); + break; + + case NAME: + contest_name = the_reader.nextString(); + break; + + case COUNTY_ID: + county = Persistence.getByID(the_reader.nextLong(), County.class); + break; + + case DESCRIPTION: + description = the_reader.nextString(); + break; + + case CHOICES: + choices = Main.GSON.fromJson(the_reader.nextString(), Choice[].class); + break; + + case VOTES_ALLOWED: + votes_allowed = the_reader.nextInt(); + break; + + case WINNERS_ALLOWED: + winners_allowed = the_reader.nextInt(); + break; + + case SEQUENCE_NUMBER: + sequence_number = the_reader.nextInt(); + break; + + default: + error = true; + break; + } + } + the_reader.endObject(); + + // check if an identically-numbered contest exists + + if (error || contest_id == null || county == null || description == null || + choices == null || votes_allowed == null || winners_allowed == null) { + throw new JsonParseException("invalid data in contest"); + } + + if (sequence_number == null) { + sequence_number = 0; + } + final Contest existing = Persistence.getByID(contest_id, Contest.class); + final Contest result; + if (existing == null) { + // we don't use the ID because it might conflict with something else + result = new Contest(contest_name, county, description, + Arrays.asList(choices), votes_allowed, + winners_allowed, sequence_number); + } else { + result = existing; + } + + return result; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/ContestToAuditJsonAdapter.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/ContestToAuditJsonAdapter.java new file mode 100644 index 00000000..fa81593b --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/ContestToAuditJsonAdapter.java @@ -0,0 +1,149 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 28, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.json; + +import java.io.IOException; + +import com.google.gson.JsonSyntaxException; +import com.google.gson.TypeAdapter; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; + +import us.freeandfair.corla.model.AuditReason; +import us.freeandfair.corla.model.AuditType; +import us.freeandfair.corla.model.Contest; +import us.freeandfair.corla.model.ContestToAudit; +import us.freeandfair.corla.persistence.Persistence; + +/** + * JSON adapter for contest to audit information. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +// the default constructor suffices for type adapters +@SuppressWarnings("PMD.AtLeastOneConstructor") +public final class ContestToAuditJsonAdapter + extends TypeAdapter { + /** + * The "contest" string (for JSON serialization). + */ + private static final String CONTEST = "contest"; + + /** + * The "reason" string (for JSON serialization). + */ + private static final String REASON = "reason"; + + /** + * The "audit type" string (for JSON serialization). + */ + private static final String AUDIT = "audit"; + + /** + * Writes a contest to audit object. + * + * @param the_writer The JSON writer. + * @param the_info The object to write. + */ + @Override + public void write(final JsonWriter the_writer, + final ContestToAudit the_contest) + throws IOException { + the_writer.beginObject(); + the_writer.name(CONTEST).value(the_contest.contest().id()); + the_writer.name(AUDIT).value(the_contest.audit().toString()); + if (the_contest.reason() != null) { + the_writer.name(REASON).value(the_contest.reason().toString()); + } + the_writer.endObject(); + } + + /** + * Checks the sanity of a contest to audit. + * + * @param the_id The contest ID. + * @param the_reason The reason for audit. + * @param the_audit_type The audit type. + * @return the corresponding contest, if the data is sane, or null if + * the data is not. + */ + private Contest sanityCheck(final Long the_id, + final AuditReason the_reason, + final AuditType the_type) { + Contest result = null; + final Contest contest = Persistence.getByID(the_id, Contest.class); + + if (contest != null && the_type != null && + (the_reason != null || the_type != AuditType.COMPARISON)) { + result = contest; + } + + return result; + } + + /** + * Reads a contest to audit object. + * + * @param the_reader The JSON reader. + * @return the object. + */ + @Override + public ContestToAudit read(final JsonReader the_reader) + throws IOException { + boolean error = false; + Long contest_id = null; + AuditReason reason = null; + AuditType type = null; + + the_reader.beginObject(); + while (the_reader.hasNext()) { + final String name = the_reader.nextName(); + switch (name) { + case CONTEST: + contest_id = the_reader.nextLong(); + break; + + case REASON: + try { + reason = AuditReason.valueOf(the_reader.nextString()); + } catch (final IllegalArgumentException e) { + throw new JsonSyntaxException(e); + } + break; + + case AUDIT: + try { + type = AuditType.valueOf(the_reader.nextString()); + } catch (final IllegalArgumentException e) { + throw new JsonSyntaxException(e); + } + break; + + default: + error = true; + break; + } + } + the_reader.endObject(); + + // check that the contest exists + + final Contest contest = sanityCheck(contest_id, reason, type); + + if (error || contest == null) { + throw new JsonSyntaxException("invalid data detected in contest to audit"); + } + + return new ContestToAudit(contest, reason, type); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/CountyDashboardRefreshResponse.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/CountyDashboardRefreshResponse.java new file mode 100644 index 00000000..d10f50cf --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/CountyDashboardRefreshResponse.java @@ -0,0 +1,456 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 12, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.json; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; + +import javax.persistence.PersistenceException; + +import us.freeandfair.corla.asm.ASMState; +import us.freeandfair.corla.asm.ASMUtilities; +import us.freeandfair.corla.asm.AuditBoardDashboardASM; +import us.freeandfair.corla.asm.CountyDashboardASM; +import us.freeandfair.corla.json.UploadedFileDTO; +import us.freeandfair.corla.model.AuditBoard; +import us.freeandfair.corla.model.AuditInfo; +import us.freeandfair.corla.model.AuditSelection; +import us.freeandfair.corla.model.AuditType; +import us.freeandfair.corla.model.Contest; +import us.freeandfair.corla.model.ContestToAudit; +import us.freeandfair.corla.model.County; +import us.freeandfair.corla.model.CountyDashboard; +import us.freeandfair.corla.model.DoSDashboard; +import us.freeandfair.corla.model.ImportStatus; +import us.freeandfair.corla.model.Round; +import us.freeandfair.corla.model.UploadedFile; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.query.ContestQueries; +import us.freeandfair.corla.util.SuppressFBWarnings; + +/** + * The response generated on a refresh of the County and Audit Board + * dashboards. + * + * @author Daniel M. Zimmerman + * @author Joseph R. Kiniry + * @version 1.0.0 + */ +@SuppressWarnings({"unused", "PMD.UnusedPrivateField", "PMD.SingularField", + "PMD.CyclomaticComplexity", "PMD.TooManyFields"}) +@SuppressFBWarnings(value = {"URF_UNREAD_FIELD"}, justification = "Field is read by Gson.") +public class CountyDashboardRefreshResponse { + /** + * The county ID. + */ + private final Long my_id; + + /** + * The ASM state. + */ + private final ASMState my_asm_state; + + /** + * The audit board ASM state. + */ + private final ASMState my_audit_board_asm_state; + + /** + * The general information. + * @todo this needs to be connected to something + */ + private final SortedMap my_general_information; + + /** + * The desired number of audit boards. + */ + private final Integer my_audit_board_count; + + /** + * The audit boards. + */ + private final Map my_audit_boards; + + /** + * The ballot manifest file. + */ + private final UploadedFileDTO my_ballot_manifest_file; + + /** + * The CVR export file. + */ + private final UploadedFileDTO my_cvr_export_file; + + /** + * The contests on the ballot (by ID). + */ + private final List my_contests; + + /** + * The contests under audit, with reasons. + */ + private final SortedMap my_contests_under_audit; + + /** + * The date and time of the audit. + */ + private final Instant my_audit_time; + + /** + * The estimated number of ballots to audit. + */ + private final Integer my_estimated_ballots_to_audit; + + /** + * The optimistic number of ballots to audit. + */ + private final Integer my_optimistic_ballots_to_audit; + + /** + * The ballots remaining in the round. + */ + private final Integer my_ballots_remaining_in_round; + + /** + * The number of ballots represented by the uploaded ballot manifest. + */ + private final Integer my_ballot_manifest_count; + + /** + * The number of cvrs in the uploaded CVR export. + */ + private final Integer my_cvr_export_count; + + /** + * The CVR import status. + */ + private final ImportStatus my_cvr_import_status; + + /** + * The number of ballots audited. + */ + private final Integer my_audited_ballot_count; + + /** + * The numbers of discrepancies found, mapped by audit selection. + */ + private final Map my_discrepancy_count; + + /** + * The number of disagreements found, mapped by audit selection. + */ + private final Map my_disagreement_count; + + /** + * The current assignment of ballots to audit boards + */ + private final List> my_ballot_sequence_assignment; + + /** + * The current ballots under audit, by audit board index. + */ + private final List my_ballot_under_audit_ids; + + /** + * The audited prefix length. + */ + private final Integer my_audited_prefix_length; + + /** + * The audit rounds. + */ + private final List my_rounds; + + /** + * The current audit round. + */ + private final Round my_current_round; + + /** + * The audit info. + */ + private final AuditInfo my_audit_info; + + /** + * Constructs a new CountyDashboardRefreshResponse. + * + * @param the_id The ID. + * @param the_asm_state The ASM state. + * @param the_audit_board_asm_state The audit board ASM state. + * @param the_general_information The general information. + * @param the_risk_limit The risk limit. + * @param the_audit_board_count The desired number of audit boards. + * @param the_audit_boards The current audit boards. + * @param the_ballot_manifest_file The ballot manifest file. + * @param the_cvr_export_file The CVR export file. + * @param the_contests The contests. + * @param the_contests_under_audit The contests under audit, with reasons. + * @param the_audit_time The audit time. + * @param the_estimated_ballots_to_audit The estimated ballots to audit. + * @param the_optimistic_ballots_to_audit The optimistic ballots to audit. + * @param the_ballots_remaining_in_round The ballots remaining in the + * current round. + * @param the_ballot_manifest_count The number of ballots represented by the + * uploaded ballot manifest. + * @param the_cvr_export_count The number of CVRs in the uploaded export file. + * @param the_cvr_import_status An indication of the status of an ongoing + * CVR import. + * @param the_audited_ballot_count The number of ballots audited. + * @param the_discrepancy_count The number of discrepencies found, + * mapped by audit reason. + * @param the_disagreement_count The number of disagreements, + * mapped by audit reason. + * @param the_ballot_under_audit_ids the IDs of the CVRs under audit, in + * positions corresponding to the audit + * board that should audit it. + * @param the_audited_prefix_length The length of the audited prefix of the + * ballots to audit list. + * @param the_rounds The list of audit rounds. + * @param the_current_round The current audit round. + * @param the_election_type The election type. + * @param the_election_date The election date. + */ + @SuppressWarnings({"PMD.ExcessiveParameterList", "checkstyle:executablestatementcount", + "checkstyle:methodlength"}) + protected CountyDashboardRefreshResponse(final Long the_id, + final ASMState the_asm_state, + final ASMState the_audit_board_asm_state, + final SortedMap + the_general_information, + final Integer the_audit_board_count, + final Map the_audit_boards, + final UploadedFile the_ballot_manifest_file, + final UploadedFile the_cvr_export_file, + final List the_contests, + final SortedMap + the_contests_under_audit, + final Instant the_audit_time, + final Integer the_estimated_ballots_to_audit, + final Integer the_optimistic_ballots_to_audit, + final Integer the_ballots_remaining_in_round, + final Integer the_ballot_manifest_count, + final Integer the_cvr_export_count, + final ImportStatus the_cvr_import_status, + final Integer the_audited_ballot_count, + final Map + the_discrepancy_count, + final Map + the_disagreement_count, + final List the_ballot_under_audit_ids, + final List> the_ballot_sequence_assignment, + final Integer the_audited_prefix_length, + final List the_rounds, + final Round the_current_round, + final AuditInfo the_audit_info) { + my_id = the_id; + my_asm_state = the_asm_state; + my_audit_board_asm_state = the_audit_board_asm_state; + my_general_information = the_general_information; + my_audit_board_count = the_audit_board_count; + my_audit_boards = the_audit_boards; + if (null == the_ballot_manifest_file) { + my_ballot_manifest_file = null; + } else { + my_ballot_manifest_file = new UploadedFileDTO(the_ballot_manifest_file); + } + if (null == the_cvr_export_file) { + my_cvr_export_file = null; + } else { + my_cvr_export_file = new UploadedFileDTO(the_cvr_export_file); + } + my_contests = the_contests; + my_contests_under_audit = the_contests_under_audit; + my_audit_time = the_audit_time; + my_estimated_ballots_to_audit = the_estimated_ballots_to_audit; + my_optimistic_ballots_to_audit = the_optimistic_ballots_to_audit; + my_ballots_remaining_in_round = the_ballots_remaining_in_round; + my_ballot_manifest_count = the_ballot_manifest_count; + my_cvr_export_count = the_cvr_export_count; + my_cvr_import_status = the_cvr_import_status; + my_audited_ballot_count = the_audited_ballot_count; + my_discrepancy_count = the_discrepancy_count; + my_disagreement_count = the_disagreement_count; + my_ballot_under_audit_ids = the_ballot_under_audit_ids; + my_ballot_sequence_assignment = the_ballot_sequence_assignment; + my_audited_prefix_length = the_audited_prefix_length; + my_rounds = the_rounds; + my_current_round = the_current_round; + my_audit_info = the_audit_info; + } + + /** + * Gets the CountyDashboardRefreshResponse for the specified County dashboard. + * + * @param the_dashboard The dashboard. + * @return the response. + * @exception NullPointerException if necessary information to construct the + * response does not exist. + */ + // this method is essentially a straight line construction of parameters, + // so we are ignoring the cyclomatic complexity checks for now + @SuppressWarnings({"PMD.NPathComplexity", "PMD.CyclomaticComplexity"}) + public static CountyDashboardRefreshResponse + createResponse(final CountyDashboard the_dashboard) { + final DoSDashboard dosd = Persistence.getByID(DoSDashboard.ID, DoSDashboard.class); + + if (dosd == null) { + throw new PersistenceException("unable to read county dashboard state"); + } + + final County county = the_dashboard.county(); + final Long county_id = county.id(); + + // general information doesn't exist yet + final SortedMap general_information = new TreeMap(); + + // contests and contests under audit + final List contests = new ArrayList(); + final SortedMap contests_under_audit = new TreeMap(); + if (the_dashboard.cvrFile() != null && + the_dashboard.manifestFile() != null) { + // only add contests if uploads are done + // TODO will this stuff be changing also? + for (final Contest c : ContestQueries.forCounty(county)) { + contests.add(c.id()); + } + + for (final ContestToAudit cta : dosd.contestsToAudit()) { + if (cta.audit() == AuditType.COMPARISON && + contests.contains(cta.contest().id())) { + contests_under_audit.put(cta.contest().id(), cta.reason().toString()); + } + } + } + + Collections.sort(contests); + + // ASM states + final CountyDashboardASM asm = ASMUtilities.asmFor(CountyDashboardASM.class, + county_id.toString()); + final AuditBoardDashboardASM audit_board_asm = + ASMUtilities.asmFor(AuditBoardDashboardASM.class, county_id.toString()); + + // sanitized rounds + + final List rounds = new ArrayList<>(); + List> ballotSequenceAssignment = null; + for (final Round round : the_dashboard.rounds()) { + rounds.add(round.withoutSequences()); + } + Round current_round = the_dashboard.currentRound(); + if (current_round != null) { + ballotSequenceAssignment = current_round.ballotSequenceAssignment(); + current_round = current_round.withoutSequences(); + } + + return new CountyDashboardRefreshResponse(county_id, + asm.currentState(), + audit_board_asm.currentState(), + general_information, + the_dashboard.auditBoardCount(), + the_dashboard.auditBoards(), + the_dashboard.manifestFile(), + the_dashboard.cvrFile(), + contests, + contests_under_audit, + the_dashboard.auditTimestamp(), + // FIXME the estimates and optimistc numbers are wrong. + the_dashboard.estimatedSamplesToAudit(), + the_dashboard.optimisticSamplesToAudit(), + + the_dashboard.ballotsRemainingInCurrentRound(), + the_dashboard.ballotsInManifest(), + the_dashboard.cvrsImported(), + the_dashboard.cvrImportStatus(), + the_dashboard.ballotsAudited(), + the_dashboard.discrepancies(), + the_dashboard.disagreements(), + the_dashboard.cvrsUnderAudit(), + ballotSequenceAssignment, + the_dashboard.auditedPrefixLength(), + rounds, + current_round, + dosd.auditInfo()); + } + + /** + * Gets the abbreviated CountyDashboardRefreshResponse for the specified County + * dashboard. The abbreviated response leaves out information about contests, + * general information, audit board information, and specific ballots to audit. + * + * @param the_dashboard The dashboard. + * @return the response. + * @exception NullPointerException if necessary information to construct the + * response does not exist. + */ + // this method is essentially a straight line construction of parameters, + // so we are ignoring the cyclomatic complexity checks for now + @SuppressWarnings({"PMD.NPathComplexity", "PMD.CyclomaticComplexity"}) + public static CountyDashboardRefreshResponse + createAbbreviatedResponse(final CountyDashboard the_dashboard) { + final Long county_id = the_dashboard.id(); + final County county = Persistence.getByID(county_id, County.class); + + if (county == null) { + throw new PersistenceException("unable to read county dashboard state"); + } + + // ASM states + final CountyDashboardASM asm = ASMUtilities.asmFor(CountyDashboardASM.class, + county_id.toString()); + final AuditBoardDashboardASM audit_board_asm = + ASMUtilities.asmFor(AuditBoardDashboardASM.class, county_id.toString()); + + // sanitized rounds + + final List rounds = new ArrayList<>(); + for (final Round round : the_dashboard.rounds()) { + rounds.add(round.withoutSequences()); + } + Round current_round = the_dashboard.currentRound(); + if (current_round != null) { + current_round = current_round.withoutSequences(); + } + + return new CountyDashboardRefreshResponse(county_id, + asm.currentState(), + audit_board_asm.currentState(), + null, + the_dashboard.auditBoardCount(), + the_dashboard.auditBoards(), + the_dashboard.manifestFile(), + the_dashboard.cvrFile(), + null, + null, + the_dashboard.auditTimestamp(), + the_dashboard.estimatedSamplesToAudit(), + the_dashboard.optimisticSamplesToAudit(), + the_dashboard.ballotsRemainingInCurrentRound(), + the_dashboard.ballotsInManifest(), + the_dashboard.cvrsImported(), + the_dashboard.cvrImportStatus(), + the_dashboard.ballotsAudited(), + the_dashboard.discrepancies(), + the_dashboard.disagreements(), + null, + null, + the_dashboard.auditedPrefixLength(), + rounds, + current_round, + null); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/DoSDashboardRefreshResponse.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/DoSDashboardRefreshResponse.java new file mode 100644 index 00000000..bcfaf7ad --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/DoSDashboardRefreshResponse.java @@ -0,0 +1,254 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * + * @created Aug 12, 2017 + * + * @copyright 2017 Colorado Department of State + * + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * + * @creator Daniel M. Zimmerman + * + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.json; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; + +import javax.persistence.PersistenceException; + +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import us.freeandfair.corla.asm.ASMState; +import us.freeandfair.corla.asm.ASMUtilities; +import us.freeandfair.corla.asm.DoSDashboardASM; +import us.freeandfair.corla.model.AuditInfo; +import us.freeandfair.corla.model.AuditReason; +import us.freeandfair.corla.model.AuditType; +import us.freeandfair.corla.model.ContestToAudit; +import us.freeandfair.corla.model.County; +import us.freeandfair.corla.model.ComparisonAudit; +import us.freeandfair.corla.model.CountyDashboard; +import us.freeandfair.corla.model.DoSDashboard; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.query.ComparisonAuditQueries; +import us.freeandfair.corla.util.SuppressFBWarnings; + +/** + * The response generated on a refresh of the DoS dashboard. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings({"unused", "PMD.UnusedPrivateField", "PMD.SingularField"}) +@SuppressFBWarnings({"URF_UNREAD_FIELD", + // Justification: Field is read by Gson. + "SF_SWITCH_NO_DEFAULT"}) +// Justification: False positive; there is a default case. + +public class DoSDashboardRefreshResponse { + /** + * Class-wide logger + */ + public static final Logger LOGGER = LogManager.getLogger(DoSDashboardRefreshResponse.class); + /** + * The ASM state. + */ + private final ASMState my_asm_state; + + /** + * A map from audited contests to audit reasons. + */ + private final SortedMap my_audited_contests; + + /** + * A map from audited contests to estimated ballots left to audit. + */ + private final SortedMap my_estimated_ballots_to_audit; + + /** + * A map from audited contests to optimistic ballots left to audit. + */ + private final SortedMap my_optimistic_ballots_to_audit; + + /** + * A map from audited contests to discrepancy count maps. + */ + private final SortedMap> my_discrepancy_count; + + /** + * A map from county IDs to county status. + */ + private final SortedMap my_county_status; + + /** + * A set of contests selected for full hand count. + */ + private final List my_hand_count_contests; + + /** + * The audit info. + */ + private final AuditInfo my_audit_info; + + /** + * The audit reasons for the contests under audit. + */ + private final SortedMap my_audit_reasons; + + /** + * The audit types for the contests under audit. + */ + private final SortedMap my_audit_types; + + /** + * Constructs a new DosDashboardRefreshResponse. + * + * @param the_asm_state The ASM state. + * @param the_audited_contests The audited contests. + * @param the_estimated_ballots_to_audit The estimated ballots to audit, by + * contest. + * @param the_optimistic_ballots_to_audit The optimistic ballots to audit, by + * contest. + * @param the_discrepancy_count The discrepancy count for each discrepancy + * type, by contest. + * @param the_county_status The county statuses. + * @param the_hand_count_contests The hand count contests. + * @param the_audit_info The election info. + */ + @SuppressWarnings("PMD.ExcessiveParameterList") + protected DoSDashboardRefreshResponse(final ASMState the_asm_state, + final SortedMap the_audited_contests, + final SortedMap the_estimated_ballots_to_audit, + final SortedMap the_optimistic_ballots_to_audit, + final SortedMap> the_discrepancy_counts, + final SortedMap the_county_status, + final List the_hand_count_contests, + final AuditInfo the_audit_info, + final SortedMap the_audit_reasons, + final SortedMap the_audit_types) { + my_asm_state = the_asm_state; + my_audited_contests = the_audited_contests; + my_estimated_ballots_to_audit = the_estimated_ballots_to_audit; + my_optimistic_ballots_to_audit = the_optimistic_ballots_to_audit; + my_discrepancy_count = the_discrepancy_counts; + my_county_status = the_county_status; + my_hand_count_contests = the_hand_count_contests; + my_audit_info = the_audit_info; + my_audit_reasons = the_audit_reasons; + my_audit_types = the_audit_types; + } + + /** + * Gets the DoSDashboardRefreshResponse for the specified DoS dashboard. + * + * @param dashboard The dashboard. + * @return the response. + * @exception NullPointerException if necessary information to construct the + * response does not exist. + */ + @SuppressWarnings("checkstyle:magicnumber") + public static DoSDashboardRefreshResponse createResponse(final DoSDashboard dashboard) { + // construct the various audit info from the contests to audit in the + // dashboard + final SortedMap audited_contests = new TreeMap(); + final SortedMap estimated_ballots_to_audit = new TreeMap(); + final SortedMap optimistic_ballots_to_audit = new TreeMap(); + final SortedMap> discrepancy_count = new TreeMap<>(); + final List hand_count_contests = new ArrayList(); + final SortedMap audit_reasons = new TreeMap(); + final SortedMap audit_types = new TreeMap(); + + for (final ContestToAudit cta : dashboard.contestsToAudit()) { + if (cta.audit() != AuditType.NONE) { + audit_reasons.put(cta.contest().id(), cta.reason()); + audit_types.put(cta.contest().id(), cta.audit()); + } + switch (cta.audit()) { + case COMPARISON: + final Map discrepancy = new HashMap<>(); + int optimistic = 0; + int estimated = 0; + audited_contests.put(cta.contest().id(), cta.reason()); + + final ComparisonAudit ca = ComparisonAuditQueries.matching(cta.contest().name()); + if (null != ca) { + optimistic = ca.optimisticRemaining(); + estimated = ca.estimatedRemaining(); + + LOGGER.debug(String + .format("[createResponse: optimistic=%d, estimated = %d, ca.optimisticSamplesToAudit()=%d, ca.estimatedSamplesToAudit()=%d, ca.getAuditedSampleCount()=%d]", + optimistic, estimated, ca.optimisticSamplesToAudit(), + ca.estimatedSamplesToAudit(), ca.getAuditedSampleCount())); + + // possible discrepancy types range from -2 to 2 inclusive, + // and we provide them all in the refresh response + for (int i = -2; i <= 2; i++) { + if (discrepancy.get(i) == null) { + discrepancy.put(i, 0); + } + discrepancy.put(i, discrepancy.get(i) + ca.discrepancyCount(i)); + } + } + + estimated_ballots_to_audit.put(cta.contest().id(), optimistic); + optimistic_ballots_to_audit.put(cta.contest().id(), estimated); + discrepancy_count.put(cta.contest().id(), discrepancy); + break; + + case HAND_COUNT: + // we list these separately for some reason + + // FIXME probably should be a set of ContestResult IDs. + hand_count_contests.add(cta.contest().id()); + break; + + default: + } + } + + Collections.sort(hand_count_contests); + + // status + final DoSDashboardASM asm = + ASMUtilities.asmFor(DoSDashboardASM.class, DoSDashboardASM.IDENTITY); + + return new DoSDashboardRefreshResponse(asm.currentState(), audited_contests, + estimated_ballots_to_audit, + optimistic_ballots_to_audit, discrepancy_count, + countyStatusMap(), hand_count_contests, + dashboard.auditInfo(), audit_reasons, audit_types); + } + + /** + * Gets the county statuses for all counties in the database. + * + * @return a map from county identifiers to statuses. + */ + private static SortedMap countyStatusMap() { + final SortedMap status_map = + new TreeMap(); + final List counties = Persistence.getAll(County.class); + + for (final County c : counties) { + final CountyDashboard db = Persistence.getByID(c.id(), CountyDashboard.class); + if (db == null) { + throw new PersistenceException("unable to read county dashboard state."); + } else { + status_map.put(db.id(), CountyDashboardRefreshResponse.createAbbreviatedResponse(db)); + } + } + + return status_map; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/FreeAndFairNamingStrategy.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/FreeAndFairNamingStrategy.java new file mode 100644 index 00000000..823d800b --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/FreeAndFairNamingStrategy.java @@ -0,0 +1,41 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 24, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.json; + +import java.lang.reflect.Field; + +import com.google.gson.FieldNamingStrategy; + +/** + * A naming strategy for Gson that takes our standard instance field names + * (prepended with "my_", separated by underscores) and translates them to + * JSON field names without the "my_". + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +// we suppress this PMD warning because this class, despite being +// stateless (and thus ideally being a utility class), is required by the +// Gson interface to be instantiable +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class FreeAndFairNamingStrategy implements FieldNamingStrategy { + /** + * Translates a Java field name to a JSON field name, by removing the + * "my_" prefix and leaving the remainder of the field name intact. + * + * @param the_field The field to translate the name of. + */ + @Override + public String translateName(final Field the_field) { + return the_field.getName().replaceFirst("^my_", ""); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/InstantTypeAdapter.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/InstantTypeAdapter.java new file mode 100644 index 00000000..ac23afd3 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/InstantTypeAdapter.java @@ -0,0 +1,70 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 23, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.json; + +import java.lang.reflect.Type; +import java.time.Instant; +import java.time.format.DateTimeParseException; + +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonParseException; +import com.google.gson.JsonPrimitive; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; + +/** + * A Gson type converter for java.time.Instant. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class InstantTypeAdapter + implements JsonSerializer, JsonDeserializer { + /** + * Renders an Instant in ISO-8601 format. + * + * @param the_instant The Instant. + * @param the_type The type (ignored). + * @param the_context The JSON serialization context (ignored). + */ + @Override + public JsonElement serialize(final Instant the_instant, + final Type the_type, + final JsonSerializationContext the_context) { + return new JsonPrimitive(the_instant.toString()); + } + + /** + * Reconstitutes an Instant from ISO-8601 format. + * + * @param the_json_element The JSON element to reconstitute. + * @param the_type The type (ignored). + * @param the_context The JSON deserialization context (ignored). + * @return the reconstituted Instant. + */ + @Override + public Instant deserialize(final JsonElement the_json_element, + final Type the_type, + final JsonDeserializationContext the_context) + throws JsonParseException { + final Instant result; + try { + result = Instant.parse(the_json_element.getAsString()); + } catch (final DateTimeParseException e) { + throw new JsonParseException(e); + } + return result; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/IntermediateAuditReportJsonAdapter.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/IntermediateAuditReportJsonAdapter.java new file mode 100644 index 00000000..f7d291fb --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/IntermediateAuditReportJsonAdapter.java @@ -0,0 +1,99 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 28, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.json; + +import java.io.IOException; +import java.time.Instant; + +import com.google.gson.JsonSyntaxException; +import com.google.gson.TypeAdapter; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.model.IntermediateAuditReportInfo; + +/** + * JSON adapter for audit investigation reports. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +// the default constructor suffices for type adapters +@SuppressWarnings("PMD.AtLeastOneConstructor") +public final class IntermediateAuditReportJsonAdapter + extends TypeAdapter { + /** + * The "contest" string (for JSON serialization). + */ + private static final String TIMESTAMP = "timestamp"; + + /** + * The "audit type" string (for JSON serialization). + */ + private static final String REPORT = "report"; + + /** + * Writes an audit interim report object. + * + * @param the_writer The JSON writer. + * @param the_info The object to write. + */ + @Override + public void write(final JsonWriter the_writer, + final IntermediateAuditReportInfo the_report) + throws IOException { + the_writer.beginObject(); + the_writer.name(TIMESTAMP).value(Main.GSON.toJson(the_report.timestamp())); + the_writer.name(REPORT).value(the_report.report()); + the_writer.endObject(); + } + + /** + * Reads an audit investigation report object. + * + * @param the_reader The JSON reader. + * @return the object. + */ + @Override + public IntermediateAuditReportInfo read(final JsonReader the_reader) + throws IOException { + boolean error = false; + String report = null; + Instant timestamp = null; + + the_reader.beginObject(); + while (the_reader.hasNext()) { + final String name = the_reader.nextName(); + switch (name) { + case TIMESTAMP: + timestamp = Main.GSON.fromJson(the_reader.nextString(), Instant.class); + break; + + case REPORT: + report = the_reader.nextString(); + break; + + default: + error = true; + break; + } + } + the_reader.endObject(); + + if (error) { + throw new JsonSyntaxException("invalid data detected in audit investigation report"); + } + + return new IntermediateAuditReportInfo(timestamp, report); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/Result.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/Result.java new file mode 100644 index 00000000..c7dd5abb --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/Result.java @@ -0,0 +1,41 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 13, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.json; + +/** + * A result returned by the server. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public class Result { + /** + * The result string. + */ + private final String my_result; + + /** + * Constructs a new Result. + * + * @param the_result The result. + */ + public Result(final String the_result) { + my_result = the_result; + } + + /** + * @return the result. + */ + public String result() { + return my_result; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/ServerASMResponse.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/ServerASMResponse.java new file mode 100644 index 00000000..a34d7132 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/ServerASMResponse.java @@ -0,0 +1,50 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 10, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.json; + +import java.util.Set; + +import us.freeandfair.corla.asm.ASMState; +import us.freeandfair.corla.asm.UIEvent; +import us.freeandfair.corla.util.SuppressFBWarnings; + +/** + * The standard response provided by the server to indicate the state of the + * server's ASM and what UI events are next permitted. + * @trace endpoints.server_response + * @author Joseph R. Kiniry + * @version 1.0.0 + */ +@SuppressWarnings({"unused", "PMD.UnusedPrivateField", "PMD.SingularField"}) +@SuppressFBWarnings(value = {"URF_UNREAD_FIELD"}, justification = "Field is read by Gson.") +public class ServerASMResponse { + /** + * The server's current state. + */ + private final ASMState my_current_state; + + /** + * The permitted next UI events. + */ + private final Set my_enabled_ui_events; + + /** + * Create a new response object. + * @param the_current_state is the current state of the ASM. + * @param the_enabled_ui_events are the UI events enabled from the current state. + */ + public ServerASMResponse(final ASMState the_current_state, + final Set the_enabled_ui_events) { + my_current_state = the_current_state; + my_enabled_ui_events = the_enabled_ui_events; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/SubmittedAuditCVR.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/SubmittedAuditCVR.java new file mode 100644 index 00000000..3edc35a9 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/SubmittedAuditCVR.java @@ -0,0 +1,102 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 13, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.json; + +import us.freeandfair.corla.model.CastVoteRecord; + +/** + * A submitted audit CVR. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public class SubmittedAuditCVR { + /** + * The original CVR ID for this audit CVR. + */ + private final Long my_cvr_id; + + /** + * The audit CVR. + */ + private final CastVoteRecord my_audit_cvr; + + + /** flag to indicate whether this is a review-and-reaudit submission + * - needs to not be final so it can be optional and have a default + **/ + private final Boolean reaudit; + + /** a comment is required to explain why a reaudit is happening **/ + private final String comment; + + /** the audit board is used for reporting **/ + private final Integer auditBoardIndex; + + /** + * Constructs a new SubmittedAuditCVR. + * + * @param the_cvr_id The original CVR ID. + * @param the_audit_cvr The audit CVR. + */ + public SubmittedAuditCVR(final Long the_cvr_id, final CastVoteRecord the_audit_cvr) { + my_cvr_id = the_cvr_id; + my_audit_cvr = the_audit_cvr; + this.reaudit = false; + this.comment = ""; + this.auditBoardIndex = -1; + } + + /** create the object with all the fields **/ + public SubmittedAuditCVR(final Long the_cvr_id, + final CastVoteRecord the_audit_cvr, + final Boolean reaudit, + final String comment, + final Integer auditBoardIndex) { + my_cvr_id = the_cvr_id; + my_audit_cvr = the_audit_cvr; + this.reaudit = reaudit; + this.comment = comment; + this.auditBoardIndex = auditBoardIndex; + } + + + /** + * @return the original CVR ID. + */ + public Long cvrID() { + return my_cvr_id; + } + + /** + * @return the audit CVR. + */ + public CastVoteRecord auditCVR() { + return my_audit_cvr; + } + + /** reaudit can be null because it is optional **/ + public Boolean isReaudit() { + return this.reaudit != null && this.reaudit; + } + + /** get the comment **/ + public String getComment() { + return this.comment; + } + + /** get which audit board is submitting this **/ + public Integer getAuditBoardIndex() { + return this.auditBoardIndex; + } + +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/SubmittedAuditRoundStart.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/SubmittedAuditRoundStart.java new file mode 100644 index 00000000..027b9c27 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/SubmittedAuditRoundStart.java @@ -0,0 +1,87 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 13, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.json; + +import java.math.BigDecimal; +import java.util.Collections; +import java.util.Map; + +/** + * Data submitted to start an audit round. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public class SubmittedAuditRoundStart { + /** + * The multiplier for the number of ballots per county. + */ + private final BigDecimal my_multiplier; + + /** + * A flag indicating whether to use the minimum estimates as the base + * for each county. + */ + private final Boolean my_use_estimates; + + /** + * A mapping from county IDs to numbers of ballots per county. + */ + private final Map my_county_ballots; + + /** + * Constructs a new SubmittedAuditRoundStart. + * + * @param the_multiplier The multiplier. Ignored if using absolute numbers + * of ballots. + * @param the_use_estimates True to use algorithm estimates, false to use + * absolute numbers of ballots. + * @param the_ballots_per_county A map from county IDs to absolute numbers + * of ballots per county. + */ + public SubmittedAuditRoundStart(final BigDecimal the_multiplier, + final Boolean the_use_estimates, + final Map the_county_ballots) { + my_multiplier = the_multiplier; + my_use_estimates = the_use_estimates; + my_county_ballots = the_county_ballots; + } + + /** + * @return the multiplier. + */ + public BigDecimal multiplier() { + return my_multiplier; + } + + /** + * @return true if we are using minimum estimates, false otherwise. + */ + public boolean useEstimates() { + if (my_use_estimates == null) { + return false; + } else { + return my_use_estimates; + } + } + + /** + * @return the map from county IDs to absolute numbers of ballots per county. + */ + public Map countyBallots() { + if (my_county_ballots == null) { + return my_county_ballots; + } else { + return Collections.unmodifiableMap(my_county_ballots); + } + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/SubmittedBallotNotFound.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/SubmittedBallotNotFound.java new file mode 100644 index 00000000..edc923f7 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/SubmittedBallotNotFound.java @@ -0,0 +1,85 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 13, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.json; + +/** + * A submitted ballot not found ID. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public class SubmittedBallotNotFound { + /** + * The id. + */ + private final Long my_id; + + /** flag to indicate whether this is a review-and-reaudit submission + * - needs to not be final so it can be optional and have a default + **/ + private final Boolean reaudit; + + /** a comment is required to explain why a reaudit is happening **/ + private final String comment; + + /** the audit board is used for reporting **/ + private final Integer auditBoardIndex; + + /** + * Constructs a new SubmittedBallotNotFound. + * + * @param the_id The id. + */ + public SubmittedBallotNotFound(final Long the_id) { + my_id = the_id; + this.reaudit = false; + this.comment = ""; + this.auditBoardIndex = -1; + } + + /** + * Constructs a new SubmittedBallotNotFound. + * + * @param the_id The id. + */ + public SubmittedBallotNotFound(final Long the_id, + final Boolean reaudit, + final String comment, + final Integer auditBoardIndex) { + my_id = the_id; + this.comment = comment; + this.reaudit = reaudit; + this.auditBoardIndex = auditBoardIndex; + } + + /** + * @return the id. + */ + public Long id() { + return my_id; + } + + /** reaudit can be null because it is optional **/ + public Boolean isReaudit() { + return this.reaudit != null && this.reaudit; + } + + /** get the comment **/ + public String getComment() { + return this.comment; + } + + /** get which audit board is submitting this **/ + public Integer getAuditBoardIndex() { + return this.auditBoardIndex; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/SubmittedCredentials.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/SubmittedCredentials.java new file mode 100644 index 00000000..cb1230ae --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/SubmittedCredentials.java @@ -0,0 +1,78 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 13, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.json; + +/** + * A submitted set of credentials, usually either a (username, password) + * pair or a (username, second factor) pair. + * + * @author Daniel M. Zimmerman + * @author Joseph R. Kiniry + * @version 1.0.0 + */ +public class SubmittedCredentials { + /** + * The username. + */ + private final String my_username; + + /** + * The password. + */ + private final String my_password; + + /** + * The two-factor authentication information. + */ + private final String my_second_factor; + + //@ private invariant my_password != null || my_second_factor != null; + + /** + * Constructs a new instance of this class. Note that most two-factor + * authentication systems preclude the ability to authenticate both factors + * simultaneously, since that often opens up a replay attack. Thus, in most + * use cases, either `the_password` or `the_second_factor` is non-null. + * + * @param the_username The username. + * @param the_password The password. + * @param the_second_factor The second factor. + */ + public SubmittedCredentials(final String the_username, + final String the_password, + final String the_second_factor) { + my_username = the_username; + my_password = the_password; + my_second_factor = the_second_factor; + } + + /** + * @return the username. + */ + public String username() { + return my_username; + } + + /** + * @return the password. + */ + public String password() { + return my_password; + } + + /** + * @return the second factor. + */ + public String secondFactor() { + return my_second_factor; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/TwoFactorResponse.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/TwoFactorResponse.java new file mode 100644 index 00000000..83bda2d5 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/TwoFactorResponse.java @@ -0,0 +1,43 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 30, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.json; + +import us.freeandfair.corla.util.SuppressFBWarnings; + +/** + * The response provided by the server during the first phase of two factor + * authentication to indicate the challenge for the second factor to the user. + * This particular response class is general purpose, encoding a generic + * string challenge used by most two-factor authentication systems. The client + * UI will simply print the challenge with little adornment. + * + * @trace authentication.two_factor_challenge + * @author Joseph R. Kiniry + * @version 1.0.0 + */ +@SuppressWarnings({"unused", "PMD.UnusedPrivateField", "PMD.SingularField"}) +@SuppressFBWarnings(value = {"URF_UNREAD_FIELD"}, + justification = "Field is read by Gson.") +public class TwoFactorResponse { + /** + * The challenge issues by the two-factor authentication system. + */ + private final String my_challenge; + + /** + * Create a new response object. + * @param the_challenge the two-factor challenge. + */ + public TwoFactorResponse(final String the_challenge) { + my_challenge = the_challenge; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/UploadedFileDTO.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/UploadedFileDTO.java new file mode 100644 index 00000000..d9bf54ee --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/UploadedFileDTO.java @@ -0,0 +1,58 @@ +package us.freeandfair.corla.json; + +import us.freeandfair.corla.model.UploadedFile; +import us.freeandfair.corla.csv.Result; +import us.freeandfair.corla.util.SuppressFBWarnings; + +@SuppressFBWarnings(value = {"URF_UNREAD_FIELD"}, justification = "Field is read by Gson.") +public class UploadedFileDTO { + private Integer approximateRecordCount; + private Long countyId; + private String fileName; + private String hash; + private Long id; + private Result result; + private String status; + private Long size; + private String timestamp; + + public UploadedFileDTO(final UploadedFile uploadedFile) { + this.approximateRecordCount = uploadedFile.approximateRecordCount(); + this.countyId = uploadedFile.county().id(); + this.fileName = uploadedFile.filename(); + this.hash = uploadedFile.getHash(); + this.id = uploadedFile.id(); + this.result = uploadedFile.getResult(); + this.size = uploadedFile.size(); + this.status = uploadedFile.getStatus().toString(); + this.timestamp = uploadedFile.timestamp().toString(); + } + + public Long getFileId() { + return this.id; + } + + public String getStatus() { + return this.status; + } + + public void setStatus(final String status ) { + this.status = status; + } + + public void setResult(final Result result) { + this.result = result; + } + + public Result getResult() { + return this.result; + } + + public Long getCountyId() { + return this.countyId; + } + + public void setCountyId(final Long countyId) { + this.countyId = countyId; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/VersionExclusionStrategy.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/VersionExclusionStrategy.java new file mode 100644 index 00000000..28a35219 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/json/VersionExclusionStrategy.java @@ -0,0 +1,45 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 27, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.json; + +import javax.persistence.Version; + +import com.google.gson.ExclusionStrategy; +import com.google.gson.FieldAttributes; + +/** + * An exclusion strategy that excludes our Hibernate version fields from Gson + * serialization. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class VersionExclusionStrategy implements ExclusionStrategy { + /** + * Don't exclude any classes. + * + * @param the_class The class, ignored. + */ + public boolean shouldSkipClass(final Class the_class) { + return false; + } + + /** + * Exclude fields with the @Version annotation. + * + * @param the_field The field attributes. + */ + public boolean shouldSkipField(final FieldAttributes the_field) { + return the_field.getAnnotation(Version.class) != null; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/math/Audit.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/math/Audit.java new file mode 100644 index 00000000..fc0469c4 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/math/Audit.java @@ -0,0 +1,249 @@ +package us.freeandfair.corla.math; + +import static java.math.MathContext.DECIMAL128; + +// this is BigDecimal land +import java.math.BigDecimal; +import static java.math.BigDecimal.*; +import java.math.RoundingMode; + +import static ch.obermuhlner.math.big.BigDecimalMath.pow; +import static ch.obermuhlner.math.big.BigDecimalMath.log; + +/** + * A static class that should grow to contain audit related mathematical + * functions that do not belong in models, controllers, or endpoint + * bodies. + */ +public final class Audit { + + /** + * Stark's gamma from the literature. As seen in a controller. + */ + public static final BigDecimal GAMMA = valueOf(1.03905); + + private Audit() { + } + + /** + * μ = V / N + * @param margin the smallest margin of winning, V votes. + * @param ballotCount N, the number of ballots cast in a contest. + * @return BigDecimal the diluted margin + */ + public static BigDecimal dilutedMargin(final Integer margin, + final Long ballotCount) { + return dilutedMargin(valueOf(margin), + valueOf(ballotCount)); + } + + /** + * μ = V / N + * @param margin the smallest margin of winning, V votes. + * @param ballotCount N, the number of ballots cast in a contest. + * @return BigDecimal the diluted margin + */ + public static BigDecimal dilutedMargin(final BigDecimal margin, + final BigDecimal ballotCount) { + if (margin == ZERO || ballotCount == ZERO) { + return ZERO; + } else { + return margin.divide(ballotCount, DECIMAL128); + } + } + + /** + * The "total error bound" defined in the literature. + * + * Usually represented as `U`, this can be found in equation (8) in Stark's + * Super-Simple Simultaneous Single-Ballot Risk Limiting Audits paper. + * + * @param dilutedMargin the diluted margin of the contest + * @param gamma the "error inflator" parameter from the literature + * + * @return the total error bound + */ + public static BigDecimal totalErrorBound(final BigDecimal dilutedMargin, + final BigDecimal gamma) { + + return gamma.multiply(valueOf(2), DECIMAL128) + .divide(dilutedMargin, DECIMAL128); + } + + /** + * Computes the expected number of ballots to audit overall, assuming + * zero over- and understatements. + * + * @param riskLimit as prescribed + * @param dilutedMargin of the contest. + * + * @return the expected number of ballots remaining to audit. + * This is the stopping sample size as defined in the literature: + * https://www.stat.berkeley.edu/~stark/Preprints/gentle12.pdf + */ + public static BigDecimal optimistic(final BigDecimal riskLimit, + final BigDecimal dilutedMargin) { + return optimistic(riskLimit, dilutedMargin, GAMMA, + 0, 0, 0, 0); + } + + /** + * Computes the expected number of ballots to audit overall given the + * specified numbers of over- and understatements. + * + * @param the_two_under The two-vote understatements. + * @param the_one_under The one-vote understatements. + * @param the_one_over The one-vote overstatements. + * @param the_two_over The two-vote overstatements. + * + * @return the expected number of ballots remaining to audit. + * This is the stopping sample size as defined in the literature: + * https://www.stat.berkeley.edu/~stark/Preprints/gentle12.pdf + */ + public static BigDecimal optimistic(final BigDecimal riskLimit, + final BigDecimal dilutedMargin, + final BigDecimal gamma, + final int twoUnder, + final int oneUnder, + final int oneOver, + final int twoOver) { + + if (dilutedMargin.compareTo(ZERO) == 0) { //hilarious + // nothing to do here, no samples will need to be audited because the + // contest is uncontested + return ZERO; + } + + final BigDecimal result; + final BigDecimal invgamma = ONE.divide(gamma, DECIMAL128); + final BigDecimal twogamma = valueOf(2).multiply(gamma); + final BigDecimal invtwogamma = + ONE.divide(twogamma, DECIMAL128); + final BigDecimal two_under_bd = valueOf(twoUnder); + final BigDecimal one_under_bd = valueOf(oneUnder); + final BigDecimal one_over_bd = valueOf(oneOver); + final BigDecimal two_over_bd = valueOf(twoOver); + + final BigDecimal over_under_sum = + two_under_bd.add(one_under_bd).add(one_over_bd).add(two_over_bd); + final BigDecimal two_under = + two_under_bd.multiply(log(ONE.add(invgamma), + DECIMAL128)); + final BigDecimal one_under = + one_under_bd.multiply(log(ONE.add(invtwogamma), + DECIMAL128)); + final BigDecimal one_over = + one_over_bd.multiply(log(ONE.subtract(invtwogamma), + DECIMAL128)); + final BigDecimal two_over = + two_over_bd.multiply(log(ONE.subtract(invgamma), + DECIMAL128)); + final BigDecimal numerator = + twogamma.negate(). + multiply(log(riskLimit, DECIMAL128). + add(two_under.add(one_under).add(one_over).add(two_over))); + final BigDecimal ceil = + numerator.divide(dilutedMargin, DECIMAL128).setScale(0, RoundingMode.CEILING); + result = ceil.max(over_under_sum); + + return result; + } + + /** + * Conservative approximation of the Kaplan-Markov P-value. + * + * The audit can stop when the P-value drops to or below the defined risk + * limit. The output of this method will never estimate a P-value that is too + * low, it will always be at or above the (more complicated to calculate) + * Kaplan-Markov P-value, but usually not by much. Therefore this method is + * safe to use as the stopping condition for the audit, even though it may be + * possible to stop the audit "a ballot or two" earlier if calculated using + * the Kaplan-Markov method. + * + * Implements equation (10) of Philip B. Stark's paper, Super-Simple + * Simultaneous Single-Ballot Risk-Limiting Audits. + * + * Translated from Stark's implementation under the heading "A simple + * approximation" at the following URL: + * + * https://github.com/pbstark/S157F17/blob/master/audit.ipynb + * + * NOTE: The ordering of the under and overstatement parameters is different + * from its cousin method `optimistic`. + * + * @param auditedBallots the number of ballots audited so far + * @param dilutedMargin the diluted margin of the contest + * @param gamma the "error inflator" parameter from the literature + * @param twoUnder the number of two-vote understatements + * @param oneUnder the number of one-vote understatements + * @param oneOver the number of one-vote overstatements + * @param twoOver the number of two-vote overstatements + * + * @return approximation of the Kaplan-Markov P-value + */ + public static BigDecimal pValueApproximation(final int auditedBallots, + final BigDecimal dilutedMargin, + final BigDecimal gamma, + final int oneUnder, + final int twoUnder, + final int oneOver, + final int twoOver) { + final BigDecimal totalErrorBound = totalErrorBound(dilutedMargin, gamma); + + return ONE.min( + pow( + ONE.subtract( + ONE.divide(totalErrorBound, DECIMAL128) + ), + auditedBallots, + DECIMAL128 + ) + .multiply( + pow( + ONE.subtract( + ONE.divide( + gamma.multiply(valueOf(2), DECIMAL128), + DECIMAL128 + ) + ), + -1 * oneOver, + DECIMAL128 + ), + DECIMAL128 + ) + .multiply( + pow( + ONE.subtract( + ONE.divide(gamma, DECIMAL128) + ), + -1 * twoOver, + DECIMAL128 + ), + DECIMAL128 + ) + .multiply( + pow( + ONE.add( + ONE.divide( + gamma.multiply(valueOf(2), DECIMAL128), + DECIMAL128 + ) + ), + -1 * oneUnder, + DECIMAL128 + ), + DECIMAL128 + ) + .multiply( + pow( + ONE.add( + ONE.divide(gamma, DECIMAL128) + ), + -1 * twoUnder, + DECIMAL128 + ), + DECIMAL128 + ) + ); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/Administrator.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/Administrator.java new file mode 100644 index 00000000..d3b23737 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/Administrator.java @@ -0,0 +1,260 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 8, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.model; + +import static us.freeandfair.corla.util.EqualsHashcodeHelper.*; + +import java.io.Serializable; +import java.time.Instant; + +import javax.persistence.Cacheable; +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.FetchType; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Index; +import javax.persistence.JoinColumn; +import javax.persistence.ManyToOne; +import javax.persistence.Table; +import javax.persistence.UniqueConstraint; +import javax.persistence.Version; + +import us.freeandfair.corla.persistence.PersistentEntity; + +/** + * An administrator in the system. + * + * @author Daniel M. Zimmerman + * @author Joseph R. Kiniry + * @version 1.0.0 + */ +@Entity +@Cacheable(true) +@Table(name = "administrator", + uniqueConstraints = { + @UniqueConstraint(columnNames = {"username"}) }, + indexes = { @Index(name = "idx_admin_username", + columnList = "username", unique = true) }) +// this class has many fields that would normally be declared final, but +// cannot be for compatibility with Hibernate and JPA. +@SuppressWarnings("PMD.ImmutableField") +public class Administrator implements PersistentEntity, Serializable { + /** + * The serialVersionUID. + */ + private static final long serialVersionUID = 1; + + /** + * The ID number. + */ + @Id + @Column(updatable = false, nullable = false) + @GeneratedValue(strategy = GenerationType.SEQUENCE) + private Long my_id; + + /** + * The version (for optimistic locking). + */ + @Version + private Long my_version; + + /** + * The username. + */ + @Column(name = "username", unique = true, nullable = false, updatable = false) + private String my_username; + + /** + * The administrator type. + */ + @Enumerated(EnumType.STRING) + @Column(nullable = false, updatable = false) + private AdministratorType my_type; + + /** + * The user's full (display) name. + */ + @Column(nullable = false, updatable = false) + private String my_full_name; + + /** + * The county. + */ + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn + private County my_county; + + /** + * The last login time. + */ + private Instant my_last_login_time; + + /** + * The last logout time. + */ + private Instant my_last_logout_time; + + /** + * Constructs a new Administrator with default values. + */ + public Administrator() { + super(); + } + + /** + * Constructs a new Administrator with the specified values, which has + * never logged in or out. + * + * @param the_username The username. + * @param the_type The type. + * @param the_full_name The full name. + * @param the_county The county. + */ + public Administrator(final String the_username, + final AdministratorType the_type, + final String the_full_name, + final County the_county) { + super(); + my_username = the_username; + my_type = the_type; + my_full_name = the_full_name; + my_county = the_county; + } + + /** + * {@inheritDoc} + */ + @Override + public Long id() { + return my_id; + } + + /** + * {@inheritDoc} + */ + @Override + public void setID(final Long the_id) { + my_id = the_id; + } + + /** + * {@inheritDoc} + */ + @Override + public Long version() { + return my_version; + } + + /** + * @return the username. + */ + public String username() { + return my_username; + } + + /** + * @return the type. + */ + public AdministratorType type() { + return my_type; + } + + /** + * @return the full name. + */ + public String fullName() { + return my_full_name; + } + + /** + * @return the county for the administrator, or null if it doesn't have one. + */ + public County county() { + return my_county; + } + + /** + * @return the last login time. + */ + public Instant lastLoginTime() { + return my_last_login_time; + } + + /** + * Updates the last login time to the current time. + */ + public void updateLastLoginTime() { + my_last_login_time = Instant.now(); + } + + /** + * @return the last logout time. + */ + public Instant lastLogoutTime() { + return my_last_logout_time; + } + + /** + * Updates the last logout time to the current time. + */ + public void updateLastLogoutTime() { + my_last_logout_time = Instant.now(); + } + + /** + * @return a String representation of this contest. + */ + @Override + public String toString() { + return "Administrator [username=" + my_username + ", type=" + + my_type + ", full_name=" + my_full_name + ", county=" + + my_county + ", last_login_time=" + my_last_login_time + + ", last_logout_time=" + my_last_logout_time + "]"; + } + + /** + * Compare this object with another for equivalence. + * + * @param the_other The other object. + * @return true if the objects are equivalent, false otherwise. + */ + @Override + public boolean equals(final Object the_other) { + boolean result = true; + if (the_other instanceof Administrator) { + final Administrator other_admin = (Administrator) the_other; + result &= nullableEquals(other_admin.username(), username()); + } else { + result = false; + } + return result; + } + + /** + * @return a hash code for this object. + */ + @Override + public int hashCode() { + return nullableHashCode(username()); + } + + /** + * Types of administrator. + */ + public enum AdministratorType { + COUNTY, STATE; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/AuditBoard.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/AuditBoard.java new file mode 100644 index 00000000..8a7f8d4d --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/AuditBoard.java @@ -0,0 +1,147 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 27, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.model; + +import static us.freeandfair.corla.util.EqualsHashcodeHelper.*; + +import java.io.Serializable; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import javax.persistence.Column; +import javax.persistence.Convert; +import javax.persistence.Embeddable; + +import us.freeandfair.corla.persistence.ElectorListConverter; + +/** + * An audit board. Contains a set of electors, a timestamp when the board + * signed in, and a timestamp when the board signed out. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@Embeddable +// this class has many fields that would normally be declared final, but +// cannot be for compatibility with Hibernate and JPA. +@SuppressWarnings("PMD.ImmutableField") +public class AuditBoard implements Serializable { + /** + * The serialVersionUID. + */ + private static final long serialVersionUID = 1; + + /** + * The audit board members. + */ + @Column(name = "members", columnDefinition = "text") + @Convert(converter = ElectorListConverter.class) + private List my_members = new ArrayList<>(); + + /** + * The time at which the audit board signed in. + */ + @Column(nullable = false, updatable = false) + private Instant my_sign_in_time; + + /** + * The time at which the audit board signed out. + */ + private Instant my_sign_out_time; + + /** + * Constructs a new, empty audit board, solely for persistence. + */ + public AuditBoard() { + // defaults + } + + /** + * Constructs a new audit board. + * + * @param the_members The set of Electors on the board. + * @param the_sign_in_time The sign in time of the board. + */ + public AuditBoard(final List the_members, final Instant the_sign_in_time) { + my_members.addAll(the_members); + my_sign_in_time = the_sign_in_time; + } + + /** + * @return the audit board members. + */ + public List members() { + return Collections.unmodifiableList(my_members); + } + + /** + * @return the sign in time. + */ + public Instant signInTime() { + return my_sign_in_time; + } + + /** + * @return the sign out time. + */ + public Instant signOutTime() { + return my_sign_out_time; + } + + /** + * Sets the sign out time. + * + * @param the_sign_out_time The time. + */ + public void setSignOutTime(final Instant the_sign_out_time) { + my_sign_out_time = the_sign_out_time; + } + + /** + * @return a String representation of this contest. + */ + @Override + public String toString() { + return "AuditBoard [members=" + my_members + ", sign_in_time=" + + my_sign_in_time + ", sign_out_time=" + my_sign_out_time + "]"; + } + + /** + * Compare this object with another for equivalence. + * + * @param the_other The other object. + * @return true if the objects are equivalent, false otherwise. + */ + @Override + public boolean equals(final Object the_other) { + boolean result = true; + if (the_other instanceof AuditBoard) { + final AuditBoard other_board = (AuditBoard) the_other; + result &= nullableEquals(other_board.members(), members()); + result &= nullableEquals(other_board.signInTime(), signInTime()); + result &= nullableEquals(other_board.signOutTime(), signOutTime()); + } else { + result = false; + } + return result; + } + + /** + * @return a hash code for this object. + */ + @Override + public int hashCode() { + return nullableHashCode(signInTime()); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/AuditInfo.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/AuditInfo.java new file mode 100644 index 00000000..96050c23 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/AuditInfo.java @@ -0,0 +1,322 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 13, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.model; + +import static us.freeandfair.corla.util.EqualsHashcodeHelper.*; + +import java.io.Serializable; +import java.math.BigDecimal; +import java.time.Instant; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import javax.persistence.Column; +import javax.persistence.Convert; +import javax.persistence.Embeddable; +import javax.persistence.MapKey; + +import us.freeandfair.corla.persistence.CountyCanonicalContestsMapConverter; + +/** + * Election information. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@Embeddable +//this class has many fields that would normally be declared final, but +//cannot be for compatibility with Hibernate and JPA. +@SuppressWarnings("PMD.ImmutableField") +public class AuditInfo implements Serializable { + /** + * The database stored precision for decimal types. + */ + public static final int PRECISION = 10; + + /** + * The database stored scale for decimal types. + */ + public static final int SCALE = 8; + + /** + * The serialVersionUID. + */ + private static final long serialVersionUID = 1L; + + /** + * The election type. + */ + private String my_election_type; + + /** + * The election date (stored as an instant). + */ + private Instant my_election_date; + + /** + * The public meeting date (stored as an instant). + */ + private Instant my_public_meeting_date; + + /** + * The random seed. + */ + private String my_seed; + + /** + * The risk limit. + */ + @Column(precision = PRECISION, scale = SCALE) + private BigDecimal my_risk_limit; + + /** + * The mapping of county name to a set of contest names within each + * county. + */ + @Convert(converter = CountyCanonicalContestsMapConverter.class) + @Column(name = "canonical_contests", columnDefinition = "text") + @MapKey(name = "my_id") + private Map> + canonicalContests = new TreeMap>(); + + /** map of contestName to choices **/ + @Convert(converter = CountyCanonicalContestsMapConverter.class) + @Column(name = "canonical_choices", columnDefinition = "text") + // @MapKey(name = "my_id") // not sure this is needed + private Map> + canonicalChoices = new TreeMap>(); + + + /** + * Constructs an empty AuditInfo using defaults + */ + public AuditInfo() { + // defaults + } + + /** + * Constructs a new AuditInfo. + * + * @param electionType The election type. + * @param electionDate The election date. + * @param publicMeetingDate The public meeting date. + * @param seed The random seed. + * @param riskLimit The risk limit + */ + public AuditInfo(final String electionType, + final Instant electionDate, + final Instant publicMeetingDate, + final String seed, + final BigDecimal riskLimit) { + my_election_type = electionType; + my_election_date = electionDate; + my_public_meeting_date = publicMeetingDate; + my_seed = seed; + my_risk_limit = riskLimit; + } + + /** + * Constructs a new AuditInfo with a collection of canonical contests. + * + * @param electionType The election type. + * @param electionDate The election date. + * @param publicMeetingDate The public meeting date. + * @param seed The random seed. + * @param riskLimit The risk limit + * @param contests The map of canonical contest names for counties + */ + public AuditInfo(final String electionType, + final Instant electionDate, + final Instant publicMeetingDate, + final String seed, + final BigDecimal riskLimit, + final Map> contests) { + my_election_type = electionType; + my_election_date = electionDate; + my_public_meeting_date = publicMeetingDate; + my_seed = seed; + my_risk_limit = riskLimit; + this.canonicalContests = contests; + } + + /** + * @return the election type. + */ + public String electionType() { + return my_election_type; + } + + /** + * @return a capitalized string for the election type. + */ + @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") + public String capitalizedElectionType() { + final String result; + + if (my_election_type == null) { + result = null; + } else if (my_election_type.length() > 1) { + result = my_election_type.substring(0, 1).toUpperCase(Locale.getDefault()) + + my_election_type.substring(1).toLowerCase(Locale.getDefault()); + } else { + result = my_election_type.toUpperCase(Locale.getDefault()); + } + + return result; + } + + /** + * @return the election date (as an instant). + */ + public Instant electionDate() { + return my_election_date; + } + + /** + * @return the public meeting date (as an instant). + */ + public Instant publicMeetingDate() { + return my_public_meeting_date; + } + + /** + * @return the random seed. + */ + public String seed() { + return my_seed; + } + + /** + * @return the risk limit. + */ + public BigDecimal riskLimit() { + return my_risk_limit; + } + + /** + * @return some kind of canonical contests thingy + */ + public Map> canonicalContests() { + return this.canonicalContests; + } + + /** + * setter of county-contest mapping + * @param m The map you want to set + */ + public void setCanonicalContests(final Map> m) { + this.canonicalContests = m; + } + + /** + * setter of contest-choices mapping + * @param m The map you want to set + */ + public void setCanonicalChoices(final Map> m) { + this.canonicalChoices = m; + } + + /** + * getter of canonicalChoices + * @return map of contestNames to choices + */ + public Map> getCanonicalChoices() { + return this.canonicalChoices; + } + + + + /** + * Updates this AuditInfo with information from the specified one. + * Any non-null fields in the specified AuditInfo replace the + * corresponding fields of this AuditInfo; any null fields in the + * specified AuditInfo are ignored. It is not possible to nullify + * a field of an AuditInfo once it has been set, only to replace + * it with a new non-null value. + * + * @param the_other_info The other ElectionInfo. + */ + public void updateFrom(final AuditInfo the_other_info) { + if (the_other_info.my_election_type != null) { + my_election_type = the_other_info.my_election_type; + } + + if (the_other_info.my_election_date != null) { + my_election_date = the_other_info.my_election_date; + } + + if (the_other_info.my_public_meeting_date != null) { + my_public_meeting_date = the_other_info.my_public_meeting_date; + } + + if (the_other_info.my_seed != null) { + my_seed = the_other_info.my_seed; + } + + if (the_other_info.my_risk_limit != null) { + my_risk_limit = the_other_info.my_risk_limit; + } + + if (the_other_info.canonicalContests().size() > 0) { + canonicalContests = the_other_info.canonicalContests; + } + if (the_other_info.getCanonicalChoices().size() > 0) { + canonicalChoices = the_other_info.canonicalChoices; + } + } + + /** + * Compare this object with another for equivalence. + * + * @param the_other The other object. + * @return true if the objects are equivalent, false otherwise. + */ + @Override + public boolean equals(final Object the_other) { + boolean result = true; + if (the_other instanceof AuditInfo) { + final AuditInfo other_info = (AuditInfo) the_other; + result &= nullableEquals(other_info.electionType(), electionType()); + result &= nullableEquals(other_info.electionDate(), electionDate()); + result &= nullableEquals(other_info.publicMeetingDate(), publicMeetingDate()); + result &= nullableEquals(other_info.seed(), seed()); + result &= nullableEquals(other_info.riskLimit(), riskLimit()); + } else { + result = false; + } + return result; + } + + /** + * @return a hash code for this object. + */ + @Override + public int hashCode() { + return nullableHashCode(seed()); + } + + /** + * a good practice + */ + @Override + public String toString() { + return + "AuditInfo[canonicalContests=" + canonicalContests() + + ", electionType=" + electionType() + + ", electionDate=" + electionDate() + + ", publicMeetingDate=" + publicMeetingDate() + + ", seed=" + seed() + + ", riskLimit" + riskLimit(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/AuditInvestigationReportInfo.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/AuditInvestigationReportInfo.java new file mode 100644 index 00000000..4f767084 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/AuditInvestigationReportInfo.java @@ -0,0 +1,146 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 2, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.model; + +import static us.freeandfair.corla.util.EqualsHashcodeHelper.*; + +import java.io.Serializable; +import java.time.Instant; + +import javax.persistence.Column; +import javax.persistence.Embeddable; + +import org.hibernate.annotations.Immutable; + +import com.google.gson.annotations.JsonAdapter; + +import us.freeandfair.corla.json.AuditInvestigationReportInfoJsonAdapter; + +/** + * An audit investigation report. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@Embeddable +@Immutable // this is a Hibernate-specific annotation, but there is no JPA alternative +//this class has many fields that would normally be declared final, but +//cannot be for compatibility with Hibernate and JPA. +@SuppressWarnings("PMD.ImmutableField") +@JsonAdapter(AuditInvestigationReportInfoJsonAdapter.class) +public class AuditInvestigationReportInfo implements Serializable { + /** + * The serialVersionUID. + */ + private static final long serialVersionUID = 1L; + + /** + * The timestamp of this report. + */ + @Column(updatable = false) + private Instant my_timestamp; + + /** + * The name for this report. + */ + @Column(updatable = false) + private String my_name; + + /** + * The report for this report. + */ + @Column(updatable = false) + private String my_report; + + /** + * Constructs an empty AuditInvestigationReport, solely for persistence. + */ + public AuditInvestigationReportInfo() { + super(); + } + + /** + * Constructs an audit investigation report with the specified + * parameters. + * + * @param the_timestamp The timestamp. + * @param the_name The name. + * @param the_report The report. + */ + public AuditInvestigationReportInfo(final Instant the_timestamp, + final String the_name, + final String the_report) { + super(); + my_timestamp = the_timestamp; + my_name = the_name; + my_report = the_report; + } + + /** + * @return the timestamp. + */ + public Instant timestamp() { + return my_timestamp; + } + + /** + * @return the name. + */ + public String name() { + return my_name; + } + + /** + * @return the report. + */ + public String report() { + return my_report; + } + + /** + * @return a String representation of this cast vote record. + */ + @Override + public String toString() { + return "AuditInvestigationReport [timestamp=" + my_timestamp + + ", name=" + my_name + ", report=" + my_report + "]"; + } + + /** + * Compare this object with another for equivalence. + * + * @param the_other The other object. + * @return true if the objects are equivalent, false otherwise. + */ + @Override + public boolean equals(final Object the_other) { + boolean result = true; + if (the_other instanceof AuditInvestigationReportInfo) { + final AuditInvestigationReportInfo other_report = + (AuditInvestigationReportInfo) the_other; + result &= nullableEquals(other_report.timestamp(), timestamp()); + result &= nullableEquals(other_report.name(), name()); + result &= nullableEquals(other_report.report(), report()); + } else { + result = false; + } + return result; + } + + /** + * @return a hash code for this object. + */ + @Override + public int hashCode() { + return nullableHashCode(timestamp()); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/AuditReason.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/AuditReason.java new file mode 100644 index 00000000..48e6825f --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/AuditReason.java @@ -0,0 +1,68 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Sep 6, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.model; + +/** + * The possible reasons for an audit. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public enum AuditReason { + STATE_WIDE_CONTEST("Statewide Contest"), + COUNTY_WIDE_CONTEST("Countywide Contest"), + CLOSE_CONTEST("Close Contest"), + TIED_CONTEST("Tied Contest"), + GEOGRAPHICAL_SCOPE("Geographical Scope"), + CONCERN_REGARDING_ACCURACY("Concern Regarding Accuracy"), + OPPORTUNISTIC_BENEFITS("Opportunistic Benefits"), + COUNTY_CLERK_ABILITY("County Clerk Ability"); + + /** + * The pretty printing string for this enum value. + */ + private final String my_pretty_string; + + /** + * Constructs a new AuditReason. + * + * @param the_pretty_string The pretty printing string. + */ + AuditReason(final String the_pretty_string) { + my_pretty_string = the_pretty_string; + } + + /** + * @return the pretty printing string for this enum value. + */ + public String prettyString() { + return my_pretty_string; + } + + /** + * @return the audit selection corresponding to this audit reason. + */ + public AuditSelection selection() { + if (this.equals(TIED_CONTEST) || this.equals(OPPORTUNISTIC_BENEFITS)) { + return AuditSelection.UNAUDITED_CONTEST; + } else { + return AuditSelection.AUDITED_CONTEST; + } + } + + /** + * we are thinking that this was selected by DOS as a targeted contest + **/ + public Boolean isTargeted() { + return selection() == AuditSelection.AUDITED_CONTEST; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/AuditSelection.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/AuditSelection.java new file mode 100644 index 00000000..19e58555 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/AuditSelection.java @@ -0,0 +1,44 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Sep 6, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.model; + +/** + * The possible reasons for an audit. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public enum AuditSelection { + AUDITED_CONTEST("Audited Contest"), + UNAUDITED_CONTEST("Unaudited Contest"); + + /** + * The pretty printing string for this enum value. + */ + private final String my_pretty_string; + + /** + * Constructs a new AuditReason. + * + * @param the_pretty_string The pretty printing string. + */ + AuditSelection(final String the_pretty_string) { + my_pretty_string = the_pretty_string; + } + + /** + * @return the pretty printing string for this enum value. + */ + public String prettyString() { + return my_pretty_string; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/AuditStatus.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/AuditStatus.java new file mode 100644 index 00000000..b60efb57 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/AuditStatus.java @@ -0,0 +1,27 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Sep 6, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.model; + +/** + * The possible statuses for an audit. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public enum AuditStatus { + NOT_STARTED, + NOT_AUDITABLE, + IN_PROGRESS, + RISK_LIMIT_ACHIEVED, + ENDED, + HAND_COUNT; +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/AuditType.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/AuditType.java new file mode 100644 index 00000000..e8fe0320 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/AuditType.java @@ -0,0 +1,22 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Sep 6, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.model; + +/** + * The possible types for an audit. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public enum AuditType { + COMPARISON, HAND_COUNT, NOT_AUDITABLE, NONE; +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/BallotManifestInfo.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/BallotManifestInfo.java new file mode 100644 index 00000000..e2607ee0 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/BallotManifestInfo.java @@ -0,0 +1,374 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 25, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joey Dodds + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.model; + +import static us.freeandfair.corla.util.EqualsHashcodeHelper.*; + +import java.io.Serializable; +import java.util.Comparator; + +import javax.persistence.Cacheable; +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Index; +import javax.persistence.Table; +import javax.persistence.Version; + +import org.hibernate.annotations.Immutable; + +import us.freeandfair.corla.persistence.PersistentEntity; + +/** + * Information about the locations of specific batches of ballots. The + * Set for a given county forms a complete ballot + * manifest. Joins to a CastVoteRecord to provide a ballot pull list. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@Entity +@Immutable // this is a Hibernate-specific annotation, but there is no JPA alternative +@Cacheable(true) +@Table(name = "ballot_manifest_info", + indexes = { @Index(name = "idx_bmi_county", columnList = "county_id"), + @Index(name = "idx_bmi_seqs", columnList = "sequence_start,sequence_end")}) +// this class has many fields that would normally be declared final, but +// cannot be for compatibility with Hibernate and JPA. +@SuppressWarnings("PMD.ImmutableField") +final public class BallotManifestInfo implements PersistentEntity, Serializable { + /** + * The serialVersionUID. + */ + private static final long serialVersionUID = 1; + + /** + * The ID number. + */ + @Id + @Column(updatable = false, nullable = false) + @GeneratedValue(strategy = GenerationType.SEQUENCE) + private Long my_id; + + /** + * The version (for optimistic locking). + */ + @Version + private Long my_version; + + /** + * The ID number of the county in which the batch was scanned. + */ + @Column(name = "county_id", updatable = false, nullable = false) + private Long my_county_id; + //@ private invariant my_county_id >= 0; + + /** + * The ID number of the scanner that scanned the batch. + */ + @Column(updatable = false, nullable = false) + private Integer my_scanner_id; + + /** + * The batch number. + */ + @Column(updatable = false, nullable = false) + private String my_batch_id; + + /** + * The size of the batch. + */ + @Column(updatable = false, nullable = false) + private Integer my_batch_size; + + /** + * The storage location for the batch. + */ + @Column(updatable = false, nullable = false) + private String my_storage_location; + + /** + * The first sequence number (of all ballots) in this batch. Used to find a batch + * based on a random sequence number. + */ + @Column(updatable = false, nullable = false, name = "sequence_start") + private Long my_sequence_start; + + /** + * The last sequence number (of all ballots) in this batch. Used to find a batch + * based on a random sequence number. + */ + @Column(updatable = false, nullable = false, name = "sequence_end") + private Long my_sequence_end; + + /** + * The unique properties as a string for fast selection + */ + private String uri; + + /** + * The projected start of this manifest chunk + */ + public Long ultimateSequenceStart; + + /** + * The projected end of this manifest chunk + */ + public Long ultimateSequenceEnd; + + /** + * @param start the adjusted sequence start + * this is like offset and limit of pages of a contest (multiple counties) + */ + public void setUltimate(final Long start) { + this.ultimateSequenceStart = start; + this.ultimateSequenceEnd = start + rangeSize(); + } + + /** from contest scope to county scope **/ + public Integer sequencePosition(final Integer rand) { + // subtraction gives a 0-based offset, adding one gives us the 1-based + // ballot position in the file, row number of the file, a.k.a.: cvr.cvrNumber() + return rand - this.ultimateSequenceStart.intValue() + 1; + } + + /** + * translate a generated random number from contest to county scope, then + * from county to batch scope + **/ + public Integer translateRand(final Integer rand) { + return sequencePosition(rand); + } + + /** + * @return Long the number of ballots in my chunk of a manifest + */ + public Long rangeSize() { + return my_sequence_end - my_sequence_start; + } + + /** get the uri for fast selection **/ + public String getUri() { + return this.uri; + } + + /** set the uri for fast selection **/ + public void setUri() { + this.uri = String.format("%s:%s:%s-%s", + "bmi", + countyID(), + scannerID(), + batchID()); + } + + /** + * @return Boolean whether this manifest section would hold the random + * nth selection + */ + public Boolean isHolding(final Long rand) { + // setUltimate(last + 1L) means the start and end is inclusive + return this.ultimateSequenceStart + <= rand && rand <= this.ultimateSequenceEnd; + } + + /** + * Constructs an empty ballot manifest information record, solely + * for persistence. + */ + public BallotManifestInfo() { + super(); + } + + /** + * Constructs a ballot manifest information record. + * + * @param the_county_id The county ID. + * @param the_scanner_id The scanner ID. + * @param the_batch_id The batch ID. + * @param the_batch_size The batch size. + * @param the_storage_location The storage location. + */ + public BallotManifestInfo(final Long the_county_id, + final Integer the_scanner_id, + final String the_batch_id, + final int the_batch_size, + final String the_storage_location, + final Long the_sequence_start, + final Long the_sequence_end) { + super(); + my_county_id = the_county_id; + my_scanner_id = the_scanner_id; + my_batch_id = the_batch_id; + my_batch_size = the_batch_size; + my_storage_location = the_storage_location; + my_sequence_start = the_sequence_start; + my_sequence_end = the_sequence_end; + this.setUri(); + } + + /** + * {@inheritDoc} + */ + @Override + public Long id() { + return my_id; + } + + /** + * {@inheritDoc} + */ + @Override + public void setID(final Long the_id) { + my_id = the_id; + } + + /** + * {@inheritDoc} + */ + @Override + public Long version() { + return my_version; + } + + /** + * @return the county ID. + */ + public Long countyID() { + return my_county_id; + } + + /** + * @return the scanner ID. + */ + public Integer scannerID() { + return my_scanner_id; + } + + /** + * @return the batch number. + */ + public String batchID() { + return my_batch_id; + } + + /** + * @return the batch size. + */ + public Integer batchSize() { + return my_batch_size; + } + + /** + * @return the storage container number. + */ + public String storageLocation() { + return my_storage_location; + } + + /** + * @return the sequence start + */ + public Long sequenceStart() { + return my_sequence_start; + } + + /** + * @return the sequence end + */ + public Long sequenceEnd() { + return my_sequence_end; + } + + /** + * computed value based on scannerId,batchID, and ballotPosition (in the bin) + * This is what the auditors will use to confirm they have the correct card. + **/ + public String imprintedID(final Long rand) { + return scannerID() + "-" + + batchID() + "-" + + ballotPosition(rand.intValue()).toString(); + } + + /** + * where the ballot sits in it's storage bin + **/ + public Integer ballotPosition(final Integer sequencePosition) { + // position is the nth (1 based) + return sequencePosition - sequenceStart().intValue() + 1; + } + + /** + * @return a String representation of this object. + */ + @Override + public String toString() { + return "BallotManifestInfo [" + ", county_id=" + my_county_id + + ", scanner_id=" + my_scanner_id + ", batch_size=" + + my_batch_size + ", storage_container=" + my_storage_location + "]"; + } + + /** + * Compare this object with another for equivalence. + * + * @param the_other The other object. + * @return true if the objects are equivalent, false otherwise. + */ + @Override + public boolean equals(final Object the_other) { + boolean result = true; + if (the_other instanceof BallotManifestInfo) { + final BallotManifestInfo other_bmi = (BallotManifestInfo) the_other; + result &= nullableEquals(other_bmi.countyID(), countyID()); + result &= nullableEquals(other_bmi.scannerID(), scannerID()); + result &= nullableEquals(other_bmi.batchID(), batchID()); + result &= nullableEquals(other_bmi.batchSize(), batchSize()); + result &= nullableEquals(other_bmi.storageLocation(), storageLocation()); + } else { + result = false; + } + return result; + } + + /** + * @return a hash code for this object. + */ + @Override + public int hashCode() { + return nullableHashCode(storageLocation()); + } + + /** + * sort across counties by comparing countyID() as well as sequenceEnd() + **/ + public static class Sort implements Comparator, Serializable { + + /** + * a good practice + */ + public static final long serialVersionUID = 1L; + + /** + * a good practice + */ + @Override + public int compare(final BallotManifestInfo bmi1, + final BallotManifestInfo bmi2) { + if (bmi1.countyID().equals(bmi2.countyID())) { + return bmi1.sequenceEnd().compareTo(bmi2.sequenceEnd()); + } else { + return bmi1.countyID().compareTo(bmi2.countyID()); + } + } + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/CVRAuditInfo.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/CVRAuditInfo.java new file mode 100644 index 00000000..b0cc5ce3 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/CVRAuditInfo.java @@ -0,0 +1,319 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 10, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.model; + +import static us.freeandfair.corla.util.EqualsHashcodeHelper.nullableEquals; + +import java.util.Collections; +import java.util.HashSet; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import javax.persistence.Cacheable; +import javax.persistence.Column; +import javax.persistence.Convert; +import javax.persistence.Entity; +import javax.persistence.FetchType; +import javax.persistence.Id; +import javax.persistence.JoinColumn; +import javax.persistence.ManyToOne; +import javax.persistence.Table; +import javax.persistence.Version; + +import us.freeandfair.corla.persistence.AuditReasonSetConverter; +import us.freeandfair.corla.persistence.LongIntegerMapConverter; +import us.freeandfair.corla.persistence.PersistentEntity; + +/** + * A class representing a contest to audit or hand count. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@Entity +@Cacheable(true) +@Table(name = "cvr_audit_info") +//this class has many fields that would normally be declared final, but +//cannot be for compatibility with Hibernate and JPA. +// note: CVRAuditInfo is not serializable because it references CountyDashboard, +// which is not serializable +@SuppressWarnings("PMD.ImmutableField") +public class CVRAuditInfo implements Comparable, + PersistentEntity { + /** + * The ID number. This is always the same as the CVR ID number. + */ + @Id + @Column(updatable = false, nullable = false) + private Long my_id; + + /** + * The version (for optimistic locking). + */ + @Version + private Long my_version; + + /** + * The CVR to audit. + */ + @ManyToOne(optional = false, fetch = FetchType.LAZY) + @JoinColumn + private CastVoteRecord my_cvr; + + /** + * The submitted audit CVR. + */ + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn + private CastVoteRecord my_acvr; + + /** + * The number of times this auditInfo's CVR appears in the selections of + * ComparisonAudits + * {Long ComparisonAuditId: Integer count} + */ + @Convert(converter = LongIntegerMapConverter.class) + @Column(columnDefinition = "text") + private Map multiplicity_by_contest = new HashMap<>(); + + /** + * The number of times this CVRAuditInfo has been counted/sampled in each + * ComparisonAudit + */ + @Convert(converter = LongIntegerMapConverter.class) + @Column(columnDefinition = "text") + private Map count_by_contest = new HashMap<>(); + + /** + * The number of discrepancies found in the audit so far. + */ + @Column(nullable = false, name = "discrepancy", columnDefinition = "text") + @Convert(converter = AuditReasonSetConverter.class) + private Set my_discrepancy = new HashSet<>(); + + /** + * The number of disagreements found in the audit so far. + */ + @Column(nullable = false, name = "disagreement", columnDefinition = "text") + @Convert(converter = AuditReasonSetConverter.class) + private Set my_disagreement = new HashSet<>(); + + /** + * Constructs an empty CVRAuditInfo, solely for persistence. + */ + public CVRAuditInfo() { + super(); + } + + /** + * Constructs a new CVRAuditInfo for the specified CVR to audit. + * + * @param the_cvr The CVR to audit. + */ + public CVRAuditInfo(final CastVoteRecord the_cvr) { + super(); + my_id = the_cvr.id(); + my_cvr = the_cvr; + } + + /** + * {@inheritDoc} + */ + @Override + public Long id() { + return my_id; + } + + /** + * {@inheritDoc} + */ + @Override + public void setID(final Long the_id) { + my_id = the_id; + } + + /** + * {@inheritDoc} + */ + @Override + public Long version() { + return my_version; + } + + /** + * @return the CVR to audit. + */ + public CastVoteRecord cvr() { + return my_cvr; + } + + /** + * @return the submitted audit CVR.. + */ + public CastVoteRecord acvr() { + return my_acvr; + } + + /** + * Sets the submitted audit CVR for this record. + * + * @param the_acvr The audit CVR. + */ + public void setACVR(final CastVoteRecord the_acvr) { + my_acvr = the_acvr; + } + + /** + * Sets the number of times this record appears in one contest. + * + * @param comparisonAuditId. + * @param count. + */ + public void setMultiplicityByContest (final Long comparisonAuditId, final Integer count) { + this.multiplicity_by_contest.put(comparisonAuditId, count); + } + + /** + * Sets the number of times this record has been counted in the + * audit calculations. + * + * @param the_counted The new value. + */ + public void setCountByContest (final Long comparisonAuditId, final Integer count) { + this.count_by_contest.put(comparisonAuditId, count); + } + + /** + * get the number of times this cvr has been counted per contest + **/ + public int getCountByContest(final Long comparisonAuditId) { + return this.count_by_contest.getOrDefault(comparisonAuditId, 0); + } + + /** + * how many times has this been counted over all contests? + */ + public int totalCounts() { + return this.count_by_contest.values().stream().mapToInt(e -> e).sum(); + } + + /** + * clear record of counts per contest, for unauditing. + */ + public void resetCounted() { + this.count_by_contest.clear(); + } + + /** + * @return a map from audit reason to whether this record was marked + * as a discrepancy in a contest audited for that reason. + */ + public Set discrepancy() { + return Collections.unmodifiableSet(my_discrepancy); + } + + /** + * Sets the audit reasons for which the record is marked as a discrepancy. + * + * @param the_reasons The reasons. + */ + public void setDiscrepancy(final Set the_reasons) { + my_discrepancy.clear(); + if (the_reasons != null) { + my_discrepancy.addAll(the_reasons); + } + } + + /** + * @return a map from audit reason to whether this record was marked + * as a disagreement in a contest audited for that reason. + */ + public Set disagreement() { + return Collections.unmodifiableSet(my_disagreement); + } + + /** + * Sets the audit reasons for which the record is marked as a disagreement. + * + * @param the_reasons The reasons. + */ + public void setDisagreement(final Set the_reasons) { + my_disagreement.clear(); + if (the_reasons != null) { + my_disagreement.addAll(the_reasons); + } + } + + /** + * @return a String representation of this contest to audit. + */ + @Override + public String toString() { + final String cvr; + final String acvr; + if (my_cvr == null) { + cvr = "null"; + } else { + cvr = my_cvr.id().toString(); + } + if (my_acvr == null) { + acvr = "null"; + } else { + acvr = my_acvr.id().toString(); + } + return "CVRAuditInfo [cvr=" + cvr + ", acvr=" + acvr + "]"; + } + + /** + * Compare this object with another for equivalence. + * + * @param the_other The other object. + * @return true if the objects are equivalent, false otherwise. + */ + @Override + public boolean equals(final Object the_other) { + boolean result = true; + if (id() != null && the_other instanceof CVRAuditInfo) { + final CVRAuditInfo other_info = (CVRAuditInfo) the_other; + // we compare by database ID + result &= nullableEquals(other_info.id(), id()); + } else { + result = false; + } + return result; + } + + /** + * @return a hash code for this object. + */ + @Override + public int hashCode() { + if (id() == null) { + return 0; + } else { + return id().hashCode(); + } + } + + /** + * Compares this CVRAuditInfo to another. + * + * Uses the underlying CVR to provide the sorting behavior. + * + * @return int + */ + @Override + public int compareTo(final CVRAuditInfo other) { + return this.cvr().compareTo(other.cvr()); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/CVRContestInfo.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/CVRContestInfo.java new file mode 100644 index 00000000..5eafd123 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/CVRContestInfo.java @@ -0,0 +1,200 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 2, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.model; + +import static us.freeandfair.corla.util.EqualsHashcodeHelper.*; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import javax.persistence.Column; +import javax.persistence.Convert; +import javax.persistence.Embeddable; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.FetchType; +import javax.persistence.ManyToOne; + +import com.google.gson.annotations.JsonAdapter; + +import us.freeandfair.corla.json.CVRContestInfoJsonAdapter; +import us.freeandfair.corla.persistence.StringListConverter; + +/** + * A cast vote record contains information about a single ballot, either + * imported from a tabulator export file or generated by auditors. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@Embeddable +//this class has many fields that would normally be declared final, but +//cannot be for compatibility with Hibernate and JPA. +@SuppressWarnings("PMD.ImmutableField") +@JsonAdapter(CVRContestInfoJsonAdapter.class) +public class CVRContestInfo implements Serializable { + /** + * The serialVersionUID. + */ + private static final long serialVersionUID = 1L; + + /** + * The contest in this record. + */ + @ManyToOne(optional = false, fetch = FetchType.EAGER) + private Contest my_contest; + + /** + * The comment for this contest. + */ + @Column(updatable = false) + private String my_comment; + + /** + * The countyId for fast bulk deletion. This is because of the apparent + * lacking feature of jpa; on delete cascade added to ddl + * the value is used in a query not in the code, hence the SuppressWarnings + */ + @SuppressWarnings({"PMD.UnusedPrivateField","PMD.SingularField"}) + @Column + private Long county_id; + + /** + * The consensus value for this contest + */ + @Column(updatable = false) + @Enumerated(EnumType.STRING) + private ConsensusValue my_consensus; + + /** + * The choices for this contest. + */ + @Column(name = "choices", columnDefinition = "character varying (1024)") + @Convert(converter = StringListConverter.class) + private List my_choices = new ArrayList<>(); + + /** + * Constructs an empty CVRContestInfo, solely for persistence. + */ + public CVRContestInfo() { + super(); + } + + /** + * Constructs a CVR contest information record with the specified + * parameters. + * + * @param the_contest The contest. + * @param the_comment The comment. + * @param the_consensus The consensus value. + * @param the_choices The choices. + * @exception IllegalArgumentException if any choice is not a valid choice + * for the specified contest. + */ + public CVRContestInfo(final Contest the_contest, final String the_comment, + final ConsensusValue the_consensus, + final List the_choices) { + super(); + my_contest = the_contest; + my_comment = the_comment; + my_consensus = the_consensus; + my_choices.addAll(the_choices); + for (final String s : my_choices) { + if (!my_contest.isValidChoice(s)) { + throw new IllegalArgumentException("invalid choice " + s + + " for contest " + my_contest); + } + } + } + + /** + * @return the contest in this record. + */ + public Contest contest() { + return my_contest; + } + + /** set the county id **/ + public void setCountyId(final Long countyId) { + this.county_id = countyId; + } + + /** + * @return the comment in this record. + */ + public String comment() { + return my_comment; + } + + /** + * @return the consensus flag in this record. + */ + public ConsensusValue consensus() { + return my_consensus; + } + + /** + * @return the choices in this record. + */ + public List choices() { + return Collections.unmodifiableList(my_choices); + } + + /** + * @return a String representation of this cast vote record. + */ + @Override + public String toString() { + return "CVRContestInfo [contest=" + my_contest.id() + ", comment=" + + my_comment + ", consensus=" + my_consensus + ", choices=" + + my_choices + "]"; + } + + /** + * Compare this object with another for equivalence. + * + * @param the_other The other object. + * @return true if the objects are equivalent, false otherwise. + */ + @Override + public boolean equals(final Object the_other) { + boolean result = true; + if (the_other instanceof CVRContestInfo) { + final CVRContestInfo other_info = (CVRContestInfo) the_other; + result &= nullableEquals(other_info.contest(), contest()); + result &= nullableEquals(other_info.comment(), comment()); + result &= nullableEquals(other_info.consensus(), consensus()); + result &= nullableEquals(other_info.choices(), choices()); + } else { + result = false; + } + return result; + } + + /** + * @return a hash code for this object. + */ + @Override + public int hashCode() { + return nullableHashCode(choices()); + } + + /** + * The possible values for consensus. + */ + public enum ConsensusValue { + YES, + NO + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/CastVoteRecord.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/CastVoteRecord.java new file mode 100644 index 00000000..2b46ac31 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/CastVoteRecord.java @@ -0,0 +1,728 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 25, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.model; + +import static us.freeandfair.corla.util.EqualsHashcodeHelper.*; + +import java.io.Serializable; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import javax.persistence.Cacheable; +import javax.persistence.CollectionTable; +import javax.persistence.Column; +import javax.persistence.ElementCollection; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.FetchType; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Index; +import javax.persistence.JoinColumn; +import javax.persistence.OrderColumn; +import javax.persistence.Table; +import javax.persistence.UniqueConstraint; +import javax.persistence.Version; + +import org.hibernate.annotations.Immutable; + +import us.freeandfair.corla.persistence.PersistentEntity; +import us.freeandfair.corla.util.NaturalOrderComparator; +import us.freeandfair.corla.util.SuppressFBWarnings; + +/** + * A cast vote record contains information about a single ballot, either + * imported from a tabulator export file or generated by auditors. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@Entity +@Immutable // this is a Hibernate-specific annotation, but there is no JPA alternative +@Cacheable(false) +@Table(name = "cast_vote_record", + uniqueConstraints = {@UniqueConstraint(columnNames = {"county_id", + "imprinted_id", + "record_type", + "revision"}, + name = "uniqueCVR")}, + indexes = { @Index(name = "idx_cvr_county_type", columnList = "county_id, record_type"), + @Index(name = "idx_cvr_county_cvr_number", + columnList = "county_id, cvr_number"), + @Index(name = "idx_cvr_county_cvr_number_type", + columnList = "county_id, cvr_number, record_type"), + @Index(name = "idx_cvr_county_sequence_number_type", + columnList = "county_id, sequence_number, record_type"), + @Index(name = "idx_cvr_county_imprinted_id_type", + columnList = "county_id, imprinted_id, record_type"), + @Index(name = "idx_cvr_uri", columnList = "uri")}) +// this class has many fields that would normally be declared final, but +// cannot be for compatibility with Hibernate and JPA. +@SuppressWarnings({"PMD.ImmutableField", + // I agreed but let's put if off for now + "PMD.TooManyMethods", + "PMD.TooManyFields", + "PMD.GodClass"}) +// this FindBugs warning is for the transient field, which we know will not be +// restored when the class is unserialized, because we intentionally made it +// transient so it wouldn't be. Since that's what "transient" means. +@SuppressFBWarnings("SE_TRANSIENT_FIELD_NOT_RESTORED") +final public class CastVoteRecord implements Comparable, + PersistentEntity, + Serializable { + /** + * The serialVersionUID. + */ + private static final long serialVersionUID = 1L; + + /** + * The ID number. + */ + @Id + @Column(updatable = false, nullable = false) + @GeneratedValue(strategy = GenerationType.SEQUENCE) + private Long my_id; + + /** + * The version (for optimistic locking). + */ + @Version + private Long my_version; + + /** + * used to store the order of edits + * made to a ballot submission + * (note: version is for hibernate) + **/ + private Long revision; + + /** + * A flag indicating whether this record was generated by auditors or + * by import. + */ + @Column(name = "record_type", nullable = false) + @Enumerated(EnumType.STRING) + private RecordType my_record_type; + + /** + * The timestamp of this cast vote record; used only for ACVRs. + */ + @Column(updatable = false) + private Instant my_timestamp; + + /** + * The county ID of this cast vote record. + */ + @Column(name = "county_id", updatable = false, nullable = false) + private Long my_county_id; + + /** + * The CVR number of this cast vote record. + */ + @Column(name = "cvr_number", updatable = false, nullable = false) + private Integer my_cvr_number; + + /** + * The sequence number of this cast vote record. Only applicable + * to imported CVRs. - nth of the county + */ + @Column(name = "sequence_number", updatable = false) + private Integer my_sequence_number; + + /** + * The scanner ID of this cast vote record. + */ + @Column(updatable = false, nullable = false) + private Integer my_scanner_id; + + /** + * The batch ID of this cast vote record. + */ + @Column(updatable = false, nullable = false) + private String my_batch_id; + + /** + * The record ID of this cast vote record. - nth of the batch + */ + @Column(updatable = false, nullable = false) + private Integer my_record_id; + + /** + * The imprinted ID of this cast vote record. + */ + @Column(name = "imprinted_id", updatable = false, nullable = false) + private String my_imprinted_id; + + /** + * The countyid + imprinted ID for fast selection + */ + private String uri; + + /** + * The ballot style of this cast vote record. + */ + @Column(updatable = false, nullable = false) + private String my_ballot_type; + + /** + * The contest information in this cast vote record. + */ + @ElementCollection(fetch = FetchType.EAGER) + @OrderColumn(name = "index") + @CollectionTable(name = "cvr_contest_info", + joinColumns = @JoinColumn(name = "cvr_id", + referencedColumnName = "my_id"), + indexes = {@Index(name = "idx_cvrci_uri", columnList = "county_id,contest_id")}) + private List my_contest_info = new ArrayList<>(); + + /** + * ACVR level comments, used for explaining why reaudit is happening + **/ + private String comment; + + /** + * who is submitting this ACVR, used for reporting + **/ + private Integer auditBoardIndex; + + /** + * A transient flag that indicates whether this CVR was audited; this is only + * used for passing information around within the RLA tool and is not serialized + * in the database; the authoritative source of information about whether a CVR + * has been audited, and in what audit, is the responsible audit information + * object. + */ + private transient boolean my_audit_flag; + + /** + * A transient flag that indicates whether this CVR was audited in a "previous + * round"; this is only used for passing information around and is not + * serialized in the database. + */ + private transient boolean my_previously_audited; + + /** + * The CVR to audit, for ACVRs only + */ + private Long cvrId; + + /** + * The round that the submission happened in, for ACVRs only + **/ + private Integer roundNumber; + + /** + * The generated random number that selected/resolves to this cvr + **/ + private Integer rand; + + /** + * both a performance optimization and work around for a feature lacking from + * hibernate: on delete cascade in the ddl + * so we tag the ContestInfo with a county so we can delete them all quickly + **/ + public static List claim(final List contestInfos, final Long countyId) { + return contestInfos.stream() + .map(ci -> {ci.setCountyId(countyId); return ci;}) + .collect(Collectors.toList()); + } + + + /** + * Constructs an empty cast vote record, solely for persistence. + */ + public CastVoteRecord() { + super(); + } + + /** + * Constructs a new cast vote record. + * + * @param the_record_type The type. + * @param the_timestamp The timestamp. + * @param the_county_id The county ID. + * @param the_cvr_number The CVR number (as imported). + * @param the_sequence_number The sequence number, if applicable. + * @param the_scanner_id The scanner ID. + * @param the_batch_id The batch ID. + * @param the_record_id The record ID. + * @param the_imprinted_id The imprinted ID. + * @param the_ballot_type The ballot type. + * @param the_contest_info A map of the choices made in each contest. + */ + @SuppressWarnings("PMD.ExcessiveParameterList") + public CastVoteRecord(final RecordType the_record_type, + final Instant the_timestamp, + final Long the_county_id, + final Integer the_cvr_number, + final Integer the_sequence_number, + final Integer the_scanner_id, + final String the_batch_id, + final Integer the_record_id, + final String the_imprinted_id, + final String the_ballot_type, + final List the_contest_info) { + super(); + my_record_type = the_record_type; + my_timestamp = the_timestamp; + my_county_id = the_county_id; + my_cvr_number = the_cvr_number; + my_sequence_number = the_sequence_number; + my_scanner_id = the_scanner_id; + my_batch_id = the_batch_id; + my_record_id = the_record_id; + my_imprinted_id = the_imprinted_id; + my_ballot_type = the_ballot_type; + if (the_contest_info != null) { + my_contest_info.addAll(CastVoteRecord.claim(the_contest_info, my_county_id)); + } + this.setUri(); + } + + /** + * {@inheritDoc} + */ + @Override + public Long id() { + return my_id; + } + + /** + * {@inheritDoc} + */ + @Override + public void setID(final Long the_id) { + my_id = the_id; + } + + /** + * {@inheritDoc} + */ + @Override + public Long version() { + return my_version; + } + + /** + * used to store the order of edits + * made to a ballot submission + * (note: version is for hibernate) + **/ + public Long getRevision() { + return this.revision; + } + + /** set revision **/ + public void setRevision(final Long rev) { + this.revision = rev; + } + + /** + * @return this record's type. + */ + public RecordType recordType() { + return my_record_type; + } + + /** set the record type to EDITED and the uri to rcvr:... **/ + public void setToReaudited() { + setRecordType(RecordType.REAUDITED); + setUri(); + } + + /** set the record type **/ + public void setRecordType(final RecordType recordType) { + this.my_record_type = recordType; + } + + /** + * set the comment + * A comment is required to explain why a reaudit happens + **/ + public void setComment(final String comment) { + this.comment = comment; + } + + /** get comment **/ + public String getComment() { + return this.comment; + } + + /** + * set the auditBoardIndex + **/ + public void setAuditBoardIndex(final Integer auditBoardIndex) { + this.auditBoardIndex = auditBoardIndex; + } + + /** get auditBoardIndex **/ + public Integer getAuditBoardIndex() { + return this.auditBoardIndex; + } + + /** + * @return the timestamp of this record. + */ + public Instant timestamp() { + return my_timestamp; + } + + /** + * @return the county ID. + */ + public Long countyID() { + return my_county_id; + } + + /** + * @return the CVR number (as imported). + */ + public Integer cvrNumber() { + return my_cvr_number; + } + + /** + * @return the CVR sequence number. + */ + public Integer sequenceNumber() { + return my_sequence_number; + } + + /** + * @return the scanner ID. + */ + public Integer scannerID() { + return my_scanner_id; + } + + /** + * @return the batch ID. + */ + public String batchID() { + return my_batch_id; + } + + /** + * @return the record ID. + */ + public Integer recordID() { + return my_record_id; + } + + /** + * @return the imprinted ID for this cast vote record. + */ + public String imprintedID() { + return my_imprinted_id; + } + + /** + * @return the ballot type for this cast vote record. + */ + public String ballotType() { + return my_ballot_type; + } + + /** get the round that this acvr is audited in **/ + public Integer getRoundNumber () { + return this.roundNumber; + } + + /** set the round that this acvr is audited in **/ + public void setRoundNumber (final Integer roundNumber) { + this.roundNumber = roundNumber ; + } + + /** get the random number **/ + public Integer getRand () { + return this.rand; + } + + /** set the random number **/ + public void setRand (final Integer rand) { + this.rand = rand; + } + + /** get the uri for fast selection **/ + public String getUri() { + return this.uri; + } + + /** set the uri for fast selection **/ + public void setUri() { + String cvrOrAcvr; + String rev = ""; + if (recordType() == RecordType.UPLOADED + || recordType() == RecordType.PHANTOM_RECORD) { + // phantoms play the role of uploaded cvrs + cvrOrAcvr = "cvr"; + } else if (recordType() == RecordType.REAUDITED){ + rev = "?rev=" + getRevision().toString(); + cvrOrAcvr = "rcvr"; + } else { + // auditor entered (or ballot not found) + cvrOrAcvr = "acvr"; + } + this.uri = String.format("%s:%s:%s-%s-%s%s", + cvrOrAcvr, + countyID(), + scannerID(), + batchID(), + recordID(), + rev); + + } + + /** link to a bmi for fast selection **/ + public String bmiUri() { + return String.format("%s:%s:%s-%s", + "bmi", + countyID(), + scannerID(), + batchID()); + } + + /** + * keep a record of what this ACVR was submitted to audit, which is lost when + * reauditing because the CVRAuditInfo join is broke when reauditing + **/ + public void setCvrId(final Long cvrId) { + this.cvrId = cvrId; + } + + /** get the cvrId **/ + public Long getCvrId() { + return this.cvrId; + } + + + /** + * @return the choices made in this cast vote record. + */ + public List contestInfo() { + return Collections.unmodifiableList(my_contest_info); + } + + /** setter **/ + public void setContestInfo (final List contestInfos) { + this.my_contest_info.clear(); + this.my_contest_info.addAll(CastVoteRecord.claim(contestInfos, this.my_county_id)); + } + + /** + * Gets the choices for the specified contest. + * + * @param the_contest The contest. + * @return the choices made in this cast vote record for the specified contest, + * or null if none were made for the specified contest. + */ + public CVRContestInfo contestInfoForContest(final Contest the_contest) { + for (final CVRContestInfo info : my_contest_info) { + if (info.contest().equals(the_contest)) { + return info; + } + } + return null; + } + + /** + * Get info about a CVR by way of a ContestResult, matching on contest + * name. + * @param cr + * @return maybe the first CVRContestInfo found, maybe nothing. + */ + public Optional contestInfoForContestResult(final ContestResult cr) { + return my_contest_info.stream() + .filter(x -> x.contest().name().equals(cr.getContestName())) + .findFirst(); + } + + /** + * @return the audit flag. This flag is meaningless unless it was explicitly set + * when this record was loaded. It is useful only for communicating information + * about a CVR within a specific computation of the tool, and is not serialized + * in the database; the authoritative source of information about whether a CVR + * has been audited, and in what audit, is the responsible audit information + * object. + */ + public boolean auditFlag() { + return my_audit_flag; + } + + /** + * Sets the audit flag. + * + * @param the_audit_flag The new flag. + */ + public void setAuditFlag(final boolean the_audit_flag) { + my_audit_flag = the_audit_flag; + } + + /** + * Whether or not the ballot was previously audited + * + * Like auditFlag(), this is not persisted to the database, and is only used + * during a single run of the tool. + */ + public boolean previouslyAudited() { + return my_previously_audited; + } + + /** + * Set whether or not the ballot was previously audited + */ + public void setPreviouslyAudited(final boolean the_previously_audited) { + my_previously_audited = the_previously_audited; + } + + /** + * @return a String representation of this cast vote record. + */ + @Override + public String toString() { + return "CastVoteRecord [record_type=" + my_record_type + ", timestamp=" + + my_timestamp + ", county_id=" + my_county_id + ", cvr_id=" + my_cvr_number + + ", scanner_id=" + my_scanner_id + ", batch_id=" + my_batch_id + ", record_id=" + + my_record_id + ", imprinted_id=" + my_imprinted_id + ", ballot_type=" + + my_ballot_type + ", contest_info=" + my_contest_info + "]"; + } + + /** + * Compare this object with another for equivalence. + * + * @param the_other The other object. + * @return true if the objects are equivalent, false otherwise. + */ + @Override + public boolean equals(final Object the_other) { + boolean result = true; + if (the_other instanceof CastVoteRecord) { + final CastVoteRecord other_cvr = (CastVoteRecord) the_other; + result &= nullableEquals(other_cvr.countyID(), countyID()); + result &= nullableEquals(other_cvr.cvrNumber(), cvrNumber()); + result &= nullableEquals(other_cvr.sequenceNumber(), sequenceNumber()); + result &= nullableEquals(other_cvr.scannerID(), scannerID()); + result &= nullableEquals(other_cvr.batchID(), batchID()); + result &= nullableEquals(other_cvr.recordID(), recordID()); + result &= nullableEquals(other_cvr.imprintedID(), imprintedID()); + result &= nullableEquals(other_cvr.ballotType(), ballotType()); + result &= nullableEquals(other_cvr.contestInfo(), contestInfo()); + result &= nullableEquals(other_cvr.recordType(), recordType()); + result &= nullableEquals(other_cvr.getRevision(), getRevision()); + result &= nullableEquals(other_cvr.getUri(), getUri()); + result &= nullableEquals(other_cvr.getAuditBoardIndex(), getAuditBoardIndex()); + result &= nullableEquals(other_cvr.getRoundNumber(), getRoundNumber()); + result &= nullableEquals(other_cvr.getRand(), getRand()); + result &= nullableEquals(other_cvr.getComment(), getComment()); + } else { + result = false; + } + return result; + } + + /** + * Compares this CVR with another to determine whether + * one is an audit CVR for the other - that is, whether they have + * the same county ID, scanner ID, batch ID, record ID, + * imprinted ID, and ballot type, and exactly one of them is an + * auditor uploaded CVR. + * + * @param the_other The other CVR. + * @return true if one CVR is an audit CVR for the other; false + * otherwise. + */ + public boolean isAuditPairWith(final CastVoteRecord the_other) { + boolean result = true; + + if (the_other == null) { + result = false; + } else { + result &= nullableEquals(the_other.countyID(), countyID()); + result &= nullableEquals(the_other.cvrNumber(), cvrNumber()); + result &= nullableEquals(the_other.scannerID(), scannerID()); + result &= nullableEquals(the_other.batchID(), batchID()); + result &= nullableEquals(the_other.recordID(), recordID()); + result &= nullableEquals(the_other.imprintedID(), imprintedID()); + result &= nullableEquals(the_other.ballotType(), ballotType()); + // if PHANTOM_RECORD, neither are auditorGenerated + // result &= recordType().isAuditorGenerated() ^ + // the_other.recordType().isAuditorGenerated(); + } + + return result; + } + + /** + * @return a hash code for this object. + */ + @Override + public int hashCode() { + return nullableHashCode(getUri()); + } + + /** + * An enumeration used to select cast vote record types. + * REAUDITED are the previous revisions, AUDITOR_ENTERED is the latest revision + */ + public enum RecordType { + UPLOADED, AUDITOR_ENTERED, REAUDITED, PHANTOM_RECORD, PHANTOM_RECORD_ACVR, PHANTOM_BALLOT; + + /** + * @return true if this record was generated by an auditor, + * false otherwise. + */ + public boolean isAuditorGenerated() { + return this == AUDITOR_ENTERED || this == REAUDITED || this == PHANTOM_BALLOT; + } + + /** + * the cvr data did not contain a cvr we looked for so we generate a + * discrepancy automatically, at least for PHANTOM_RECORD + **/ + public boolean isSystemGenerated() { + return this == PHANTOM_RECORD || this == PHANTOM_RECORD_ACVR; + } + } + + /** + * Compares this object to another. + * + * The sorting happens by the triple (scannerID(), batchID(), recordID()) and + * will return a negative, positive, or 0-valued result if this should come + * before, after, or at the same point as the other object, respectively. + * + * @return int + */ + @Override + public int compareTo(final CastVoteRecord other) { + final int scanner = this.scannerID() - other.scannerID(); + + if (scanner != 0) { + return scanner; + } + + final int batch = NaturalOrderComparator.INSTANCE.compare( + this.batchID(), other.batchID()); + + if (batch != 0) { + return batch; + } + + return this.recordID() - other.recordID(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/Choice.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/Choice.java new file mode 100644 index 00000000..9fe62735 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/Choice.java @@ -0,0 +1,159 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 25, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joey Dodds + * @model_review Joseph R. Kiniry + * @design In the formal model this concept is currently called "option". + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.model; + +import static us.freeandfair.corla.util.EqualsHashcodeHelper.*; + +import java.io.Serializable; + +import javax.persistence.Embeddable; + +/** + * A contest choice; has a name and a description. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@Embeddable +public class Choice implements Serializable { + /** + * The serialVersionUID. + */ + private static final long serialVersionUID = 1L; + + /** + * The choice name. + */ + private String my_name; + + /** + * The choice description. + */ + private String my_description; + + /** + * A flag that indicates whether or not a choice is a qualified write-in. + */ + private boolean my_qualified_write_in; + + /** + * A flag that indicates whether or not a choice is "fictitious" (i.e., whether + * its votes should be counted and it should be displayed). This is to + * handle cases where specific "fake" choice names are used to delineate + * sections of a ballot, as with Dominion and qualified write-ins. + */ + private boolean my_fictitious; + + /** + * Constructs a choice with default values, solely for persistence. + */ + public Choice() { + // defaults + } + + /** + * Constructs a choice with the specified parameters. + * + * @param the_name The choice name. + * @param the_description The choice description. + * @param the_qualified_write_in True if this choice is a qualified + * write-in candidate, false otherwise. + * @param the_fictitious True of this choice is fictitious (should not be + * counted), false otherwise. + */ + public Choice(final String the_name, final String the_description, + final boolean the_qualified_write_in, + final boolean the_fictitious) { + my_name = the_name; + my_description = the_description; + my_qualified_write_in = the_qualified_write_in; + my_fictitious = the_fictitious; + } + + /** + * @return the name. + */ + public String name() { + return my_name; + } + + /** set the name **/ + public void setName(final String name) { + this.my_name = name; + } + + + /** + * @return the description. + */ + public String description() { + return my_description; + } + + /** + * @return true if this choice is a qualified write-in, false otherwise. + */ + public boolean qualifiedWriteIn() { + return my_qualified_write_in; + } + + /** + * @return true if this choice is fictitious, false otherwise. + */ + public boolean fictitious() { + return my_fictitious; + } + + /** + * @return a String representation of this contest. + */ + @Override + public String toString() { + return "Choice [name=" + my_name + ", description=" + + my_description + "]"; + } + + public String shortToString() { + return "[" + my_name + "],"; + } + + /** + * Compare this object with another for equivalence. + * + * @param the_other The other object. + * @return true if the objects are equivalent, false otherwise. + */ + @Override + public boolean equals(final Object the_other) { + boolean result = true; + if (the_other instanceof Choice) { + final Choice other_choice = (Choice) the_other; + result &= nullableEquals(other_choice.name(), name()); + result &= nullableEquals(other_choice.description(), description()); + result &= nullableEquals(other_choice.qualifiedWriteIn(), qualifiedWriteIn()); + result &= nullableEquals(other_choice.fictitious(), fictitious()); + } else { + result = false; + } + return result; + } + + /** + * @return a hash code for this object. + */ + @Override + public int hashCode() { + return nullableHashCode(name()); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/ComparisonAudit.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/ComparisonAudit.java new file mode 100644 index 00000000..bb35f39c --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/ComparisonAudit.java @@ -0,0 +1,1166 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 19, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.model; + +import java.math.BigDecimal; +import java.math.MathContext; +import java.math.RoundingMode; +import java.util.Collections; +import java.util.List; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import javax.persistence.Cacheable; +import javax.persistence.CollectionTable; +import javax.persistence.Column; +import javax.persistence.Convert; +import javax.persistence.ElementCollection; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.FetchType; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.JoinColumn; +import javax.persistence.JoinTable; +import javax.persistence.ManyToMany; +import javax.persistence.ManyToOne; +import javax.persistence.MapKeyJoinColumn; +import javax.persistence.Table; +import javax.persistence.Version; + +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import us.freeandfair.corla.math.Audit; +import us.freeandfair.corla.model.CVRContestInfo.ConsensusValue; +import us.freeandfair.corla.model.CastVoteRecord.RecordType; +import us.freeandfair.corla.persistence.PersistentEntity; +import us.freeandfair.corla.persistence.LongListConverter; + +/** + * A class representing the state of a single audited contest for + * across multiple counties + * + */ +@Entity +@Cacheable(true) +@Table(name = "comparison_audit") + +@SuppressWarnings({"PMD.ImmutableField", "PMD.ExcessiveClassLength", + "PMD.CyclomaticComplexity", "PMD.GodClass", "PMD.ModifiedCyclomaticComplexity", + "PMD.StdCyclomaticComplexity", "PMD.TooManyFields", "PMD.TooManyMethods", + "PMD.ExcessiveImports"}) +public class ComparisonAudit implements PersistentEntity { + + /** + * Class-wide logger + */ + public static final Logger LOGGER = + LogManager.getLogger(ComparisonAudit.class); + + /** + * The database stored precision for decimal types. + */ + public static final int PRECISION = 10; + + /** + * The database stored scale for decimal types. + */ + public static final int SCALE = 8; + + /** + * The ID number. + */ + @Id + @Column(updatable = false, nullable = false) + @GeneratedValue(strategy = GenerationType.SEQUENCE) + private Long my_id; + + /** + * The version (for optimistic locking). + */ + @Version + private Long my_version; + + /** + * The contest result for this audit state. + */ + @ManyToOne(optional = false, fetch = FetchType.LAZY) + @JoinColumn + private ContestResult my_contest_result; + + /** + * The reason for this audit. + */ + @Column(updatable = false, nullable = false) + @Enumerated(EnumType.STRING) + private AuditReason my_audit_reason; + + /** + * The status of this audit. + */ + @Column(nullable = false) + @Enumerated(EnumType.STRING) + private AuditStatus my_audit_status = AuditStatus.NOT_STARTED; + + /** + * The gamma. + */ + @Column(updatable = false, nullable = false, + precision = PRECISION, scale = SCALE) + private BigDecimal my_gamma = Audit.GAMMA; + + /** + * The diluted margin + */ + @Column(updatable = false, nullable = false, + precision = PRECISION, scale = SCALE) + private BigDecimal diluted_margin = BigDecimal.ONE; + + /** + * The risk limit. + */ + @Column(updatable = false, nullable = false, + precision = PRECISION, scale = SCALE) + private BigDecimal my_risk_limit = BigDecimal.ONE; + + /** + * The number of samples audited. + */ + @Column(nullable = false) + private Integer my_audited_sample_count = 0; + + /** + * The number of samples to audit overall assuming no further overstatements. + */ + @Column(nullable = false) + private Integer my_optimistic_samples_to_audit = 0; + + /** + * The expected number of samples to audit overall assuming overstatements + * continue at the current rate. + */ + @Column(nullable = false) + private Integer my_estimated_samples_to_audit = 0; + + /** + * The number of two-vote understatements recorded so far. + */ + @Column(nullable = false) + private Integer my_two_vote_under_count = 0; + + /** + * The number of one-vote understatements recorded so far. + */ + @Column(nullable = false) + private Integer my_one_vote_under_count = 0; + + /** + * The number of one-vote overstatements recorded so far. + */ + @Column(nullable = false) + private Integer my_one_vote_over_count = 0; + + /** + * The number of two-vote overstatements recorded so far. + */ + @Column(nullable = false) + private Integer my_two_vote_over_count = 0; + + /** + * The number of discrepancies recorded so far that are neither + * understatements nor overstatements. + */ + @Column(nullable = false) + private Integer my_other_count = 0; + + /** + * The number of disagreements. + */ + @Column(nullable = false) + private Integer my_disagreement_count = 0; + + /** + * gets incremented + */ + @Column(nullable = true) // true for migration + private BigDecimal overstatements = BigDecimal.ZERO; + + /** + * A flag that indicates whether the optimistic ballots to audit + * estimate needs to be recalculated. + */ + @Column(nullable = false) + private Boolean my_optimistic_recalculate_needed = true; + + /** + * A flag that indicates whether the non-optimistic ballots to + * audit estimate needs to be recalculated + */ + @Column(nullable = false) + private Boolean my_estimated_recalculate_needed = true; + + /** + * The sequence of CastVoteRecord ids for this contest ordered by County id + */ + @Column(name = "contest_cvr_ids", columnDefinition = "text") + @Convert(converter = LongListConverter.class) + private List contestCVRIds = new ArrayList(); + + /** + * A map from CVRAuditInfo objects to their discrepancy values for this + * audited contest. + */ + @ElementCollection + @CollectionTable(name = "contest_comparison_audit_discrepancy", + joinColumns = @JoinColumn(name = "contest_comparison_audit_id", + referencedColumnName = "my_id")) + @MapKeyJoinColumn(name = "cvr_audit_info_id") + @Column(name = "discrepancy") + private Map my_discrepancies = new HashMap<>(); + + /** + * A map from CVRAuditInfo objects to their discrepancy values for this + * audited contest. + */ + @ManyToMany + @JoinTable(name = "contest_comparison_audit_disagreement", + joinColumns = @JoinColumn(name = "contest_comparison_audit_id", + referencedColumnName = "my_id"), + inverseJoinColumns = @JoinColumn(name = "cvr_audit_info_id", + referencedColumnName = "my_id")) + private Set my_disagreements = new HashSet<>(); + + /** + * Constructs a new, empty ComparisonAudit (solely for persistence). + */ + public ComparisonAudit() { + super(); + } + + /** + * Constructs a ComparisonAudit for the given params + * + * @param contestResult The contest result. + * @param riskLimit The risk limit. + * @param dilutedMargin μ + * @param gamma γ + * @param auditReason The audit reason. + */ + // FIXME estimatedSamplesToAudit / optimisticSamplesToAudit have side + // effects, so we should call that out + // + // FIXME Remove the warning by not calling overridable methods :D + @SuppressWarnings({"PMD.ConstructorCallsOverridableMethod"}) + public ComparisonAudit(final ContestResult contestResult, + final BigDecimal riskLimit, + final BigDecimal dilutedMargin, + final BigDecimal gamma, + final AuditReason auditReason) { + + super(); + my_contest_result = contestResult; + my_risk_limit = riskLimit; + this.diluted_margin = dilutedMargin; + my_gamma = gamma; + my_audit_reason = auditReason; + // compute initial sample size + optimisticSamplesToAudit(); + estimatedSamplesToAudit(); + + if (contestResult.getDilutedMargin().equals(BigDecimal.ZERO)) { + // the diluted margin is 0, so this contest is not auditable + my_audit_status = AuditStatus.NOT_AUDITABLE; + } + } + + /** + * {@inheritDoc} + */ + @Override + public Long id() { + return my_id; + } + + /** + * {@inheritDoc} + */ + @Override + public void setID(final Long the_id) { + my_id = the_id; + } + + /** + * {@inheritDoc} + */ + @Override + public Long version() { + return my_version; + } + + /** + * @return the counties related to this contestresult. + */ + public Set getCounties() { + return Collections.unmodifiableSet(this.contestResult().getCounties()); + } + + /** + * @return the contest result associated with this audit. + */ + public ContestResult contestResult() { + return my_contest_result; + } + + /** + * @return the gamma associated with this audit. + */ + public BigDecimal getGamma() { + return my_gamma; + } + + /** + * @return the risk limit associated with this audit. + */ + public BigDecimal getRiskLimit() { + return my_risk_limit; + } + + /** + * @return the diluted margin from the ContestResult. + */ + public BigDecimal getDilutedMargin() { + return this.diluted_margin; + } + + public String getContestName() { + return this.contestResult().getContestName(); + } + + /** + * @return the audit reason associated with this audit. + */ + public AuditReason auditReason() { + return my_audit_reason; + } + + /** + * @return the audit status associated with this audit. + */ + public AuditStatus auditStatus() { + return my_audit_status; + } + + /** set audit status **/ + public void setAuditStatus(final AuditStatus auditStatus) { + my_audit_status = auditStatus; + } + + /** see if the county is participating in this audit(contest) **/ + public boolean isForCounty(final Long countyId) { + return getCounties().stream() + .filter(c -> c.id().equals(countyId)) + .findFirst() + .isPresent(); + } + + /** + * Does this audit belong to only a single county? + */ + public boolean isSingleCountyFor(final County c) { + return getCounties().equals(Stream.of(c) + .collect(Collectors.toSet())); + } + + /** + * Updates the audit status based on the current risk limit. If the audit + * has already been ended or the contest is not auditable, this method has + * no effect on its status. + * Fix: RLA-00450 + */ + public void updateAuditStatus() { + LOGGER.debug(String.format("[updateAuditStatus: %s for contest=%s " + + "my_optimistic_samples_to_audit=%d my_audited_sample_count=%d my_optimistic_recalculate_needed=%s my_estimated_recalculate_needed=%s]", + my_audit_status, contestResult().getContestName(), + my_optimistic_samples_to_audit, my_audited_sample_count, + my_optimistic_recalculate_needed, my_estimated_recalculate_needed)); + + if (my_audit_status == AuditStatus.ENDED || + my_audit_status == AuditStatus.HAND_COUNT || + my_audit_status == AuditStatus.NOT_AUDITABLE) { + return; + } + + if (Boolean.TRUE.equals(my_optimistic_recalculate_needed) || + Boolean.TRUE.equals(my_estimated_recalculate_needed)) { + recalculateSamplesToAudit(); + } //below calculation needs recalculate RLA-00450 + + if (my_optimistic_samples_to_audit - my_audited_sample_count <= 0) { + LOGGER.debug(String.format("[updateAuditStatus: RISK_LIMIT_ACHIEVED for contest=%s]", + contestResult().getContestName())); + my_audit_status = AuditStatus.RISK_LIMIT_ACHIEVED; + } else { + // risk limit has not been achieved + // note that it _is_ possible to go from RISK_LIMIT_ACHIEVED to + // IN_PROGRESS if a sample or set of samples is "unaudited" + if (my_audit_status.equals(AuditStatus.RISK_LIMIT_ACHIEVED)) { + LOGGER.warn("[updateAuditStatus: Moving from RISK_LIMIT_ACHIEVED -> IN_PROGRESS!]"); + } + + my_audit_status = AuditStatus.IN_PROGRESS; + } + } + + /** + * Ends this audit; if the audit has already reached its risk limit, + * or the contest is not auditable, this call has no effect on its status. + */ + public void endAudit() { + if (my_audit_status != AuditStatus.RISK_LIMIT_ACHIEVED && + my_audit_status != AuditStatus.NOT_AUDITABLE) { + my_audit_status = AuditStatus.ENDED; + } + } + + /** + * @return the initial expected number of samples to audit. + */ + @SuppressWarnings({"checkstyle:magicnumber", "PMD.AvoidDuplicateLiterals"}) + public int initialSamplesToAudit() { + return computeOptimisticSamplesToAudit(0, 0, 0, 0). + setScale(0, RoundingMode.CEILING).intValue(); + } + + /** + * @return the expected overall number of ballots to audit, assuming no + * further overstatements occur. + */ + public Integer optimisticSamplesToAudit() { + if (my_optimistic_recalculate_needed) { + recalculateSamplesToAudit(); + } + return my_optimistic_samples_to_audit; + } + + /** estimatedSamplesToAudit minus getAuditedSampleCount **/ + public final Integer estimatedRemaining() { + return Math.max(0, estimatedSamplesToAudit() - getAuditedSampleCount()); + } + + /** optimisticSamplesToAudit minus getAuditedSampleCount **/ + public final Integer optimisticRemaining() { + return Math.max(0, optimisticSamplesToAudit() - getAuditedSampleCount()); + } + + /** + * @return the expected overall number of ballots to audit, assuming + * overstatements continue to occur at the current rate. + */ + public final Integer estimatedSamplesToAudit() { + if (my_estimated_recalculate_needed) { + LOGGER.debug("[estimatedSampleToAudit: recalculate needed]"); + recalculateSamplesToAudit(); + } + return my_estimated_samples_to_audit; + } + + /** + * + * The number of one-vote and two-vote overstatements across the set + * of counties participating in this audit. + * + * TODO collect the number of 1 and 2 vote overstatements across + * participating counties. + */ + public BigDecimal getOverstatements() { + return this.overstatements; // FIXME + } + + /** the number of ballots audited **/ + public Integer getAuditedSampleCount() { + return this.my_audited_sample_count; + } + + /** + * A scaling factor for the estimate, from 1 (when no samples have + * been audited) upward.41 + The scaling factor grows as the ratio of + * overstatements to samples increases. + */ + private BigDecimal scalingFactor() { + final BigDecimal auditedSamples = BigDecimal.valueOf(getAuditedSampleCount()); + if (auditedSamples.equals(BigDecimal.ZERO)) { + return BigDecimal.ONE; + } else { + return BigDecimal.ONE.add(getOverstatements() + .divide(auditedSamples, MathContext.DECIMAL128)); + } + } + + /** + * Recalculates the overall numbers of ballots to audit, setting this + * object's `my_optimistic_samples_to_audit` and + * `my_estimates_samples_to_audit` fields. + */ + private void recalculateSamplesToAudit() { + LOGGER.debug(String.format("[recalculateSamplestoAudit start contestName=%s, " + + "twoUnder=%d, oneUnder=%d, oneOver=%d, twoOver=%d" + + " optimistic=%d, estimated=%d]", + contestResult().getContestName(), + my_two_vote_under_count, my_one_vote_under_count, + my_one_vote_over_count, my_two_vote_over_count, + my_optimistic_samples_to_audit, my_estimated_samples_to_audit)); + + if (my_optimistic_recalculate_needed) { + LOGGER.debug("[recalculateSamplesToAudit: calling computeOptimisticSamplesToAudit]"); + final BigDecimal optimistic = computeOptimisticSamplesToAudit(my_two_vote_under_count, + my_one_vote_under_count, + my_one_vote_over_count, + my_two_vote_over_count); + my_optimistic_samples_to_audit = optimistic.intValue(); + my_optimistic_recalculate_needed = false; + } + + if (my_one_vote_over_count + my_two_vote_over_count == 0) { + LOGGER.debug("[recalculateSamplesToAudit: zero overcounts]"); + my_estimated_samples_to_audit = my_optimistic_samples_to_audit; + } else { + LOGGER.debug(String.format("[recalculateSamplesToAudit: non-zero overcounts, using scaling factor %s]", scalingFactor())); + my_estimated_samples_to_audit = + BigDecimal.valueOf(my_optimistic_samples_to_audit) + .multiply(scalingFactor()) + .setScale(0, RoundingMode.CEILING) + .intValue(); + } + + LOGGER.debug(String.format("[recalculateSamplestoAudit end contestName=%s, " + + "twoUnder=%d, oneUnder=%d, oneOver=%d, twoOver=%d" + + " optimistic=%d, estimated=%d]", + contestResult().getContestName(), + my_two_vote_under_count, my_one_vote_under_count, + my_one_vote_over_count, my_two_vote_over_count, + my_optimistic_samples_to_audit, my_estimated_samples_to_audit)); + my_estimated_recalculate_needed = false; + } + + /** + * Computes the expected number of ballots to audit overall given the + * specified numbers of over- and understatements. + * + * @param the_two_under The two-vote understatements. + * @param the_one_under The one-vote understatements. + * @param the_one_over The one-vote overstatements. + * @param the_two_over The two-vote overstatements. + * + * @return the expected number of ballots remaining to audit. + * This is the stopping sample size as defined in the literature: + * https://www.stat.berkeley.edu/~stark/Preprints/gentle12.pdf + */ + private BigDecimal computeOptimisticSamplesToAudit(final int twoUnder, + final int oneUnder, + final int oneOver, + final int twoOver) { + return Audit.optimistic(getRiskLimit(), getDilutedMargin(), getGamma(), + twoUnder, oneUnder, oneOver, twoOver) ; + } + + /** + * Signals that a sample has been audited. This ensures that estimates + * are recalculated correctly and states are updated. + * + * @param count The count of samples that have been audited simultaneously + * (for duplicates). + */ + public void signalSampleAudited(final int count) { + my_estimated_recalculate_needed = true; + my_audited_sample_count = my_audited_sample_count + count; + + // this may not be needed, but I'm not sure + if (my_audit_status == AuditStatus.RISK_LIMIT_ACHIEVED) { + LOGGER.warn("RESETTING AuditStatus from RISK_LIMIT_ACHIEVED to IN_PROGRESS"); + my_audit_status = AuditStatus.IN_PROGRESS; + } + } + + /** + * Signals that a sample has been audited, if the CVR was selected for + * this audit and this audit is targeted (i.e., not for opportunistic + * benefits.) + * + * @param count The count of samples that have been audited simultaneously + * @param cvrID ID of the CVR being audited + */ + public void signalSampleAudited(final int count, final Long cvrID) { + final boolean covered = isCovering(cvrID); + final boolean targeted = isTargeted(); + + if (targeted && !covered) { + LOGGER.debug + (String.format("[signalSampleAudited: %s is targeted, but cvrID (%d) not selected for audit.]", + contestResult().getContestName(), cvrID)); + } + + if (targeted && covered) { + LOGGER.debug + (String.format + ("[signalSampleAudited: targeted and covered! " + + "contestName=%s, cvrID=%d, auditedSamples=%d, count=%d]", + contestResult().getContestName(), cvrID, getAuditedSampleCount(), count)); + signalSampleAudited(count); + } + } + + /** + * Signals that a sample has been unaudited. This ensures that estimates + * are recalculated correctly and states are updated. + * + * @param the_count The count of samples that have been unaudited simultaneously + * (for duplicates). + */ + public void signalSampleUnaudited(final int count) { + my_estimated_recalculate_needed = true; + my_audited_sample_count = my_audited_sample_count - count; + + // this may not be needed, but I'm not sure + if (my_audit_status == AuditStatus.RISK_LIMIT_ACHIEVED) { + LOGGER.warn("RESETTING AuditStatus from RISK_LIMIT_ACHIEVED to IN_PROGRESS"); + my_audit_status = AuditStatus.IN_PROGRESS; + } + } + + + /** + * Signals that a sample has been unaudited, if the CVR was selected + * for this audit. + * + * @param count The count of samples that have been unaudited simultaneously + * (for duplicates). + * @parma cvrID The ID of the CVR to unaudit + */ + public void signalSampleUnaudited(final int count, final Long cvrID) { + LOGGER.debug + (String.format + ("[signalSampleUnaudited: start " + + "contestName=%s, cvrID=%d, auditedSamples=%d, count=%d]", + contestResult().getContestName(), cvrID, getAuditedSampleCount(), count)); + + final boolean covered = isCovering(cvrID); + final boolean targeted = isTargeted(); + + if (targeted && !covered) { + LOGGER.debug + (String.format("[signalSampleUnaudited: Targeted contest, but cvrID (%d) not selected.]", + cvrID)); + } + + if (targeted && covered) { + LOGGER.debug(String.format("[signalSampleUnaudited: CVR ID [%d] is interesting to %s]", + cvrID, contestResult().getContestName())); + signalSampleUnaudited(count); + } + } + + + /** + * Records a disagreement with the specified CVRAuditInfo. + * + * @param the_record The CVRAuditInfo record that generated the disagreement. + */ + public void recordDisagreement(final CVRAuditInfo the_record) { + my_disagreements.add(the_record); + my_disagreement_count = my_disagreement_count + 1; + } + + /** + * Removes a disagreement with the specified CVRAuditInfo. + * + * @param the_record The CVRAuditInfo record that generated the disagreement. + */ + public void removeDisagreement(final CVRAuditInfo the_record) { + my_disagreements.remove(the_record); + my_disagreement_count = my_disagreement_count - 1; + } + + /** + * @return the disagreement count. + */ + public int disagreementCount() { + return my_disagreement_count; + } + + /** was the given cvrid selected for this contest? **/ + public boolean isCovering(final Long cvrId) { + return getContestCVRIds().contains(cvrId); + } + + /** + * Adds to the current collection of Contest CVR IDs + * @param contestCVRIds a list + */ + public void addContestCVRIds (final List contestCVRIds) { + this.contestCVRIds.addAll(contestCVRIds); + } + + /** + * getter + */ + public List getContestCVRIds() { + return this.contestCVRIds; + } + + /** + * Is this audit because of a targeted contest? + */ + public boolean isTargeted() { + return this.contestResult().getAuditReason().isTargeted() + && !isHandCount(); + } + + /** + * Is an audit finished, or should we find more samples to compare? + * + */ + public boolean isFinished() { + return + this.auditStatus().equals(AuditStatus.NOT_AUDITABLE) || + this.auditStatus().equals(AuditStatus.RISK_LIMIT_ACHIEVED) || + this.auditStatus().equals(AuditStatus.HAND_COUNT) || + this.auditStatus().equals(AuditStatus.ENDED); + } + + public boolean isHandCount() { + return this.auditStatus().equals(AuditStatus.HAND_COUNT); + } + + /** calculate the number of times the given cvrId appears in the selection + * (across all rounds) + **/ + public int multiplicity(final Long cvrId) { + return Collections.frequency(getContestCVRIds(), cvrId); + } + + /** + * Records the specified discrepancy. If the discrepancy is for this Contest + * but from a CVR/ballot that was not selected for this Contest (selected for + * another Contest), is does not contribute to the counts and calculations. It + * is still recorded, though, for informational purposes. The valid range is + * -2 .. 2: -2 and -1 are understatements, 0 is a discrepancy that doesn't + * affect the RLA calculations, and 1 and 2 are overstatements). + * + * @param the_record The CVRAuditInfo record that generated the discrepancy. + * @param the_type The type of discrepancy to add. + * @exception IllegalArgumentException if an invalid discrepancy type is + * specified. + */ + @SuppressWarnings("checkstyle:magicnumber") + public void recordDiscrepancy(final CVRAuditInfo the_record, + final int the_type) { + // we never trigger an estimated recalculate here; it is + // triggered by signalBallotAudited() regardless of whether there is + // a discrepancy or not + + if (isCovering(the_record.cvr().id())) { + switch (the_type) { + case -2: + my_two_vote_under_count = my_two_vote_under_count + 1; + my_optimistic_recalculate_needed = true; + break; + + case -1: + my_one_vote_under_count = my_one_vote_under_count + 1; + my_optimistic_recalculate_needed = true; + break; + + case 0: + my_other_count = my_other_count + 1; + // no optimistic recalculate needed + break; + + case 1: + my_one_vote_over_count = my_one_vote_over_count + 1; + my_optimistic_recalculate_needed = true; + break; + + case 2: + my_two_vote_over_count = my_two_vote_over_count + 1; + my_optimistic_recalculate_needed = true; + break; + + default: + throw new IllegalArgumentException("invalid discrepancy type: " + the_type); + } + } + + LOGGER.info(String.format("[recordDiscrepancy type=%s, record=%s]", + the_type, the_record)); + my_discrepancies.put(the_record, the_type); + } + + /** + * get the discrepancy value that was recorded for this + * ComparisonAudit(contest) on the given CVRAuditInfo(ballot). used for + * reporting. + **/ + public Integer getDiscrepancy(final CVRAuditInfo cai) { + return my_discrepancies.get(cai); + } + + /** + * Removes the specified over/understatement (the valid range is -2 .. 2: + * -2 and -1 are understatements, 0 is a discrepancy that doesn't affect the + * RLA calculations, and 1 and 2 are overstatements). This is typically done + * when a new interpretation is submitted for a ballot that had already been + * interpreted. + * + * @param the_record The CVRAuditInfo record that generated the discrepancy. + * @param the_type The type of discrepancy to remove. + * @exception IllegalArgumentException if an invalid discrepancy type is + * specified. + */ + @SuppressWarnings("checkstyle:magicnumber") + public void removeDiscrepancy(final CVRAuditInfo the_record, final int the_type) { + // we never trigger an estimated recalculate here; it is + // triggered by signalBallotAudited() regardless of whether there is + // a discrepancy or not + switch (the_type) { + case -2: + my_two_vote_under_count = my_two_vote_under_count - 1; + my_optimistic_recalculate_needed = true; + break; + + case -1: + my_one_vote_under_count = my_one_vote_under_count - 1; + my_optimistic_recalculate_needed = true; + break; + + case 0: + my_other_count = my_other_count - 1; + // no recalculate needed + break; + + case 1: + my_one_vote_over_count = my_one_vote_over_count - 1; + my_optimistic_recalculate_needed = true; + break; + + case 2: + my_two_vote_over_count = my_two_vote_over_count - 1; + my_optimistic_recalculate_needed = true; + break; + + default: + throw new IllegalArgumentException("invalid discrepancy type: " + the_type); + } + + my_discrepancies.remove(the_record); + } + + /** + * Returns the count of the specified type of discrepancy. -2 and -1 represent + * understatements, 0 represents a discrepancy that doesn't affect the RLA + * calculations, and 1 and 2 represent overstatements. + * + * @param the_type The type of discrepancy. + * @exception IllegalArgumentException if an invalid discrepancy type is + * specified. + */ + @SuppressWarnings("checkstyle:magicnumber") + public int discrepancyCount(final int the_type) { + final int result; + + switch (the_type) { + case -2: + result = my_two_vote_under_count; + break; + + case -1: + result = my_one_vote_under_count; + break; + + case 0: + result = my_other_count; + break; + + case 1: + result = my_one_vote_over_count; + break; + + case 2: + result = my_two_vote_over_count; + break; + + default: + throw new IllegalArgumentException("invalid discrepancy type: " + the_type); + } + + return result; + } + + /** + * Computes the over/understatement represented by the CVR/ACVR pair stored in + * the specified CVRAuditInfo. This method returns an optional int that, if + * present, indicates a discrepancy. There are 5 possible types of + * discrepancy: -1 and -2 indicate 1- and 2-vote understatements; 1 and 2 + * indicate 1- and 2- vote overstatements; and 0 indicates a discrepancy that + * does not count as either an under- or overstatement for the RLA algorithm, + * but nonetheless indicates a difference between ballot interpretations. + * + * @param the_info The CVRAuditInfo. + * @return an optional int that is present if there is a discrepancy and absent + * otherwise. + */ + public OptionalInt computeDiscrepancy(final CVRAuditInfo the_info) { + if (the_info.acvr() == null || the_info.cvr() == null) { + throw new IllegalArgumentException("null CVR or ACVR in pair " + the_info); + } else { + return computeDiscrepancy(the_info.cvr(), the_info.acvr()); + } + } + + /** + * Computes the over/understatement represented by the specified CVR and ACVR. + * This method returns an optional int that, if present, indicates a discrepancy. + * There are 5 possible types of discrepancy: -1 and -2 indicate 1- and 2-vote + * understatements; 1 and 2 indicate 1- and 2- vote overstatements; and 0 + * indicates a discrepancy that does not count as either an under- or + * overstatement for the RLA algorithm, but nonetheless indicates a difference + * between ballot interpretations. + * + * @param cvr The CVR that the machine saw + * @param auditedCVR The ACVR that the human audit board saw + * @return an optional int that is present if there is a discrepancy and absent + * otherwise. + */ + @SuppressWarnings("checkstyle:magicnumber") + // FIXME Should we point to the ContestResult instead? + public OptionalInt computeDiscrepancy(final CastVoteRecord cvr, + final CastVoteRecord auditedCVR) { + OptionalInt result = OptionalInt.empty(); + + // FIXME this needs to get this stuff from the ContestResult + // - a CastVoteRecord belongs to a county. + // - a CVRContestInfo belongs to a Contest, which belongs to a county. + // - should we change the CVRContestInfo to belong to a ContestResult instead? + // + // The CVRContestInfo has teh list of choices. we need this for + // winners and loser of the contest......BUT the ContestResult also + // has a set of winners and losers, which is now the MOST ACCURATE + // version of this, since we're now out of the county context... + final Optional cvr_info = cvr.contestInfoForContestResult(my_contest_result); + final Optional acvr_info = auditedCVR.contestInfoForContestResult(my_contest_result); + + if (auditedCVR.recordType() == RecordType.PHANTOM_BALLOT) { + if (cvr_info.isPresent()) { + result = OptionalInt.of(computePhantomBallotDiscrepancy(cvr_info.get(), my_contest_result)); + } else { + //not sure why exactly, but that is what computePhantomBallotDiscrepancy + //returns if winner_votes is empty, which it is, in this case, if it is + //not present + result = OptionalInt.of(1); + } + } else if (cvr.recordType() == RecordType.PHANTOM_RECORD){ + // similar to the phantom ballot, we use the worst case scenario, a 2-vote + // overstatement, except here, we don't have a CVR to check anything on. + result = OptionalInt.of(2); + } else if (cvr_info.isPresent() && acvr_info.isPresent()) { + if (acvr_info.get().consensus() == ConsensusValue.NO) { + // a lack of consensus for this contest is treated + // identically to a phantom ballot + result = OptionalInt.of(computePhantomBallotDiscrepancy(cvr_info.get(), my_contest_result)); + } else { + result = computeAuditedBallotDiscrepancy(cvr_info.get(), acvr_info.get()); + } + } + + return result; + } + + /** + * Computes the discrepancy between two ballots. This method returns an optional + * int that, if present, indicates a discrepancy. There are 5 possible types of + * discrepancy: -1 and -2 indicate 1- and 2-vote understatements; 1 and 2 indicate + * 1- and 2- vote overstatements; and 0 indicates a discrepancy that does not + * count as either an under- or overstatement for the RLA algorithm, but + * nonetheless indicates a difference between ballot interpretations. + * + * @param the_cvr_info The CVR info. + * @param the_acvr_info The ACVR info. + * @return an optional int that is present if there is a discrepancy and absent + * otherwise. + */ + @SuppressWarnings({"PMD.ModifiedCyclomaticComplexity", "PMD.StdCyclomaticComplexity", + "PMD.NPathComplexity", "PMD.ExcessiveMethodLength", + "checkstyle:methodlength"}) + + private OptionalInt computeAuditedBallotDiscrepancy(final CVRContestInfo the_cvr_info, + final CVRContestInfo the_acvr_info) { + // Check for overvotes. + // + // Overvotes are represented, perhaps confusingly, in the CVR as "all + // zeroes" for the given contest - it will look indistinguishable from a + // contest in which no selections were made. We therefore have to check if + // the number of selections the audit board found is less than or equal to + // the allowed votes for the given contest. If it is, then the audit board + // found a valid selection and we can proceed with the rest of the math as + // usual. If not, then the audit board recorded an overvote which we must + // now make match the way the CVR format records overvotes: we must record + // *no* selections. The code below does that by excluding the selections + // submitted by the audit board. + // + // If the CVR does show an overvote (no selections counted) then our + // zero-selection ACVR will match it and we will find no discrepancies. If, + // however, the CVR *did* show a selection but the audit board recorded an + // overvote, then we will be able to calculate the discrepancy - the CVR + // will have a choice (or choices) marked as selected, but the ACVR will + // not. The converse is also true: if the CVR shows an overvote but the + // audit board records a valid selection, we will calculate an expected + // discrepancy. + final Set acvr_choices = new HashSet<>(); + if (the_acvr_info.choices().size() <= my_contest_result.winnersAllowed()) { + acvr_choices.addAll(the_acvr_info.choices()); + } + + // avoid linear searches on CVR choices + final Set cvr_choices = new HashSet<>(the_cvr_info.choices()); + // if the choices in the CVR and ACVR are identical now, we can simply return the + // fact that there's no discrepancy + if (cvr_choices.equals(acvr_choices)) { + return OptionalInt.empty(); + } + + // we want to get the maximum pairwise update delta, because that's the "worst" + // change in a pairwise margin, and the discrepancy we record; we start with + // Integer.MIN_VALUE so our maximization algorithm works. it is also the case + // that _every_ pairwise margin must be increased for an understatement to be + // reported + + int raw_result = Integer.MIN_VALUE; + + boolean possible_understatement = true; + // FIXME my_contest_result is global to this object. I'd rather it + // be an argument to this function. + for (final String winner : my_contest_result.getWinners()) { + final int winner_change; + if (!cvr_choices.contains(winner) && acvr_choices.contains(winner)) { + // this winner gained a vote + winner_change = 1; + } else if (cvr_choices.contains(winner) && !acvr_choices.contains(winner)) { + // this winner lost a vote + winner_change = -1; + } else { + // this winner's votes didn't change + winner_change = 0; + } + if (my_contest_result.getLosers().isEmpty()) { + // if there are no losers, we'll just negate this number - even though in + // real life, we wouldn't be auditing the contest at all + raw_result = Math.max(raw_result, -winner_change); + } else { + for (final String loser : my_contest_result.getLosers()) { + final int loser_change; + if (!cvr_choices.contains(loser) && acvr_choices.contains(loser)) { + // this loser gained a vote + loser_change = 1; + } else if (cvr_choices.contains(loser) && !acvr_choices.contains(loser)) { + // this loser lost a vote + loser_change = -1; + } else { + // this loser's votes didn't change + loser_change = 0; + } + // the discrepancy is the loser change minus the winner change (i.e., if this + // loser lost a vote (-1) and this winner gained a vote (1), that's a 2-vote + // understatement (-1 - 1 = -2). Overstatements are worse than understatements, + // as far as the audit is concerned, so we keep the highest discrepancy + final int discrepancy = loser_change - winner_change; + + // taking the max here does not cause a loss of information even if the + // discrepancy is 0; if the discrepancy is 0 we can no longer report an + // understatement, and we still know there was a discrepancy because we + // didn't short circuit earlier + raw_result = Math.max(raw_result, discrepancy); + + // if this discrepancy indicates a narrowing of, or no change in, this pairwise + // margin, then an understatement is no longer possible because that would require + // widening _every_ pairwise margin + if (discrepancy >= 0) { + possible_understatement = false; + } + } + } + } + + if (raw_result == Integer.MIN_VALUE) { + // this should only be possible if something went horribly wrong (like the contest + // has no winners) + throw new IllegalStateException("unable to compute discrepancy in contest " + + contestResult().getContestName()); + } + + final OptionalInt result; + + if (possible_understatement) { + // we return the raw result unmodified + result = OptionalInt.of(raw_result); + } else { + // we return the raw result with a floor of 0, because we can't report an + // understatement + result = OptionalInt.of(Math.max(0, raw_result)); + } + + return result; + } + + /** + * Computes the discrepancy between a phantom ballot and the specified + * CVRContestInfo. + * @return The number of discrepancies + */ + private Integer computePhantomBallotDiscrepancy(final CVRContestInfo cvrInfo, + final ContestResult contestResult) { + int result = 2; + // the second predicate means "no contest winners had votes on the + // original CVR" + final Set winner_votes = new HashSet<>(cvrInfo.choices()); + winner_votes.removeAll(contestResult.getLosers()); + if (winner_votes.isEmpty()) { + result = 1; + } + return result; + } + + /** + * a good idea + */ + @Override + public String toString() { + return String.format("[ComparisonAudit for %s: counties=%s, auditedSampleCount=%d, overstatements=%f," + + " contestResult.contestCvrIds=%s, status=%s, reason=%s]", + this.contestResult().getContestName(), + this.contestResult().getCounties(), + this.getAuditedSampleCount(), + this.getOverstatements(), + this.getContestCVRIds(), + my_audit_status, + this.auditReason()); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/Contest.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/Contest.java new file mode 100644 index 00000000..e0315cf8 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/Contest.java @@ -0,0 +1,320 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 25, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joey Dodds + * @model_review Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.model; + +import static us.freeandfair.corla.util.EqualsHashcodeHelper.*; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import javax.persistence.Cacheable; +import javax.persistence.CollectionTable; +import javax.persistence.Column; +import javax.persistence.ElementCollection; +import javax.persistence.Entity; +import javax.persistence.FetchType; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Index; +import javax.persistence.JoinColumn; +import javax.persistence.ManyToOne; +import javax.persistence.OrderColumn; +import javax.persistence.Table; +import javax.persistence.UniqueConstraint; +import javax.persistence.Version; + +import com.google.gson.annotations.JsonAdapter; + +import us.freeandfair.corla.json.ContestJsonAdapter; +import us.freeandfair.corla.persistence.PersistentEntity; + +/** + * The definition of a contest; comprises a contest name and a set of + * possible choices. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@Entity +@Cacheable(true) +@Table(name = "contest", + uniqueConstraints = { + @UniqueConstraint(columnNames = {"name", "county_id", "description", "votes_allowed"}) }, + indexes = { @Index(name = "idx_contest_name", columnList = "name"), + @Index(name = "idx_contest_name_county_description_votes_allowed", + columnList = "name, county_id, description, votes_allowed") }) +@JsonAdapter(ContestJsonAdapter.class) +//this class has many fields that would normally be declared final, but +//cannot be for compatibility with Hibernate and JPA. +@SuppressWarnings("PMD.ImmutableField") +public class Contest implements PersistentEntity, Serializable { + /** + * The serialVersionUID. + */ + private static final long serialVersionUID = 1L; + + /** + * The ID number. + */ + @Id + @Column(updatable = false, nullable = false) + @GeneratedValue(strategy = GenerationType.SEQUENCE) + private Long my_id; + + /** + * The version (for optimistic locking). + */ + @Version + private Long my_version; + + /** + * The contest name. + */ + @Column(name = "name", nullable = false) + private String my_name; + + /** + * The county to which this contest result set belongs. + */ + @ManyToOne(optional = false, fetch = FetchType.LAZY) + @JoinColumn + private County my_county; + + /** + * The contest description. + */ + @Column(name = "description", updatable = false, nullable = false) + private String my_description; + + /** + * The contest choices. + */ + @ElementCollection(fetch = FetchType.EAGER) + @OrderColumn(name = "index") + @CollectionTable(name = "contest_choice", + uniqueConstraints= @UniqueConstraint(columnNames={"contest_id","my_name"}), + joinColumns = @JoinColumn(name = "contest_id", + referencedColumnName = "my_id")) + private List my_choices = new ArrayList<>(); + + /** + * The maximum number of votes that can be made in this contest. + */ + @Column(name = "votes_allowed", updatable = false, nullable = false) + private Integer my_votes_allowed; + + /** + * The maximum number of winners in this contest. + */ + @Column(name = "winners_allowed", updatable = false, nullable = false) + private Integer my_winners_allowed; + + /** + * The import sequence number. + */ + @Column(updatable = false, nullable = false) + private Integer my_sequence_number; + + /** + * Constructs an empty contest, solely for persistence. + */ + public Contest() { + super(); + // default values for everything + } + + /** + * Constructs a contest with the specified parameters. + * + * @param the_name The contest name. + * @param the_county The county for this contest. + * @param the_description The contest description. + * @param the_choices The set of contest choices. + * @param the_votes_allowed The maximum number of votes that can + * be made in this contest. + * @param the_winners_allowed The maximum number of winners for + * this contest. + * @param the_sequence_number The sequence number. + */ + //@ requires 1 <= the_votes_allowed; + //@ requires the_votes_allowed <= the_choices.size(); + public Contest(final String the_name, final County the_county, + final String the_description, final List the_choices, + final int the_votes_allowed, final int the_winners_allowed, + final int the_sequence_number) { + super(); + my_name = the_name; + my_county = the_county; + my_description = the_description; + my_choices.addAll(the_choices); + my_votes_allowed = the_votes_allowed; + my_winners_allowed = the_winners_allowed; + my_sequence_number = the_sequence_number; + } + + /** + * {@inheritDoc} + */ + @Override + public Long id() { + return my_id; + } + + /** + * {@inheritDoc} + */ + @Override + public void setID(final Long the_id) { + my_id = the_id; + } + + /** + * {@inheritDoc} + */ + @Override + public Long version() { + return my_version; + } + + /** + * @return the contest name. + */ + public String name() { + return my_name; + } + + /** set the name **/ + public void setName(final String name) { + this.my_name = name; + } + + /** + * @return the contest description. + */ + public String description() { + return my_description; + } + + /** + * @return the county ID. + */ + public County county() { + return my_county; + } + + /** + * Checks to see if the specified choice is valid for this contest. + * + * @param the_choice The choice. + * @return true if the choice is valid, false otherwise. + */ + public boolean isValidChoice(final String the_choice) { + for (final Choice c : my_choices) { + if (c.name().equals(the_choice)) { + return true; + } + } + return false; + } + + /** + * Change a choice name as part of Canonicalization. + */ + public void updateChoiceName(final String oldName, + final String newName) { + for (final Choice choice : my_choices) { + if (choice.name().equals(oldName)) { + choice.setName(newName); + } + } + } + + /** + * @return the contest choices. + */ + public List choices() { + return Collections.unmodifiableList(my_choices); + } + + /** + * @return the maximum number of votes that can be made in this contest. + */ + public Integer votesAllowed() { + return my_votes_allowed; + } + + /** + * @return the maximum number of winners in this contest. + */ + public Integer winnersAllowed() { + return my_winners_allowed; + } + + /** + * @return the sequence number of this contest. + */ + public Integer sequenceNumber() { + return my_sequence_number; + } + + /** + * @return a String representation of this contest. + */ + @Override + public String toString() { + return "Contest [name=" + my_name + ", description=" + + my_description + ", choices=" + choices() + + ", votes_allowed=" + my_votes_allowed + "]"; + } + + public String shortToString() { + return "Contest [name=" + my_name + ", choices=" + + choices().stream().map(c->c.shortToString()).collect(Collectors.joining()) + "]"; + } + + + /** + * Compare this object with another for equivalence. + * + * @param the_other The other object. + * @return true if the objects are equivalent, false otherwise. + */ + @Override + public boolean equals(final Object the_other) { + boolean result = true; + if (the_other instanceof Contest) { + final Contest other_contest = (Contest) the_other; + result &= nullableEquals(other_contest.name(), name()); + result &= nullableEquals(other_contest.county(), county()); + result &= nullableEquals(other_contest.description(), description()); + result &= nullableEquals(other_contest.choices(), choices()); + result &= nullableEquals(other_contest.votesAllowed(), votesAllowed()); + result &= nullableEquals(other_contest.sequenceNumber(), sequenceNumber()); + } else { + result = false; + } + return result; + } + + /** + * @return a hash code for this object. + */ + @Override + public int hashCode() { + return nullableHashCode(name().hashCode()); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/ContestResult.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/ContestResult.java new file mode 100644 index 00000000..3e1c76b5 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/ContestResult.java @@ -0,0 +1,462 @@ +package us.freeandfair.corla.model; + +import static us.freeandfair.corla.util.EqualsHashcodeHelper.nullableEquals; + +import java.io.Serializable; +import java.math.BigDecimal; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import javax.persistence.Cacheable; +import javax.persistence.CollectionTable; +import javax.persistence.Column; +import javax.persistence.Convert; +import javax.persistence.ElementCollection; +import javax.persistence.Entity; +import javax.persistence.FetchType; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Index; +import javax.persistence.JoinColumn; +import javax.persistence.JoinTable; +import javax.persistence.ManyToMany; +import javax.persistence.MapKeyColumn; +import javax.persistence.OneToMany; +import javax.persistence.Table; +import javax.persistence.UniqueConstraint; +import javax.persistence.Version; + +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import us.freeandfair.corla.persistence.PersistentEntity; +import us.freeandfair.corla.persistence.StringSetConverter; + +/** + * A class representing the results for a contest across counties. + * A roll-up of CountyContestResults + */ +@Entity +@Cacheable(true) +@Table(name = "contest_result", + uniqueConstraints = { + @UniqueConstraint(columnNames = {"contest_name"}) }, + indexes = { @Index(name = "idx_cr_contest", + columnList = "contest_name", + unique = true)}) +@SuppressWarnings({"PMD.ExcessiveImports"}) // you complain if we import x.y.z.*, so.... +public class ContestResult implements PersistentEntity, Serializable { + + /** + * Class-wide logger + */ + public static final Logger LOGGER = + LogManager.getLogger(ContestResult.class); + + /** + * text + */ + private static final String TEXT = "text"; + + /** + * The "id" string. + */ + private static final String ID = "id"; + + /** + * The "result_id" string. + */ + private static final String RESULT_ID = "result_id"; + + /** + * The serialVersionUID. + */ + private static final long serialVersionUID = 1L; + + /** + * The unique identifier. + */ + @Id + @Column(updatable = false, nullable = false) + @GeneratedValue(strategy = GenerationType.SEQUENCE) + private Long id; + + /** + * The version (for optimistic locking). + */ + @Version + private Long version; + + /** + * The winners allowed. + */ + @Column(name = "winners_allowed") + private Integer winnersAllowed; + + /** + * The set of contest winners. + */ + @Column(name = "winners", columnDefinition = TEXT) + @Convert(converter = StringSetConverter.class) + private final Set winners = new HashSet<>(); + + /** + * The set of contest losers. + */ + @Column(name = "losers", columnDefinition = TEXT) + @Convert(converter = StringSetConverter.class) + private final Set losers = new HashSet<>(); + + /** + * A map from choices to vote totals. + */ + @ElementCollection(fetch = FetchType.EAGER) + @CollectionTable(name = "contest_vote_total", + joinColumns = @JoinColumn(name = RESULT_ID, + referencedColumnName = ID)) + @MapKeyColumn(name = "choice") + @Column(name = "vote_total") + private Map vote_totals = new HashMap<>(); + + /** + * A ContestResult has many counties - supporting auditing multi-county + * contests. Counties have many ContestResults (and many Contests) + **/ + @ManyToMany() + @JoinTable(name = "counties_to_contest_results", + joinColumns = { @JoinColumn(name = "contest_result_id") }, + inverseJoinColumns = { @JoinColumn(name = "county_id") }) + private final Set counties = new HashSet<>(); + + + /** + * A ContestResult has many Contests through "contests_to_contest_results" + * Contests are many in the db because each county has their own, just 'cause + */ + @OneToMany() + @JoinTable(name = "contests_to_contest_results", + joinColumns = { @JoinColumn(name = "contest_result_id") }, + inverseJoinColumns = { @JoinColumn(name = "contest_id") }) + private final Set contests = new HashSet<>(); + + /** + * The contest name. + */ + @Column(name = "contest_name", nullable = false) + private String contestName; + + /** + * The margin divided by total number of ballots cast. + */ + @Column(name = "diluted_margin") + private BigDecimal dilutedMargin; + + /** + * The smallest margin between any winner and loser of the contest + */ + @Column(name = "min_margin") + private Integer minMargin; + + /** + * The largest margin between any winner and loser of the contest. + */ + @Column(name = "max_margin") + private Integer maxMargin; + + /** + * The number of ballots cast for this contest + */ + @Column(name = "ballot_count") + private Long ballotCount; + + /** + * AuditReason + */ + @Column(name = "audit_reason") + private AuditReason auditReason; + + /** + * Constructs a new empty ContestResult (solely for persistence). + */ + public ContestResult() { + super(); + } + + /** + * Constructs a new ContestResult with the specified contestName. The + * contestName is what links Contests together (along with one + * ContestResult). + * + * @param contestName The contest. + */ + public ContestResult(final String contestName) { + super(); + this.contestName = contestName; + } + + /** + * {@inheritDoc} + */ + @Override + public Long id() { + return id; + } + + /** + * {@inheritDoc} + */ + @Override + public void setID(final Long the_id) { + this.id = the_id; + } + + /** + * {@inheritDoc} + */ + @Override + public Long version() { + return version; + } + + /** + * @return the contest name. + */ + public String getContestName() { + return this.contestName; + } + + /** + * @return the AuditReason. + */ + public AuditReason getAuditReason() { + return this.auditReason; + } + + /** set it **/ + public void setAuditReason(final AuditReason auditReason) { + this.auditReason = auditReason; + } + + /** + * set the set of counties + */ + public boolean addCounties(final Set cs) { + return this.counties.addAll(cs); + } + + /** + * @return the counties related to this contestresult. + */ + public Set getCounties() { + return Collections.unmodifiableSet(this.counties); + } + + /** + * set the set of contests + */ + public boolean addContests(final Set cs) { + return this.contests.addAll(cs); + } + + /** + * @return the contests related to this ContestResult + * (should be contests that have the same name as this.contestName) + **/ + public Set getContests() { + return Collections.unmodifiableSet(this.contests); + } + + /** + * getter + */ + public int winnersAllowed() { + return this.winnersAllowed; + } + + /** + * setter + */ + public void setWinnersAllowed(final int n) { + this.winnersAllowed = n; + } + + /** + * @param county the county owning the contest you want + * @return the contest belonging to county + */ + public Contest contestFor(final County county) { + final Optional contestMaybe = getContests().stream() + .filter(c -> c.county().id().equals(county.id())) + .findFirst(); // should only be one? + + if (contestMaybe.isPresent()) { + return contestMaybe.get(); + } else { + return null; + } + } + + /** + * @param winners a set of the choices that won the contest + */ + public void setWinners(final Set winners) { + this.winners.clear(); + this.winners.addAll(winners); + } + + /** + * @return the winners for thie ContestResult. + */ + public Set getWinners() { + return Collections.unmodifiableSet(this.winners); + } + + /** + * @param losers a set of the choices that did not win the contest + */ + public void setLosers(final Set losers) { + this.losers.clear(); + this.losers.addAll(losers); + } + + /** + * @return the losers for this ContestResult. + */ + public Set getLosers() { + return Collections.unmodifiableSet(this.losers); + } + + /** + * @return a map from choices to vote totals. + */ + public Map getVoteTotals() { + return Collections.unmodifiableMap(this.vote_totals); + } + + /** data access helper **/ + public Integer totalVotes() { + return getVoteTotals().values().stream().reduce(0, Integer::sum); + } + + /** + * @param voteTotals a map from choices to vote totals. + */ + public void setVoteTotals(final Map voteTotals) { + this.vote_totals = voteTotals; + } + + /** + * set dilutedMargin. + */ + public void setDilutedMargin(final BigDecimal dilutedMargin) { + this.dilutedMargin = dilutedMargin; + } + + /** + * The diluted margin (μ) of this ContestResult + */ + public BigDecimal getDilutedMargin() { + return this.dilutedMargin; + } + + /** + * set minMargin. + */ + public void setMinMargin(final Integer minMargin) { + this.minMargin = minMargin; + } + + /** + * The smallest margin between any winner and loser of the contest + */ + public Integer getMinMargin() { + return this.minMargin; + } + + /** + * set maxMargin. + */ + public void setMaxMargin(final Integer maxMargin) { + this.maxMargin = maxMargin; + } + + /** + * The largest margin between any winner and loser of the contest + */ + public Integer getMaxMargin() { + return this.maxMargin; + } + + /** + * set ballotCount + */ + public void setBallotCount(final Long n) { + this.ballotCount = n; + } + + /** + * what is the ballotCount? + */ + public Long getBallotCount() { + return this.ballotCount; + } + + /** + * The set of county ids related to this ContestResult + */ + public Set countyIDs() { + return this.getCounties().stream() + .map(x -> x.id()) + .collect(Collectors.toSet()); + } + + /** + * The set of contest ids related to this ContestResult + */ + public Set contestIDs() { + return this.getContests().stream() + .map(x -> x.id()) + .collect(Collectors.toSet()); + } + + /** + * @return a String representation of this contest. + */ + @Override + public String toString() { + return "ContestResult [id=" + id() + " contestName=" + getContestName() + "]"; + } + + /** + * Compare this object with another for equivalence. + * + * @param the_other The other object. + * @return true if the objects are equivalent, false otherwise. + */ + @Override + public boolean equals(final Object the_other) { + boolean result = true; + if (the_other instanceof ContestResult) { + final ContestResult other_result = (ContestResult) the_other; + // compare by database ID, since that is the only + // context in which they can reasonably be compared + result &= nullableEquals(other_result.id(), id()); + } else { + result = false; + } + return result; + } + + /** + * @return a hash code for this object. + */ + @Override + public int hashCode() { + return id().hashCode(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/ContestToAudit.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/ContestToAudit.java new file mode 100644 index 00000000..fd438a51 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/ContestToAudit.java @@ -0,0 +1,158 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 10, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.model; + +import static us.freeandfair.corla.util.EqualsHashcodeHelper.*; + +import java.io.Serializable; + +import javax.persistence.Column; +import javax.persistence.Embeddable; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.FetchType; +import javax.persistence.ManyToOne; + +import com.google.gson.annotations.JsonAdapter; + +import us.freeandfair.corla.json.ContestToAuditJsonAdapter; + +/** + * A class representing a contest to audit or hand count. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@Embeddable +//this class has many fields that would normally be declared final, but +//cannot be for compatibility with Hibernate and JPA. +@SuppressWarnings("PMD.ImmutableField") +@JsonAdapter(ContestToAuditJsonAdapter.class) +public class ContestToAudit implements Serializable { + /** + * The serialVersionUID. + */ + private static final long serialVersionUID = 1L; + + /** + * The contest to audit. + */ + @ManyToOne(optional = false, fetch = FetchType.LAZY) + private Contest my_contest; + + /** + * The audit reason. + */ + @Enumerated(EnumType.STRING) + @Column(updatable = false) + private AuditReason my_reason; + + /** + * A value that determines whether to audit or hand count the contest. + */ + @Enumerated(EnumType.STRING) + @Column(updatable = false) + private AuditType my_audit; + + /** + * Constructs an empty ContestToAudit, solely for persistence. + */ + public ContestToAudit() { + super(); + } + + /** + * Constructs a new ContestToAudit. + * + * @param the_contest The contest ID. + * @param the_reason The reason. + * @param the_audit The audit type. + */ + public ContestToAudit(final Contest the_contest, final AuditReason the_reason, + final AuditType the_audit) { + super(); + my_contest = the_contest; + my_reason = the_reason; + my_audit = the_audit; + } + + /** + * @return the contest. + */ + public Contest contest() { + return my_contest; + } + + /** + * @return the reason. + */ + public AuditReason reason() { + return my_reason; + } + + /** + * @return the audit type. + */ + public AuditType audit() { + return my_audit; + } + + /** + * @return boolean + */ + public Boolean isAuditable() { + // should match SelectContestsPageContainer.select + return my_audit != AuditType.HAND_COUNT + && my_audit != AuditType.NOT_AUDITABLE; + } + + /** + * @return a String representation of this contest to audit. + */ + @Override + public String toString() { + Long id = null; + if (my_contest != null) { + id = my_contest.id(); + } + return "ContestToAudit [contest=" + id + ", reason=" + + my_reason + ", audit_type=" + my_audit + "]"; + } + + /** + * Compare this object with another for equivalence. + * + * @param the_other The other object. + * @return true if the objects are equivalent, false otherwise. + */ + @Override + public boolean equals(final Object the_other) { + boolean result = true; + if (the_other instanceof ContestToAudit) { + final ContestToAudit other_cta = (ContestToAudit) the_other; + result &= nullableEquals(other_cta.contest(), contest()); + result &= nullableEquals(other_cta.reason(), reason()); + result &= nullableEquals(other_cta.audit(), audit()); + } else { + result = false; + } + return result; + } + + /** + * @return a hash code for this object. + */ + @Override + public int hashCode() { + return nullableHashCode(contest()); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/County.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/County.java new file mode 100644 index 00000000..a1c6c8fa --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/County.java @@ -0,0 +1,192 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 27, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @model_review Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.model; + +import static us.freeandfair.corla.util.EqualsHashcodeHelper.*; + +import java.io.Serializable; +import java.util.Comparator; + +import javax.persistence.Cacheable; +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Version; + +import org.hibernate.annotations.Immutable; + +import us.freeandfair.corla.persistence.PersistentEntity; + +/** + * A county involved in an audit. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@Entity +@Immutable // this is a Hibernate-specific annotation, but there is no JPA alternative +@Cacheable(true) +@Table(name = "county") +// this class has many fields that would normally be declared final, but +// cannot be for compatibility with Hibernate and JPA. +@SuppressWarnings("PMD.ImmutableField") +public class County implements PersistentEntity, Serializable { + /** + * The serialVersionUID. + */ + private static final long serialVersionUID = 1L; + + /** + * The database, and county, ID. + */ + @Id + private Long my_id; + + /** + * The version (for optimistic locking). + */ + @Version + private Long my_version; + + /** + * The county name. + */ + @Column(nullable = false, updatable = false, unique = true) + private String my_name; + + /** + * Constructs an empty county, solely for persistence. + */ + public County() { + super(); + } + + /** + * Constructs a county with the specified parameters. + * + * @param the_name The county name. + * @param the_identifier The county ID. + * @param the_administrators The administrators. + */ + public County(final String the_name, final Long the_identifier) { + super(); + my_name = the_name; + my_id = the_identifier; + } + + /** + * @return the county name. + */ + public String name() { + return my_name; + } + + /** + * @return the county ID. + */ + @Override + public Long id() { + return my_id; + } + + /** + * @return the version for this county. + */ + @Override + public Long version() { + return my_version; + } + + /** + * Sets the ID of this county. + * + * @param the_id The ID. + */ + @Override + public void setID(final Long the_id) { + my_id = the_id; + } + + /** + * @return a String representation of this contest. + */ + @Override + public String toString() { + return "County [name=" + my_name + ", id=" + + my_id + "]"; + } + + /** + * Compare this object with another for equivalence. + * + * @param the_other The other object. + * @return true if the objects are equivalent, false otherwise. + */ + @Override + public boolean equals(final Object the_other) { + boolean result = true; + if (the_other instanceof County) { + final County other_county = (County) the_other; + result &= nullableEquals(other_county.name(), name()); + } else { + result = false; + } + return result; + } + + /** + * @return a hash code for this object. + */ + @Override + public int hashCode() { + return nullableHashCode(name()); + } + + + /** + * A comparator to sort County objects alphabetically by county name. + */ + @SuppressWarnings("PMD.AtLeastOneConstructor") + public static class NameComparator + implements Serializable, Comparator { + /** + * The serialVersionUID. + */ + private static final long serialVersionUID = 1; + + /** + * Orders two County objects lexicographically by county name. + * + * @param the_first The first response. + * @param the_second The second response. + * @return a positive, negative, or 0 value as the first response is + * greater than, equal to, or less than the second, respectively. + */ + @SuppressWarnings("PMD.ConfusingTernary") + public int compare(final County the_first, + final County the_second) { + final int result; + if (the_first == null && the_second == null) { + result = 0; + } else if (the_first == null || the_first.name() == null) { + result = -1; + } else if (the_second == null || the_second.name() == null) { + result = 1; + } else { + result = the_first.name().compareTo(the_second.name()); + } + return result; + } + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/CountyContestComparisonAudit.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/CountyContestComparisonAudit.java new file mode 100644 index 00000000..fb3af885 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/CountyContestComparisonAudit.java @@ -0,0 +1,947 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 19, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.model; + +import java.math.BigDecimal; +import java.math.MathContext; +import java.math.RoundingMode; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.OptionalInt; +import java.util.Set; + +import javax.persistence.Cacheable; +import javax.persistence.CollectionTable; +import javax.persistence.Column; +import javax.persistence.ElementCollection; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.FetchType; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Index; +import javax.persistence.JoinColumn; +import javax.persistence.JoinTable; +import javax.persistence.ManyToMany; +import javax.persistence.ManyToOne; +import javax.persistence.MapKeyJoinColumn; +import javax.persistence.Table; +import javax.persistence.Version; + +import us.freeandfair.corla.math.Audit; +import us.freeandfair.corla.model.CVRContestInfo.ConsensusValue; +import us.freeandfair.corla.model.CastVoteRecord.RecordType; +import us.freeandfair.corla.persistence.PersistentEntity; + +/** + * A class representing the state of a single audited contest for + * a single county. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@Entity +@Cacheable(true) +@Table(name = "county_contest_comparison_audit", + indexes = { @Index(name = "idx_ccca_dashboard", columnList = "dashboard_id") }) +@SuppressWarnings({"PMD.ImmutableField", "PMD.CyclomaticComplexity", "PMD.GodClass", + "PMD.ModifiedCyclomaticComplexity", "PMD.StdCyclomaticComplexity", "PMD.TooManyFields", + "PMD.TooManyMethods", "PMD.ExcessiveImports"}) +// note: CountyContestComparisionAudit is not serializable because it references +// CountyDashboard, which is not serializable. +public class CountyContestComparisonAudit implements PersistentEntity { + /** + * The database stored precision for decimal types. + */ + public static final int PRECISION = 10; + + /** + * The database stored scale for decimal types. + */ + public static final int SCALE = 8; + + /** + * Gamma, as presented in the literature: + * https://www.stat.berkeley.edu/~stark/Preprints/gentle12.pdf + */ + public static final BigDecimal STARK_GAMMA = BigDecimal.valueOf(1.03905); + + /** + * Gamma, as recommended by Neal McBurnett for use in Colorado. + */ + public static final BigDecimal COLORADO_GAMMA = BigDecimal.valueOf(1.1); + + /** + * Conservative estimate of error rates for one-vote over- and understatements. + */ + public static final BigDecimal CONSERVATIVE_ONES_RATE = BigDecimal.valueOf(0.01); + + /** + * Conservative estimate of error rates for two-vote over- and understatements. + */ + public static final BigDecimal CONSERVATIVE_TWOS_RATE = BigDecimal.valueOf(0.01); + + /** + * Conservative rounding up of 1-vote over/understatements for the initial + * estimate of error rates. + */ + public static final boolean CONSERVATIVE_ROUND_ONES_UP = true; + + /** + * Conservative rounding up of 2-vote over/understatements for the initial + * estimate of error rates. + */ + public static final boolean CONSERVATIVE_ROUND_TWOS_UP = true; + + /** + * The gamma to use. + */ + public static final BigDecimal GAMMA = STARK_GAMMA; + + /** + * The initial estimate of error rates for one-vote over- and understatements. + */ + public static final BigDecimal ONES_RATE = BigDecimal.ZERO; + + /** + * The initial estimate of error rates for two-vote over- and understatements. + */ + public static final BigDecimal TWOS_RATE = BigDecimal.ZERO; + + /** + * The initial rounding up of 1-vote over/understatements. + */ + public static final boolean ROUND_ONES_UP = false; + + /** + * The initial rounding up of 2-vote over/understatements. + */ + public static final boolean ROUND_TWOS_UP = false; + + /** + * The ID number. + */ + @Id + @Column(updatable = false, nullable = false) + @GeneratedValue(strategy = GenerationType.SEQUENCE) + private Long my_id; + + /** + * The version (for optimistic locking). + */ + @Version + private Long my_version; + + /** + * The county dashboard to which this audit state belongs. + */ + @ManyToOne(optional = false, fetch = FetchType.LAZY) + @JoinColumn + private CountyDashboard my_dashboard; + + /** + * The contest to which this audit state belongs. + */ + @ManyToOne(optional = false, fetch = FetchType.LAZY) + @JoinColumn + private Contest my_contest; + + /** + * The contest result for this audit state. + */ + @ManyToOne(optional = false, fetch = FetchType.LAZY) + @JoinColumn + private ContestResult my_contest_result; + + /** + * The reason for this audit. + */ + @Column(updatable = false, nullable = false) + @Enumerated(EnumType.STRING) + private AuditReason my_audit_reason; + + /** + * The status of this audit. + */ + @Column(nullable = false) + @Enumerated(EnumType.STRING) + private AuditStatus my_audit_status = AuditStatus.NOT_STARTED; + + /** + * The gamma. + */ + @Column(updatable = false, nullable = false, + precision = PRECISION, scale = SCALE) + private BigDecimal my_gamma = GAMMA; + + /** + * The risk limit. + */ + @Column(updatable = false, nullable = false, + precision = PRECISION, scale = SCALE) + private BigDecimal diluted_margin = BigDecimal.ONE; + + /** + * The risk limit. + */ + @Column(updatable = false, nullable = false, + precision = PRECISION, scale = SCALE) + private BigDecimal my_risk_limit = BigDecimal.ONE; + + /** + * The number of samples audited. + */ + @Column(nullable = false) + private Integer my_audited_sample_count = 0; + + /** + * The number of samples to audit overall assuming no further overstatements. + */ + @Column(nullable = false) + private Integer my_optimistic_samples_to_audit = 0; + + /** + * The expected number of samples to audit overall assuming overstatements + * continue at the current rate. + */ + @Column(nullable = false) + private Integer my_estimated_samples_to_audit = 0; + + /** + * The number of two-vote understatements recorded so far. + */ + @Column(nullable = false) + private Integer my_two_vote_under_count = 0; + + /** + * The number of one-vote understatements recorded so far. + */ + @Column(nullable = false) + private Integer my_one_vote_under_count = 0; + + /** + * The number of one-vote overstatements recorded so far. + */ + @Column(nullable = false) + private Integer my_one_vote_over_count = 0; + + /** + * The number of two-vote overstatements recorded so far. + */ + @Column(nullable = false) + private Integer my_two_vote_over_count = 0; + + /** + * The number of discrepancies recorded so far that are neither + * understatements nor overstatements. + */ + @Column(nullable = false) + private Integer my_other_count = 0; + + /** + * The number of disagreements. + */ + @Column(nullable = false) + private Integer my_disagreement_count = 0; + + /** + * A flag that indicates whether the optimistic ballots to audit + * estimate needs to be recalculated. + */ + @Column(nullable = false) + private Boolean my_optimistic_recalculate_needed = true; + + /** + * A flag that indicates whether the non-optimistic ballots to + * audit estimate needs to be recalculated + */ + @Column(nullable = false) + private Boolean my_estimated_recalculate_needed = true; + + /** + * A map from CVRAuditInfo objects to their discrepancy values for this + * audited contest. + */ + @ElementCollection + @CollectionTable(name = "county_contest_comparison_audit_discrepancy", + joinColumns = @JoinColumn(name = "county_contest_comparison_audit_id", + referencedColumnName = "my_id")) + @MapKeyJoinColumn(name = "cvr_audit_info_id") + @Column(name = "discrepancy") + private Map my_discrepancies = new HashMap<>(); + + /** + * A map from CVRAuditInfo objects to their discrepancy values for this + * audited contest. + */ + @ManyToMany + @JoinTable(name = "county_contest_comparison_audit_disagreement", + joinColumns = @JoinColumn(name = "county_contest_comparison_audit_id", + referencedColumnName = "my_id"), + inverseJoinColumns = @JoinColumn(name = "cvr_audit_info_id", + referencedColumnName = "my_id")) + private Set my_disagreements = new HashSet<>(); + + /** + * Constructs a new, empty CountyContestAudit (solely for persistence). + */ + public CountyContestComparisonAudit() { + super(); + } + + /** + * Constructs a CountyContestAudit for the specified dashboard, contest result, + * risk limit, and audit reason. + * + * @param cdb The dashboard. + * @param contestResult The contest result. + * @param riskLimit The risk limit. + * @param auditReason The audit reason. + */ + public CountyContestComparisonAudit(final CountyDashboard cdb, + final ContestResult contestResult, + final Contest contest, + final BigDecimal riskLimit, + final BigDecimal dilutedMargin, + final BigDecimal gamma, + final AuditReason auditReason) { + super(); + my_dashboard = cdb; + my_contest_result = contestResult; + my_contest = contest; + my_risk_limit = riskLimit; + this.diluted_margin = dilutedMargin; + my_gamma = gamma; + my_audit_reason = auditReason; + + if (contestResult.getDilutedMargin().equals(BigDecimal.ZERO)) { + // the diluted margin is 0, so this contest is not auditable + my_audit_status = AuditStatus.NOT_AUDITABLE; + } + } + + /** + * {@inheritDoc} + */ + @Override + public Long id() { + return my_id; + } + + /** + * {@inheritDoc} + */ + @Override + public void setID(final Long the_id) { + my_id = the_id; + } + + /** + * {@inheritDoc} + */ + @Override + public Long version() { + return my_version; + } + + /** + * @return the county dashboard associated with this audit. + */ + public CountyDashboard dashboard() { + return my_dashboard; + } + + /** + * @return the contest associated with this audit. + */ + public Contest contest() { + return my_contest; + } + + /** + * @return the contest result associated with this audit. + */ + public ContestResult contestResult() { + return my_contest_result; + } + + /** + * @return the gamma associated with this audit. + */ + public BigDecimal getGamma() { + return my_gamma; + } + + /** + * @return the risk limit associated with this audit. + */ + public BigDecimal getRiskLimit() { + return my_risk_limit; + } + + /** + * @return the risk limit associated with this audit. + */ + public BigDecimal getDilutedMargin() { + return this.diluted_margin; + } + + /** + * @return the audit reason associated with this audit. + */ + public AuditReason auditReason() { + return my_audit_reason; + } + + /** + * @return the audit status associated with this audit. + */ + public AuditStatus auditStatus() { + return my_audit_status; + } + + /** + * Updates the audit status based on the current risk limit. If the audit + * has already been ended or the contest is not auditable, this method has + * no effect on its status. + */ + public void updateAuditStatus() { + if (my_audit_status == AuditStatus.ENDED || + my_audit_status == AuditStatus.NOT_AUDITABLE) { + return; + } + + if (my_optimistic_samples_to_audit - my_audited_sample_count <= 0) { + my_audit_status = AuditStatus.RISK_LIMIT_ACHIEVED; + } else { + // risk limit has not been achieved + // note that it _is_ possible to go from RISK_LIMIT_ACHIEVED to + // IN_PROGRESS if a sample or set of samples is "unaudited" + my_audit_status = AuditStatus.IN_PROGRESS; + } + } + + /** + * Ends this audit; if the audit has already reached its risk limit, + * or the contest is not auditable, this call has no effect on its status. + */ + public void endAudit() { + if (my_audit_status != AuditStatus.RISK_LIMIT_ACHIEVED && + my_audit_status != AuditStatus.NOT_AUDITABLE) { + my_audit_status = AuditStatus.ENDED; + } + } + + /** + * @return the initial expected number of samples to audit. + */ + @SuppressWarnings({"checkstyle:magicnumber", "PMD.AvoidDuplicateLiterals"}) + public int initialSamplesToAudit() { + return computeOptimisticSamplesToAudit(0, 0, 0, 0). + setScale(0, RoundingMode.CEILING).intValue(); + } + + /** + * @return the expected overall number of ballots to audit, assuming no + * further overstatements occur. + */ + public Integer optimisticSamplesToAudit() { + if (my_optimistic_recalculate_needed) { + recalculateSamplesToAudit(); + } + + return my_optimistic_samples_to_audit; + } + + /** + * @return the expected overall number of ballots to audit, assuming + * overstatements continue to occur at the current rate. + */ + public Integer estimatedSamplesToAudit() { + if (my_estimated_recalculate_needed) { + recalculateSamplesToAudit(); + } + return my_estimated_samples_to_audit; + } + + /** + * Recalculates the overall numbers of ballots to audit. + */ + private void recalculateSamplesToAudit() { + if (my_optimistic_recalculate_needed) { + final BigDecimal optimistic = computeOptimisticSamplesToAudit(my_two_vote_under_count, + my_one_vote_under_count, + my_one_vote_over_count, + my_two_vote_over_count); + my_optimistic_samples_to_audit = optimistic.intValue(); + my_optimistic_recalculate_needed = false; + } + + if (my_one_vote_over_count + my_two_vote_over_count == 0) { + my_estimated_samples_to_audit = my_optimistic_samples_to_audit; + } else { + // compute the "fudge factor" for the estimate + final BigDecimal audited_samples = BigDecimal.valueOf(my_dashboard.auditedSampleCount()); + final BigDecimal overstatements = + BigDecimal.valueOf(my_one_vote_over_count + my_two_vote_over_count); + final BigDecimal fudge_factor; + if (audited_samples.equals(BigDecimal.ZERO)) { + fudge_factor = BigDecimal.ONE; + } else { + fudge_factor = + BigDecimal.ONE.add(overstatements.divide(audited_samples, MathContext.DECIMAL128)); + } + final BigDecimal estimated = + BigDecimal.valueOf(my_optimistic_samples_to_audit).multiply(fudge_factor); + my_estimated_samples_to_audit = estimated.setScale(0, RoundingMode.CEILING).intValue(); + } + my_estimated_recalculate_needed = false; + } + + /** + * Computes the expected number of ballots to audit overall given the + * specified numbers of over- and understatements. + * + * @param the_two_under The two-vote understatements. + * @param the_one_under The one-vote understatements. + * @param the_one_over The one-vote overstatements. + * @param the_two_over The two-vote overstatements. + * + * @return the expected number of ballots remaining to audit. + * This is the stopping sample size as defined in the literature: + * https://www.stat.berkeley.edu/~stark/Preprints/gentle12.pdf + */ + private BigDecimal computeOptimisticSamplesToAudit(final int twoUnder, + final int oneUnder, + final int oneOver, + final int twoOver) { + return Audit.optimistic(getRiskLimit(), getDilutedMargin(), getGamma(), + twoUnder, oneUnder, oneOver, twoOver); + } + + /** + * Signals that a sample has been audited. This ensures that estimates + * are recalculated correctly and states are updated. + * + * @param the_count The count of samples that have been audited simultaneously + * (for duplicates). + */ + public void signalSampleAudited(final int the_count) { + my_estimated_recalculate_needed = true; + my_audited_sample_count = my_audited_sample_count + the_count; + + if (my_audit_status != AuditStatus.ENDED && + my_audit_status != AuditStatus.NOT_AUDITABLE) { + my_audit_status = AuditStatus.IN_PROGRESS; + } + } + + /** + * Signals that a sample has been unaudited. This ensures that estimates + * are recalculated correctly and states are updated. + * + * @param the_count The count of samples that have been unaudited simultaneously + * (for duplicates). + */ + public void signalSampleUnaudited(final int the_count) { + my_estimated_recalculate_needed = true; + my_audited_sample_count = my_audited_sample_count - the_count; + + if (my_audit_status != AuditStatus.ENDED && + my_audit_status != AuditStatus.NOT_AUDITABLE) { + my_audit_status = AuditStatus.IN_PROGRESS; + } + } + + /** + * Records a disagreement with the specified CVRAuditInfo. + * + * @param the_record The CVRAuditInfo record that generated the disagreement. + */ + public void recordDisagreement(final CVRAuditInfo the_record) { + my_disagreements.add(the_record); + my_disagreement_count = my_disagreement_count + 1; + } + + /** + * Removes a disagreement with the specified CVRAuditInfo. + * + * @param the_record The CVRAuditInfo record that generated the disagreement. + */ + public void removeDisagreement(final CVRAuditInfo the_record) { + my_disagreements.remove(the_record); + my_disagreement_count = my_disagreement_count - 1; + } + + /** + * @return the disagreement count. + */ + public int disagreementCount() { + return my_disagreement_count; + } + + /** + * Records the specified discrepancy (the valid range is -2 .. 2: -2 and -1 are + * understatements, 0 is a discrepancy that doesn't affect the RLA calculations, + * and 1 and 2 are overstatements). + * + * @param the_record The CVRAuditInfo record that generated the discrepancy. + * @param the_type The type of discrepancy to add. + * @exception IllegalArgumentException if an invalid discrepancy type is + * specified. + */ + @SuppressWarnings("checkstyle:magicnumber") + public void recordDiscrepancy(final CVRAuditInfo the_record, + final int the_type) { + // we never trigger an estimated recalculate here; it is + // triggered by signalBallotAudited() regardless of whether there is + // a discrepancy or not + switch (the_type) { + case -2: + my_two_vote_under_count = my_two_vote_under_count + 1; + my_optimistic_recalculate_needed = true; + break; + + case -1: + my_one_vote_under_count = my_one_vote_under_count + 1; + my_optimistic_recalculate_needed = true; + break; + + case 0: + my_other_count = my_other_count + 1; + // no optimistic recalculate needed + break; + + case 1: + my_one_vote_over_count = my_one_vote_over_count + 1; + my_optimistic_recalculate_needed = true; + break; + + case 2: + my_two_vote_over_count = my_two_vote_over_count + 1; + my_optimistic_recalculate_needed = true; + break; + + default: + throw new IllegalArgumentException("invalid discrepancy type: " + the_type); + } + + my_discrepancies.put(the_record, the_type); + } + + /** + * Removes the specified over/understatement (the valid range is -2 .. 2: + * -2 and -1 are understatements, 0 is a discrepancy that doesn't affect the + * RLA calculations, and 1 and 2 are overstatements). This is typically done + * when a new interpretation is submitted for a ballot that had already been + * interpreted. + * + * @param the_record The CVRAuditInfo record that generated the discrepancy. + * @param the_type The type of discrepancy to remove. + * @exception IllegalArgumentException if an invalid discrepancy type is + * specified. + */ + @SuppressWarnings("checkstyle:magicnumber") + public void removeDiscrepancy(final CVRAuditInfo the_record, final int the_type) { + // we never trigger an estimated recalculate here; it is + // triggered by signalBallotAudited() regardless of whether there is + // a discrepancy or not + switch (the_type) { + case -2: + my_two_vote_under_count = my_two_vote_under_count - 1; + my_optimistic_recalculate_needed = true; + break; + + case -1: + my_one_vote_under_count = my_one_vote_under_count - 1; + my_optimistic_recalculate_needed = true; + break; + + case 0: + my_other_count = my_other_count - 1; + // no recalculate needed + break; + + case 1: + my_one_vote_over_count = my_one_vote_over_count - 1; + my_optimistic_recalculate_needed = true; + break; + + case 2: + my_two_vote_over_count = my_two_vote_over_count - 1; + my_optimistic_recalculate_needed = true; + break; + + default: + throw new IllegalArgumentException("invalid discrepancy type: " + the_type); + } + + my_discrepancies.remove(the_record); + } + + /** + * Returns the count of the specified type of discrepancy. -2 and -1 represent + * understatements, 0 represents a discrepancy that doesn't affect the RLA + * calculations, and 1 and 2 represent overstatements. + * + * @param the_type The type of discrepancy. + * @exception IllegalArgumentException if an invalid discrepancy type is + * specified. + */ + @SuppressWarnings("checkstyle:magicnumber") + public int discrepancyCount(final int the_type) { + final int result; + + switch (the_type) { + case -2: + result = my_two_vote_under_count; + break; + + case -1: + result = my_one_vote_under_count; + break; + + case 0: + result = my_other_count; + break; + + case 1: + result = my_one_vote_over_count; + break; + + case 2: + result = my_two_vote_over_count; + break; + + default: + throw new IllegalArgumentException("invalid discrepancy type: " + the_type); + } + + return result; + } + + /** + * Computes the over/understatement represented by the CVR/ACVR pair stored in + * the specified CVRAuditInfo. This method returns an optional int that, if + * present, indicates a discrepancy. There are 5 possible types of + * discrepancy: -1 and -2 indicate 1- and 2-vote understatements; 1 and 2 + * indicate 1- and 2- vote overstatements; and 0 indicates a discrepancy that + * does not count as either an under- or overstatement for the RLA algorithm, + * but nonetheless indicates a difference between ballot interpretations. + * + * @param the_info The CVRAuditInfo. + * @return an optional int that is present if there is a discrepancy and absent + * otherwise. + */ + public OptionalInt computeDiscrepancy(final CVRAuditInfo the_info) { + if (the_info.acvr() == null || the_info.cvr() == null) { + throw new IllegalArgumentException("null CVR or ACVR in pair " + the_info); + } else { + return computeDiscrepancy(the_info.cvr(), the_info.acvr()); + } + } + + /** + * Computes the over/understatement represented by the specified CVR and ACVR. + * This method returns an optional int that, if present, indicates a discrepancy. + * There are 5 possible types of discrepancy: -1 and -2 indicate 1- and 2-vote + * understatements; 1 and 2 indicate 1- and 2- vote overstatements; and 0 + * indicates a discrepancy that does not count as either an under- or + * overstatement for the RLA algorithm, but nonetheless indicates a difference + * between ballot interpretations. + * + * @param cvr The CVR that the machine saw + * @param auditedCVR The ACVR that the human audit board saw + * @return an optional int that is present if there is a discrepancy and absent + * otherwise. + */ + @SuppressWarnings("checkstyle:magicnumber") + // FIXME Should we point to the ContestResult instead? + public OptionalInt computeDiscrepancy(final CastVoteRecord cvr, + final CastVoteRecord auditedCVR) { + OptionalInt result = OptionalInt.empty(); + final CVRContestInfo cvr_info = cvr.contestInfoForContest(contest()); + final CVRContestInfo acvr_info = auditedCVR.contestInfoForContest(contest()); + + if (auditedCVR.recordType() == RecordType.PHANTOM_BALLOT) { + result = OptionalInt.of(computePhantomBallotDiscrepancy(cvr_info)); + } else if (cvr.recordType() == RecordType.PHANTOM_RECORD){ + // similar to the phantom ballot, we use the worst case scenario, a 2-vote + // overstatement, except here, we don't have a CVR to check anything on. + result = OptionalInt.of(2); + } + else if (cvr_info != null && acvr_info != null) { + if (acvr_info.consensus() == ConsensusValue.NO) { + // a lack of consensus for this contest is treated + // identically to a phantom ballot + result = OptionalInt.of(computePhantomBallotDiscrepancy(cvr_info)); + } else { + result = computeAuditedBallotDiscrepancy(cvr_info, acvr_info); + } + } + + return result; + } + + /** + * Computes the discrepancy between two ballots. This method returns an optional + * int that, if present, indicates a discrepancy. There are 5 possible types of + * discrepancy: -1 and -2 indicate 1- and 2-vote understatements; 1 and 2 indicate + * 1- and 2- vote overstatements; and 0 indicates a discrepancy that does not + * count as either an under- or overstatement for the RLA algorithm, but + * nonetheless indicates a difference between ballot interpretations. + * + * @param the_cvr_info The CVR info. + * @param the_acvr_info The ACVR info. + * @return an optional int that is present if there is a discrepancy and absent + * otherwise. + */ + @SuppressWarnings({"PMD.ModifiedCyclomaticComplexity", "PMD.StdCyclomaticComplexity", + "PMD.NPathComplexity", "PMD.ExcessiveMethodLength", + "checkstyle:methodlength"}) + private OptionalInt computeAuditedBallotDiscrepancy(final CVRContestInfo the_cvr_info, + final CVRContestInfo the_acvr_info) { + // Check for overvotes. + // + // See the ComparisonAudit class for more details. In short, if we have an + // overvoted ACVR we record no selections for that contest, matching the + // CVR format. + final Set acvr_choices = new HashSet<>(); + if (the_acvr_info.choices().size() <= my_contest_result.winnersAllowed()) { + acvr_choices.addAll(the_acvr_info.choices()); + } + + // avoid linear searches on CVR choices + final Set cvr_choices = new HashSet<>(the_cvr_info.choices()); + + // if the choices in the CVR and ACVR are identical now, we can simply return the + // fact that there's no discrepancy + if (cvr_choices.equals(acvr_choices)) { + return OptionalInt.empty(); + } + + // we want to get the maximum pairwise update delta, because that's the "worst" + // change in a pairwise margin, and the discrepancy we record; we start with + // Integer.MIN_VALUE so our maximization algorithm works. it is also the case + // that _every_ pairwise margin must be increased for an understatement to be + // reported + + int raw_result = Integer.MIN_VALUE; + + boolean possible_understatement = true; + + for (final String winner : my_contest_result.getWinners()) { + final int winner_change; + if (!cvr_choices.contains(winner) && acvr_choices.contains(winner)) { + // this winner gained a vote + winner_change = 1; + } else if (cvr_choices.contains(winner) && !acvr_choices.contains(winner)) { + // this winner lost a vote + winner_change = -1; + } else { + // this winner's votes didn't change + winner_change = 0; + } + if (my_contest_result.getLosers().isEmpty()) { + // if there are no losers, we'll just negate this number - even though in + // real life, we wouldn't be auditing the contest at all + raw_result = Math.max(raw_result, -winner_change); + } else { + for (final String loser : my_contest_result.getLosers()) { + final int loser_change; + if (!cvr_choices.contains(loser) && acvr_choices.contains(loser)) { + // this loser gained a vote + loser_change = 1; + } else if (cvr_choices.contains(loser) && !acvr_choices.contains(loser)) { + // this loser lost a vote + loser_change = -1; + } else { + // this loser's votes didn't change + loser_change = 0; + } + // the discrepancy is the loser change minus the winner change (i.e., if this + // loser lost a vote (-1) and this winner gained a vote (1), that's a 2-vote + // understatement (-1 - 1 = -2). Overstatements are worse than understatements, + // as far as the audit is concerned, so we keep the highest discrepancy + final int discrepancy = loser_change - winner_change; + + // taking the max here does not cause a loss of information even if the + // discrepancy is 0; if the discrepancy is 0 we can no longer report an + // understatement, and we still know there was a discrepancy because we + // didn't short circuit earlier + raw_result = Math.max(raw_result, discrepancy); + + // if this discrepancy indicates a narrowing of, or no change in, this pairwise + // margin, then an understatement is no longer possible because that would require + // widening _every_ pairwise margin + if (discrepancy >= 0) { + possible_understatement = false; + } + } + } + } + + if (raw_result == Integer.MIN_VALUE) { + // this should only be possible if something went horribly wrong (like the contest + // has no winners) + throw new IllegalStateException("unable to compute discrepancy in contest " + + contest().name()); + } + + final OptionalInt result; + + if (possible_understatement) { + // we return the raw result unmodified + result = OptionalInt.of(raw_result); + } else { + // we return the raw result with a floor of 0, because we can't report an + // understatement + result = OptionalInt.of(Math.max(0, raw_result)); + } + + return result; + } + + /** + * Computes the discrepancy between a phantom ballot and the specified + * CVRContestInfo. + * + * @param the_info The CVRContestInfo. + * @return the discrepancy. + */ + private Integer computePhantomBallotDiscrepancy(final CVRContestInfo the_info) { + final int result; + + // if the ACVR is a phantom ballot, we need to assume that it was a vote + // for all the losers; so if any winners had votes on the original CVR + // it's a 2-vote overstatement, otherwise a 1-vote overstatement + + if (the_info == null) { + // this contest doesn't appear in the CVR, so we assume the worst + result = 2; + } else { + // this contest does appear in the CVR, so we can actually check + final Set winner_votes = new HashSet<>(the_info.choices()); + winner_votes.removeAll(my_contest_result.getLosers()); + if (winner_votes.isEmpty()) { + result = 1; + } else { + result = 2; + } + } + + return result; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/CountyContestResult.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/CountyContestResult.java new file mode 100644 index 00000000..4ff97a3b --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/CountyContestResult.java @@ -0,0 +1,624 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 19, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.model; + +import static us.freeandfair.corla.util.EqualsHashcodeHelper.nullableEquals; + +import java.io.Serializable; +import java.math.BigDecimal; +import java.math.MathContext; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.OptionalInt; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; + +import javax.persistence.Cacheable; +import javax.persistence.CollectionTable; +import javax.persistence.Column; +import javax.persistence.Convert; +import javax.persistence.ElementCollection; +import javax.persistence.Entity; +import javax.persistence.FetchType; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Index; +import javax.persistence.JoinColumn; +import javax.persistence.ManyToOne; +import javax.persistence.MapKeyColumn; +import javax.persistence.Table; +import javax.persistence.UniqueConstraint; +import javax.persistence.Version; + +import us.freeandfair.corla.persistence.PersistentEntity; +import us.freeandfair.corla.persistence.StringSetConverter; +import us.freeandfair.corla.util.SuppressFBWarnings; + +/** + * A class representing the results for a single contest for a single county. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@Entity +@Cacheable(true) +@Table(name = "county_contest_result", + uniqueConstraints = { + @UniqueConstraint(columnNames = {"county_id", "contest_id"}) }, + indexes = { @Index(name = "idx_ccr_county_contest", + columnList = "county_id, contest_id", + unique = true), + @Index(name = "idx_ccr_county", columnList = "county_id"), + @Index(name = "idx_ccr_contest", columnList = "contest_id") }) +@SuppressWarnings({"PMD.TooManyMethods", "PMD.ImmutableField", "PMD.ExcessiveImports", + "PMD.GodClass"}) +public class CountyContestResult implements PersistentEntity, Serializable { + /** + * The "my_id" string. + */ + private static final String MY_ID = "my_id"; + + /** + * The "result_id" string. + */ + private static final String RESULT_ID = "result_id"; + + /** + * The serialVersionUID. + */ + private static final long serialVersionUID = 1L; + + /** + * The ID number. + */ + @Id + @Column(updatable = false, nullable = false) + @GeneratedValue(strategy = GenerationType.SEQUENCE) + private Long my_id; + + /** + * The version (for optimistic locking). + */ + @Version + private Long my_version; + + /** + * The county to which this contest result set belongs. + */ + @ManyToOne(optional = false, fetch = FetchType.LAZY) + @JoinColumn + private County my_county; + + /** + * The contest. + */ + @ManyToOne(optional = false, fetch = FetchType.LAZY) + @JoinColumn + private Contest my_contest; + + /** + * The winners allowed. + */ + @Column(updatable = false, nullable = false) + private Integer my_winners_allowed; + + /** + * The set of contest winners. + */ + @Column(name = "winners", columnDefinition = "text") + @Convert(converter = StringSetConverter.class) + private Set my_winners = new HashSet<>(); + + /** + * The set of contest losers. + */ + @Column(name = "losers", columnDefinition = "text") + @Convert(converter = StringSetConverter.class) + private Set my_losers = new HashSet<>(); + + /** + * A map from choices to vote totals. + */ + @ElementCollection(fetch = FetchType.EAGER) + @CollectionTable(name = "county_contest_vote_total", + joinColumns = @JoinColumn(name = RESULT_ID, + referencedColumnName = MY_ID)) + @MapKeyColumn(name = "choice") + @Column(name = "vote_total") + private Map my_vote_totals = new HashMap<>(); + + /** + * The minimum pairwise margin between a winner and a loser. + */ + private Integer my_min_margin; + + /** + * The maximum pairwise margin between a winner and a loser. + */ + private Integer my_max_margin; + + /** + * The total number of ballots cast in this county. + */ + private Integer my_county_ballot_count = 0; + + /** + * The total number of ballots cast in this county that contain this contest. + */ + private Integer my_contest_ballot_count = 0; + + /** + * Constructs a new empty CountyContestResult (solely for persistence). + */ + public CountyContestResult() { + super(); + } + + /** + * Constructs a new CountyContestResult for the specified county ID and + * contest. + * + * @param the_county The county. + * @param the_contest The contest. + */ + public CountyContestResult(final County the_county, final Contest the_contest) { + super(); + my_county = the_county; + my_contest = the_contest; + my_winners_allowed = the_contest.winnersAllowed(); + for (final Choice c : the_contest.choices()) { + if (!c.fictitious()) { + my_vote_totals.put(c.name(), 0); + } + } + } + + /** + * {@inheritDoc} + */ + @Override + public Long id() { + return my_id; + } + + /** + * {@inheritDoc} + */ + @Override + public void setID(final Long the_id) { + my_id = the_id; + } + + /** + * {@inheritDoc} + */ + @Override + public Long version() { + return my_version; + } + + /** + * @return the county for this CountyContestResult. + */ + public County county() { + return my_county; + } + + /** + * @return the contest for this CountyContestResult. + */ + public Contest contest() { + return my_contest; + } + + /** + * @return the winners for thie CountyContestResult. + */ + public Set winners() { + return Collections.unmodifiableSet(my_winners); + } + + /** + * @return the losers for this CountyContestResult. + */ + public Set losers() { + return Collections.unmodifiableSet(my_losers); + } + + /** + * @return a map from choices to vote totals. + */ + public Map voteTotals() { + return Collections.unmodifiableMap(my_vote_totals); + } + + /** + * @return a list of the choices in descending order by number of votes + * received. + */ + public List rankedChoices() { + final List result = new ArrayList(); + + final SortedMap> sorted_totals = + new TreeMap>(new ReverseIntegerComparator()); + for (final Entry e : my_vote_totals.entrySet()) { + final List list = sorted_totals.get(e.getValue()); + if (list == null) { + sorted_totals.put(e.getValue(), new ArrayList<>()); + } + sorted_totals.get(e.getValue()).add(e.getKey()); + } + + final Iterator>> iterator = + sorted_totals.entrySet().iterator(); + while (iterator.hasNext()) { + final Entry> entry = iterator.next(); + result.addAll(entry.getValue()); + } + return result; + } + + /** + * Change a choice name as part of Canonicalization. + */ + public void updateChoiceName(final String oldName, + final String newName) { + final Integer vote_total = my_vote_totals.remove(oldName); + my_vote_totals.put(newName, vote_total); + } + + /** + * Compute the pairwise margin between the specified choices. + * If the first choice has more votes than the second, the + * result will be positive; if the second choie has more + * votes than the first, the result will be negative; if they + * have the same number of votes, the result will be 0. + * + * @param the_first_choice The first choice. + * @param the_second_choice The second choice. + * @return the pairwise margin between the two choices, as + * an OptionalInt (empty if the margin cannot be calculated). + */ + public OptionalInt pairwiseMargin(final String the_first_choice, + final String the_second_choice) { + final Integer first_votes = my_vote_totals.get(the_first_choice); + final Integer second_votes = my_vote_totals.get(the_second_choice); + final OptionalInt result; + + if (first_votes == null || second_votes == null) { + result = OptionalInt.empty(); + } else { + result = OptionalInt.of(first_votes - second_votes); + } + + return result; + } + + /** + * Computes the margin between the specified choice and the next choice. + * If the specified choice is the last choice, or is not a valid choice, + * the margin is empty. + * + * @param the_choice The choice. + * @return the margin. + */ + public OptionalInt marginToNearestLoser(final String the_choice) { + final OptionalInt result; + final List choices = rankedChoices(); + int index = choices.indexOf(the_choice); + + if (index < 0 || index == choices.size() - 1) { + result = OptionalInt.empty(); + } else { + // find the nearest loser + String loser = ""; + index = index + 1; + while (index < choices.size() && !losers().contains(loser)) { + loser = choices.get(index); + index = index + 1; + } + if (losers().contains(loser)) { + result = OptionalInt.of(voteTotals().get(the_choice) - + voteTotals().get(loser)); + } else { + // there was no nearest loser, maybe there are only winners + result = OptionalInt.empty(); + } + } + + return result; + } + + /** + * Computes the diluted margin between the specified choice and the nearest + * loser. If the specified choice is the last choice or is not a valid + * choice, or the margin is undefined, the result is null. + * + * @param the_choice The choice. + * @return the margin. + */ + public BigDecimal countyDilutedMarginToNearestLoser(final String the_choice) { + BigDecimal result = null; + final OptionalInt margin = marginToNearestLoser(the_choice); + + if (margin.isPresent() && my_county_ballot_count > 0) { + result = BigDecimal.valueOf(margin.getAsInt()). + divide(BigDecimal.valueOf(my_county_ballot_count), + MathContext.DECIMAL128); + } + + return result; + } + + /** + * Computes the diluted margin between the specified choice and the nearest + * loser. If the specified choice is the last choice or is not a valid + * choice, or the margin is undefined, the result is null. + * + * @param the_choice The choice. + * @return the margin. + */ + public BigDecimal contestDilutedMarginToNearestLoser(final String the_choice) { + BigDecimal result = null; + final OptionalInt margin = marginToNearestLoser(the_choice); + + if (margin.isPresent() && my_contest_ballot_count > 0) { + result = BigDecimal.valueOf(margin.getAsInt()). + divide(BigDecimal.valueOf(my_contest_ballot_count), + MathContext.DECIMAL128); + } + + return result; + } + + /** + * @return the number of winners allowed in this contest. + */ + public Integer winnersAllowed() { + return my_winners_allowed; + } + + /** + * @return the number of ballots cast in this county that include this contest. + */ + public Integer contestBallotCount() { + return my_contest_ballot_count; + } + + /** + * @return the number of ballots cast in this county. + */ + public Integer countyBallotCount() { + return my_county_ballot_count; + } + + /** + * @return the maximum margin between a winner and a loser. + */ + public Integer maxMargin() { + return my_max_margin; + } + + /** + * @return the minimum margin between a winner and a loser. + */ + public Integer minMargin() { + return my_min_margin; + } + + /** + * @return the county diluted margin for this contest, defined as the + * minimum margin divided by the number of ballots cast in the county. + * @exception IllegalStateException if no ballots have been counted. + */ + public BigDecimal countyDilutedMargin() { + BigDecimal result; + if (my_county_ballot_count > 0) { + result = BigDecimal.valueOf(my_min_margin). + divide(BigDecimal.valueOf(my_county_ballot_count), + MathContext.DECIMAL128); + if (my_losers.isEmpty()) { + // if we only have winners, there is no margin + result = BigDecimal.ONE; + } + + // TODO: how do we handle a tie? + } else { + throw new IllegalStateException("attempted to calculate diluted margin with no ballots"); + } + + return result; + } + + /** + * @return the diluted margin for this contest, defined as the + * minimum margin divided by the number of ballots cast in this county + * that contain this contest. + * @exception IllegalStateException if no ballots have been counted. + */ + public BigDecimal contestDilutedMargin() { + BigDecimal result; + if (my_contest_ballot_count > 0) { + result = BigDecimal.valueOf(my_min_margin). + divide(BigDecimal.valueOf(my_contest_ballot_count), + MathContext.DECIMAL128); + if (my_losers.isEmpty()) { + // if we only have winners, there is no margin + result = BigDecimal.ONE; + } + + // TODO: how do we handle a tie? + } else { + throw new IllegalStateException("attempted to calculate diluted margin with no ballots"); + } + + return result; + } + + /** + * Reset the vote totals and all related data in this CountyContestResult. + */ + public void reset() { + my_winners.clear(); + my_losers.clear(); + for (final String s : my_vote_totals.keySet()) { + my_vote_totals.put(s, 0); + } + updateResults(); + } + + /** + * Update the vote totals using the data from the specified CVR. + * + * @param the_cvr The CVR. + */ + public void addCVR(final CastVoteRecord the_cvr) { + final CVRContestInfo ci = the_cvr.contestInfoForContest(my_contest); + if (ci != null) { + for (final String s : ci.choices()) { + my_vote_totals.put(s, my_vote_totals.get(s) + 1); + } + my_contest_ballot_count = Integer.valueOf(my_contest_ballot_count + 1); + } + my_county_ballot_count = Integer.valueOf(my_county_ballot_count + 1); + } + + /** + * Updates the stored results. + */ + public void updateResults() { + // first, sort the vote totals + final SortedMap> sorted_totals = + new TreeMap>(new ReverseIntegerComparator()); + for (final Entry e : my_vote_totals.entrySet()) { + final List list = sorted_totals.get(e.getValue()); + if (list == null) { + sorted_totals.put(e.getValue(), new ArrayList<>()); + } + sorted_totals.get(e.getValue()).add(e.getKey()); + } + // next, get the winners and losers + final Iterator>> vote_total_iterator = + sorted_totals.entrySet().iterator(); + Entry> entry = null; + while (vote_total_iterator.hasNext() && my_winners.size() < my_winners_allowed) { + entry = vote_total_iterator.next(); + final List choices = entry.getValue(); + if (choices.size() + my_winners.size() <= my_winners_allowed) { + my_winners.addAll(choices); + } else { + // we are arbitrarily making the first choices in the list "winners" and + // the last choices in the list "losers", but since it's a tie, it really + // doesn't matter + final int to_add = my_winners_allowed - my_winners.size(); + my_winners.addAll(choices.subList(0, to_add)); + my_losers.addAll(choices.subList(to_add, choices.size())); + } + } + while (vote_total_iterator.hasNext()) { + // all the other choices count as losers + my_losers.addAll(vote_total_iterator.next().getValue()); + } + + calculateMargins(); + } + + /** + * Calculates all the pairwise margins using the vote totals. + */ + private void calculateMargins() { + my_min_margin = Integer.MAX_VALUE; + my_max_margin = Integer.MIN_VALUE; + for (final String w : my_winners) { + if (my_losers.isEmpty()) { + // this could be either uncontested or tied (I think) and it means that the + // ContestToAudit will have an AuditType of NOT_AUDITABLE + my_min_margin = 0; + my_max_margin = 0; + } else { + for (final String l : my_losers) { + final int margin = my_vote_totals.get(w) - my_vote_totals.get(l); + my_min_margin = Math.min(my_min_margin, margin); + my_max_margin = Math.max(my_max_margin, margin); + } + } + } + } + + /** + * @return a String representation of this contest. + */ + @Override + public String toString() { + return "CountyContestResult [id=" + id() + "]"; + } + + /** + * Compare this object with another for equivalence. + * + * @param the_other The other object. + * @return true if the objects are equivalent, false otherwise. + */ + @Override + public boolean equals(final Object the_other) { + boolean result = true; + if (the_other instanceof CountyContestResult) { + final CountyContestResult other_result = (CountyContestResult) the_other; + // compare by database ID, since that is the only + // context in which they can reasonably be compared + result &= nullableEquals(other_result.id(), id()); + } else { + result = false; + } + return result; + } + + /** + * @return a hash code for this object. + */ + @Override + public int hashCode() { + return id().hashCode(); + } + + /** + * A reverse integer comparator, for sorting lists of integers in reverse. + */ + @SuppressFBWarnings("RV_NEGATING_RESULT_OF_COMPARETO") + public static class ReverseIntegerComparator + implements Comparator, Serializable { + /** + * The serialVersionUID. + */ + private static final long serialVersionUID = 1L; + + /** + * Compares two integers. Returns the negation of the regular integer + * comparator result. + * + * @param the_first The first integer. + * @param the_second The second integer. + */ + public int compare(final Integer the_first, final Integer the_second) { + return -(the_first.compareTo(the_second)); + } + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/CountyDashboard.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/CountyDashboard.java new file mode 100644 index 00000000..ef714954 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/CountyDashboard.java @@ -0,0 +1,1022 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 27, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @model_review Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.model; + +import static us.freeandfair.corla.util.EqualsHashcodeHelper.nullableEquals; + +import java.time.Instant; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.Comparator; +import java.util.stream.Collectors; + +import javax.persistence.AttributeOverride; +import javax.persistence.AttributeOverrides; +import javax.persistence.Cacheable; +import javax.persistence.CollectionTable; +import javax.persistence.Column; +import javax.persistence.Convert; +import javax.persistence.ElementCollection; +import javax.persistence.Embedded; +import javax.persistence.Entity; +import javax.persistence.FetchType; +import javax.persistence.Id; +import javax.persistence.JoinColumn; +import javax.persistence.JoinTable; +import javax.persistence.ManyToMany; +import javax.persistence.MapKeyColumn; +import javax.persistence.OneToOne; +import javax.persistence.OrderColumn; +import javax.persistence.Table; +import javax.persistence.Version; + +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import us.freeandfair.corla.model.ImportStatus.ImportState; + +import us.freeandfair.corla.persistence.AuditSelectionIntegerMapConverter; +import us.freeandfair.corla.persistence.PersistentEntity; +import us.freeandfair.corla.persistence.StringSetConverter; + +/** + * The county dashboard. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@Entity +@Cacheable(true) +@Table(name = "county_dashboard") +@SuppressWarnings({"PMD.ImmutableField", "PMD.TooManyMethods", "PMD.TooManyFields", + "PMD.GodClass", "PMD.ExcessiveImports", "checkstyle:methodcount", + "PMD.ExcessivePublicCount", "PMD.CyclomaticComplexity"}) +// note: county dashboard is not serializable because it contains an uploaded file +public class CountyDashboard implements PersistentEntity { + /** + * Class-wide logger + */ + public static final Logger LOGGER = + LogManager.getLogger(CountyDashboard.class); + + /** + * The "text" constant. + */ + private static final String TEXT = "text"; + + /** + * The minimum number of members on an audit board. + */ + public static final int MIN_AUDIT_BOARD_MEMBERS = 2; + + /** + * The minimum number of members on an audit round sign-off. + */ + public static final int MIN_ROUND_SIGN_OFF_MEMBERS = 2; + + /** + * The "no content" constant. + */ + private static final Integer NO_CONTENT = null; + + /** + * The "index" string. + */ + private static final String INDEX = "index"; + + /** + * The "my_id" string. + */ + private static final String MY_ID = "my_id"; + + /** + * The "dashboard_id" string. + */ + private static final String DASHBOARD_ID = "dashboard_id"; + + /** + * The database ID; this is always the county ID. + */ + @Id + private Long my_id; + + /** + * The county. + */ + @OneToOne(optional = false, fetch = FetchType.LAZY) + @JoinColumn + private County my_county; + + /** + * The version (for optimistic locking). + */ + @Version + private Long my_version; + + /** + * The file containing the most recent set of uploaded CVRs. + */ + @OneToOne(fetch = FetchType.EAGER) + @JoinColumn + private UploadedFile my_cvr_file; + + /** + * The number of CVRs imported. + */ + @Column(nullable = false) + private Integer my_cvrs_imported = 0; + + /** + * The CVR import status. + */ + @Embedded + @AttributeOverrides({ + @AttributeOverride(name = "my_import_state", + column = @Column(name = "cvr_import_state")), + @AttributeOverride(name = "my_error_message", + column = @Column(name = "cvr_import_error_message")), + @AttributeOverride(name = "my_timestamp", + column = @Column(name = "cvr_import_timestamp")) + }) + private ImportStatus my_cvr_import_status = + new ImportStatus(ImportState.NOT_ATTEMPTED, null); + + /** + * The timestamp of the most recent uploaded ballot manifest. + */ + @OneToOne(fetch = FetchType.EAGER) + @JoinColumn + private UploadedFile my_manifest_file; + + /** + * The number of ballots described in the ballot manifest. + */ + @Column(nullable = false) + private Integer my_ballots_in_manifest = 0; + + /** + * The timestamp for the start of the audit. + */ + private Instant my_audit_timestamp; + + /** + * The number of audit boards. + */ + @Column(name="audit_board_count") + private Integer auditBoardCount; + + /** + * The audit boards. + */ + @ElementCollection(fetch = FetchType.LAZY) + @MapKeyColumn(name = INDEX) + @CollectionTable(name = "audit_board", + joinColumns = @JoinColumn(name = DASHBOARD_ID, + referencedColumnName = MY_ID)) + private Map my_audit_boards = new HashMap<>(); + + /** + * The audit rounds. + */ + @ElementCollection(fetch = FetchType.LAZY) + @OrderColumn(name = INDEX) + @CollectionTable(name = "round", + joinColumns = @JoinColumn(name = DASHBOARD_ID, + referencedColumnName = MY_ID)) + private List my_rounds = new ArrayList<>(); + + /** + * The current audit round. + */ + private Integer my_current_round_index; + + /** + * The set of contests that drive our audits. Strings, not "fancy" + * Abstract Data Types + */ + @Column(name = "driving_contests", columnDefinition = TEXT) + @Convert(converter = StringSetConverter.class) + private Set drivingContestNames = new HashSet<>(); + + + /** + * The audit data. + */ + @ManyToMany(fetch = FetchType.LAZY) + @JoinTable(name = "county_dashboard_to_comparison_audit", + joinColumns = { @JoinColumn(name = DASHBOARD_ID, + referencedColumnName = MY_ID) }, + inverseJoinColumns = { @JoinColumn(name = "comparison_audit_id", + referencedColumnName = MY_ID) }) + private Set audits = new HashSet<>(); + + /** + * The audit investigation reports. + */ + @ElementCollection(fetch = FetchType.LAZY) + @OrderColumn(name = INDEX) + @CollectionTable(name = "audit_investigation_report", + joinColumns = @JoinColumn(name = DASHBOARD_ID, + referencedColumnName = MY_ID)) + private List my_investigation_reports = + new ArrayList<>(); + + /** + * The audit interim reports. + */ + @ElementCollection(fetch = FetchType.LAZY) + @OrderColumn(name = INDEX) + @CollectionTable(name = "audit_intermediate_report", + joinColumns = @JoinColumn(name = DASHBOARD_ID, + referencedColumnName = MY_ID)) + private List my_intermediate_reports = + new ArrayList<>(); + + /** + * The number of ballots audited. + */ + @Column(nullable = false) + private Integer my_ballots_audited = 0; + + /** + * The length of the audited prefix of the list of samples to audit; + * equivalent to the index of the CVR currently under audit. + */ + private Integer my_audited_prefix_length; + + /** + * The number of samples that have been audited so far. + */ + private Integer my_audited_sample_count; + + /** + * The number of discrepancies found in the audit so far. + */ + @Column(nullable = false, name = "discrepancies", columnDefinition = "text") + @Convert(converter = AuditSelectionIntegerMapConverter.class) + private Map my_discrepancies = new HashMap<>(); + + /** + * The number of disagreements found in the audit so far. + */ + @Column(nullable = false, name = "disagreements", columnDefinition = "text") + @Convert(converter = AuditSelectionIntegerMapConverter.class) + private Map my_disagreements = new HashMap<>(); + + /** + * Constructs an empty county dashboard, solely for persistence. + */ + public CountyDashboard() { + super(); + } + + /** + * Constructs a new county dashboard for the specified county. + * + * @param the_county The county. + */ + public CountyDashboard(final County the_county) { + super(); + my_county = the_county; + my_id = the_county.id(); + } + + /** + * @return the database ID for this dashboard, which is the same as + * its county ID. + */ + @Override + public Long id() { + return my_id; + } + + /** + * Sets the database ID for this dashboard. This operation is unsupported on + * this class. + * + * @param the_id The ID. + * @exception UnsupportedOperationException always. + */ + @Override + public final void setID(final Long the_id) { + throw new UnsupportedOperationException("setID() not supported on county dashboard"); + } + + /** + * @return the version for this dashboard. + */ + @Override + public Long version() { + return my_version; + } + + /** + * @return the county for this dashboard. + */ + public County county() { + return my_county; + } + + /** + * @return the CVR file. A return value of null means + * that no CVRs have been uploaded for this county. + */ + public UploadedFile cvrFile() { + return my_cvr_file; + } + + /** + * Sets a new CVR file, replacing the previous one. + * + * @param the_file The CVR file. + */ + public void setCVRFile(final UploadedFile the_file) { + my_cvr_file = the_file; + } + + /** + * @return the ballot manifest file. A return value of null means + * that no ballot manifest has been uploaded for this county. + */ + public UploadedFile manifestFile() { + return my_manifest_file; + } + + /** + * Sets a new ballot manifest file, replacing the previous one. + * + * @param the_file The manifest file. + */ + public void setManifestFile(final UploadedFile the_file) { + my_manifest_file = the_file; + } + + /** + * @return the audit timestamp. A return value of null means + * that no audit has been started. + */ + public Instant auditTimestamp() { + return my_audit_timestamp; + } + + /** + * Sets a new audit timestamp, replacing the previous one. + * + * @param the_timestamp The new audit timestamp. + */ + public void setAuditTimestamp(final Instant the_timestamp) { + my_audit_timestamp = the_timestamp; + } + + /** + * @return the number of audit boards. + */ + public Integer auditBoardCount() { + return this.auditBoardCount; + } + + /** + * Set the expected number of audit boards. + * + * @param count number of audit boards + */ + public void setAuditBoardCount(final Integer count) { + this.auditBoardCount = count; + } + + /** + * @return the entire list of audit boards. + */ + public Map auditBoards() { + return Collections.unmodifiableMap(my_audit_boards); + } + + /** + * Signs in the specified audit board as of the present time; + * the supplied set of electors must be the full set of electors on + * the board. The previous audit board, if any, is signed out if it + * had not yet been signed out. + * + * @param the_members The members. + */ + public void signInAuditBoard(final Integer index, + final List the_members) { + final AuditBoard currentBoard = my_audit_boards.get(index); + final AuditBoard newBoard = new AuditBoard(the_members, Instant.now()); + + if (currentBoard != null) { + this.signOutAuditBoard(index); + } + + my_audit_boards.put(index, newBoard); + } + + /** + * Signs out the audit board at index. + * + * If no audit board is present at the given index, nothing is changed. + */ + public void signOutAuditBoard(final Integer index) { + final AuditBoard currentBoard = my_audit_boards.get(index); + + if (currentBoard != null) { + currentBoard.setSignOutTime(Instant.now()); + my_audit_boards.remove(index); + } + } + + /** + * Signs out all audit boards. + */ + public void signOutAllAuditBoards() { + final Set ks = new HashSet(my_audit_boards.keySet()); + + for (final Integer i : ks) { + this.signOutAuditBoard(i); + } + } + + /** + * Test if the desired number of audit boards have signed in. + * + * Note: Only works properly for indexes less than the current audit board + * count in case there are orphaned boards outside of the current expected + * key range, because just counting the number of keys in the audit board map + * might yield the wrong answer if there are orphaned audit boards. + * + * Use signOutAllAuditBoards to properly clear out the data structure holding + * all audit boards, signing out audit boards as necessary. + * + * @return boolean + */ + public boolean areAuditBoardsSignedIn() { + boolean result = true; + + for (int i = 0; i < this.auditBoardCount(); i++) { + if (my_audit_boards.get(i) == null) { + result = false; + break; + } + } + + return result; + } + + /** + * Test if all audit boards are signed out. + * + * @return boolean + */ + public boolean areAuditBoardsSignedOut() { + boolean result = true; + + for (int i = 0; i < this.auditBoardCount(); i++) { + if (my_audit_boards.get(i) != null) { + result = false; + break; + } + } + + return result; + } + + /** + * @return all the audit rounds. + */ + public List rounds() { + return Collections.unmodifiableList(my_rounds); + } + + /** + * @return the current audit round, or null if no round is in progress. + */ + public Round currentRound() { + if (my_current_round_index == null) { + return null; + } else { + return my_rounds.get(my_current_round_index); + } + } + + /** + * Begins a new round with the specified number of ballots to audit + * and expected achieved prefix length, starting at the specified index + * in the random audit sequence. + * + * @param numberOfBallots The number of ballots in this round + * @param prefixLength The expected audited prefix length at the round's end. + * @param startIndex The start index. + * @param ballotSequence The ballots to audit in the round, in the order + * in which they should be presented. + * @param auditSubsequence The audit subsequence for the round. + * @exception IllegalStateException if a round is currently ongoing. + */ + public void startRound(final int numberOfBallots, + final int prefixLength, + final int startIndex, + final List ballotSequence, + final List auditSubsequence) { + if (my_current_round_index == null) { + my_current_round_index = my_rounds.size(); + } else { + throw new IllegalStateException("cannot start a round while one is running"); + } + + // note UI round indexing is from 1, not 0 + final Round round = new Round(my_current_round_index + 1, + Instant.now(), + numberOfBallots, + my_ballots_audited, + prefixLength, + startIndex, + ballotSequence, + auditSubsequence); + my_rounds.add(round); + } + + /** + * Ends the current round. + * + * Signs out all audit boards, and performs any bookkeeping necessary to end + * the round. + * + * @exception IllegalStateException if there is no current round. + */ + public void endRound() { + if (my_current_round_index == null) { + throw new IllegalStateException("no round to end"); + } else { + this.setAuditBoardCount(null); + this.signOutAllAuditBoards(); + + final Round round = my_rounds.get(my_current_round_index); + round.setEndTime(Instant.now()); + my_current_round_index = NO_CONTENT; + } + } + + /** + * @return the number of ballots remaining in the current round, or 0 + * if there is no current round. + */ + public int ballotsRemainingInCurrentRound() { + final int result; + + if (my_current_round_index == null) { + result = 0; + } else { + + final Round round = currentRound(); + + result = round.ballotSequence().size() - round.actualCount(); + + LOGGER.debug(String.format("[ballotsRemainingInCurrentRound:" + + " index=%d, result=%d," + + " ballotSequence=%s" + + " ballotSequence.size=%d" + + " cdb.auditedSampleCount()=%d]", + my_current_round_index, + result, + round.ballotSequence(), + round.ballotSequence().size(), + this.auditedSampleCount())); + } + return result; + } + + /** + * @return the set of comparison audits being performed. + */ + public Set getAudits() { + return Collections.unmodifiableSet(audits); + } + + + /** + * @return the set of comparison audits being performed. + */ + public Set comparisonAudits() { + return Collections.unmodifiableSet(audits); + } + + /** + * Sets the comparison audits being performed. + * + * @param audits The comparison audits. + */ + public void setAudits(final Set audits) { + this.audits.clear(); + this.audits.addAll(audits); + } + + /** + * @return the set of contest names driving the audit. + */ + public Set drivingContestNames() { + return Collections.unmodifiableSet(drivingContestNames); + } + + /** + * Sets the contests driving the audit. + * + * @param the_driving_contests The contests. + */ + public void setDrivingContestNames(final Set the_driving_contests) { + drivingContestNames.clear(); + drivingContestNames.addAll(the_driving_contests); + } + + /** + * Submits an audit investigation report. + * + * @param the_report The audit investigation report. + */ + public void submitInvestigationReport(final AuditInvestigationReportInfo the_report) { + my_investigation_reports.add(the_report); + } + + /** + * @return the list of submitted audit investigation reports. + */ + public List investigationReports() { + return Collections.unmodifiableList(my_investigation_reports); + } + + /** + * Submits an audit investigation report. + * + * @param the_report The audit investigation report. + */ + public void submitIntermediateReport(final IntermediateAuditReportInfo the_report) { + my_intermediate_reports.add(the_report); + } + + /** + * @return the list of submitted audit interim reports. + */ + public List intermediateReports() { + return Collections.unmodifiableList(my_intermediate_reports); + } + + /** + * Returns the a list of CVR IDs under audit for the assigned audit boards. + * + * Delegates the actual calculation to the current Round, if one exists. + * + * @return a list of CVR IDs assigned to each audit board, where the list + * offset matches the audit board offset. + */ + public List cvrsUnderAudit() { + final Round round = this.currentRound(); + + if (round == null) { + return null; + } + + return round.cvrsUnderAudit(); + } + + /** + * @return the number of ballots audited. + */ + public Integer ballotsAudited() { + return my_ballots_audited; + } + + /** + * Adds an audited ballot. This adds it both to the total and to + * the current audit round. If no round is ongoing, this method + * does nothing. + */ + public void addAuditedBallot() { + if (my_current_round_index != null) { + my_ballots_audited = my_ballots_audited + 1; + my_rounds.get(my_current_round_index).addAuditedBallot(); + } + } + + /** + * Removes an audited ballot. This removes it both from the total and + * from the current audit round, if one is ongoing. + */ + public void removeAuditedBallot() { + if (my_current_round_index != null) { + my_ballots_audited = my_ballots_audited - 1; + my_rounds.get(my_current_round_index).removeAuditedBallot(); + } + } + + /** + * @return the number of CVRs in the CVR import. + */ + public Integer cvrsImported() { + return my_cvrs_imported; + } + + /** + * Sets the number of CVRs imported. + * + * @param the_cvrs_imported The number. + */ + public void setCVRsImported(final Integer the_cvrs_imported) { + my_cvrs_imported = the_cvrs_imported; + } + + /** + * @return the CVR import status. + */ + public ImportStatus cvrImportStatus() { + return my_cvr_import_status; + } + + /** + * Sets the CVR import status. + * + * @param the_cvr_import_status The new status. + */ + public void setCVRImportStatus(final ImportStatus the_cvr_import_status) { + my_cvr_import_status = the_cvr_import_status; + } + + /** + * @return the number of ballots described in the ballot manifest. + */ + public Integer ballotsInManifest() { + return my_ballots_in_manifest; + } + + /** + * Sets the number of ballots described in the ballot manifest. + * + * @param the_ballots_in_manifest The number. + */ + public void setBallotsInManifest(final Integer the_ballots_in_manifest) { + my_ballots_in_manifest = the_ballots_in_manifest; + } + + /** + * @return the numbers of discrepancies found in the audit so far, + * categorized by contest audit selection. + */ + public Map discrepancies() { + return Collections.unmodifiableMap(my_discrepancies); + } + + /** + * Adds a discrepancy for the specified audit reasons. This adds it both to the + * total and to the current audit round, if one is ongoing. + * + * @param the_reasons The reasons. + */ + public void addDiscrepancy(final Set the_reasons) { + LOGGER.debug(String.format("[addDiscrepancy for %s County: the_reasons=%s", county().name(), the_reasons)); + final Set selections = new HashSet<>(); + for (final AuditReason r : the_reasons) { + selections.add(r.selection()); + } + for (final AuditSelection s : selections) { + my_discrepancies.put(s, my_discrepancies.getOrDefault(s, 0) + 1); + } + if (my_current_round_index != null) { + my_rounds.get(my_current_round_index).addDiscrepancy(the_reasons); + } + } + + /** + * Removes a discrepancy for the specified audit reasons. This removes it + * both from the total and from the current audit round, if one is ongoing. + * + * + * @param the_reasons The reasons. + */ + public void removeDiscrepancy(final Set the_reasons) { + final Set selections = new HashSet<>(); + for (final AuditReason r : the_reasons) { + selections.add(r.selection()); + } + for (final AuditSelection s : selections) { + my_discrepancies.put(s, my_discrepancies.getOrDefault(s, 0) - 1); + } + if (my_current_round_index != null) { + my_rounds.get(my_current_round_index).removeDiscrepancy(the_reasons); + } + } + + + /** + * @return the numbers of disagreements found in the audit so far, + * categorized by contest audit selection. + */ + public Map disagreements() { + return my_disagreements; + } + + /** + * Adds a disagreement for the specified audit reasons. This adds it both to the + * total and to the current audit round, if one is ongoing. + * + * @param the_reasons The reasons. + */ + public void addDisagreement(final Set the_reasons) { + final Set selections = new HashSet<>(); + for (final AuditReason r : the_reasons) { + selections.add(r.selection()); + } + for (final AuditSelection s : selections) { + my_disagreements.put(s, my_disagreements.getOrDefault(s, 0) + 1); + } + if (my_current_round_index != null) { + my_rounds.get(my_current_round_index).addDisagreement(the_reasons); + } + } + + /** + * Removes a disagreement for the specified audit reasons. This removes it + * both from the total and from the current audit round, if one is ongoing. + * + * + * @param the_reasons The reasons. + */ + public void removeDisagreement(final Set the_reasons) { + final Set selections = new HashSet<>(); + for (final AuditReason r : the_reasons) { + selections.add(r.selection()); + } + for (final AuditSelection s : selections) { + my_disagreements.put(s, my_disagreements.getOrDefault(s, 0) - 1); + } + if (my_current_round_index != null) { + my_rounds.get(my_current_round_index).removeDisagreement(the_reasons); + } + } + + /** + * takes the targeted contests/ComparisonAudits and checks them for + * completion/RiskLimitAchieved + **/ + public Boolean allAuditsComplete() { + return comparisonAudits().stream() + // .filter(ca -> ca.isTargeted()) // FIXME This might be better? + .filter(ca -> ca.auditReason() != AuditReason.OPPORTUNISTIC_BENEFITS) + .allMatch(ca -> ca.isFinished()); + } + + /** + * @return the estimated number of samples to audit. + */ + public Integer estimatedSamplesToAudit() { + return comparisonAudits().stream() + .filter(ca -> ca.auditReason() != AuditReason.OPPORTUNISTIC_BENEFITS) + .map(ca -> ca.estimatedRemaining()) + .mapToInt(Integer::intValue) + .sum(); + } + + /** + * @return the optimistic number of samples to audit. + */ + public Integer optimisticSamplesToAudit() { + // NOTE: there could be race conditions between audit boards across counties + final Optional maybe = comparisonAudits().stream() + .filter(ca -> ca.auditReason() != AuditReason.OPPORTUNISTIC_BENEFITS) + .map(ca -> ca.optimisticSamplesToAudit()) + .max(Comparator.naturalOrder()); + // NOTE: we may be asking for this when we don't need to; when there are no + // audits setup yet + if (maybe.isPresent()) { + return maybe.get(); + } else { + return 0; + } + } + + /** + * @return the length of the audited prefix of the sequence of + * ballots to audit (i.e., the number of audited ballots that + * "count"). + */ + public Integer auditedPrefixLength() { + return my_audited_prefix_length; + } + + /** + * Sets the length of the audited prefix of the sequence of + * ballots to audit. If there is no active round, this method does + * nothing. + * + * @param the_audited_prefix_length The audited prefix length. + */ + public void setAuditedPrefixLength(final int the_audited_prefix_length) { + if (my_current_round_index != null) { + my_audited_prefix_length = the_audited_prefix_length; + my_rounds.get(my_current_round_index). + setActualAuditedPrefixLength(the_audited_prefix_length); + } + } + + /** + * @return the number of samples that have been included in the + * audit calculations so far. + */ + public Integer auditedSampleCount() { + return my_audited_sample_count; + } + + /** + * Sets the number of samples that have been included in the + * audit calculations so far. + * + * @param the_audited_sample_count The audited sample count. + */ + public void setAuditedSampleCount(final int the_audited_sample_count) { + my_audited_sample_count = the_audited_sample_count; + } + + /** + * Ends all audits in the county. This changes the status of any audits + * that have not achieved their risk limit to ENDED. + * + * You should not use this lightly, some of the audits might be shared + * with others! + */ + public void endAudits() { + for (final ComparisonAudit ca : audits) { + ca.endAudit(); + } + } + + /** + * End all audits that only belong to this county. + */ + public List endSingleCountyAudits() { + return comparisonAudits().stream() + .filter(a -> a.isSingleCountyFor(this.county())) + .map(a -> { + a.endAudit(); + return a;}) + .collect(Collectors.toList()); + } + + /** + * Updates the status for all audits in the county. This changes their statuses + * based on whether they have achieved their risk limits. + */ + public void updateAuditStatus() { + for (final ComparisonAudit ca : audits) { + ca.updateAuditStatus(); + } + } + + /** + * @return a String representation of this contest. + */ + @Override + public String toString() { + return "CountyDashboard [county=" + id() + "]"; + } + + /** + * Compare this object with another for equivalence. + * + * @param the_other The other object. + * @return true if the objects are equivalent, false otherwise. + */ + @Override + public boolean equals(final Object the_other) { + boolean result = true; + if (the_other instanceof CountyDashboard) { + final CountyDashboard other_cdb = (CountyDashboard) the_other; + // there can only be one county dashboard in the system for each + // ID, so we check their equivalence by ID + result &= nullableEquals(other_cdb.id(), id()); + } else { + result = false; + } + return result; + } + + /** + * @return a hash code for this object. + */ + @Override + public int hashCode() { + return id().hashCode(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/DoSDashboard.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/DoSDashboard.java new file mode 100644 index 00000000..0408994c --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/DoSDashboard.java @@ -0,0 +1,324 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 27, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @model_review Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.model; + +import static us.freeandfair.corla.util.EqualsHashcodeHelper.*; + +import java.io.Serializable; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import javax.persistence.Cacheable; +import javax.persistence.CollectionTable; +import javax.persistence.ElementCollection; +import javax.persistence.Embedded; +import javax.persistence.Entity; +import javax.persistence.FetchType; +import javax.persistence.Id; +import javax.persistence.JoinColumn; +import javax.persistence.Table; +import javax.persistence.Version; + +import us.freeandfair.corla.persistence.PersistentEntity; + +/** + * The Department of State dashboard. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +// this is an unusual entity, in that it is a singleton; it thus has only one +// possible id (0). +@Entity +@Cacheable(true) +@Table(name = "dos_dashboard") +// this class has many fields that would normally be declared final, but +// cannot be for compatibility with Hibernate and JPA. +@SuppressWarnings("PMD.ImmutableField") +public class DoSDashboard implements PersistentEntity, Serializable { + /** + * The DoS dashboard ID (it is a singleton). + */ + public static final Long ID = Long.valueOf(0); + + /** + * The minimum number of random seed characters. + */ + public static final int MIN_SEED_LENGTH = 20; + + /** + * The serialVersionUID. + */ + private static final long serialVersionUID = 1; + + /** + * The ID. This is always 0, because this object is a singleton. + */ + @Id + private Long my_id = ID; + + /** + * The version (for optimistic locking). + */ + @Version + private Long my_version; + + /** + * The contests to be audited and the reasons for auditing. + */ + @ElementCollection(fetch = FetchType.EAGER) + @CollectionTable(name = "contest_to_audit", + joinColumns = @JoinColumn(name = "dashboard_id", + referencedColumnName = "my_id")) + private Set my_contests_to_audit = new HashSet<>(); + + /** + * The election info. + */ + @Embedded + private AuditInfo my_audit_info = new AuditInfo(); + + /** + * Constructs a new Department of State dashboard with default values. + */ + // if we delete this constructor, we get warned that each class should + // define at least one constructor; we can't win in this situation. + @SuppressWarnings("PMD.UnnecessaryConstructor") + public DoSDashboard() { + super(); + } + + /** + * @return the database ID for this dashboard, which is the same as + * its county ID. + */ + @Override + public Long id() { + return my_id; + } + + /** + * Sets the database ID for this dashboard. + * + * @param the_id The ID, effectively ignored; the database ID for a DoS + * dashboard is always 0. + * @exception IllegalArgumentException if the ID is not 0. + */ + @Override + public final void setID(final Long the_id) { + if (!ID.equals(the_id)) { + throw new IllegalArgumentException("the only valid ID for a DoSDashboard is 0"); + } + my_id = ID; + } + + /** + * @return the version for this dashboard. + */ + @Override + public Long version() { + return my_version; + } + + /** + * Checks the validity of a random seed. To be valid, a random seed must + * have at least MIN_SEED_CHARACTERS characters, and all characters must + * be digits. + * + * @param the_seed The seed. + * @return true if the seed meets the validity requirements, false otherwise. + */ + public static boolean isValidSeed(final String the_seed) { + boolean result = true; + + if (the_seed != null && the_seed.length() >= MIN_SEED_LENGTH) { + for (final char c : the_seed.toCharArray()) { + if (!Character.isDigit(c)) { + result = false; + break; + } + } + } else { + result = false; + } + + return result; + } + + /** + * @return the audit info. + */ + public AuditInfo auditInfo() { + return my_audit_info; + } + + /** + * Updates the audit info, using the non-null fields of the specified + * AuditInfo. This method does not do any sanity checks on the fields; + * it is assumed that they are checked by the caller. + * + * @param the_new_info The new info. + */ + public void updateAuditInfo(final AuditInfo the_new_info) { + my_audit_info.updateFrom(the_new_info); + } + + /** + * Removes all contests to audit for the specified county. This is + * typically done if the county re-uploads their CVRs (generating new + * contest information). + * + * @param the_county The county. + * @return true if any contests to audit were removed, false otherwise. + */ + public boolean removeContestsToAuditForCounty(final County the_county) { + boolean result = false; + + final Set contests_to_remove = new HashSet<>(); + for (final ContestToAudit c : my_contests_to_audit) { + if (c.contest().county().equals(the_county)) { + contests_to_remove.add(c); + result = true; + } + } + my_contests_to_audit.removeAll(contests_to_remove); + + return result; + } + + /** + * Remove all ContestsToAudit that are auditable from this dashboard because + * they may have been unchecked in the ui. The checked ones should be added + * back in a following step. Unaditable contests are not able to be checked in + * the ui so they can stay. + * + * note: an alternative approach would be to set a hidden field for every + * checkbox in the ui + **/ + public void removeAuditableContestsToAudit() { + my_contests_to_audit.removeAll(my_contests_to_audit.stream() + .filter(c -> c.isAuditable()) + .collect(Collectors.toList())); + } + + /** remove a contest by name, supports the hand count button **/ + public void removeContestToAuditByName(final String contestName){ + final Set contests_to_remove = new HashSet<>(); + for (final ContestToAudit c : my_contests_to_audit) { + if (c.contest().name().equals(contestName)) { + contests_to_remove.add(c); + } + } + my_contests_to_audit.removeAll(contests_to_remove); + } + + /** + * Update the audit status of a contest. + * + * @param the_contest_to_audit The new status of the contest to audit. + * @return true if the contest was already being audited or hand counted, + * false otherwise. + */ + //@ requires the_contest_to_audit != null; + public boolean updateContestToAudit(final ContestToAudit the_contest_to_audit) { + boolean auditable = true; + + // check to see if the contest is in our set + ContestToAudit contest_to_remove = null; + for (final ContestToAudit c : my_contests_to_audit) { + if (c.contest().equals(the_contest_to_audit.contest())) { + // check if the entry is auditable; if so, it will be removed later + auditable = !c.audit().equals(AuditType.NOT_AUDITABLE); + contest_to_remove = c; + break; + } + } + + if (auditable) { + my_contests_to_audit.remove(contest_to_remove); + if (the_contest_to_audit.audit() != AuditType.NONE) { + my_contests_to_audit.add(the_contest_to_audit); + } + } + + return auditable; + } + + /** + * @return the current set of contests to audit. This is an unmodifiable + * set; to update, use updateContestToAudit(). + */ + public Set contestsToAudit() { + return Collections.unmodifiableSet(my_contests_to_audit); + } + + /** + * data access helper + * @return contests of contestsToAudit a.k.a selected contests, targeted contests + */ + public Stream targetedContests() { + return contestsToAudit().stream() + .map(a -> a.contest()); + } + + /** + * data access helper + * @return contests names of contestsToAudit a.k.a selected contests, targeted + * contests + */ + public Set targetedContestNames() { + return targetedContests() + .map(c -> c.name()) + .collect(Collectors.toSet()); + } + + /** + * @return a String representation of this contest. + */ + @Override + public String toString() { + return "DoSDashboard [county=" + id() + "]"; + } + + /** + * Compare this object with another for equivalence. + * + * @param the_other The other object. + * @return true if the objects are equivalent, false otherwise. + */ + @Override + public boolean equals(final Object the_other) { + boolean result = true; + if (the_other instanceof DoSDashboard) { + final DoSDashboard other_ddb = (DoSDashboard) the_other; + // there can only be one DoS dashboard in the system for each + // ID, so we check their equivalence by ID + result &= nullableEquals(other_ddb.contestsToAudit(), contestsToAudit()); + result &= nullableEquals(other_ddb.auditInfo(), auditInfo()); + } else { + result = false; + } + return result; + } + + /** + * @return a hash code for this object. + */ + @Override + public int hashCode() { + return nullableHashCode(auditInfo()); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/Elector.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/Elector.java new file mode 100644 index 00000000..1c43c7c2 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/Elector.java @@ -0,0 +1,139 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 25, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joey Dodds + * @model_review Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.model; + +import static us.freeandfair.corla.util.EqualsHashcodeHelper.*; + +import java.io.Serializable; + +import javax.persistence.Column; +import javax.persistence.Embeddable; + +import org.hibernate.annotations.Immutable; + +/** + * An elector; has a first name, a last name, and a political party. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@Embeddable +@Immutable // this is a Hibernate-specific annotation, but there is no JPA alternative +// this class has many fields that would normally be declared final, but +// cannot be for compatibility with Hibernate and JPA. +@SuppressWarnings("PMD.ImmutableField") +public class Elector implements Serializable { + /** + * The serialVersionUID. + */ + private static final long serialVersionUID = 1L; + + /** + * The first name. + */ + @Column(nullable = false, updatable = false) + private String my_first_name; + + /** + * The last name. + */ + @Column(nullable = false, updatable = false) + private String my_last_name; + + /** + * The political party + */ + @Column(nullable = false, updatable = false) + private String my_political_party; + + /** + * Constructs an empty elector, solely for persistence. + */ + public Elector() { + super(); + } + + /** + * Constructs an elector with the specified parameters. + * + * @param the_first_name The first name. + * @param the_last_name The last name. + * @param the_political_party The political party. + */ + public Elector(final String the_first_name, + final String the_last_name, + final String the_political_party) { + super(); + my_first_name = the_first_name; + my_last_name = the_last_name; + my_political_party = the_political_party; + } + + /** + * @return the first name. + */ + public String firstName() { + return my_first_name; + } + + /** + * @return the last name. + */ + public String lastName() { + return my_last_name; + } + + /** + * @return the political party. + */ + public String politicalParty() { + return my_political_party; + } + + /** + * @return a String representation of this elector. + */ + @Override + public String toString() { + return "Elector [first_name=" + my_first_name + ", last_name=" + + my_last_name + ", political_party=" + my_political_party + "]"; + } + + /** + * Compare this object with another for equivalence. + * + * @param the_other The other object. + * @return true if the objects are equivalent, false otherwise. + */ + @Override + public boolean equals(final Object the_other) { + boolean result = true; + if (the_other instanceof Elector) { + final Elector other_elector = (Elector) the_other; + result &= nullableEquals(other_elector.firstName(), firstName()); + result &= nullableEquals(other_elector.lastName(), lastName()); + result &= nullableEquals(other_elector.politicalParty(), politicalParty()); + } else { + result = false; + } + return result; + } + + /** + * @return a hash code for this object. + */ + @Override + public int hashCode() { + return nullableHashCode(lastName()); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/ImportStatus.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/ImportStatus.java new file mode 100644 index 00000000..b357d6dd --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/ImportStatus.java @@ -0,0 +1,128 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Sep 6, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.model; + +import java.io.Serializable; +import java.time.Instant; + +import javax.persistence.Embeddable; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; + +/** + * Status information for a file import. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@Embeddable +//this class has many fields that would normally be declared final, but +//cannot be for compatibility with Hibernate and JPA. +@SuppressWarnings("PMD.ImmutableField") +public class ImportStatus implements Serializable { + /** + * The serialVersionUID. + */ + private static final long serialVersionUID = 1L; + + /** + * The import state. + */ + @Enumerated(EnumType.STRING) + private ImportState my_import_state; + + /** + * The error message, if any. + */ + private String my_error_message; + + /** + * The timestamp of the status update. + */ + private Instant my_timestamp; + + /** + * Constructs an empty ImportStatus, solely for persistence. + */ + public ImportStatus() { + super(); + } + + /** + * Constructs a new ImportStatus with the specified contents. + * + * @param the_import_state The import state. + * @param the_error_message The error message, or null if there has been no error. + * @param the_timestamp The timestamp. + */ + public ImportStatus(final ImportState the_import_state, + final String the_error_message, + final Instant the_timestamp) { + my_import_state = the_import_state; + my_error_message = the_error_message; + my_timestamp = the_timestamp; + } + + /** + * Constructs a new ImportStatus with the specified contents, using the + * current time as a timestamp. + * + * @param the_import_state The import state. + * @param the_error_message The error message, or null if there has been no error. + */ + public ImportStatus(final ImportState the_import_state, + final String the_error_message) { + this(the_import_state, the_error_message, Instant.now()); + } + + /** + * Constructs a new ImportStatus with the specified import state, no error + * message, and the current time as a timestamp. + * + * @param the_import_state The import state. + * @param the_error_message The error message, or null if there has been no error. + */ + public ImportStatus(final ImportState the_import_state) { + this(the_import_state, null, Instant.now()); + } + + /** + * @return the import state. + */ + public ImportState importState() { + return my_import_state; + } + + /** + * @return the error message. + */ + public String errorMessage() { + return my_error_message; + } + + /** + * @return the timestamp. + */ + public Instant timestamp() { + return my_timestamp; + } + + /** + * The state of an import. + */ + public enum ImportState { + NOT_ATTEMPTED, + IN_PROGRESS, + SUCCESSFUL, + FAILED; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/IntermediateAuditReportInfo.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/IntermediateAuditReportInfo.java new file mode 100644 index 00000000..7d234275 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/IntermediateAuditReportInfo.java @@ -0,0 +1,127 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 2, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.model; + +import static us.freeandfair.corla.util.EqualsHashcodeHelper.*; + +import java.io.Serializable; +import java.time.Instant; + +import javax.persistence.Column; +import javax.persistence.Embeddable; + +import com.google.gson.annotations.JsonAdapter; + +import us.freeandfair.corla.json.IntermediateAuditReportJsonAdapter; + +/** + * An audit investigation report. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@Embeddable +//this class has many fields that would normally be declared final, but +//cannot be for compatibility with Hibernate and JPA. +@SuppressWarnings("PMD.ImmutableField") +@JsonAdapter(IntermediateAuditReportJsonAdapter.class) +public class IntermediateAuditReportInfo implements Serializable { + /** + * The serialVersionUID. + */ + private static final long serialVersionUID = 1L; + + /** + * The timestamp of this report. + */ + @Column(updatable = false) + private Instant my_timestamp; + + /** + * The report for this report. + */ + @Column(updatable = false) + private String my_report; + + /** + * Constructs an empty AuditInvestigationReport, solely for persistence. + */ + public IntermediateAuditReportInfo() { + super(); + } + + /** + * Constructs an audit investigation report with the specified + * parameters. + * + * @param the_timestamp The timestamp. + * @param the_name The name. + * @param the_report The report. + */ + public IntermediateAuditReportInfo(final Instant the_timestamp, + final String the_report) { + super(); + my_timestamp = the_timestamp; + my_report = the_report; + } + + /** + * @return the timestamp. + */ + public Instant timestamp() { + return my_timestamp; + } + + /** + * @return the report. + */ + public String report() { + return my_report; + } + + /** + * @return a String representation of this cast vote record. + */ + @Override + public String toString() { + return "AuditInterimReport [timestamp=" + my_timestamp + + ", report=" + my_report + "]"; + } + + /** + * Compare this object with another for equivalence. + * + * @param the_other The other object. + * @return true if the objects are equivalent, false otherwise. + */ + @Override + public boolean equals(final Object the_other) { + boolean result = true; + if (the_other instanceof IntermediateAuditReportInfo) { + final IntermediateAuditReportInfo other_report = + (IntermediateAuditReportInfo) the_other; + result &= nullableEquals(other_report.timestamp(), timestamp()); + result &= nullableEquals(other_report.report(), report()); + } else { + result = false; + } + return result; + } + + /** + * @return a hash code for this object. + */ + @Override + public int hashCode() { + return nullableHashCode(timestamp()); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/LogEntry.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/LogEntry.java new file mode 100644 index 00000000..79266611 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/LogEntry.java @@ -0,0 +1,331 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 16, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.model; + +import static us.freeandfair.corla.util.EqualsHashcodeHelper.*; + +import java.io.Serializable; +import java.math.BigInteger; +import java.nio.charset.Charset; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Instant; + +import javax.persistence.Cacheable; +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.FetchType; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.JoinColumn; +import javax.persistence.ManyToOne; +import javax.persistence.Table; +import javax.persistence.Version; + +import org.hibernate.annotations.Immutable; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.persistence.PersistentEntity; + +/** + * A log entry that is stored in the database. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@Entity +@Immutable // this is a Hibernate-specific annotation, but there is no JPA alternative +@Cacheable(true) +@Table(name = "log") +//this class has many fields that would normally be declared final, but +//cannot be for compatibility with Hibernate and JPA. +@SuppressWarnings("PMD.ImmutableField") +public class LogEntry implements PersistentEntity, Serializable { + /** + * The root hash for the hash chain (a 256-bit block of zeros). + */ + public static final String ROOT_HASH = + "0000000000000000000000000000000000000000000000000000000000000000"; + + /** + * The serialVersionUID. + */ + private static final long serialVersionUID = 1L; + + /** + * The ID number. + */ + @Id + @Column(updatable = false, nullable = false) + @GeneratedValue(strategy = GenerationType.SEQUENCE) + private Long my_id; + + /** + * The version (for optimistic locking). + */ + @Version + private Long my_version; + + /** + * The result code of this log entry, if any. In most cases, this will be an HTTP + * result code, as enumerated in HttpStatus. + */ + @Column(updatable = false) + private Integer my_result_code; + + /** + * The informational string of this log entry. + */ + @Column(updatable = false, nullable = false) + private String my_information; + + /** + * Information about the authentication status at the time of this log entry, + * if any. + */ + @Column(updatable = false) + private String my_authentication_data; + + /** + * Information about the client host that generated this log entry, if any. + */ + @Column(updatable = false) + private String my_client_host; + + /** + * The timestamp of this log entry. + */ + @Column(updatable = false, nullable = false) + private Instant my_timestamp; + + /** + * The previous log entry for this log entry. + */ + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "previous_entry") + private LogEntry my_previous_entry; + + /** + * The hash chain entry of this log entry. + */ + @Column(updatable = false, nullable = false) + private String my_hash; + + /** + * Constructs a new empty log entry, solely for persistence. + */ + public LogEntry() { + super(); + } + + /** + * Constructs a new log entry with the specified information. If the previous + * entry is null, it is assumed that this is the beginning of a new log hash + * chain. + * + * @param the_result_code The result code, if any. + * @param the_information The information. + * @param the_authentication_data The authentication data, if any. + * @param the_client_host The client host, if any. + * @param the_timestamp The timestamp. + * @param the_previous_entry The previous log entry. + */ + public LogEntry(final Integer the_result_code, final String the_information, + final String the_authentication_data, final String the_client_host, + final Instant the_timestamp, final LogEntry the_previous_entry) { + super(); + my_result_code = the_result_code; + my_information = the_information; + my_authentication_data = the_authentication_data; + my_client_host = the_client_host; + my_timestamp = the_timestamp; + my_previous_entry = the_previous_entry; + my_hash = calculateHash(the_previous_entry); + } + + /** + * Constructs a new, unhashed log entry with the specified information; + * such a log entry cannot be persisted, and is useful only for subsequently + * building persistable log entries (as, for example, at the end of request + * processing). + * + * @param the_result_code The result code. + * @param the_information The information. + * @param the_timestamp The timestamp. + */ + public LogEntry(final Integer the_result_code, final String the_information, + final Instant the_timestamp) { + super(); + my_result_code = the_result_code; + my_information = the_information; + my_timestamp = the_timestamp; + } + + /** + * {@inheritDoc} + */ + @Override + public Long id() { + return my_id; + } + + /** + * {@inheritDoc} + */ + @Override + public void setID(final Long the_id) { + my_id = the_id; + } + + /** + * {@inheritDoc} + */ + @Override + public Long version() { + return my_version; + } + + /** + * Generates a hash from the previous log entry and the contents of this + * log entry. If the previous entry is null, the hash is based on the + * root hash. + * + * @param the_previous_entry The previous log entry. + * @return the hash. If the hash cannot be calculated, this method + * returns the root hash. + */ + private String calculateHash(final LogEntry the_previous_entry) { + String result = ROOT_HASH; + final StringBuilder hash_input = new StringBuilder(hashString()); + if (the_previous_entry == null) { + hash_input.append(ROOT_HASH); + } else { + hash_input.append(the_previous_entry.hash()); + } + try { + final MessageDigest md = MessageDigest.getInstance("SHA-256"); + final BigInteger bi = + new BigInteger(1, md.digest(hash_input.toString(). + getBytes(Charset.forName("UTF-8")))); + result = String.format("%0" + (md.digest().length << 1) + "X", bi); + } catch (final NoSuchAlgorithmException e) { + Main.LOGGER.error("could not use SHA-256"); + } + return result; + } + + /** + * @return the result code in this log entry. + */ + public Integer resultCode() { + return my_result_code; + } + + /** + * @return the information in this log entry. + */ + public String information() { + return my_information; + } + + /** + * @return the authentication data of this log entry. + */ + public String authenticationData() { + return my_authentication_data; + } + + /** + * @return the client host of this log entry. + */ + public String clientHost() { + return my_client_host; + } + + /** + * @return the timestamp of this log entry. + */ + public Instant timestamp() { + return my_timestamp; + } + + /** + * @return the previous log entry. + */ + public LogEntry previousEntry() { + return my_previous_entry; + } + + /** + * @return the hash of this log entry. + */ + public String hash() { + return my_hash; + } + + /** + * Returns a String based on the data in this log entry and used as part + * of the hash computation. + * + * @return the String. + */ + public final String hashString() { + final StringBuilder hash_input = new StringBuilder(); + hash_input.append(my_result_code.toString()); + hash_input.append(my_information); + hash_input.append(my_timestamp.toString()); + return hash_input.toString(); + } + + /** + * @return a String representation of this elector. + */ + @Override + public String toString() { + return "LogEntry [information=" + my_information + ", timestamp=" + + my_timestamp + ", hash=" + my_hash + "]"; + } + + /** + * Compare this object with another for equivalence. + * + * @param the_other The other object. + * @return true if the objects are equivalent, false otherwise. + */ + @Override + public boolean equals(final Object the_other) { + boolean result = true; + if (the_other instanceof LogEntry) { + final LogEntry other_entry = (LogEntry) the_other; + result &= nullableEquals(other_entry.resultCode(), resultCode()); + result &= nullableEquals(other_entry.information(), information()); + result &= nullableEquals(other_entry.authenticationData(), + authenticationData()); + result &= nullableEquals(other_entry.clientHost(), clientHost()); + result &= nullableEquals(other_entry.timestamp(), timestamp()); + // we don't include the previous entry because it would be very recursive + result &= nullableEquals(other_entry.hash(), hash()); + } else { + result = false; + } + return result; + } + + /** + * @return a hash code for this object. + */ + @Override + public int hashCode() { + return nullableHashCode(hash()); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/Round.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/Round.java new file mode 100644 index 00000000..c4ff6c9a --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/Round.java @@ -0,0 +1,579 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 25, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Joey Dodds + * @model_review Joseph R. Kiniry + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.model; + +import static us.freeandfair.corla.util.EqualsHashcodeHelper.*; + +import java.io.Serializable; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.persistence.Column; +import javax.persistence.Convert; +import javax.persistence.Embeddable; + +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; +import org.hibernate.Session; +import org.hibernate.query.Query; + +import us.freeandfair.corla.persistence.AuditSelectionIntegerMapConverter; +import us.freeandfair.corla.persistence.BallotSequenceAssignmentConverter; +import us.freeandfair.corla.persistence.SignatoriesConverter; +import us.freeandfair.corla.persistence.LongListConverter; +import us.freeandfair.corla.persistence.Persistence; + +/** + * Information about an audit round. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@Embeddable +@SuppressWarnings({"PMD.ImmutableField", "PMD.TooManyMethods"}) +public class Round implements Serializable { + /** + * Class-wide logger + */ + public static final Logger LOGGER = LogManager.getLogger(Round.class); + + /** + * The serialVersionUID. + */ + private static final long serialVersionUID = 1L; + + /** + * The "text" constant. + */ + private static final String TEXT = "text"; + + /** + * The round number. + */ + @Column(nullable = false, updatable = false) + private Integer my_number; + + /** + * The start time. + */ + @Column(nullable = false, updatable = false) + private Instant my_start_time; + + /** + * The end time. + */ + private Instant my_end_time; + + /** + * The expected number of ballots to audit in this round. + */ + @Column(nullable = false, updatable = false) + private Integer my_expected_count; + + /** + * The actual number of ballots audited in this round. + */ + @Column(nullable = false) + private Integer my_actual_count; + + /** + * The audited prefix length expected to be achieved by the end of + * this round. + */ + @Column(nullable = false, updatable = false) + private Integer my_expected_audited_prefix_length; + + /** + * The audited prefix length actually achieved by the end of this round. + */ + @Column + private Integer my_actual_audited_prefix_length; + + /** + * The index of the audit random sequence where the round starts. + */ + @Column(nullable = false, updatable = false) + private Integer my_start_audited_prefix_length; + + /** + * The number of previously-audited ballots when the round starts. + */ + @Column(nullable = false, updatable = false) + private Integer my_previous_ballots_audited; + + /** + * The sequence of CVR IDs for ballots to audit in this round, + * in the order they are to be presented. + */ + @Column(nullable = false, updatable = false, + name = "ballot_sequence", columnDefinition = TEXT) + @Convert(converter = LongListConverter.class) + private List my_ballot_sequence; + + /** + * The assignment of work from the ballot sequence to each audit board. + * + * Audit boards are represented by the indices of the list, and each entry in + * the list is a data structure as follows: + * + * ({"index": 0, "count": 5}, {"index": 5, "count": 6} ...) + * + * where "index" represents the index into the ballot sequence list, and + * "count" represents the number of ballots assigned to that audit board. + */ + @Column(nullable = false, updatable = false, + name = "ballot_sequence_assignment", columnDefinition = TEXT) + @Convert(converter = BallotSequenceAssignmentConverter.class) + private List> ballotSequenceAssignment; + + /** + * The CVR IDs for the audit subsequence to audit in this + * round, in audit sequence order. + */ + @Column(nullable = false, updatable = false, + name = "audit_subsequence", columnDefinition = TEXT) + @Convert(converter = LongListConverter.class) + private List my_audit_subsequence; + + /** + * The number of discrepancies found in the audit so far. + */ + @Column(nullable = false, name = "discrepancies", columnDefinition = TEXT) + @Convert(converter = AuditSelectionIntegerMapConverter.class) + private Map my_discrepancies = new HashMap<>(); + + /** + * The number of disagreements found in the audit so far. + */ + @Column(nullable = false, name = "disagreements", columnDefinition = TEXT) + @Convert(converter = AuditSelectionIntegerMapConverter.class) + private Map my_disagreements = new HashMap<>(); + + /** + * The signatories for round sign-off. + * + * This is a map from audit board index to list of signatories that were part + * of that audit board. + */ + @Column(name = "signatories", columnDefinition = TEXT) + @Convert(converter = SignatoriesConverter.class) + private Map> my_signatories = new HashMap<>(); + + /** + * Constructs an empty round, solely for persistence. + */ + public Round() { + super(); + } + + /** + * Constructs a round with the specified parameters. + * + * @param the_number The round number. + * @param the_start_time The start time. + * @param the_expected_count The expected number of ballots to audit. + * @param the_previous_ballots_audited The number of ballots audited when the + * round starts. + * @param the_expected_audited_prefix_length The audit random sequence index + * where the round is expected to end. + * @param the_start_audited_prefix_length The index of the audit random sequence + * where the round starts. + * @param the_ballot_sequence The sequence of ballots to audit in this round. + * @param the_audit_subsequence The subsequence of the audit sequence for + * this round. + */ + public Round(final Integer the_number, + final Instant the_start_time, + final Integer the_expected_count, + final Integer the_previous_ballots_audited, + final Integer the_expected_audited_prefix_length, + final Integer the_start_audited_prefix_length, + final List the_ballot_sequence, + final List the_audit_subsequence) { + super(); + my_number = the_number; + my_start_time = the_start_time; + my_expected_count = the_expected_count; + my_expected_audited_prefix_length = the_expected_audited_prefix_length; + my_actual_count = 0; + my_start_audited_prefix_length = the_start_audited_prefix_length; + my_actual_audited_prefix_length = the_start_audited_prefix_length; + my_previous_ballots_audited = the_previous_ballots_audited; + my_ballot_sequence = the_ballot_sequence; + my_audit_subsequence = the_audit_subsequence; + } + + /** + * @return the round number. + */ + public Integer number() { + return my_number; + } + + /** + * @return the start time. + */ + public Instant startTime() { + return my_start_time; + } + + /** + * @return the end time. + */ + public Instant endTime() { + return my_end_time; + } + + /** + * Sets the end time. + * + * @param the_end_time The end time. + */ + public void setEndTime(final Instant the_end_time) { + my_end_time = the_end_time; + } + + /** + * @return the expected number of ballots to audit. + */ + public Integer expectedCount() { + return my_expected_count; + } + + /** + * @return the actual number of ballots audited. + */ + public Integer actualCount() { + return my_actual_count; + } + + /** + * Sets the actual number of ballots audited. + * + * @param the_actual_count The count. + */ + public void setActualCount(final Integer the_actual_count) { + my_actual_count = the_actual_count; + } + + /** + * @return the number of ballots audited prior to this round. + */ + public Integer previousBallotsAudited() { + return my_previous_ballots_audited; + } + + /** + * @return the expected audit sequence prefix length to be + * achieved by the end of this round. + */ + public Integer expectedAuditedPrefixLength() { + return my_expected_audited_prefix_length; + } + + /** + * @return the audit sequence prefix length achieved. + */ + public Integer actualAuditedPrefixLength() { + return my_actual_audited_prefix_length; + } + + /** + * Sets the audit prefix sequence length achieved. + * + * @param the_audited_prefix_length The prefix length achieved. + */ + public void setActualAuditedPrefixLength(final int the_audited_prefix_length) { + my_actual_audited_prefix_length = the_audited_prefix_length; + } + + /** + * @return the ballot sequence for this round. + */ + public List ballotSequence() { + return my_ballot_sequence; + } + + + /** + * @return the ballot sequence assignment + */ + public List> ballotSequenceAssignment() { + return this.ballotSequenceAssignment; + } + + /** + * Set the ballot sequence assignment. + * + * @param l the list of audit board assignment maps + */ + public void setBallotSequenceAssignment(final List> l) { + this.ballotSequenceAssignment = l; + } + + /** + * Returns the list of CVRs under audit in this round. + * + * @return a list whose indices correspond to audit board indices and values + * being the next CVR for the given audit board to audit. + */ + // TODO: Extract into query class + // FIXME did we duplicate this ever? + public List cvrsUnderAudit() { + final List> bsa = this.ballotSequenceAssignment(); + + if (bsa == null) { + return new ArrayList<>(); + } + + final List bs = this.ballotSequence(); + + if (bs.isEmpty()) { + // avoid psql exception + return new ArrayList<>(); + } + + // All CVR IDs that have no corresponding ACVR + final Session s = Persistence.currentSession(); + final Query q = s.createQuery( + "select cvrai.my_cvr.my_id from CVRAuditInfo cvrai " + + "where cvrai.my_cvr.my_id in (:ids) " + + "and cvrai.my_acvr is null"); + q.setParameterList("ids", bs); + // Put them in a set for quick membership testing + final Set unauditedIds = new HashSet(q.getResultList()); + + // Walk the sequence assignments getting the audit boards' index and count + // values, finding the first CVR with no corresponding ACVR *in ballot audit + // sequence order*. Any board that has finished the audit will get a null + // instead of a CVR ID. + final List result = new ArrayList(); + for (int i = 0; i < bsa.size(); i++) { + final Map m = bsa.get(i); + + final Integer index = m.get("index"); + final Integer count = m.get("count"); + + result.add(null); + for (int j = index; j < index + count; j++) { + final Long cvrId = bs.get(j); + + if (unauditedIds.contains(cvrId)) { + result.set(i, cvrId); + break; + } + } + } + + return result; + } + + /** + * @return the audit subsequence for this round. + */ + public List auditSubsequence() { + return my_audit_subsequence; + } + + /** + * Adds an audited ballot. + */ + public void addAuditedBallot() { + my_actual_count = my_actual_count + 1; + } + + /** + * Removes an audited ballot. + */ + public void removeAuditedBallot() { + my_actual_count = my_actual_count - 1; + } + + /** + * @return the index of the audit random sequence where this round + * starts. + */ + public Integer startAuditedPrefixLength() { + return my_start_audited_prefix_length; + } + + /** + * @return the numbers of discrepancies found in the audit so far, + * categorized by contest audit selection. + */ + public Map discrepancies() { + return Collections.unmodifiableMap(my_discrepancies); + } + + /** + * Adds a discrepancy for the specified audit reasons. This adds it both to the + * total and to the current audit round, if one is ongoing. + * + * @param the_reasons The reasons. + */ + public void addDiscrepancy(final Set the_reasons) { + final Set selections = new HashSet<>(); + for (final AuditReason r : the_reasons) { + selections.add(r.selection()); + } + for (final AuditSelection s : selections) { + my_discrepancies.put(s, my_discrepancies.getOrDefault(s, 0) + 1); + } + + LOGGER.info(String.format("[addDiscrepancy: the_reasons= %s, my_discrepancies=%s]", + the_reasons, my_discrepancies)); + } + + /** + * Removes a discrepancy for the specified audit reasons. This removes it + * both from the total and from the current audit round, if one is ongoing. + * + * @param the_reasons The reasons. + */ + public void removeDiscrepancy(final Set the_reasons) { + final Set selections = new HashSet<>(); + for (final AuditReason r : the_reasons) { + selections.add(r.selection()); + } + for (final AuditSelection s : selections) { + my_discrepancies.put(s, my_discrepancies.getOrDefault(s, 0) - 1); + } + } + + /** + * @return the numbers of disagreements found in the audit so far, + * categorized by contest audit reason. + */ + public Map disagreements() { + return my_disagreements; + } + + /** + * Adds a disagreement for the specified audit reasons. This adds it both to the + * total and to the current audit round, if one is ongoing. + * + * @param the_reasons The reasons. + */ + public void addDisagreement(final Set the_reasons) { + final Set selections = new HashSet<>(); + for (final AuditReason r : the_reasons) { + selections.add(r.selection()); + } + for (final AuditSelection s : selections) { + my_disagreements.put(s, my_disagreements.getOrDefault(s, 0) + 1); + } + } + + /** + * Removes a disagreement for the specified audit reasons. This removes it + * both from the total and from the current audit round, if one is ongoing. + * + * @param the_reasons The reasons. + */ + public void removeDisagreement(final Set the_reasons) { + final Set selections = new HashSet<>(); + for (final AuditReason r : the_reasons) { + selections.add(r.selection()); + } + for (final AuditSelection s : selections) { + my_disagreements.put(s, my_disagreements.getOrDefault(s, 0) - 1); + } + } + + /** + * @return the signatories. + */ + public Map> signatories() { + return Collections.unmodifiableMap(my_signatories); + } + + /** + * Sets the signatories for a particular audit board. + */ + public void setSignatories(final Integer auditBoardIndex, + final List signatories) { + my_signatories.put(auditBoardIndex, signatories); + } + + /** + * @return a String representation of this round. + */ + @Override + public String toString() { + return + String.format("Round [number=%d, start_time=%s, end_time=%s, expected_count=%d," + + " actual_count=%d, start_index=%d, discrepancies=%s," + + " disagreements=%s, signatories=%s]", + my_number, my_start_time, my_end_time, my_expected_count, + my_actual_count, my_start_audited_prefix_length, my_discrepancies, + my_disagreements, my_signatories); + } + + /** + * Compare this object with another for equivalence. + * + * @param the_other The other object. + * @return true if the objects are equivalent, false otherwise. + */ + @Override + public boolean equals(final Object the_other) { + boolean result = true; + if (the_other instanceof Round) { + final Round other_round = (Round) the_other; + result &= nullableEquals(other_round.startTime(), startTime()); + result &= nullableEquals(other_round.endTime(), endTime()); + result &= nullableEquals(other_round.expectedCount(), expectedCount()); + result &= nullableEquals(other_round.actualCount(), actualCount()); + result &= nullableEquals(other_round.startAuditedPrefixLength(), + startAuditedPrefixLength()); + result &= nullableEquals(other_round.discrepancies(), discrepancies()); + result &= nullableEquals(other_round.disagreements(), disagreements()); + result &= nullableEquals(other_round.signatories(), signatories()); + } else { + result = false; + } + return result; + } + + /** + * @return a hash code for this object. + */ + @Override + public int hashCode() { + return nullableHashCode(startTime()); + } + + /** + * @return a version of this Round with no ballot/cvr sequences + */ + public Round withoutSequences() { + final Round result = + new Round(my_number, my_start_time, my_expected_count, my_previous_ballots_audited, + my_expected_audited_prefix_length, my_start_audited_prefix_length, + null, null); + result.my_actual_count = my_actual_count; + result.my_actual_audited_prefix_length = my_actual_audited_prefix_length; + result.my_discrepancies = my_discrepancies; + result.my_disagreements = my_disagreements; + result.my_signatories = my_signatories; + result.my_end_time = my_end_time; + + return result; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/Tribute.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/Tribute.java new file mode 100644 index 00000000..1646fa0f --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/Tribute.java @@ -0,0 +1,117 @@ +package us.freeandfair.corla.model; + +import java.io.Serializable; + +import javax.persistence.Entity; +import javax.persistence.Version; +import javax.persistence.Column; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; + +import us.freeandfair.corla.persistence.PersistentEntity; + +/** + * I volunteer as tribute, + * to be randomly selected and audited. + * A tribute is a theoretical cvr that may or may not exist. + **/ +@Entity +public class Tribute implements PersistentEntity, Serializable { + /** + * The serialVersionUID. + */ + private static final long serialVersionUID = 1; + + /** + * The version (for optimistic locking). + */ + @Version + private Long version; + + /** + * The ID number. + */ + @Id + @Column(updatable = false, nullable = false) + @GeneratedValue(strategy = GenerationType.SEQUENCE) + private Long my_id; + + /** + * A county id + */ + public Long countyId; + + /** + * A scanner id + */ + public Integer scannerId; + + /** + * A batch id + */ + public String batchId; + + /** + * A ballot's position as an offest + */ + public Integer ballotPosition; + + /** + * The generated random number that selects/resolves to this Tribute + */ + public Integer rand; + + /** + * to preserve the order of randomly selected cvrs + **/ + public Integer randSequencePosition; + + /** + * the contest this tribute was selected for + **/ + public String contestName; + + /** + * combine attributes to form a uri for fast selection + */ + public String uri; + + /** + * {@inheritDoc} + */ + @Override + public Long version() { + return this.version; + } + + /** + * {@inheritDoc} + */ + @Override + public Long id() { + return my_id; + } + + /** + * {@inheritDoc} + */ + @Override + public void setID(final Long the_id) { + my_id = the_id; + } + + /** get the uri **/ + public String getUri() { + return this.uri; + } + + /** + * set the uri for the cvr that is to be selected + * this is used to find the cvr later + **/ + public void setUri() { + this.uri = String.format("%s:%s:%s-%s-%s", "cvr", countyId, scannerId, batchId, ballotPosition); + } + +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/UploadedFile.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/UploadedFile.java new file mode 100644 index 00000000..6cecaa90 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/model/UploadedFile.java @@ -0,0 +1,335 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 1, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.model; + +import static us.freeandfair.corla.util.EqualsHashcodeHelper.*; + +import java.sql.Blob; +import java.time.Instant; + +import javax.persistence.Cacheable; +import javax.persistence.Column; +import javax.persistence.Convert; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.FetchType; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Index; +import javax.persistence.JoinColumn; +import javax.persistence.Lob; +import javax.persistence.ManyToOne; +import javax.persistence.Table; +import javax.persistence.Version; + +import us.freeandfair.corla.persistence.PersistentEntity; +import us.freeandfair.corla.persistence.ResultConverter; +import us.freeandfair.corla.csv.Result; + +/** + * An uploaded file, kept in persistent storage for archival. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +// note that unlike our other entities, uploaded files are not Serializable +@Entity +@Cacheable(false) // uploaded files are explicitly not cacheable +@Table(name = "uploaded_file", + indexes = { @Index(name = "idx_uploaded_file_county", columnList = "county_id") }) +// this class has many fields that would normally be declared final, but +// cannot be for compatibility with Hibernate and JPA. +@SuppressWarnings("PMD.ImmutableField") +public class UploadedFile implements PersistentEntity { + /** + * The database ID. + */ + @Id + @Column(updatable = false, nullable = false) + @GeneratedValue(strategy = GenerationType.SEQUENCE) + private Long my_id; + + /** + * The version (for optimistic locking). + */ + @Version + private Long my_version; + + /** + * The timestamp for this ballot manifest info, in milliseconds since the epoch. + */ + @Column(updatable = false, nullable = false) + private Instant my_timestamp; + + /** + * The county that uploaded the file. + */ + @ManyToOne(optional = false, fetch = FetchType.LAZY) + @JoinColumn + private County my_county; + + /** + * The status of the file. + */ + @Column(nullable = false) + @Enumerated(EnumType.STRING) + private FileStatus status; + + /** + * The orignal filename. + */ + @Column(updatable = false) + private String my_filename; + + /** + * The computed hash of the file blob. + */ + @Column(updatable = false, nullable = false) + private String computed_hash; + + /** + * The hash submitted with file upload. + */ + @Column(updatable = false, nullable = false) + private String submitted_hash; + + /** the parse result **/ + @Column(length = 65535, columnDefinition = "text") + @Convert(converter = ResultConverter.class) + private Result result; + + /** + * The uploaded file. + */ + @Lob + @Column(updatable = false, nullable = false) + private Blob my_file; + + /** + * The file size. + */ + @Column(updatable = false, nullable = false) + private Long my_size; + + /** + * The approximate number of records in the file. + */ + @Column(updatable = false, nullable = false) + private Integer my_approximate_record_count; + + /** + * Constructs an empty uploaded file, solely for persistence. + */ + public UploadedFile() { + super(); + } + + /** + * Constructs an uploaded file with the specified information. + * + * @param the_timestamp The timestamp. + * @param the_county The county that uploaded the file. + * @param the_filename The original filename. + * @param the_status The file status. + * @param computed_hash The computed hash of the file blob. + * @param submitted_hash The hash entered at upload time. + * @param the_file The file (as a Blob). + * @param the_size The file size (in bytes). + * @param the_approximate_record_count The approximate record count. + */ + public UploadedFile(final Instant the_timestamp, + final County the_county, + final String the_filename, + final FileStatus status, + final String computed_hash, + final String submitted_hash, + final Blob the_file, + final Long the_size, + final Integer the_approximate_record_count) { + super(); + my_timestamp = the_timestamp; + my_county = the_county; + my_filename = the_filename; + this.status = status; + this.computed_hash = computed_hash; + this.submitted_hash = submitted_hash; + my_file = the_file; + my_size = the_size; + my_approximate_record_count = the_approximate_record_count; + } + + /** + * {@inheritDoc} + */ + @Override + public Long id() { + return my_id; + } + + /** + * {@inheritDoc}. + */ + @Override + public final void setID(final Long the_id) { + my_id = the_id; + } + + /** + * {@inheritDoc} + */ + @Override + public Long version() { + return my_version; + } + + /** + * @return the timestamp of this file. + */ + public Instant timestamp() { + return my_timestamp; + } + + /** + * @return the county that uploaded this file. + */ + public County county() { + return my_county; + } + + /** + * @return the original filename of this file. + */ + public String filename() { + return my_filename; + } + + /** + * @return the status of this file. + */ + public FileStatus getStatus() { + return this.status; + } + /** + * Sets the file status. + * + * @param the_status The new status. + */ + public void setStatus(final FileStatus status) { + this.status = status; + } + + /** + * Set the parse result + * @param errorMessage of this file. + */ + public void setResult(final Result result) { + this.result = result; + } + + /** + * @return the parse result of this file. + */ + public Result getResult() { + return this.result; + } + + /** + * @return the computed hash of the file blob. + */ + public String getHash() { + return this.computed_hash; + } + + /** + * @return the computed hash of the file blob. + */ + public String getSubmittedHash() { + return this.submitted_hash; + } + + /** + * @return the file, as a binary blob. + */ + public Blob file() { + return my_file; + } + + /** + * @return the file size (in bytes). + */ + public Long size() { + return my_size; + } + + /** + * @return the approximate record count. + */ + public Integer approximateRecordCount() { + return my_approximate_record_count; + } + + /** + * @return a String representation of this elector. + */ + @Override + public String toString() { + return "UploadedFile [id=" + my_id + + ", county=" + my_county + + ", filename=" + my_filename + + "]"; + } + + /** + * Compare this object with another for equivalence. + * + * @param the_other The other object. + * @return true if the objects are equivalent, false otherwise. + */ + @Override + public boolean equals(final Object the_other) { + boolean result = true; + if (the_other instanceof UploadedFile) { + final UploadedFile other_file = (UploadedFile) the_other; + result &= nullableEquals(other_file.timestamp(), timestamp()); + result &= nullableEquals(other_file.county(), county()); + result &= nullableEquals(other_file.filename(), filename()); + result &= nullableEquals(other_file.getStatus(), getStatus()); + result &= nullableEquals(other_file.getHash(), getHash()); + result &= nullableEquals(other_file.getSubmittedHash(), getSubmittedHash()); + result &= nullableEquals(other_file.size(), size()); + result &= nullableEquals(other_file.approximateRecordCount(), approximateRecordCount()); + } else { + result = false; + } + return result; + } + + /** + * @return a hash code for this object. + */ + @Override + public int hashCode() { + return nullableHashCode(getHash()); + } + + /** the possible statuses **/ + public enum FileStatus { + HASH_VERIFIED, + HASH_MISMATCH, + IMPORTING, + IMPORTED, + FAILED + } + +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/AuditReasonSetConverter.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/AuditReasonSetConverter.java new file mode 100644 index 00000000..9994bac8 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/AuditReasonSetConverter.java @@ -0,0 +1,68 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 26, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.persistence; + +import java.lang.reflect.Type; +import java.util.Set; + +import javax.persistence.AttributeConverter; +import javax.persistence.Converter; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; + +import us.freeandfair.corla.model.AuditReason; + +/** + * A converter between maps from audit reasons to booleans and JSON + * representations of those maps, for database efficiency. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@Converter +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class AuditReasonSetConverter + implements AttributeConverter, String> { + /** + * The type information for a map from AuditReason to Boolean. + */ + private static final Type AUDIT_REASON_SET = new TypeToken>() { }.getType(); + + /** + * Our Gson instance, which does not do pretty-printing (unlike the global + * one defined in Main). + */ + private static final Gson GSON = + new GsonBuilder().serializeNulls().disableHtmlEscaping().create(); + + /** + * Converts the specified list of Strings to a database column entry. + * + * @param the_set The list of Strings. + */ + @Override + public String convertToDatabaseColumn(final Set the_map) { + return GSON.toJson(the_map); + } + + /** + * Converts the specified database column entry to a list of strings. + * + * @param the_column The column entry. + */ + @Override + public Set convertToEntityAttribute(final String the_column) { + return GSON.fromJson(the_column, AUDIT_REASON_SET); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/AuditSelectionIntegerMapConverter.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/AuditSelectionIntegerMapConverter.java new file mode 100644 index 00000000..977a29ad --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/AuditSelectionIntegerMapConverter.java @@ -0,0 +1,69 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 26, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.persistence; + +import java.lang.reflect.Type; +import java.util.Map; + +import javax.persistence.AttributeConverter; +import javax.persistence.Converter; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; + +import us.freeandfair.corla.model.AuditSelection; + +/** + * A converter between maps from audit reasons to integers and JSON + * representations of those maps, for database efficiency. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@Converter +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class AuditSelectionIntegerMapConverter + implements AttributeConverter, String> { + /** + * The type information for a map from AuditSelection to Integer. + */ + private static final Type AUDIT_SELECTION_INTEGER_MAP = + new TypeToken>() { }.getType(); + + /** + * Our Gson instance, which does not do pretty-printing (unlike the global + * one defined in Main). + */ + private static final Gson GSON = + new GsonBuilder().serializeNulls().disableHtmlEscaping().create(); + + /** + * Converts the specified list of Strings to a database column entry. + * + * @param the_set The list of Strings. + */ + @Override + public String convertToDatabaseColumn(final Map the_map) { + return GSON.toJson(the_map); + } + + /** + * Converts the specified database column entry to a list of strings. + * + * @param the_column The column entry. + */ + @Override + public Map convertToEntityAttribute(final String the_column) { + return GSON.fromJson(the_column, AUDIT_SELECTION_INTEGER_MAP); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/BallotSequenceAssignmentConverter.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/BallotSequenceAssignmentConverter.java new file mode 100644 index 00000000..0b625ed4 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/BallotSequenceAssignmentConverter.java @@ -0,0 +1,68 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 26, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.persistence; + +import java.lang.reflect.Type; + +import java.util.List; +import java.util.Map; + +import javax.persistence.AttributeConverter; +import javax.persistence.Converter; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +import com.google.gson.reflect.TypeToken; + +/** + * A converter between ballot sequence assignment data and the database. + * + * @author Democracy Works, Inc. + */ +@Converter +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class BallotSequenceAssignmentConverter + implements AttributeConverter>, String> { + /** + * The required type information. + */ + private static final Type BALLOT_SEQUENCE_ASSIGNMENT_TYPE = + new TypeToken>>() { }.getType(); + + /** + * Our Gson instance, which does not do pretty-printing (unlike the global + * one defined in Main). + */ + private static final Gson GSON = + new GsonBuilder().serializeNulls().disableHtmlEscaping().create(); + + /** + * Convert the type into JSON for database storage. + * + * @param l the list to persist. + */ + @Override + public String convertToDatabaseColumn(final List> l) { + return GSON.toJson(l); + } + + /** + * Converts a type stored as JSON in the database to a Java object. + * + * @param s the JSON-encoded string + */ + @Override + public List> convertToEntityAttribute(final String s) { + return GSON.fromJson(s, BALLOT_SEQUENCE_ASSIGNMENT_TYPE); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/CountyCanonicalContestsMapConverter.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/CountyCanonicalContestsMapConverter.java new file mode 100644 index 00000000..0e287b1f --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/CountyCanonicalContestsMapConverter.java @@ -0,0 +1,60 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 26, 2018 + * @copyright 2018 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.persistence; + +import java.lang.reflect.Type; +import java.util.Map; +import java.util.Set; +import javax.persistence.AttributeConverter; +import javax.persistence.Converter; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; + +/** + * A converter between maps from String to Sets of Strings + * @author Democracy Works + * @version 1.3.2 + */ +@Converter +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class CountyCanonicalContestsMapConverter + implements AttributeConverter>, String> { + + /** + * The Type of the thing + */ + private static final Type COUNTY_CONTEST_MAP = + new TypeToken>>() { }.getType(); + + /** + * A serializer for the thing + */ + private static final Gson GSON = + new GsonBuilder().serializeNulls().disableHtmlEscaping().create(); + + /** + * We can use the default JSON serialization for this type of thing + */ + @Override + public String convertToDatabaseColumn(final Map> m) { + return GSON.toJson(m); + } + + /** + * When deserializing, we'll use the Java collection type + */ + @Override + public Map> convertToEntityAttribute(final String column) { + return GSON.fromJson(column, COUNTY_CONTEST_MAP); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/ElectorListConverter.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/ElectorListConverter.java new file mode 100644 index 00000000..55ad1520 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/ElectorListConverter.java @@ -0,0 +1,73 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 26, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.persistence; + +import java.lang.reflect.Type; +import java.util.List; + +import javax.persistence.AttributeConverter; +import javax.persistence.Converter; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; + +import us.freeandfair.corla.json.FreeAndFairNamingStrategy; +import us.freeandfair.corla.model.Elector; + +/** + * A converter between lists of AuditBoard objects and JSON representations + * of AuditBoard objects, for database efficiency. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@Converter +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class ElectorListConverter implements AttributeConverter, String> { + /** + * The type information for a set of String. + */ + private static final Type ELECTOR_LIST = + new TypeToken>() { }.getType(); + + /** + * Our Gson instance, which does not do pretty-printing (unlike the global + * one defined in Main). + */ + private static final Gson GSON = + new GsonBuilder(). + setFieldNamingStrategy(new FreeAndFairNamingStrategy()). + serializeNulls(). + disableHtmlEscaping(). + create(); + + /** + * Converts the specified list of Strings to a database column entry. + * + * @param the_set The list of Strings. + */ + @Override + public String convertToDatabaseColumn(final List the_set) { + return GSON.toJson(the_set); + } + + /** + * Converts the specified database column entry to a list of strings. + * + * @param the_column The column entry. + */ + @Override + public List convertToEntityAttribute(final String the_column) { + return GSON.fromJson(the_column, ELECTOR_LIST); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/FreeAndFairNamingStrategy.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/FreeAndFairNamingStrategy.java new file mode 100644 index 00000000..b9f29817 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/FreeAndFairNamingStrategy.java @@ -0,0 +1,86 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 24, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.persistence; + +import java.util.Locale; + +import org.hibernate.boot.model.naming.Identifier; +import org.hibernate.boot.model.naming.PhysicalNamingStrategyStandardImpl; +import org.hibernate.engine.jdbc.env.spi.JdbcEnvironment; + +/** + * A naming strategy for Hibernate that takes our standard instance field names + * (prepended with "my_", separated by underscores) and translates them to + * column names without the "my_". + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +// we suppress this PMD warning because this class, despite being +// stateless (and thus ideally being a utility class), is required by the +// Hibernate interface to be instantiable +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class FreeAndFairNamingStrategy extends PhysicalNamingStrategyStandardImpl { + /** + * The serialVersionUID. + */ + private static final long serialVersionUID = 1L; + + /** + * Translates a name from Java conventions by adding underscores and undoing + * camel case. + * + * @param the_name The name to translate. + */ + private static String addUnderscores(final String the_name) { + final StringBuilder buf = new StringBuilder(the_name.replace('.', '_')); + int i = 1; + while (i < buf.length()) { + if (Character.isLowerCase(buf.charAt(i - 1)) && + Character.isUpperCase(buf.charAt(i)) && + Character.isLowerCase(buf.charAt(i + 1))) { + buf.insert(i, '_'); + i = i + 1; + } + i = i + 1; + } + return buf.toString().toLowerCase(Locale.ROOT); + } + + /** + * Translates a Java identifier to a Hibernate table name, by removing all + * "my_" occurrences and leaving the remainder of the field name intact. + * + * @param the_identifier The identifier to translate. + * @param the_context The context (ignored). + */ + @Override + public Identifier toPhysicalTableName(final Identifier the_identifier, + final JdbcEnvironment the_context) { + return new Identifier(addUnderscores(the_identifier.getText()).replaceAll("my_", ""), + the_identifier.isQuoted()); + } + + /** + * Translates a Java identifier to a Hibernate column name, by removing all + * "my_" occurrences and leaving the remainder of the field name intact. + * + * @param the_identifier The identifier to translate. + * @param the_context The context (ignored). + */ + @Override + public Identifier toPhysicalColumnName(final Identifier the_identifier, + final JdbcEnvironment the_context) { + return new Identifier(addUnderscores(the_identifier.getText()).replaceAll("my_", ""), + the_identifier.isQuoted()); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/IntegerListConverter.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/IntegerListConverter.java new file mode 100644 index 00000000..4577353b --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/IntegerListConverter.java @@ -0,0 +1,64 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created September 12, 2018 + * @copyright 2018 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Democracy Works, Inc + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.persistence; + +import java.lang.reflect.Type; +import java.util.List; + +import javax.persistence.AttributeConverter; +import javax.persistence.Converter; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; + +/** + * A converter between lists of Integers and JSON representations of such lists, + * for database efficiency. + * + * @author Democracy Works, Inc + */ +@Converter +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class IntegerListConverter implements AttributeConverter, String> { + /** + * The type information for a list of Integer + */ + private static final Type INTEGER_LIST = new TypeToken>() { }.getType(); + + /** + * Our Gson instance, which does not do pretty-printing (unlike the global + * one defined in Main). + */ + private static final Gson GSON = + new GsonBuilder().serializeNulls().disableHtmlEscaping().create(); + + /** + * Converts the specified list of Strings to a database column entry. + * + * @param the_list The list of Strings. + */ + @Override + public String convertToDatabaseColumn(final List the_list) { + return GSON.toJson(the_list); + } + + /** + * Converts the specified database column entry to a list of strings. + * + * @param the_column The column entry. + */ + @Override + public List convertToEntityAttribute(final String the_column) { + return GSON.fromJson(the_column, INTEGER_LIST); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/LongIntegerMapConverter.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/LongIntegerMapConverter.java new file mode 100644 index 00000000..f87f770b --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/LongIntegerMapConverter.java @@ -0,0 +1,54 @@ +package us.freeandfair.corla.persistence; + +import java.lang.reflect.Type; +import java.util.Map; + +import javax.persistence.AttributeConverter; +import javax.persistence.Converter; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; + +/** + * A converter between maps from longs to integers and JSON + * representations of those maps, for database efficiency. + * + */ +@Converter +public class LongIntegerMapConverter + implements AttributeConverter, String> { + + /** + * The type information for a map. + */ + private static final Type LONG_INTEGER_MAP = + new TypeToken>() { }.getType(); + + /** + * Our Gson instance, which does not do pretty-printing (unlike the global + * one defined in Main). + */ + private static final Gson GSON = + new GsonBuilder().serializeNulls().disableHtmlEscaping().create(); + + /** + * Converts the specified list of Strings to a database column entry. + * + * @param the_set The list of Strings. + */ + @Override + public String convertToDatabaseColumn(final Map the_map) { + return GSON.toJson(the_map); + } + + /** + * Converts the specified database column entry to a list of strings. + * + * @param the_column The column entry. + */ + @Override + public Map convertToEntityAttribute(final String the_column) { + return GSON.fromJson(the_column, LONG_INTEGER_MAP); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/LongListConverter.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/LongListConverter.java new file mode 100644 index 00000000..43f3338e --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/LongListConverter.java @@ -0,0 +1,65 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 26, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.persistence; + +import java.lang.reflect.Type; +import java.util.List; + +import javax.persistence.AttributeConverter; +import javax.persistence.Converter; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; + +/** + * A converter between lists of Integers and JSON representations of such lists, + * for database efficiency. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@Converter +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class LongListConverter implements AttributeConverter, String> { + /** + * The type information for a list of Long. + */ + private static final Type LONG_LIST = new TypeToken>() { }.getType(); + + /** + * Our Gson instance, which does not do pretty-printing (unlike the global + * one defined in Main). + */ + private static final Gson GSON = + new GsonBuilder().serializeNulls().disableHtmlEscaping().create(); + + /** + * Converts the specified list of Strings to a database column entry. + * + * @param the_list The list of Strings. + */ + @Override + public String convertToDatabaseColumn(final List the_list) { + return GSON.toJson(the_list); + } + + /** + * Converts the specified database column entry to a list of strings. + * + * @param the_column The column entry. + */ + @Override + public List convertToEntityAttribute(final String the_column) { + return GSON.fromJson(the_column, LONG_LIST); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/Persistence.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/Persistence.java new file mode 100644 index 00000000..72999515 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/Persistence.java @@ -0,0 +1,753 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 27, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.persistence; + +import java.io.IOException; +import java.io.InputStream; +import java.io.Serializable; +import java.sql.Blob; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Scanner; +import java.util.stream.Stream; + +import javax.persistence.PersistenceException; +import javax.persistence.RollbackException; +import javax.persistence.TypedQuery; +import javax.persistence.criteria.CriteriaBuilder; +import javax.persistence.criteria.CriteriaQuery; +import javax.persistence.criteria.Root; + +import org.hibernate.Hibernate; +import org.hibernate.HibernateException; +import org.hibernate.ObjectNotFoundException; +import org.hibernate.Session; +import org.hibernate.SessionFactory; +import org.hibernate.Transaction; +import org.hibernate.boot.Metadata; +import org.hibernate.boot.MetadataSources; +import org.hibernate.boot.registry.StandardServiceRegistry; +import org.hibernate.boot.registry.StandardServiceRegistryBuilder; +import org.hibernate.cfg.Environment; +import org.hibernate.query.Query; +import org.hibernate.resource.transaction.spi.TransactionStatus; + +import us.freeandfair.corla.Main; + +/** + * Manages persistence through Hibernate, and provides several utility methods. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings({"PMD.ExcessiveImports", "PMD.GodClass"}) +public final class Persistence { + /** + * The path to the resource containing the list of entity classes. + */ + public static final String ENTITY_CLASSES = + "us/freeandfair/corla/persistence/entity_classes"; + + /** + * The "true" constant. + */ + public static final String TRUE = "true"; + + /** + * The "false" constant. + */ + public static final String FALSE = "false"; + + /** + * The "NO SESSION" constant. + */ + public static final Session NO_SESSION = null; + + /** + * The "NO TRANSACTION" constant. + */ + public static final Transaction NO_TRANSACTION = null; + + /** + * The system properties. + */ + private static Properties system_properties; + + /** + * The service registry for Hibernate. + */ + private static StandardServiceRegistry service_registry; + + /** + * The session factory for Hibernate. + */ + private static SessionFactory session_factory; + + /** + * A thread-local containing the active session on this thread. + */ + private static ThreadLocal session_info = new ThreadLocal(); + + /** + * A flag indicating whether persistence has failed to start or not. + */ + private static boolean failed; + + /** + * Private constructor to prevent instantiation. + */ + private Persistence() { + // do nothing + } + + /** + * @return true if database persistence is enabled, false otherwise. + */ + public static synchronized boolean hasDB() { + return !failed && (session_info.get() != null || openSession() != null); + } + + /** + * Sets the properties for the system. + * + * @param the_properties The properties. + */ + public static synchronized void setProperties(final Properties the_properties) { + system_properties = the_properties; + final Map env = System.getenv(); + + if (env.containsKey("HIBERNATE_URL")) { + system_properties.setProperty("hibernate.url", env.get("HIBERNATE_URL")); + } + } + + /** + * Creates a new Session. Note that this method should typically only be called + * by the Persistence methods that control transactions. + * + * @return the new Session. + * @exception IllegalStateException if a Session is already open on this thread. + */ + public static synchronized Session openSession() { + Session session = session_info.get(); + if (session != null && session.isOpen()) { + throw new IllegalStateException("session is already open on this thread"); + } + + if (!failed && session == null) { + if (session_factory == null) { + setupSessionFactory(); + } + + if (session_factory == null) { + failed = true; + } else { + try { + session = session_factory.openSession(); + session_info.set(session); + } catch (final HibernateException e) { + Main.LOGGER.error("Exception getting Hibernate session: " + e); + failed = true; + } + } + } + return session; + } + + /** + * @return the currently open session. + * @exception PersistenceException if there is no currently open session. + */ + public static Session currentSession() { + final Session session = session_info.get(); + if (session == null || !session.isOpen()) { + throw new PersistenceException("no open session"); + } else { + return session; + } + } + + /** + * Sets up the session factory from the properties in the properties file. + */ + @SuppressWarnings({"PMD.AvoidCatchingGenericException", "PMD.ExcessiveMethodLength", + "checkstyle:magicnumber", "checkstyle:executablestatementcount", + "checkstyle:methodlength"}) + private static synchronized void setupSessionFactory() { + Main.LOGGER.info("attempting to create Hibernate session factory"); + + try { + final StandardServiceRegistryBuilder rb = new StandardServiceRegistryBuilder(); + final Map settings = new HashMap<>(); + + // database settings + settings.put(Environment.DRIVER, system_properties.getProperty("hibernate.driver", "")); + settings.put(Environment.URL, system_properties.getProperty("hibernate.url", "")); + settings.put(Environment.USER, system_properties.getProperty("hibernate.user", "")); + settings.put(Environment.PASS, system_properties.getProperty("hibernate.pass", "")); + settings.put(Environment.DIALECT, + system_properties.getProperty("hibernate.dialect", "")); + + // C3P0 connection pooling + settings.put(Environment.C3P0_MIN_SIZE, + system_properties.getProperty("hibernate.c3p0.min_size", "20")); + settings.put(Environment.C3P0_MAX_SIZE, + system_properties.getProperty("hibernate.c3p0.max_size", "20")); + settings.put(Environment.C3P0_IDLE_TEST_PERIOD, + system_properties.getProperty("hibernate.c3p0.idle_test_period", "0")); + settings.put(Environment.C3P0_MAX_STATEMENTS, + system_properties.getProperty("hibernate.c3p0.max_statements", "0")); + settings.put(Environment.C3P0_TIMEOUT, + system_properties.getProperty("hibernate.c3p0.timeout", "300")); + settings.put("hibernate.c3p0.numHelperThreads", + system_properties.getProperty("hibernate.c3p0.numHelperThreads", "3")); + settings.put("hibernate.c3p0.privilegeSpawnedThreads", TRUE); + settings.put("hibernate.c3p0.contextClassLoaderSource", "none"); + + // automatic schema generation + settings.put(Environment.HBM2DDL_AUTO, + system_properties.getProperty("hibernate.hbm2ddl.auto", "")); + + // sql debugging + settings.put(Environment.SHOW_SQL, + system_properties.getProperty("hibernate.show_sql", FALSE)); + settings.put(Environment.FORMAT_SQL, + system_properties.getProperty("hibernate.format_sql", FALSE)); + settings.put(Environment.USE_SQL_COMMENTS, + system_properties.getProperty("hibernate.use_sql_comments", FALSE)); + + // table and column naming + settings.put(Environment.PHYSICAL_NAMING_STRATEGY, + "us.freeandfair.corla.persistence.FreeAndFairNamingStrategy"); + + // concurrency and isolation + settings.put(Environment.CURRENT_SESSION_CONTEXT_CLASS, "thread"); + settings.put(Environment.USE_STREAMS_FOR_BINARY, TRUE); + settings.put(Environment.AUTOCOMMIT, FALSE); + settings.put(Environment.CONNECTION_PROVIDER_DISABLES_AUTOCOMMIT, TRUE); + settings.put(Environment.ISOLATION, "REPEATABLE_READ"); + + // caching + settings.put(Environment.JPA_SHARED_CACHE_MODE, "ENABLE_SELECTIVE"); + settings.put(Environment.CACHE_PROVIDER_CONFIG, "org.hibernate.cache.EhCacheProvider"); + settings.put(Environment.CACHE_REGION_FACTORY, + "org.hibernate.cache.ehcache.EhCacheRegionFactory"); + settings.put(Environment.USE_SECOND_LEVEL_CACHE, FALSE); + settings.put(Environment.USE_QUERY_CACHE, FALSE); + // IMPORTANT: the USE_DIRECT_REFERENCE_CACHE_ENTRIES setting is FALSE to address + // Hibernate bug HHH-11169, and must not be changed until/unless that bug is + // resolved + settings.put(Environment.USE_DIRECT_REFERENCE_CACHE_ENTRIES, FALSE); + settings.put(Environment.DEFAULT_CACHE_CONCURRENCY_STRATEGY, "read-write"); + + // other performance + settings.put(Environment.ORDER_INSERTS, TRUE); + settings.put(Environment.ORDER_UPDATES, TRUE); + settings.put(Environment.BATCH_VERSIONED_DATA, TRUE); + settings.put(Environment.STATEMENT_BATCH_SIZE, + system_properties.getProperty("hibernate.jdbc.batch_size", "100")); + settings.put(Environment.BATCH_FETCH_STYLE, "DYNAMIC"); + settings.put(Environment.VALIDATE_QUERY_PARAMETERS, FALSE); + settings.put(Environment.DEFAULT_BATCH_FETCH_SIZE, "16"); + settings.put(Environment.MAX_FETCH_DEPTH, "3"); + + // empty composite objects + settings.put(Environment.CREATE_EMPTY_COMPOSITES_ENABLED, TRUE); + + // statistics + settings.put(Environment.GENERATE_STATISTICS, FALSE); + + // apply settings + rb.applySettings(settings); + + // create registry + service_registry = rb.build(); + + // create metadata sources and metadata + final MetadataSources sources = new MetadataSources(service_registry); + try (InputStream entity_stream = + ClassLoader.getSystemResourceAsStream(ENTITY_CLASSES)) { + if (entity_stream == null) { + Main.LOGGER.error("could not load list of entity classes"); + } else { + final Scanner scanner = new Scanner(entity_stream, "UTF-8"); + while (scanner.hasNextLine()) { + final String entity_class = scanner.nextLine(); + try { + sources.addAnnotatedClass(Class.forName(entity_class)); + } catch (final ClassNotFoundException e) { + Main.LOGGER.error("could not add entity, no such class: " + entity_class); + } + Main.LOGGER.debug("added entity class " + entity_class); + } + scanner.close(); + } + } catch (final IOException e) { + Main.LOGGER.error("error reading list of entity classes: " + e); + } + final Metadata metadata = sources.getMetadataBuilder().build(); + + // create session factory + session_factory = metadata.getSessionFactoryBuilder().build(); + Main.LOGGER.debug("started Hibernate"); + } catch (final RuntimeException e) { + Main.LOGGER.error("could not start Hibernate, persistence is disabled: " + e); + if (service_registry != null) { + StandardServiceRegistryBuilder.destroy(service_registry); + } + } + } + + /** + * @return true if a session is open on this thread, false otherwise. + * @exception IllegalStateException if the database isn't running. + */ + public static boolean isSessionOpen() + throws PersistenceException { + checkForDatabase(); + + final Session session = session_info.get(); + return session != null && session.isOpen(); + } + + /** + * @return true if a long-lived transaction is running in this thread, + * false otherwise. + * @exception IllegalStateException if the database isn't running. + */ + public static boolean isTransactionActive() + throws PersistenceException { + checkForDatabase(); + + final Session session = session_info.get(); + Transaction transaction = null; + if (session != null) { + try { + transaction = session.getTransaction(); + } catch (final HibernateException e) { + // the session did not have a transaction + } + } + return session != null && transaction != null && + transaction.getStatus() == TransactionStatus.ACTIVE; + } + + /** + * @return true if a long-lived transaction can be rolled back, + * false otherwise. + * @exception IllegalStateException if the database isn't running. + */ + public static boolean canTransactionRollback() + throws PersistenceException { + checkForDatabase(); + + final Session session = currentSession(); + Transaction transaction = null; + try { + transaction = session.getTransaction(); + } catch (final HibernateException e) { + // the session did not have a transaction + } + return transaction != null && transaction.getStatus().canRollback(); + } + + /** + * Begins a long-lived transaction in this thread that will span several + * operations, opening a session if necessary. If an existing transaction + * is in a non-active state, it is rolled back (if possible) and a new transaction + * is started. + * + * @return true if a new transaction is started, false if a transaction was + * already active. + * @exception IllegalStateException if the database isn't running. + * @exception PersistenceException if a transaction cannot be started or + * continued. + */ + public static boolean beginTransaction() + throws PersistenceException { + checkForDatabase(); + + boolean result = true; + Session session = currentSession(); + + if (isTransactionActive()) { + result = false; + } else if (canTransactionRollback()) { + rollbackTransaction(); + // session is explicitly closed by rollback + // interesting note: isOpen() on the session _still returns true_ even though + // it is closed! + session = openSession(); + session.beginTransaction(); + } else { + // we don't have an active or rollback-able transaction, so we just + // start a new one + session.beginTransaction(); + } + + return result; + } + + /** + * Commits the active long-lived transaction. This also closes the current + * session, regardless of the transaction's success (it is rolled back if + * it does not succeed). + * + * @exception IllegalStateException if no such transaction is running. + * @exception PersistenceException if there is a problem with persistent storage. + * @exception RollbackException if the commit fails. + */ + public static void commitTransaction() + throws IllegalStateException, PersistenceException, RollbackException { + checkForRunningTransaction(); + try { + currentSession().getTransaction().commit(); + } finally { + currentSession().close(); + session_info.remove(); + } + + } + + /** + * Rolls back the active long lived transaction. This also closes the current + * session, regardless of the rollback's success. + * + * @exception IllegalStateException if no such transaction is running, or if the + * running transaction cannot be rolled back. + * @exception PersistenceException if there is a problem with persistent storage. + */ + public static void rollbackTransaction() + throws IllegalStateException, PersistenceException { + if (canTransactionRollback()) { + try { + currentSession().getTransaction().rollback(); + } finally { + currentSession().close(); + session_info.remove(); + } + } else { + throw new IllegalStateException("no active transaction to roll back"); + } + } + + /** + * Saves or updates the specified object in persistent storage. This + * method must be called within a transaction. + * + * @param the_object The object to save or update. + * @return true if the save/update was successful, false otherwise + * @exception IllegalStateException if no database is available or no + * transaction is running. + */ + public static boolean saveOrUpdate(final PersistentEntity the_object) + throws IllegalStateException { + checkForRunningTransaction(); + + boolean result = true; + + try { + currentSession().saveOrUpdate(the_object); + } catch (final PersistenceException e) { + Main.LOGGER.error("could not save/update object " + the_object + ": " + e); + result = false; + } + + return result; + } + + /** + * Like saveOrUpdate, but returns the thing you wanted to save instead + * of a boolean. + * @param obj the thing to persist + */ + public static T persist(final T obj) { + currentSession().saveOrUpdate(obj); + return obj; + } + + /** + * Saves the specified object in persistent storage. This will cause an + * exception if there is already an object in persistent storage with the same + * class and ID. This method must be called within a transaction. + * + * @param the_object The object to save. + * @exception IllegalStateException if no database is available or no + * transaction is running. + * @exception PersistenceException if the object cannot be saved. + */ + public static void save(final PersistentEntity the_object) + throws IllegalStateException, PersistenceException { + checkForRunningTransaction(); + currentSession().save(the_object); + } + + /** + * Updates the specified object in persistent storage. This will cause an + * exception if there is no object in persistent storage with the same class + * and ID. This method must be called within a transaction. + * + * @param the_object The object to save. + * @exception IllegalStateException if no database is available or no + * transaction is running. + * @exception PersistenceException if the object cannot be updated. + */ + public static void update(final PersistentEntity the_object) + throws IllegalStateException, PersistenceException { + checkForRunningTransaction(); + currentSession().update(the_object); + } + + /** + * Deletes the specified object from persistent storage, if it exists. This + * method must be called within a transaction. + * + * @param the_object The object to delete. + * @return true if the deletion was successful, false otherwise (if + * the object did not exist, false is returned). + * @exception IllegalStateException if no database is available or no + * transaction is running. + */ + public static boolean delete(final PersistentEntity the_object) + throws IllegalStateException { + checkForRunningTransaction(); + + boolean result = true; + + try { + currentSession().delete(the_object); + } catch (final PersistenceException e) { + result = false; + Main.LOGGER.debug("could not delete object " + the_object + ": " + e); + } + + return result; + } + + /** + * Deletes the object of the specified class with the specified ID from + * persistent storage, if it exists. This method must be called within + * a transaction. + * + * @param the_class The class of the object to delete. + * @param the_id The ID of the object to delete. + * @return true if the deletion was successful, false otherwise (if + * the object did not exist, false is returned). + * @exception IllegalStateException if no database is available or no + * transaction is running. + */ + public static boolean delete(final Class the_class, + final Long the_id) + throws IllegalStateException { + checkForRunningTransaction(); + + boolean result = true; + + try { + final PersistentEntity instance = currentSession().load(the_class, the_id); + if (instance == null) { + result = false; + } else { + try { + currentSession().delete(instance); + } catch (final ObjectNotFoundException e) { + // no object with this ID to delete + result = false; + } + } + } catch (final PersistenceException e) { + Main.LOGGER.debug("error deleting object of class " + the_class + + "with ID " + the_id + ": " + e); + result = false; + } + + return result; + } + + /** + * Gets the entity in the current session that has the specified ID and class. + * This method must be called within a transaction. + * + * @param the_id The ID. + * @param the_class The class. + * @return the result entity, or null if no such entity exists. + * @exception IllegalStateException if no database is available or no + * transaction is running. + */ + public static T getByID(final Serializable the_id, + final Class the_class) + throws IllegalStateException { + checkForRunningTransaction(); + + T result = null; + Main.LOGGER.debug("searching session for object " + the_class + "/" + the_id); + try { + result = currentSession().get(the_class, the_id); + } catch (final PersistenceException e) { + Main.LOGGER.error("exception when searching for " + the_class + "/" + the_id + + ": " + e); + } + return result; + } + + /** + * Gets all the entities of the specified class. This method must be called + * within a transaction. + * + * @param the_class The class. + * @return a list containing all the entities of the_class. + * @exception IllegalStateException if no database is available or no + * transaction is running. + */ + public static List getAll(final Class the_class) + throws IllegalStateException { + checkForRunningTransaction(); + + final List result = new ArrayList<>(); + + try { + final Session s = Persistence.currentSession(); + final CriteriaBuilder cb = s.getCriteriaBuilder(); + final CriteriaQuery cq = cb.createQuery(the_class); + final Root root = cq.from(the_class); + cq.select(root); + final TypedQuery query = s.createQuery(cq); + result.addAll(query.getResultList()); + } catch (final PersistenceException e) { + Main.LOGGER.error("could not query database"); + } + + return result; + } + + /** + * Gets a stream of all the entities of the specified class. This method + * must be called within a transaction, and the result stream must be used + * within the same transaction. + * + * @param the_class The class. + * @return a stream containing all the entities of the_class, or null if + * one could not be acquired. + * @exception IllegalStateException if no database is available or no + * transaction is running. + */ + public static Stream + getAllAsStream(final Class the_class) throws IllegalStateException { + checkForRunningTransaction(); + + Stream result = null; + + try { + final Session s = Persistence.currentSession(); + final CriteriaBuilder cb = s.getCriteriaBuilder(); + final CriteriaQuery cq = cb.createQuery(the_class); + final Root root = cq.from(the_class); + cq.select(root); + final Query query = s.createQuery(cq); + result = query.stream(); + } catch (final PersistenceException e) { + Main.LOGGER.error("could not query database"); + } + + return result; + } + + /** + * Flushes the current session, if one exists. If no session is open, this + * method is equivalent to a skip. + * + * @exception PersistenceException if there is a problem flushing the session. + */ + public static void flush() throws PersistenceException { + final Session session = session_info.get(); + if (session != null) { + session.flush(); + } + } + + /** + * Evicts the specified object from the current session, if one exists. If no + * session is open, this method is equivalent to a skip. + * + * @exception NullPointerException if a null object is specified. + * @exception IllegalArgumentException if the specified object is not an entity. + */ + public static void evict(final PersistentEntity the_entity) { + final Session session = session_info.get(); + if (session != null) { + session.evict(the_entity); + } + } + + /** + * Clears all entities from the current session, if one exists. This also + * causes a flush to occur, to ensure that no previous state changes are lost + * (for a "naked" clear, use currentSession().clear()). If no session is open, + * this method is equivalent to a skip. + * + * @exception PersistenceException if there is a problem flushing or clearing + * the session. + */ + public static void flushAndClear() { + final Session session = session_info.get(); + if (session != null) { + session.flush(); + session.clear(); + } + } + + /** + * Gets a streaming Blob for the specified input stream and file size. This method + * must be called within a running transaction. + * + * @param the_stream The input stream. + * @param the_size The file size. + * @exception IllegalStateException if there is no running transaction. + */ + public static Blob blobFor(final InputStream the_stream, final long the_size) { + checkForRunningTransaction(); + return currentSession().getLobHelper().createBlob(the_stream, the_size); + } + + /** + * Unwraps an object from its proxy object, if any; typically used before + * converting the entity to JSON for wire transmission. + * + * @param the_object The object. + * @return the unwrapped object; if the object is not a proxy, it is returned + * unchanged. + */ + public static Object unproxy(final Object the_object) { + return Hibernate.unproxy(the_object); + } + + /** + * Throws an IllegalStateException if there is no running transaction. + */ + private static void checkForRunningTransaction() throws IllegalStateException { + if (!isTransactionActive()) { + throw new IllegalStateException("no running transaction"); + } + } + + /** + * Throws an IllegalStateException if there is no database. + */ + private static void checkForDatabase() throws IllegalStateException { + if (!hasDB()) { + throw new IllegalStateException("no database"); + } + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/PersistentEntity.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/PersistentEntity.java new file mode 100644 index 00000000..ffc8838a --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/PersistentEntity.java @@ -0,0 +1,38 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 7, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.persistence; + +/** + * A persistable entity with an ID number. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public interface PersistentEntity { + /** + * @return the ID number of this entity. + */ + Long id(); + + /** + * Set the ID number of this entity. This method should typically not be used + * except by the persistence system. + * + * @param the_id The new ID number. + */ + void setID(Long the_id); + + /** + * @return the version number of this entity. This is primarily for debugging. + */ + Long version(); +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/ResultConverter.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/ResultConverter.java new file mode 100644 index 00000000..2e8fa705 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/ResultConverter.java @@ -0,0 +1,53 @@ +package us.freeandfair.corla.persistence; + +import java.lang.reflect.Type; + +import javax.persistence.AttributeConverter; +import javax.persistence.Converter; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; + +import us.freeandfair.corla.csv.Result; + +/** + * A converter for the Result class to json. + * + */ +@Converter +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class ResultConverter + implements AttributeConverter { + /** + * The type information for a map from AuditReason to Boolean. + */ + private static final Type RESULT_TYPE = new TypeToken() { }.getType(); + + /** + * Our Gson instance, which does not do pretty-printing (unlike the global + * one defined in Main). + */ + private static final Gson GSON = + new GsonBuilder().serializeNulls().disableHtmlEscaping().create(); + + /** + * Converts the specified list of Strings to a database column entry. + * + * @param the_set The list of Strings. + */ + @Override + public String convertToDatabaseColumn(final Result result) { + return GSON.toJson(result); + } + + /** + * Converts the specified database column entry to a list of strings. + * + * @param the_column The column entry. + */ + @Override + public Result convertToEntityAttribute(final String column) { + return GSON.fromJson(column, RESULT_TYPE); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/SignatoriesConverter.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/SignatoriesConverter.java new file mode 100644 index 00000000..608140ef --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/SignatoriesConverter.java @@ -0,0 +1,66 @@ +/* + * Free & Fair Colorado RLA System + */ + +package us.freeandfair.corla.persistence; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +import com.google.gson.reflect.TypeToken; + +import java.lang.reflect.Type; + +import java.util.List; +import java.util.Map; + +import javax.persistence.AttributeConverter; +import javax.persistence.Converter; + +import us.freeandfair.corla.model.Elector; + +/** + * A converter between a Signatories object and JSON stored in a database. + * + * @author Democracy Works, Inc. + */ +@Converter +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class SignatoriesConverter + implements AttributeConverter>, String> { + /** + * The type information for a set of String. + */ + private static final Type SIGNATORIES_TYPE = + new TypeToken>>() { }.getType(); + + /** + * Our Gson instance, which does not do pretty-printing (unlike the global + * one defined in Main). + */ + private static final Gson GSON = + new GsonBuilder(). + serializeNulls(). + disableHtmlEscaping(). + create(); + + /** + * Converts the Java object to JSON for the database. + * + * @param o The specified type we are converting + */ + @Override + public String convertToDatabaseColumn(final Map> o) { + return GSON.toJson(o); + } + + /** + * Converts the JSON from the database into a Java object. + * + * @param s the JSON string + */ + @Override + public Map> convertToEntityAttribute(final String s) { + return GSON.fromJson(s, SIGNATORIES_TYPE); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/StringListConverter.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/StringListConverter.java new file mode 100644 index 00000000..e47b3cad --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/StringListConverter.java @@ -0,0 +1,74 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * + * @created Aug 26, 2017 + * + * @copyright 2017 Colorado Department of State + * + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * + * @creator Daniel M. Zimmerman + * + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.persistence; + +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.List; + +import javax.persistence.AttributeConverter; +import javax.persistence.Converter; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; + +/** + * A converter between lists of Strings and JSON representations of such lists, + * for database efficiency. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@Converter +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class StringListConverter implements AttributeConverter, String> { + /** + * The type information for a list of String. + */ + private static final Type STRING_LIST = new TypeToken>() { + }.getType(); + + /** + * Our Gson instance, which does not do pretty-printing (unlike the global one + * defined in Main). + */ + // private static final Gson GSON = + // new GsonBuilder().serializeNulls().create(); + private static final Gson GSON = + new GsonBuilder().serializeNulls().disableHtmlEscaping().create(); + + /** + * Converts the specified list of Strings to a database column entry. + * + * @param the_list The list of Strings. + */ + @Override + public String convertToDatabaseColumn(final List the_list) { + return GSON.toJson(the_list); + } + + /** + * Converts the specified database column entry to a list of strings. + * + * @param the_column The column entry. + */ + @Override + public List convertToEntityAttribute(final String the_column) { + return GSON.fromJson(the_column, STRING_LIST); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/StringSetConverter.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/StringSetConverter.java new file mode 100644 index 00000000..594340ec --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/persistence/StringSetConverter.java @@ -0,0 +1,65 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 26, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.persistence; + +import java.lang.reflect.Type; +import java.util.Set; + +import javax.persistence.AttributeConverter; +import javax.persistence.Converter; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; + +/** + * A converter between sets of Strings and JSON representations of those sets, + * for database efficiency. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@Converter +@SuppressWarnings("PMD.AtLeastOneConstructor") +public class StringSetConverter implements AttributeConverter, String> { + /** + * The type information for a set of String. + */ + private static final Type STRING_SET = new TypeToken>() { }.getType(); + + /** + * Our Gson instance, which does not do pretty-printing (unlike the global + * one defined in Main). + */ + private static final Gson GSON = + new GsonBuilder().serializeNulls().disableHtmlEscaping().create(); + + /** + * Converts the specified list of Strings to a database column entry. + * + * @param the_set The list of Strings. + */ + @Override + public String convertToDatabaseColumn(final Set the_set) { + return GSON.toJson(the_set); + } + + /** + * Converts the specified database column entry to a list of strings. + * + * @param the_column The column entry. + */ + @Override + public Set convertToEntityAttribute(final String the_column) { + return GSON.fromJson(the_column, STRING_SET); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/AdministratorQueries.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/AdministratorQueries.java new file mode 100644 index 00000000..b58322e5 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/AdministratorQueries.java @@ -0,0 +1,77 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 8, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.query; + +import java.util.List; + +import javax.persistence.PersistenceException; +import javax.persistence.TypedQuery; +import javax.persistence.criteria.CriteriaBuilder; +import javax.persistence.criteria.CriteriaQuery; +import javax.persistence.criteria.Root; + +import org.hibernate.Session; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.model.Administrator; +import us.freeandfair.corla.persistence.Persistence; + +/** + * Queries having to do with Administrator entities. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public final class AdministratorQueries { + /** + * Private constructor to prevent instantiation. + */ + private AdministratorQueries() { + // do nothing + } + + /** + * Obtains the Administrator object with the specified username, if one exists. + * + * @param the_username The string. + * @return the matched Administrator. If the results are ambiguous or empty + * (more than one match, or no match), returns null. + */ + // we are checking to see if exactly one result is in a list, and + // PMD doesn't like it + @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") + public static Administrator byUsername(final String the_username) { + Administrator result = null; + + try { + final Session s = Persistence.currentSession(); + final CriteriaBuilder cb = s.getCriteriaBuilder(); + final CriteriaQuery cq = cb.createQuery(Administrator.class); + final Root root = cq.from(Administrator.class); + cq.select(root).where(cb.equal(root.get("my_username"), the_username)); + final TypedQuery query = s.createQuery(cq); + final List query_results = query.getResultList(); + // if there's exactly one result, return that + if (query_results.size() == 1) { + result = query_results.get(0); + } + } catch (final PersistenceException e) { + Main.LOGGER.error("could not query database for administrator"); + } + if (result == null) { + Main.LOGGER.debug("found no administrator for username " + the_username); + } else { + Main.LOGGER.debug("found administrator " + result + " for string " + the_username); + } + return result; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/BallotManifestInfoQueries.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/BallotManifestInfoQueries.java new file mode 100644 index 00000000..ee4e8473 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/BallotManifestInfoQueries.java @@ -0,0 +1,261 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 8, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.query; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; + +import javax.persistence.PersistenceException; +import javax.persistence.TypedQuery; +import javax.persistence.criteria.CriteriaBuilder; +import javax.persistence.criteria.CriteriaQuery; +import javax.persistence.criteria.Predicate; +import javax.persistence.criteria.Root; + +import org.hibernate.Session; +import org.hibernate.query.Query; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.model.BallotManifestInfo; +import us.freeandfair.corla.model.CastVoteRecord; +import us.freeandfair.corla.persistence.Persistence; + +/** + * Queries having to do with BallotManfestInfo entities. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public final class BallotManifestInfoQueries { + + /** + * The "county ID" field. + */ + private static final String COUNTY_ID = "my_county_id"; + + + /** + * Private constructor to prevent instantiation. + */ + private BallotManifestInfoQueries() { + // do nothing + } + + /** + * Returns the set of ballot manifests matching the specified county IDs. + * + * @param the_county_ids The set of county IDs. + * @return the ballot manifests matching the specified set of county IDs, + * or null if the query fails. + */ + public static Set getMatching(final Set the_county_ids) { + final Set result = + new TreeSet(new BallotManifestInfo.Sort()); + + try { + final Session s = Persistence.currentSession(); + final CriteriaBuilder cb = s.getCriteriaBuilder(); + final CriteriaQuery cq = + cb.createQuery(BallotManifestInfo.class); + final Root root = cq.from(BallotManifestInfo.class); + final List disjuncts = new ArrayList(); + for (final Long county_id : the_county_ids) { + disjuncts.add(cb.equal(root.get(COUNTY_ID), county_id)); + } + cq.select(root).where(cb.or(disjuncts.toArray(new Predicate[disjuncts.size()]))); + final TypedQuery query = s.createQuery(cq); + result.addAll(query.getResultList()); + } catch (final PersistenceException e) { + Main.LOGGER.error("Exception when reading ballot manifests from database: " + e); + } + + return result; + } + + + + /** select ballot_manifest_info where uri in :uris **/ + public static List locationFor(final Set uris) { + if (uris.isEmpty()) { + return new ArrayList(); + } + final Session s = Persistence.currentSession(); + final CriteriaBuilder cb = s.getCriteriaBuilder(); + final CriteriaQuery cq = cb.createQuery(BallotManifestInfo.class); + final Root root = cq.from(BallotManifestInfo.class); + cq.where(root.get("uri").in(uris)); + final TypedQuery query = s.createQuery(cq); + return query.getResultList(); + } + + /** + * Returns the location for the specified CVR, assuming one can be found. + * + * @param the_cvr The CVR. + * @return the location for the CVR, or null if no location can be found. + */ + public static Optional locationFor(final CastVoteRecord the_cvr) { + Optional result = Optional.empty(); + + try { + final Session s = Persistence.currentSession(); + final CriteriaBuilder cb = s.getCriteriaBuilder(); + final CriteriaQuery cq = cb.createQuery(BallotManifestInfo.class); + final Root root = cq.from(BallotManifestInfo.class); + cq.where(cb.and(cb.equal(root.get("my_county_id"), the_cvr.countyID()), + cb.equal(root.get("my_scanner_id"), the_cvr.scannerID()), + cb.equal(root.get("my_batch_id"), the_cvr.batchID()))); + final TypedQuery query = s.createQuery(cq); + final List query_result = query.getResultList(); + // there should never be more than one result, but if there is, we'll + // return the first one + if (!query_result.isEmpty()) { + result = Optional.of(query_result.get(0)); + } + } catch (final PersistenceException e) { + Main.LOGGER.error("Exception when finding ballot location: " + e); + } + + return result; + } + + /** + * Deletes the set of ballot manifests for the specified county ID. + * + * @param the_county_id The county ID. + * @exception PersistenceException if the ballot manifests cannot be deleted. + */ + public static int deleteMatching(final Long the_county_id) { + final AtomicInteger count = new AtomicInteger(); + final Session s = Persistence.currentSession(); + final CriteriaBuilder cb = s.getCriteriaBuilder(); + final CriteriaQuery cq = cb.createQuery(BallotManifestInfo.class); + final Root root = cq.from(BallotManifestInfo.class); + cq.where(cb.equal(root.get(COUNTY_ID), the_county_id)); + final Query query = s.createQuery(cq); + final Stream to_delete = query.stream(); + to_delete.forEach((the_bmi) -> { + Persistence.delete(the_bmi); + count.incrementAndGet(); + }); + return count.get(); + } + + /** + * Count the uploaded ballot manifest info records in storage. + * + * @return the number of uploaded records. + */ + public static OptionalLong count() { + OptionalLong result = OptionalLong.empty(); + + try { + final Session s = Persistence.currentSession(); + final CriteriaBuilder cb = s.getCriteriaBuilder(); + final CriteriaQuery cq = cb.createQuery(Long.class); + final Root root = cq.from(BallotManifestInfo.class); + cq.select(cb.count(root)); + final TypedQuery query = s.createQuery(cq); + result = OptionalLong.of(query.getSingleResult()); + } catch (final PersistenceException e) { + // ignore + } + + return result; + } + + /** + Find the batch(bmi) that would hold the sequence number given. + */ + public static Optional + holdingSequencePosition(final Long rand, final Long countyId) { + Set result = null; + + try { + final Session s = Persistence.currentSession(); + final CriteriaBuilder cb = s.getCriteriaBuilder(); + final CriteriaQuery cq = + cb.createQuery(BallotManifestInfo.class); + final Root root = cq.from(BallotManifestInfo.class); + final List disjuncts = new ArrayList(); + final Predicate start = cb.lessThanOrEqualTo(root.get("my_sequence_start"), rand); + final Predicate end = cb.greaterThanOrEqualTo(root.get("my_sequence_end"), rand); + final Predicate county = cb.equal(root.get(COUNTY_ID), countyId); + disjuncts.add(start); + disjuncts.add(end); + disjuncts.add(county); + cq.select(root).where(cb.and(disjuncts.toArray(new Predicate[disjuncts.size()]))); + final TypedQuery query = s.createQuery(cq); + result = new HashSet(query.getResultList()); + } catch (final PersistenceException e) { + Main.LOGGER.error("Exception when reading ballot manifests from database: ", e); + } + return result.stream().findFirst(); + } + + /** + * Get the max sequence number which is the total number of CVRs there should be + */ + public static OptionalLong maxSequence(final Long countyId) { + OptionalLong result = OptionalLong.empty(); + + try { + final Session s = Persistence.currentSession(); + final CriteriaBuilder cb = s.getCriteriaBuilder(); + final CriteriaQuery cq = cb.createQuery(Long.class); + final Root root = cq.from(BallotManifestInfo.class); + cq.select(cb.max(root.get("my_sequence_end"))); + cq.where(cb.equal(root.get(COUNTY_ID), countyId)); + + final TypedQuery query = s.createQuery(cq); + result = OptionalLong.of(query.getSingleResult()); + } catch (final PersistenceException e) { + Main.LOGGER.error("Exception when reading ballot manifests from database: ", e); + } + return result; + } + + /** + * Get the number of ballots for a given set of counties. + */ + public static Long totalBallots(final Set countyIds) { + if (countyIds.isEmpty()) { + return 0L; + } + final Session s = Persistence.currentSession(); + final Query q = + s.createNativeQuery("with county_ballots as " + + "(select max(sequence_end) as ballots " + + "from ballot_manifest_info " + + "where county_id in (:countyIds) " + + "group by county_id) " + + "select sum(ballots) from county_ballots"); + q.setParameter("countyIds", countyIds); + + final Optional res = q.uniqueResultOptional(); + + if (res.isPresent()) { + return res.get().longValue(); + } else { + return 0L; + } + + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/CastVoteRecordQueries.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/CastVoteRecordQueries.java new file mode 100644 index 00000000..d53635a7 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/CastVoteRecordQueries.java @@ -0,0 +1,661 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * + * @created Aug 8, 2017 + * + * @copyright 2017 Colorado Department of State + * + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * + * @creator Daniel M. Zimmerman + * + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.query; + +import java.text.MessageFormat; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.stream.Stream; +import java.util.stream.Collectors; + +import javax.persistence.PersistenceException; +import javax.persistence.TypedQuery; +import javax.persistence.criteria.CriteriaBuilder; +import javax.persistence.criteria.CriteriaQuery; +import javax.persistence.criteria.Predicate; +import javax.persistence.criteria.Root; + +import org.hibernate.Session; +import org.hibernate.dialect.function.TemplateRenderer; +import org.hibernate.query.Query; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.model.CastVoteRecord; +import us.freeandfair.corla.model.CastVoteRecord.RecordType; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.model.Tribute; + +/** + * Queries having to do with CastVoteRecord entities. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings({"PMD.GodClass"}) +public final class CastVoteRecordQueries { + /** + * + */ + //@ invariant I; + private static final int _chunkOf1000 = 1000; + + /** + * The "county ID" field. + */ + private static final String COUNTY_ID = "my_county_id"; + + /** + * The "cvr number" field. + */ + private static final String SEQUENCE_NUMBER = "my_sequence_number"; + + /** + * The "record type" field. + */ + private static final String RECORD_TYPE = "my_record_type"; + + /** + * The "could not query database for CVRs error message. + */ + private static final String COULD_NOT_QUERY_DATABASE = "could not query database for CVRs"; + + private static final Gson GSON = + new GsonBuilder().serializeNulls().disableHtmlEscaping().create(); + + /** + * Private constructor to prevent instantiation. + */ + private CastVoteRecordQueries() { + // do nothing + } + + /** + * Obtain a stream of CastVoteRecord objects with the specified type. + * + * @param the_type The type. + * @return the stream of CastVoteRecord objects, or null if one could not be + * acquired. + * @exception IllegalStateException if this method is called outside a + * transaction. + */ + @SuppressWarnings("PMD.AvoidDuplicateLiterals") + public static Stream getMatching(final RecordType the_type) { + if (!Persistence.isTransactionActive()) { + throw new IllegalStateException("no running transaction"); + } + + Stream result = null; + + try { + final Session s = Persistence.currentSession(); + final CriteriaBuilder cb = s.getCriteriaBuilder(); + final CriteriaQuery cq = cb.createQuery(CastVoteRecord.class); + final Root root = cq.from(CastVoteRecord.class); + cq.select(root).where(cb.equal(root.get(RECORD_TYPE), the_type)); + final Query query = s.createQuery(cq); + result = query.stream(); + } catch (final PersistenceException e) { + Main.LOGGER.error(COULD_NOT_QUERY_DATABASE); + } + if (result == null) { + Main.LOGGER.debug("found no CVRs for type " + the_type); + } else { + Main.LOGGER.debug("query succeeded, returning CVR stream"); + } + return result; + } + + /** + * Counts the CastVoteRecord objects with the specified type. + * + * @param the_type The type. + * @return the count, empty if the query could not be completed successfully. + */ + @SuppressWarnings("PMD.AvoidDuplicateLiterals") + public static OptionalLong countMatching(final RecordType the_type) { + if (!Persistence.isTransactionActive()) { + throw new IllegalStateException("no running transaction"); + } + + OptionalLong result = OptionalLong.empty(); + + try { + final Session s = Persistence.currentSession(); + final CriteriaBuilder cb = s.getCriteriaBuilder(); + final CriteriaQuery cq = cb.createQuery(Long.class); + final Root root = cq.from(CastVoteRecord.class); + cq.select(cb.count(root)).where(cb.equal(root.get(RECORD_TYPE), the_type)); + final Query query = s.createQuery(cq); + result = OptionalLong.of(query.getSingleResult()); + } catch (final PersistenceException e) { + Main.LOGGER.error(COULD_NOT_QUERY_DATABASE); + } + if (result == null) { + Main.LOGGER.debug("found no CVRs for type " + the_type); + } else { + Main.LOGGER.debug("query succeeded, returning CVR stream"); + } + return result; + } + + /** + * Obtain a stream of CastVoteRecord objects with the specified county and + * type, ordered by their sequence number. + * + * @param the_county The county. + * @param the_type The type. + * @return the stream of CastVoteRecord objects, or null if one could not be + * acquired. + * @exception IllegalStateException if this method is called outside a + * transaction. + */ + @SuppressWarnings("PMD.AvoidDuplicateLiterals") + public static Stream getMatching(final Long the_county, + final RecordType the_type) { + Stream result = null; + + try { + final Session s = Persistence.currentSession(); + final CriteriaBuilder cb = s.getCriteriaBuilder(); + final CriteriaQuery cq = cb.createQuery(CastVoteRecord.class); + final Root root = cq.from(CastVoteRecord.class); + final List conjuncts = new ArrayList(); + conjuncts.add(cb.equal(root.get(COUNTY_ID), the_county)); + conjuncts.add(cb.equal(root.get(RECORD_TYPE), the_type)); + cq.select(root).where(cb.and(conjuncts.toArray(new Predicate[conjuncts.size()]))); + cq.orderBy(cb.asc(root.get(SEQUENCE_NUMBER))); + final Query query = s.createQuery(cq); + result = query.stream(); + } catch (final PersistenceException e) { + Main.LOGGER.error(COULD_NOT_QUERY_DATABASE); + } + if (result == null) { + Main.LOGGER.debug("found no CVRs for county " + the_county + ", type " + the_type); + } else { + Main.LOGGER.debug("query succeeded, returning CVR stream"); + } + return result; + } + + /** + * Counts the CastVoteRecord objects with the specified county and type. + * + * @param the_county The county. + * @param the_type The type. + * @return the count, empty if the query could not be completed successfully. + */ + @SuppressWarnings("PMD.AvoidDuplicateLiterals") + public static OptionalLong countMatching(final Long the_county, final RecordType the_type) { + OptionalLong result = OptionalLong.empty(); + + try { + final Session s = Persistence.currentSession(); + final CriteriaBuilder cb = s.getCriteriaBuilder(); + final CriteriaQuery cq = cb.createQuery(Long.class); + final Root root = cq.from(CastVoteRecord.class); + final List conjuncts = new ArrayList(); + conjuncts.add(cb.equal(root.get(COUNTY_ID), the_county)); + conjuncts.add(cb.equal(root.get(RECORD_TYPE), the_type)); + cq.select(cb.count(root)); + cq.where(cb.and(conjuncts.toArray(new Predicate[conjuncts.size()]))); + final Query query = s.createQuery(cq); + result = OptionalLong.of(query.getSingleResult()); + } catch (final PersistenceException e) { + Main.LOGGER.error(COULD_NOT_QUERY_DATABASE); + } + if (result == null) { + Main.LOGGER.debug("found no CVRs for county " + the_county + ", type " + the_type); + } else { + Main.LOGGER.debug("query succeeded, returning CVR stream"); + } + return result; + } + + /** + * change the votes from the export as if the cvr expost file headers had + * contained the newChoice rather than the oldChoice + **/ + public static int updateCVRContestInfos(final Long countyId, final Long contestId, + final String oldChoice, String newChoice) { + + final Session s = Persistence.currentSession(); + + newChoice = GSON.toJson(newChoice); + newChoice = (newChoice.substring(0, newChoice.length() - 1)); + newChoice = (newChoice.substring(1, newChoice.length())); + + String escapedOldChoice = oldChoice.replaceAll("\"", Matcher.quoteReplacement("\\\\\"")); + + final Query q = s + // this will only fix the first match, which is what we want, because + // this + // will make it possible to fix mistakes that create duplicates + .createNativeQuery("update cvr_contest_info set choices = " + + "regexp_replace(choices, :oldChoice , :newChoice) " + + // "regexp_replace(choices, cast( :oldChoice as + // varchar), cast( :newChoice as varchar)) " + + " where county_id = :county_id " + + " and contest_id = :contest_id " + + " and choices like :oldChoiceLike") + .setParameter("oldChoice", escapedOldChoice) + .setParameter("oldChoiceLike", "%" + escapedOldChoice + "%") + .setParameter("newChoice", newChoice) + .setParameter("county_id", countyId) + .setParameter("contest_id", contestId); + + return q.executeUpdate(); + } + + /** + * CVRContestInfo has a required foreign key to CastVoteRecord so they must be + * deleted first + **/ + public static int deleteCVRContestInfos(final Long countyId) { + final Session s = Persistence.currentSession(); + final Query q = + s.createNativeQuery("delete from cvr_contest_info ci where ci.county_id = :county_id"); + q.setParameter("county_id", countyId); + + return q.executeUpdate(); + + } + + /** delete all cvrs for a county, this supports the delete-file feature **/ + public static int deleteAll(final Long county_id) { + final Session s = Persistence.currentSession(); + // CVRContestInfo has a required foreign key to CastVoteRecord so they must + // be deleted first + deleteCVRContestInfos(county_id); + + final Query query = s + .createNativeQuery("delete from cast_vote_record cvr where cvr.county_id = :county_id"); + query.setParameter("county_id", county_id); + + return query.executeUpdate(); + } + + /** + * Obtain the CastVoteRecord object with the specified county, type, and + * sequence number. + * + * @param the_county_id The county. + * @param the_type The type. + * @param the_sequence_number The sequence number. + * @return the matching CastVoteRecord object, or null if no objects match or + * the query fails. + */ + // we are checking to see if exactly one result is in a list, and + // PMD doesn't like it + @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") + public static CastVoteRecord get(final Long the_county_id, final RecordType the_type, + final Integer the_sequence_number) { + CastVoteRecord result = null; + + try { + final Session s = Persistence.currentSession(); + final CriteriaBuilder cb = s.getCriteriaBuilder(); + final CriteriaQuery cq = cb.createQuery(CastVoteRecord.class); + final Root root = cq.from(CastVoteRecord.class); + final List conjuncts = new ArrayList(); + conjuncts.add(cb.equal(root.get(COUNTY_ID), the_county_id)); + conjuncts.add(cb.equal(root.get(RECORD_TYPE), the_type)); + conjuncts.add(cb.equal(root.get(SEQUENCE_NUMBER), the_sequence_number)); + cq.select(root).where(cb.and(conjuncts.toArray(new Predicate[conjuncts.size()]))); + final TypedQuery query = s.createQuery(cq); + final List query_results = query.getResultList(); + // if there's exactly one result, return that + if (query_results.size() == 1) { + result = query_results.get(0); + } + } catch (final PersistenceException e) { + Main.LOGGER.error(COULD_NOT_QUERY_DATABASE); + } + if (result == null) { + Main.LOGGER.debug("found no CVR for county " + the_county_id + ", type " + the_type + + ", sequence " + the_sequence_number); + } else { + Main.LOGGER.debug("found CVR " + result); + } + + return result; + } + + /** + * Obtain the CastVoteRecord objects with the specified county, type, and + * sequence numbers. + * + * @param the_county_id The county. + * @param the_type The type. + * @param the_sequence_numbers The sequence numbers. + * @return the matching CastVoteRecord objects, mapped by sequence number, an + * empty map if no records match, or null if the query fails. + */ + public static Map get(final Long the_county_id, + final RecordType the_type, + final List the_sequence_numbers) { + Map result = null; + final Set unique_numbers = new HashSet<>(the_sequence_numbers); + + try { + final Session s = Persistence.currentSession(); + final CriteriaBuilder cb = s.getCriteriaBuilder(); + final CriteriaQuery cq = cb.createQuery(CastVoteRecord.class); + final Root root = cq.from(CastVoteRecord.class); + final List conjuncts = new ArrayList<>(); + conjuncts.add(cb.equal(root.get(COUNTY_ID), the_county_id)); + conjuncts.add(cb.equal(root.get(RECORD_TYPE), the_type)); + conjuncts.add(root.get("my_sequence_number").in(unique_numbers)); + cq.select(root).where(cb.and(conjuncts.toArray(new Predicate[conjuncts.size()]))); + final TypedQuery query = s.createQuery(cq); + final List query_results = query.getResultList(); + result = new HashMap<>(); + for (final CastVoteRecord cvr : query_results) { + result.put(cvr.sequenceNumber(), cvr); + } + } catch (final PersistenceException e) { + Main.LOGGER.error(COULD_NOT_QUERY_DATABASE); + } + if (result == null) { + Main.LOGGER.debug("found no CVRs for county " + the_county_id + ", type " + the_type + + ", sequence " + unique_numbers); + } else { + Main.LOGGER.debug("found " + result.keySet().size() + "CVRs "); + } + + return result; + } + + /** + * Obtain the CastVoteRecord objects with the specified IDs. + * + * @param the_ids The IDs. + * @return the matching CastVoteRecord objects, an empty list if none are + * found, or null if the query fails. + */ + public static List get(final List the_ids) { + List result = new ArrayList<>(); + + if (the_ids.isEmpty()) { + return result; + } + + try { + final Session s = Persistence.currentSession(); + final CriteriaBuilder cb = s.getCriteriaBuilder(); + final CriteriaQuery cq = cb.createQuery(CastVoteRecord.class); + final Root root = cq.from(CastVoteRecord.class); + final List conjuncts = new ArrayList<>(); + conjuncts.add(root.get("my_id").in(the_ids)); + cq.select(root).where(cb.and(conjuncts.toArray(new Predicate[conjuncts.size()]))); + final TypedQuery query = s.createQuery(cq); + result = query.getResultList(); + } catch (final PersistenceException e) { + Main.LOGGER.error(COULD_NOT_QUERY_DATABASE); + } + if (result == null) { + Main.LOGGER.debug("found no CVRs with ids " + the_ids); + return new ArrayList<>(); + } else { + Main.LOGGER.debug("found " + result.size() + "CVRs "); + } + + return result; + } + + /** + * Find a CVR by it's Ballot Manifest position + * + * @parms tribute the ADT wrapping countyId, scannerId, batchId, and + * ballotPosition + * @return a CastVoteRecord at some position in a manifest + */ + public static CastVoteRecord atPosition(final Tribute tribute) { + return atPosition(tribute.countyId, tribute.scannerId, tribute.batchId, + tribute.ballotPosition); + } + + /** select cast_vote_record where uri in :uris **/ + public static List atPosition(final List tributes) { + + if (tributes.isEmpty()) { + return new ArrayList(); + } + + final List uris = tributes.stream().map(Persistence::persist).map(t -> t.getUri()) + .collect(Collectors.toList()); + + final Session s = Persistence.currentSession(); + final Query q = + s.createQuery("select cvr from CastVoteRecord cvr " + " where uri in (:uris) "); + + java.util.Spliterator split = uris.stream().spliterator(); + + final List results = new ArrayList<>(uris.size()); + + while (true) { + + List chunk = new ArrayList<>(_chunkOf1000); + for (int i = 0; i < _chunkOf1000 && split.tryAdvance(chunk::add); i++) + ; + if (chunk.isEmpty()) + break; + q.setParameter("uris", chunk); + final List tempResults = q.getResultList(); + results.addAll(tempResults); + Main.LOGGER.info(MessageFormat + .format("Total URIs {0} chunk size {1} tempResults size {2} results size {3}", + uris.size(), chunk.size(), tempResults.size(), results.size())); + } + + final Set foundUris = + results.stream().map(cvr -> (String) cvr.getUri()).collect(Collectors.toSet()); + + final Set phantomRecords = + tributes.stream().filter(distinctByKey((Tribute t) -> { + return t.getUri(); + })) + // is it faster to let the db do this with an except query? + .filter(t -> !foundUris.contains(t.getUri())).map(t -> phantomRecord(t)) + .map(Persistence::persist).collect(Collectors.toSet()); + + results.addAll(phantomRecords); + + // this is a dummy list so we can add a cvr at a particular position(that of + // the tributes uris) + final List randomOrder = + new ArrayList(Collections.nCopies(uris.size(), null)); + + // line the cvrs back up into the random order + for (final CastVoteRecord cvr : results) { + int index = 0; + for (final String uri : uris) { + if (uri.equals(cvr.getUri())) { + randomOrder.add(index, cvr); + } + index++; + } + } + + final List returnList = + randomOrder.stream().filter(cvr -> null != cvr).collect(Collectors.toList()); + if (returnList.size() != uris.size()) { + // we got a problem here + Main.LOGGER + .error("something went wrong with atPosition - returnList.size() != uris.size()"); + } + + return returnList; + } + + /** + * join query + **/ + public static CastVoteRecord atPosition(final Long county_id, final Integer scanner_id, + final String batch_id, final Integer position) { + final Session s = Persistence.currentSession(); + final CriteriaBuilder cb = s.getCriteriaBuilder(); + final CriteriaQuery cq = cb.createQuery(CastVoteRecord.class); + final Root root = cq.from(CastVoteRecord.class); + cq.select(root) + .where(cb.and(cb.equal(root.get("my_county_id"), county_id), + cb.equal(root.get("my_scanner_id"), scanner_id), + cb.equal(root.get("my_batch_id"), batch_id), + cb.equal(root.get("my_record_id"), position), + cb.or(cb.equal(root.get("my_record_type"), RecordType.UPLOADED), + // in case of duplicate selections on a phantom + // record + cb.equal(root.get("my_record_type"), RecordType.PHANTOM_RECORD)))); + + final Query q = s.createQuery(cq); + final Optional resultMaybe = q.uniqueResultOptional(); + + if (resultMaybe.isPresent()) { + return resultMaybe.get(); + } else { + // hmm performance no good, prevents bulk queries + return phantomRecord(county_id, scanner_id, batch_id, position); + } + } + + /** PHANTOM_RECORD conspiracy theory time **/ + public static CastVoteRecord phantomRecord(final Tribute tribute) { + return phantomRecord(tribute.countyId, tribute.scannerId, tribute.batchId, + tribute.ballotPosition); + } + + /** PHANTOM_RECORD conspiracy theory time **/ + public static CastVoteRecord phantomRecord(final Long county_id, final Integer scanner_id, + final String batch_id, final Integer position) { + final String imprintedID = String.format("%d-%s-%d", scanner_id, batch_id, position); + final CastVoteRecord cvr = + new CastVoteRecord(CastVoteRecord.RecordType.PHANTOM_RECORD, null, county_id, 0, // cvrNumber + // N/A + 0, // sequenceNumber N/A + scanner_id, batch_id, position, imprintedID, "PHANTOM RECORD", + null); + Persistence.save(cvr); + return cvr; + } + + /** + * Find max revision looks for RCVRs that are old versions of a given CVR or + * ACVR + **/ + public static Long maxRevision(final CastVoteRecord cvr) { + final Session s = Persistence.currentSession(); + final Query q = + s.createQuery("select max(revision) from CastVoteRecord cvr " + + " where revision is not null" + " and my_county_id = :countyId" + + " and my_imprinted_id = :imprintedId"); + + q.setLong("countyId", cvr.countyID()); + q.setString("imprintedId", cvr.imprintedID()); + + final Long result = (Long) q.getSingleResult(); + + if (null == result) { + return 0L; + } else { + return result; + } + } + + /** + * workaround. hibernate was ignoring the update of the object passed to the + * method for some unknown reason + **/ + public static Long forceUpdate(final CastVoteRecord cvr) { + final Session s = Persistence.currentSession(); + final Query q = + s.createNativeQuery("update cast_vote_record " + "set record_type = :recordType, " + + " revision = :revision, " + " uri = :uri " + " where id = :id "); + q.setParameter("recordType", cvr.recordType().toString()); + q.setParameter("revision", cvr.getRevision()); + q.setParameter("uri", cvr.getUri()); + q.setParameter("id", cvr.id()); + final int result = q.executeUpdate(); + return Long.valueOf(result); + } + + /** + * select every acvr which has been submitted for the the given cvr ids, + * including revisions(reaudits) + **/ + public static List activityReport(final List contestCVRIds) { + final Session s = Persistence.currentSession(); + final Query q = + s.createQuery("select acvr from CastVoteRecord acvr " + + " where acvr.cvrId in (:cvrIds)" + " order by acvr.my_timestamp asc"); + + Spliterator split = contestCVRIds.stream().spliterator(); + + final List results = new ArrayList<>(contestCVRIds.size()); + + while (true) { + List chunk = new ArrayList<>(_chunkOf1000); + for (int i = 0; i < _chunkOf1000 && split.tryAdvance(chunk::add); i++) + ; + if (chunk.isEmpty()) + break; + q.setParameter("cvrIds", chunk); + final List tempResults = q.getResultList(); + results.addAll(tempResults); + } + + + return results; + } + + /** + * select every acvr which has been submitted for the the given cvr ids, + * excluding revisions(reaudits) + **/ + public static List resultsReport(final List contestCVRIds) { + final Session s = Persistence.currentSession(); + final Query q = s.createQuery("select acvr from CastVoteRecord acvr " + + " where acvr.cvrId in (:cvrIds)" + + " and acvr.my_record_type != 'REAUDITED' "); + + Spliterator split = contestCVRIds.stream().spliterator(); + + final List results = new ArrayList<>(contestCVRIds.size()); + + while (true) { + List chunk = new ArrayList<>(_chunkOf1000); + for (int i = 0; i < _chunkOf1000 && split.tryAdvance(chunk::add); i++) + ; + if (chunk.isEmpty()) + break; + q.setParameter("cvrIds", chunk); + final List tempResults = q.getResultList(); + results.addAll(tempResults); + } + + return results; + } + + /** Utility function **/ + public static java.util.function.Predicate distinctByKey(final Function keyExtractor) { + final Map map = new ConcurrentHashMap<>(); + return t -> map.putIfAbsent(keyExtractor.apply(t), Boolean.TRUE) == null; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/ComparisonAuditQueries.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/ComparisonAuditQueries.java new file mode 100644 index 00000000..8a0677c2 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/ComparisonAuditQueries.java @@ -0,0 +1,113 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @copyright 2018 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Democracy Works, Inc + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.query; + +import java.util.List; +import java.util.Collections; +import java.util.Comparator; + +import org.hibernate.query.Query; +import org.hibernate.Session; + +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import us.freeandfair.corla.model.AuditStatus; +import us.freeandfair.corla.model.ComparisonAudit; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.util.SuppressFBWarnings; + +/** + * Queries having to do with ComparisonAudit entities. + */ +public final class ComparisonAuditQueries { + /** + * Class-wide logger + */ + public static final Logger LOGGER = + LogManager.getLogger(ComparisonAuditQueries.class); + + /** + * Private constructor to prevent instantiation. + */ + private ComparisonAuditQueries() { + // do nothing + } + + /** sort by targeted, then contest name **/ + // I disagree, FB + @SuppressFBWarnings({"RV_NEGATING_RESULT_OF_COMPARETO", "SE_COMPARATOR_SHOULD_BE_SERIALIZABLE"}) + private static class TargetedSort implements Comparator { + + /** sort by targeted, then contest name **/ + @Override + public int compare(final ComparisonAudit a, final ComparisonAudit b) { + // negative to put true first + final int t = -Boolean.compare(a.isTargeted(), b.isTargeted()); + if (0 == t) { + return a.contestResult().getContestName().compareTo(b.contestResult().getContestName()); + } else { + return t; + } + } + } + + /** All the comparison audits for all the contests, sorted by targeted, then + * alphabetical **/ + public static List sortedList() { + // sorting by db doesn't stick for some reason + final List results = Persistence.getAll(ComparisonAudit.class); + Collections.sort(results, new TargetedSort()); + return results; + } + + /** + * Obtain the ComparisonAudit object for the specified contest name. + * + * @param contestName The contest name + * @return the matched object + */ + public static ComparisonAudit matching(final String contestName) { + final Session s = Persistence.currentSession(); + final Query q = + s.createQuery("select ca from ComparisonAudit ca " + + " join ContestResult cr " + + " on ca.my_contest_result = cr " + + " where cr.contestName = :contestName"); + + q.setParameter("contestName", contestName); + + try { + return (ComparisonAudit) q.getSingleResult(); + } catch (javax.persistence.NoResultException e ) { + return null; + } + } + + + /** + * Return the ContestResult with the contestName given or create a new + * ContestResult with the contestName. + **/ + public static Integer count() { + final Session s = Persistence.currentSession(); + final Query q = s.createQuery("select count(ca) from ComparisonAudit ca"); + return ((Long)q.uniqueResult()).intValue(); + } + + /** setAuditStatus on matching contestName **/ + public static void updateStatus(final String contestName, final AuditStatus auditStatus) { + final ComparisonAudit ca = matching(contestName); + if (null != ca) { + ca.setAuditStatus(auditStatus); + } + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/ContestQueries.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/ContestQueries.java new file mode 100644 index 00000000..65d8a6ca --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/ContestQueries.java @@ -0,0 +1,119 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 8, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.query; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import javax.persistence.PersistenceException; +import javax.persistence.TypedQuery; +import javax.persistence.criteria.CriteriaBuilder; +import javax.persistence.criteria.CriteriaQuery; +import javax.persistence.criteria.Predicate; +import javax.persistence.criteria.Root; + +import org.hibernate.Session; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.model.Contest; +import us.freeandfair.corla.model.County; +import us.freeandfair.corla.persistence.Persistence; + +/** + * Queries having to do with Contest entities. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public final class ContestQueries { + /** + * Private constructor to prevent instantiation. + */ + private ContestQueries() { + // do nothing + } + + /** + * Gets contests that are in the specified set of counties. + * + * @param the_counties The counties. + * @return the matching contests, or null if the query fails. + */ + public static List forCounties(final Set the_counties) { + List result = null; + + try { + final Session s = Persistence.currentSession(); + final CriteriaBuilder cb = s.getCriteriaBuilder(); + final CriteriaQuery cq = cb.createQuery(Contest.class); + final Root root = cq.from(Contest.class); + final List disjuncts = new ArrayList(); + for (final County county : the_counties) { + disjuncts.add(cb.equal(root.get("my_county"), county)); + } + cq.select(root); + cq.where(cb.or(disjuncts.toArray(new Predicate[disjuncts.size()]))); + cq.orderBy(cb.asc(root.get("my_county").get("my_id")), + cb.asc(root.get("my_sequence_number"))); + final TypedQuery query = s.createQuery(cq); + result = query.getResultList(); + } catch (final PersistenceException e) { + Main.LOGGER.error("Exception when reading contests from database: " + e); + } + + return result; + } + + /** + * Gets contests that are in the specified county. + * + * @param the_county The county. + * @return the matching contests, or null if the query fails. + */ + public static Set forCounty(final County the_county) { + Set result = null; + + try { + final Session s = Persistence.currentSession(); + final CriteriaBuilder cb = s.getCriteriaBuilder(); + final CriteriaQuery cq = cb.createQuery(Contest.class); + final Root root = cq.from(Contest.class); + cq.select(root); + cq.where(cb.equal(root.get("my_county"), the_county)); + cq.orderBy(cb.asc(root.get("my_sequence_number"))); + final TypedQuery query = s.createQuery(cq); + result = new HashSet(query.getResultList()); + } catch (final PersistenceException e) { + Main.LOGGER.error("Exception when reading contests from database: " + e); + } + + return result; + } + + /** + * Deletes all the contests for the county with the specified ID. + * + * @param the_id The county ID. + */ + public static void deleteForCounty(final Long the_county_id) { + final Set contests = + forCounty(Persistence.getByID(the_county_id, County.class)); + if (contests != null) { + for (final Contest c : contests) { + Persistence.delete(c); + } + } + Persistence.flush(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/ContestResultQueries.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/ContestResultQueries.java new file mode 100644 index 00000000..764bb173 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/ContestResultQueries.java @@ -0,0 +1,50 @@ +/** Copyright (C) 2018 the Colorado Department of State **/ + +package us.freeandfair.corla.query; + +import java.util.Optional; + +import org.hibernate.query.Query; +import org.hibernate.Session; + +import us.freeandfair.corla.model.ContestResult; +import us.freeandfair.corla.persistence.Persistence; + + +public final class ContestResultQueries { + /** + * prevent construction + */ + private ContestResultQueries() { + } + + /** + * Return the ContestResult with the contestName given or create a new + * ContestResult with the contestName. + **/ + public static ContestResult findOrCreate(final String contestName) { + final Session s = Persistence.currentSession(); + final Query q = s.createQuery("select cr from ContestResult cr " + + "where cr.contestName = :contestName"); + q.setParameter("contestName", contestName); + final Optional contestResultMaybe = q.uniqueResultOptional(); + if (contestResultMaybe.isPresent()) { + return contestResultMaybe.get(); + } else { + final ContestResult cr = new ContestResult(contestName); + Persistence.save(cr); + return cr; + } + } + + /** + * Return the ContestResult with the contestName given or create a new + * ContestResult with the contestName. + **/ + public static Integer count() { + final Session s = Persistence.currentSession(); + final Query q = s.createQuery("select count(cr) from ContestResult cr "); + return ((Long)q.uniqueResult()).intValue(); + } + +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/CountyContestComparisonAuditQueries.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/CountyContestComparisonAuditQueries.java new file mode 100644 index 00000000..6e14b807 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/CountyContestComparisonAuditQueries.java @@ -0,0 +1,74 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 8, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.query; + +import java.util.List; + +import javax.persistence.PersistenceException; +import javax.persistence.TypedQuery; +import javax.persistence.criteria.CriteriaBuilder; +import javax.persistence.criteria.CriteriaQuery; +import javax.persistence.criteria.Root; + +import org.hibernate.Session; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.model.Contest; +import us.freeandfair.corla.model.CountyContestComparisonAudit; +import us.freeandfair.corla.persistence.Persistence; + +/** + * Queries having to do with CountyContestComparisonAudit entities. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public final class CountyContestComparisonAuditQueries { + /** + * Private constructor to prevent instantiation. + */ + private CountyContestComparisonAuditQueries() { + // do nothing + } + + /** + * Obtain all CountyContestComparisonAudit objects for the specified Contest. + * + * @param the_contest The contest. + * @return the matched objects. + */ + public static List matching(final Contest the_contest) { + List result = null; + + try { + final Session s = Persistence.currentSession(); + final CriteriaBuilder cb = s.getCriteriaBuilder(); + final CriteriaQuery cq = + cb.createQuery(CountyContestComparisonAudit.class); + final Root root = + cq.from(CountyContestComparisonAudit.class); + cq.select(root).where(cb.equal(root.get("my_contest"), the_contest)); + cq.orderBy(cb.asc(root.get("my_dashboard").get("my_county").get("my_id")), + cb.asc(root.get("my_contest").get("my_sequence_number"))); + final TypedQuery query = s.createQuery(cq); + result = query.getResultList(); + } catch (final PersistenceException e) { + Main.LOGGER.error("could not query database for county comparison audit"); + } + if (result == null) { + Main.LOGGER.debug("found no county comparison audit matching + " + the_contest); + } else { + Main.LOGGER.debug("found county comparison audits " + result); + } + return result; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/CountyContestResultQueries.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/CountyContestResultQueries.java new file mode 100644 index 00000000..b248a763 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/CountyContestResultQueries.java @@ -0,0 +1,186 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 8, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.query; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import javax.persistence.PersistenceException; +import javax.persistence.TypedQuery; +import javax.persistence.criteria.CriteriaBuilder; +import javax.persistence.criteria.CriteriaQuery; +import javax.persistence.criteria.Predicate; +import javax.persistence.criteria.Root; + +import org.hibernate.Session; +import org.hibernate.query.Query; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.model.Contest; +import us.freeandfair.corla.model.County; +import us.freeandfair.corla.model.CountyContestResult; +import us.freeandfair.corla.persistence.Persistence; + +/** + * Queries having to do with CountyContestResult entities. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public final class CountyContestResultQueries { + /** + * Private constructor to prevent instantiation. + */ + private CountyContestResultQueries() { + // do nothing + } + + /** + * Obtain a persistent CountyContestResult object for the specified + * county ID and contest. If there is not already a matching persistent + * object, one is created and returned. + * + * @param the_county The county. + * @param the_contest The contest. + * @return the matched CountyContestResult object, if one exists. + */ + // we are checking to see if exactly one result is in a list, and + // PMD doesn't like it + @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") + public static CountyContestResult matching(final County the_county, + final Contest the_contest) { + CountyContestResult result = null; + + try { + @SuppressWarnings("PMD.PrematureDeclaration") + final Session s = Persistence.currentSession(); + final CriteriaBuilder cb = s.getCriteriaBuilder(); + final CriteriaQuery cq = + cb.createQuery(CountyContestResult.class); + final Root root = cq.from(CountyContestResult.class); + final List conjuncts = new ArrayList<>(); + conjuncts.add(cb.equal(root.get("my_county"), the_county)); + conjuncts.add(cb.equal(root.get("my_contest"), the_contest)); + cq.select(root).where(cb.and(conjuncts.toArray(new Predicate[conjuncts.size()]))); + final TypedQuery query = s.createQuery(cq); + final List query_results = query.getResultList(); + // there should only be one, if one exists + if (query_results.size() == 1) { + result = query_results.get(0); + } else if (query_results.isEmpty()) { + result = new CountyContestResult(the_county, the_contest); + Persistence.saveOrUpdate(result); + } else { + throw new IllegalStateException("unique constraint violated on CountyContestResult"); + } + } catch (final PersistenceException e) { + e.printStackTrace(System.out); + Main.LOGGER.error("could not query database for contest results"); + throw e; + } + if (result == null) { + Main.LOGGER.debug("found no contest results matching + " + the_contest); + } else { + Main.LOGGER.debug("found contest results " + result); + } + return result; + } + + /** + * Gets CountyContestResults that have the contestName. + * + * @param contestName The name to match Contest#name. + * @return the matching CountyContestResults, or an empty set. + */ + public static List withContestName(final String contestName) { + final Session s = Persistence.currentSession(); + final Query q = s.createQuery("select ccr from CountyContestResult ccr " + + "inner join Contest c " + + "on ccr.my_contest = c " + + "where c.my_name = :contestName"); + q.setParameter("contestName", contestName); + return q.list(); + } + + /** + * Gets CountyContestResults that are in the specified county. + * + * @param the_county The county. + * @return the matching CountyContestResults, or null if the query fails. + */ + public static Set forCounty(final County the_county) { + Set result = null; + + try { + final Session s = Persistence.currentSession(); + final CriteriaBuilder cb = s.getCriteriaBuilder(); + final CriteriaQuery cq = + cb.createQuery(CountyContestResult.class); + final Root root = cq.from(CountyContestResult.class); + cq.select(root); + cq.where(cb.equal(root.get("my_county"), the_county)); + final TypedQuery query = s.createQuery(cq); + result = new HashSet(query.getResultList()); + } catch (final PersistenceException e) { + Main.LOGGER.error("Exception when reading contests from database: " + e); + } + + return result; + } + + /** + * Deletes all the contest results for the county with the specified ID. + * + * @param the_id The county ID. + */ + public static Integer deleteForCounty(final Long the_county_id) { + final Session s = Persistence.currentSession(); + + final Set results = + forCounty(Persistence.getByID(the_county_id, County.class)); + + final List contestIds = new ArrayList(); + + if (results.size() > 0) { + for (final CountyContestResult c : results) { + contestIds.add(c.contest().id()); + } + + // optimizing for speed, over a network connection + final Query q = s + .createNativeQuery("begin;" + +"delete from county_contest_comparison_audit where contest_id in (:contest_ids);" + +"delete from contests_to_contest_results where contest_id in (:contest_ids);" + +"delete from contest_to_audit where contest_id in (:contest_ids);" + +"delete from contest_choice where contest_id in (:contest_ids);" + + +"delete from county_contest_vote_total " + +" where result_id in (select id from county_contest_result " + +" where contest_id in (:contest_ids));" + + +"delete from county_contest_result where contest_id in (:contest_ids);" + + +"delete from contest where id in (:contest_ids);" + + +"commit" + + ); + q.setParameter("contest_ids", contestIds); + q.executeUpdate(); + + } + Persistence.flush(); + return contestIds.size(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/CountyQueries.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/CountyQueries.java new file mode 100644 index 00000000..164872bf --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/CountyQueries.java @@ -0,0 +1,105 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 8, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.query; + +import java.util.List; + +import javax.persistence.PersistenceException; +import javax.persistence.TypedQuery; +import javax.persistence.Query; +import javax.persistence.criteria.CriteriaBuilder; +import javax.persistence.criteria.CriteriaQuery; +import javax.persistence.criteria.Root; + +import org.hibernate.Session; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.model.County; +import us.freeandfair.corla.persistence.Persistence; + +/** + * Queries having to do with County entities. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public final class CountyQueries { + /** + * Private constructor to prevent instantiation. + */ + private CountyQueries() { + // do nothing + } + + /** + * Obtains a County object from a county name or ID string. If the string is + * numeric, we assume it is an ID; otherwise, we assume it is a name and match + * it as closely as we can to a county in the database. + * + * @param the_string The string. + * @return the matched County. If the results are ambiguous or empty (more than + * one match, or no match), returns null. + */ + // we are checking to see if exactly one result is in a list, and + // PMD doesn't like it + @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") + public static County fromString(final String the_string) { + County result = null; + Integer parsed_id; + + try { + parsed_id = Integer.parseInt(the_string); + } catch (final NumberFormatException e) { + parsed_id = -1; + } + + try { + final Session s = Persistence.currentSession(); + final CriteriaBuilder cb = s.getCriteriaBuilder(); + final CriteriaQuery cq = cb.createQuery(County.class); + final Root root = cq.from(County.class); + cq.select(root).where(cb.or(cb.like(root.get("my_name"), "%" + the_string + "%"), + cb.equal(root.get("my_id"), parsed_id))); + final TypedQuery query = s.createQuery(cq); + final List query_results = query.getResultList(); + // if there's exactly one result, return that + if (query_results.size() == 1) { + result = query_results.get(0); + } + } catch (final PersistenceException e) { + Main.LOGGER.error("could not query database for county"); + } + if (result == null) { + Main.LOGGER.debug("found no county for string " + the_string); + } else { + Main.LOGGER.debug("found county " + result + " for string " + the_string); + } + + return result; + } + + + /** get name for the county id, which should never change **/ + public static String getName(final Long countyId) { + final Session s = Persistence.currentSession(); + final Query q = + s.createQuery("select c.my_name from County c " + + " where c.my_id = :countyId"); + q.setParameter("countyId", countyId); + + try { + return (String) q.getSingleResult(); + } catch (javax.persistence.NoResultException e ) { + return null; + } + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/DatabaseResetQueries.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/DatabaseResetQueries.java new file mode 100644 index 00000000..785f9a06 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/DatabaseResetQueries.java @@ -0,0 +1,96 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 8, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.query; + +import javax.persistence.Cache; + +import org.hibernate.Session; + +import us.freeandfair.corla.persistence.Persistence; + +/** + * Queries for resetting the database. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public final class DatabaseResetQueries { + /** + * Private constructor to prevent instantiation. + */ + private DatabaseResetQueries() { + // do nothing + } + + /** + * Deletes everything from the database except authentication + * information. This query is very dangerous. + * + * @exception PersistenceException if the delete is unsuccessful. + */ + public static void resetDatabase() { + final Session s = Persistence.currentSession(); + + // NOTE: this is done with native queries, because otherwise it would be + // interminably slow (deleting one entity at a time) + + // the records in the following list of tables will be deleted, in order: + + final String[] tables = { + "log", + "audit_board", + "audit_intermediate_report", + "audit_investigation_report", + "ballot_manifest_info", + "county_dashboard_to_comparison_audit", + "counties_to_contest_results", + "contests_to_contest_results", + "contest_choice", + "contest_to_audit", + "county_contest_vote_total", + "contest_vote_total", + "contest_comparison_audit_discrepancy", + "contest_comparison_audit_disagreement", + "comparison_audit", + "county_contest_comparison_audit_discrepancy", + "county_contest_comparison_audit_disagreement", + "county_contest_comparison_audit", + "county_contest_result", + "contest_result", + "cvr_contest_info", + "contest", + "contest_result", + "cvr_audit_info", + "cast_vote_record", + "dos_dashboard", + "round", + "audit_board", + "county_dashboard", + "uploaded_file" + }; + + for (final String t : tables) { + s.createNativeQuery("truncate table " + t + " cascade").executeUpdate(); + } + + // delete all the no-longer-referenced LOBs + + s.createNativeQuery("select lo_unlink(l.oid) " + + "from pg_largeobject_metadata l").getResultList(); + + // empty all the Hibernate caches + final Cache cache = s.getSessionFactory().getCache(); + if (cache != null) { + cache.evictAll(); + } + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/ExportQueries.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/ExportQueries.java new file mode 100644 index 00000000..4b358d96 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/ExportQueries.java @@ -0,0 +1,275 @@ + +package us.freeandfair.corla.query; + +import java.io.OutputStream; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.BufferedReader; +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.stream.Stream; + +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import org.postgresql.copy.CopyManager; +import org.postgresql.copy.CopyOut; +import org.postgresql.core.BaseConnection; + +import org.hibernate.Session; +import org.hibernate.jdbc.Work; +import java.sql.Connection; +import java.sql.SQLException; + +import org.hibernate.query.Query; + +import us.freeandfair.corla.persistence.Persistence; + +/** export queries **/ +public class ExportQueries { + + /** + * Class-wide logger + */ + public static final Logger LOGGER = LogManager.getLogger(ExportQueries.class); + + /** to use the hibernate jdbc connection **/ + public static class CSVWork implements Work { + + /** pg query string **/ + private final String query; + + /** where to send the csv data **/ + private final OutputStream os; + + /** instantiation **/ + public CSVWork(final String query, final OutputStream os) { + + this.query = query; + this.os = os; + } + + /** do the work **/ + @SuppressWarnings("PMD.PreserveStackTrace") + public void execute(final Connection conn) throws java.sql.SQLException { + try { + final CopyManager cm = new CopyManager(conn.unwrap(BaseConnection.class)); + final String q = String.format("COPY (%s) TO STDOUT WITH CSV HEADER", this.query); + // cm.copyOut(q, this.os); + custCopyOut(q, this.os, cm); + } catch (java.io.IOException e) { + throw new java.sql.SQLException(e.getMessage()); + } + } + } + + /** no instantiation **/ + private ExportQueries() { + }; + + /** + * write the resulting rows from the query, as json objects, to the + * OutputStream + **/ + public static void customOut(final String query, final OutputStream os) { + final Session s = Persistence.currentSession(); + final String withoutSemi = query.replace(";", ""); + + final String jsonQuery = + String.format("SELECT cast(row_to_json(r) as text)" + " FROM (%s) r", withoutSemi); + final Query q = s.createNativeQuery(jsonQuery).setReadOnly(true).setFetchSize(1000); + + // interleave an object separator (the comma and line break) into the stream + // of json objects to create valid json thx! + // https://stackoverflow.com/a/25624818 + final Stream results = + q.stream().flatMap(i -> Stream.of(new String[] {",\n"}, i)).skip(1); // remove + // the + // first + // separator + + // write json by hand to preserve streaming writes in case of big data + try { + os.write("[".getBytes(StandardCharsets.UTF_8)); + results.forEach(line -> { + try { + // the object array is the columns, but in this case there is only + // one, so we take it at index 0 + os.write(line[0].toString().getBytes(StandardCharsets.UTF_8)); + } catch (java.io.IOException e) { + LOGGER.error(e.getMessage()); + } + }); + + os.write("]".getBytes(StandardCharsets.UTF_8)); + } catch (java.io.IOException e) { + // log it + LOGGER.error(e.getMessage()); + } + } + + /** + * write the resulting rows from the query, as json objects, to the + * OutputStream + **/ + public static void jsonOut(final String query, final OutputStream os) { + final Session s = Persistence.currentSession(); + final String withoutSemi = query.replace(";", ""); + final String jsonQuery = + String.format("SELECT cast(row_to_json(r) as text)" + " FROM (%s) r", withoutSemi); + final Query q = s.createNativeQuery(jsonQuery).setReadOnly(true).setFetchSize(1000); + + // interleave an object separator (the comma and line break) into the stream + // of json objects to create valid json thx! + // https://stackoverflow.com/a/25624818 + final Stream results = + q.stream().flatMap(i -> Stream.of(new String[] {",\n"}, i)).skip(1); // remove + // the + // first + // separator + + // write json by hand to preserve streaming writes in case of big data + try { + os.write("[".getBytes(StandardCharsets.UTF_8)); + results.forEach(line -> { + try { + // the object array is the columns, but in this case there is only + // one, so we take it at index 0 + os.write(line[0].toString().getBytes(StandardCharsets.UTF_8)); + } catch (java.io.IOException e) { + LOGGER.error(e.getMessage()); + } + }); + + os.write("]".getBytes(StandardCharsets.UTF_8)); + } catch (java.io.IOException e) { + // log it + LOGGER.error(e.getMessage()); + } + } + + /** send query results to output stream as csv **/ + public static void csvOut(final String query, final OutputStream os) { + final Session s = Persistence.currentSession(); + final String withoutSemi = query.replace(";", ""); + s.doWork(new CSVWork(withoutSemi, os)); + } + + /** + * The directory listing of the sql resource directory on the classpath, + * hopefully! I couldn't figure out how to do this from within a deployed jar, + * so here we are + **/ + public static List getSqlFolderFiles() { + final List paths = new ArrayList(); + final String folder = "sql"; + final String[] fileNames = {"batch_count_comparison.sql", "contest.sql", + "contest_comparison.sql", "contest_selection.sql", "contests_by_county.sql", + "tabulate.sql", "tabulate_county.sql", "upload_status.sql", "seed.sql"}; + for (final String f : fileNames) { + paths.add(String.format("%s/%s", folder, f)); + } + return paths; + } + + /** remove path and ext leaving the file name **/ + private static String fileName(final String path) { + final int slash = path.lastIndexOf('/') + 1; + final int dot = path.lastIndexOf('.'); + return path.substring(slash, dot); + } + + /** file contents to string **/ + // I respectfully disagree + @SuppressWarnings({"PMD.AssignmentInOperand"}) + public static String fileContents(final String path) throws java.io.IOException { + + final StringBuilder contents = new StringBuilder(); + final ClassLoader loader = Thread.currentThread().getContextClassLoader(); + try (final InputStream is = loader.getResourceAsStream(path); + final InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8); + final BufferedReader br = new BufferedReader(isr);) { + String line; + while ((line = br.readLine()) != null) { + contents.append(line); + contents.append('\n'); + } + } + return contents.toString(); + } + + /** + * read files from resources/sql/ and return map with keys as file names + * without extension and value as the file contents + **/ + public static Map sqlFiles() throws java.io.IOException { + final Map files = new HashMap(); + List paths = getSqlFolderFiles(); + for (final String path : paths) { + if (path.endsWith(".sql")) { + files.put(fileName(path), fileContents(path)); + } + } + return files; + } + + public static long custCopyOut(final String sql, OutputStream to, CopyManager cm) + throws SQLException, IOException { + byte[] buf; + CopyOut cp = cm.copyOut(sql); + try { + while ((buf = cp.readFromCopy()) != null) { + String s = new String(buf,Charset.forName("ASCII")); + if (s.contains("\\\"")) { + List newBuf = new ArrayList<>(); + for (int i = 0; i < buf.length; i++) { + byte b1 = buf[i]; + Character ch = (char) b1; + if (!ch.equals('\\')) { + newBuf.add(b1); + } + } + byte[] result = new byte[newBuf.size()]; + for (int i = 0; i < newBuf.size(); i++) { + result[i] = newBuf.get(i).byteValue(); + } + to.write(result); + } else { + List newBuf = new ArrayList<>(); + for (int i = 0; i < buf.length; i++) { + byte b1 = buf[i]; + newBuf.add(b1); + } + byte[] result = new byte[newBuf.size()]; + for (int i = 0; i < newBuf.size(); i++) { + result[i] = newBuf.get(i).byteValue(); + } + to.write(result); + } + } + return cp.getHandledRowCount(); + } catch (IOException ioEX) { + // if not handled this way the close call will hang, at least in 8.2 + if (cp.isActive()) { + cp.cancelCopy(); + } + try { // read until exhausted or operation cancelled SQLException + while ((buf = cp.readFromCopy()) != null) { + } + } catch (SQLException sqlEx) { + } // typically after several kB + throw ioEX; + } finally { // see to it that we do not leave the connection locked + if (cp.isActive()) { + cp.cancelCopy(); + } + } + } + +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/LogEntryQueries.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/LogEntryQueries.java new file mode 100644 index 00000000..deb06836 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/LogEntryQueries.java @@ -0,0 +1,87 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 8, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.query; + +import java.util.List; + +import javax.persistence.PersistenceException; +import javax.persistence.TypedQuery; +import javax.persistence.criteria.CriteriaBuilder; +import javax.persistence.criteria.CriteriaQuery; +import javax.persistence.criteria.Root; +import javax.persistence.criteria.Subquery; + +import org.hibernate.Session; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.model.LogEntry; +import us.freeandfair.corla.persistence.Persistence; + +/** + * Queries having to do with LogEntry entities. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public final class LogEntryQueries { + /** + * Private constructor to prevent instantiation. + */ + private LogEntryQueries() { + // do nothing + } + + /** + * Obtains the last LogEntry object in the database. By definition, this is + * the one with the largest ID. + * + * @return the corresponding LogEntry object, or null if no such object exists. + */ + // we are checking to see if exactly one result is in a list, and + // PMD doesn't like it + @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") + public static LogEntry last() { + LogEntry result = null; + + try { + final Session s = Persistence.currentSession(); + final CriteriaBuilder cb = s.getCriteriaBuilder(); + final CriteriaQuery cq = cb.createQuery(LogEntry.class); + final Subquery sq = cq.subquery(Long.class); + final Root c_root = cq.from(LogEntry.class); + final Root s_root = sq.from(LogEntry.class); + + sq.select(cb.max(s_root.get("my_id"))); + cq.where(cb.equal(c_root.get("my_id"), sq)); + final TypedQuery query = s.createQuery(cq); + // there should never be more than one result for max, but there could be + // zero, so let's be safe + final List query_results = query.getResultList(); + // if there's exactly one result, return that + if (query_results.size() == 1) { + result = query_results.get(0); + } else if (query_results.size() > 1) { + // there should never be more than one result + throw new PersistenceException("more than one max unique log entry id"); + } + } catch (final PersistenceException e) { + Main.LOGGER.error("could not query database for log entry: " + e.getMessage()); + } + if (result == null) { + Main.LOGGER.debug("found no log entries"); + } else { + Main.LOGGER.debug("found last log entry " + result); + } + + return result; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/PersistentASMStateQueries.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/PersistentASMStateQueries.java new file mode 100644 index 00000000..832fc75d --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/PersistentASMStateQueries.java @@ -0,0 +1,92 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 11, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.query; + +import java.util.List; + +import javax.persistence.PersistenceException; +import javax.persistence.TypedQuery; +import javax.persistence.criteria.CriteriaBuilder; +import javax.persistence.criteria.CriteriaQuery; +import javax.persistence.criteria.Predicate; +import javax.persistence.criteria.Root; + +import org.hibernate.Session; + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.asm.AbstractStateMachine; +import us.freeandfair.corla.asm.PersistentASMState; +import us.freeandfair.corla.persistence.Persistence; + +/** + * Queries having to do with persistent ASM state. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public final class PersistentASMStateQueries { + /** + * Private constructor to prevent instantiation. + */ + private PersistentASMStateQueries() { + // do nothing + } + + /** + * Retrieves the persistent ASM state from the database matching the specified + * ASM class and identity, if one exists. + * + * @param the_class The class of ASM to retrieve. + * @param the_identity The identity of the ASM to retrieve, or null if the ASM + * is a singleton. + * @return the persistent ASM state, or null if it does not exist. + * @exception PersistenceException if there is more than one matching ASM state + * in the database. + */ + // we are checking to see if exactly one result is in a list, and + // PMD doesn't like it + @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") + public static PersistentASMState get(final Class the_class, + final String the_identity) + throws PersistenceException { + PersistentASMState result = null; + try { + final String class_name = the_class.getName(); + final Session s = Persistence.currentSession(); + final CriteriaBuilder cb = s.getCriteriaBuilder(); + final CriteriaQuery cq = cb.createQuery(PersistentASMState.class); + final Root root = cq.from(PersistentASMState.class); + Predicate predicate = cb.equal(root.get("my_asm_class"), class_name); + if (the_identity != null) { + predicate = cb.and(predicate, cb.equal(root.get("my_asm_identity"), the_identity)); + } + cq.select(root).where(predicate); + final TypedQuery query = s.createQuery(cq); + final List query_results = query.getResultList(); + PersistentASMState asm = null; + if (query_results.size() > 1) { + Main.LOGGER.error("multiple ASM states found"); + throw new PersistenceException("multiple ASM states found for " + + the_class.getName() + ", identity " + the_identity); + } else if (!query_results.isEmpty()) { + asm = query_results.get(0); + Main.LOGGER.debug("found ASM state " + asm + " for class " + the_class.getName() + + ", identity " + the_identity); + } + result = asm; + } catch (final PersistenceException e) { + Main.LOGGER.error("could not query database for persistent ASM state" + e); + } + return result; + } + +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/TributeQueries.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/TributeQueries.java new file mode 100644 index 00000000..fe1de607 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/TributeQueries.java @@ -0,0 +1,35 @@ +package us.freeandfair.corla.query; + +import java.util.List; + +import org.hibernate.Session; +import org.hibernate.query.Query; + +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.model.Tribute; + + +/** find the tributes! **/ +public final class TributeQueries { + + /** + * Private constructor to prevent instantiation. + */ + private TributeQueries() { + // do nothing + } + + /** select every acvr which has been submitted for the the given cvr ids, + * excluding revisions(reaudits) **/ + public static List forContest(final String contestName) { + final Session s = Persistence.currentSession(); + final Query q = + s.createQuery("select t from Tribute t " + + " where t.contestName =:contestName"); + + q.setParameter("contestName", contestName); + + return q.getResultList(); + } + +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/UploadedFileQueries.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/UploadedFileQueries.java new file mode 100644 index 00000000..f7c857c4 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/query/UploadedFileQueries.java @@ -0,0 +1,205 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 8, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.query; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +import javax.persistence.PersistenceException; +import javax.persistence.TypedQuery; +import javax.persistence.criteria.CriteriaBuilder; +import javax.persistence.criteria.CriteriaQuery; +import javax.persistence.criteria.Predicate; +import javax.persistence.criteria.Root; + +import com.google.gson.Gson; + +import org.hibernate.Session; +import org.hibernate.query.Query; + + +import us.freeandfair.corla.Main; +import us.freeandfair.corla.json.UploadedFileDTO; +import us.freeandfair.corla.model.UploadedFile; +import us.freeandfair.corla.persistence.Persistence; + +/** + * Queries having to do with UploadedFile entities. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public final class UploadedFileQueries { + /** + * Private constructor to prevent instantiation. + */ + private UploadedFileQueries() { + // do nothing + } + + /** + * Obtain the UploadedFile object with a specific database ID and county + * identifier, if one exists. + * + * @param the_county_id The county identifier. + * @param the_database_id The database ID. + */ + // we are checking to see if exactly one result is in a list, and + // PMD doesn't like it + @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") + public static UploadedFile matching(final Long the_county_id, + final Long the_database_id) { + UploadedFile result = null; + + try { + final Session s = Persistence.currentSession(); + final CriteriaBuilder cb = s.getCriteriaBuilder(); + final CriteriaQuery cq = cb.createQuery(UploadedFile.class); + final Root root = cq.from(UploadedFile.class); + final List conjuncts = new ArrayList(); + conjuncts.add(cb.equal(root.get("my_county_id"), the_county_id)); + conjuncts.add(cb.equal(root.get("my_id"), the_database_id)); + cq.select(root).where(cb.and(conjuncts.toArray(new Predicate[conjuncts.size()]))); + final TypedQuery query = s.createQuery(cq); + final List query_results = query.getResultList(); + // if there's exactly one result, return that + if (query_results.size() == 1) { + result = query_results.get(0); + } + } catch (final PersistenceException e) { + Main.LOGGER.error("could not query database for uploaded file"); + } + if (result == null) { + Main.LOGGER.debug("found no uploaded file for county " + the_county_id + + ", id " + the_database_id); + } else { + Main.LOGGER.debug("found uploaded file " + result); + } + return result; + } + + /** + * Obtain the UploadedFile object with a specific county identifier, + * timestamp, and status, if one exists. + * + * @param the_county_id The county ID. + * @param the_timestamp The timestamp. + * @param the_status The status. + * @return the matched UploadedFile, if one exists, or null otherwise. + */ + // we are checking to see if exactly one result is in a list, and + // PMD doesn't like it + @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") + public static UploadedFile matching(final Long the_county_id, + final Instant the_timestamp, + final UploadedFile.FileStatus the_status) { + UploadedFile result = null; + + try { + final Session s = Persistence.currentSession(); + final CriteriaBuilder cb = s.getCriteriaBuilder(); + final CriteriaQuery cq = cb.createQuery(UploadedFile.class); + final Root root = cq.from(UploadedFile.class); + final List conjuncts = new ArrayList(); + conjuncts.add(cb.equal(root.get("my_county_id"), the_county_id)); + conjuncts.add(cb.equal(root.get("my_timestamp"), the_timestamp)); + conjuncts.add(cb.equal(root.get("my_status"), the_status)); + cq.select(root).where(cb.and(conjuncts.toArray(new Predicate[conjuncts.size()]))); + final TypedQuery query = s.createQuery(cq); + final List query_results = query.getResultList(); + // if there's exactly one result, return that + if (query_results.size() == 1) { + result = query_results.get(0); + } + } catch (final PersistenceException e) { + Main.LOGGER.error("could not query database for uploaded file"); + } + if (result == null) { + Main.LOGGER.debug("found no uploaded file for county " + the_county_id + + ", timestamp " + the_timestamp + ", status " + + the_status); + } else { + Main.LOGGER.debug("found uploaded file " + result); + } + return result; + } + + + public static UploadedFileDTO getAttrs(final UploadedFileDTO upF) { + final Session s = Persistence.currentSession(); + final Query q = + s.createNativeQuery("select id, status, county_id " + + " from uploaded_file up " + + " where up.id = :id "); + + q.setParameter("id", upF.getFileId()); + Object[] row = (Object[])q.getSingleResult(); + + if (null == row) { + return null; + } else { + upF.setStatus((String)row[1]); + upF.setCountyId(((java.math.BigInteger)row[2]).longValue()); + return upF; + } + } + + /** having to go around hibernate for cross-thread updates **/ + public static int updateStatus(final UploadedFileDTO upF) { + final Session s = Persistence.currentSession(); + final Query q = + s.createNativeQuery("update uploaded_file up " + + " set status = :status" + + " where up.id = :id"); + + q.setParameter("id", upF.getFileId()); + q.setParameter("status", upF.getStatus()); + + return q.executeUpdate(); + } + + /** having to go around hibernate for cross-thread updates **/ + public static int updateStatusAndResult(final UploadedFileDTO upF) { + final Session s = Persistence.currentSession(); + final Query q = + s.createNativeQuery("update uploaded_file up " + + " set status = :status," + + " result = :result, " + + " version = version + 1 " + + " where up.id = :id"); + + q.setParameter("id", upF.getFileId()); + q.setParameter("status", upF.getStatus()); + q.setParameter("result", (new Gson()).toJson(upF.getResult())); + + return q.executeUpdate(); + } + + /** having to go around hibernate for cross-thread updates **/ + public static int setCVRFileOnCounty(final UploadedFileDTO upF) { + final Session s = Persistence.currentSession(); + final Query q = + s.createNativeQuery("update county_dashboard cdb " + + " set cvr_file_id = :id " + + " where cdb.id = :countyId"); + + q.setParameter("id", upF.getFileId()); + q.setParameter("countyId", upF.getCountyId()); + + return q.executeUpdate(); + } + + + + +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/report/CountyReport.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/report/CountyReport.java new file mode 100644 index 00000000..59d53e3e --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/report/CountyReport.java @@ -0,0 +1,941 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 30, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.report; + +import static us.freeandfair.corla.util.PrettyPrinter.booleanYesNo; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.OptionalInt; +import java.util.TimeZone; + +import org.apache.poi.ss.usermodel.BorderStyle; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.CellStyle; +import org.apache.poi.ss.usermodel.CellType; +import org.apache.poi.ss.usermodel.DataFormat; +import org.apache.poi.ss.usermodel.Font; +import org.apache.poi.ss.usermodel.HorizontalAlignment; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; + +import org.apache.poi.ss.util.CellRangeAddress; +import org.apache.poi.ss.util.CellUtil; +import org.apache.poi.ss.util.RegionUtil; + +import org.apache.poi.xssf.usermodel.XSSFWorkbook; + +import us.freeandfair.corla.controller.ComparisonAuditController; +import us.freeandfair.corla.model.AuditSelection; +import us.freeandfair.corla.model.CVRAuditInfo; +import us.freeandfair.corla.model.CastVoteRecord.RecordType; +import us.freeandfair.corla.model.County; +import us.freeandfair.corla.model.CountyContestResult; +import us.freeandfair.corla.model.CountyDashboard; +import us.freeandfair.corla.model.DoSDashboard; +import us.freeandfair.corla.model.Elector; +import us.freeandfair.corla.model.Round; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.query.CountyContestResultQueries; + +/** + * All the data required for a county audit report. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings({"PMD.CyclomaticComplexity", "PMD.StdCyclomaticComplexity", + "PMD.ModifiedCyclomaticComplexity", "PMD.ExcessiveImports", "PMD.GodClass"}) +public class CountyReport { + /** + * The affirmation statement. + */ + public static final String AFFIRMATION_STATEMENT = + "We hereby affirm that the results presented in this report are" + + "\naccurate to the best of our knowledge."; + + /** + * The font size for Excel. + */ + // POI interop requires a short here + @SuppressWarnings("PMD.AvoidUsingShortType") + public static final short FONT_SIZE = 12; + + /** + * The date formatter. + */ + private static final DateTimeFormatter DATE_FORMATTER = + DateTimeFormatter.ofPattern("MM/dd/yyyy"); + + /** + * The date/time formatter. + */ + private static final DateTimeFormatter DATE_TIME_FORMATTER = + DateTimeFormatter.ofPattern("MM/dd/yyyy hh:mm a"); + + /** + * The DoS dashboard used to generate this report. + */ + private final DoSDashboard my_dosdb; + + /** + * The county dashboard used to generate this report. + */ + private final CountyDashboard my_cdb; + + /** + * The county for which this report was generated. + */ + private final County my_county; + + /** + * The date and time this report was generated. + */ + private final Instant my_timestamp; + + /** + * The CVRs to audit for each round. + */ + private final Map> my_cvrs_to_audit_by_round; + + /** + * The contests driving the audit, and their results. + */ + private final List my_driving_contest_results; + + /** + * The data for each audit round. + */ + private final List my_rounds; + + /** + * Initialize a county report for the specified county, timestamped + * with the current time. + * + * @param the_county The county. + */ + public CountyReport(final County the_county) { + this(the_county, Instant.now()); + } + /** + * Initialize a county report object for the specified county, with the + * specified timestamp. + * + * @param the_county The county. + * @param the_timestamp The timestamp. + */ + public CountyReport(final County the_county, final Instant the_timestamp) { + my_county = the_county; + my_timestamp = the_timestamp; + my_driving_contest_results = new ArrayList(); + my_cdb = Persistence.getByID(my_county.id(), CountyDashboard.class); + + for (final CountyContestResult ccr : + CountyContestResultQueries.forCounty(my_county)) { + if (my_cdb.drivingContestNames().contains(ccr.contest().name())) { + my_driving_contest_results.add(ccr); + } + } + + my_rounds = my_cdb.rounds(); + my_cvrs_to_audit_by_round = new HashMap<>(); + + for (final Round r : my_rounds) { + final List cvrs_to_audit = + ComparisonAuditController.cvrsToAuditInRound(my_cdb, r.number()); + cvrs_to_audit.sort(null); + my_cvrs_to_audit_by_round.put(r.number(), cvrs_to_audit); + } + + my_dosdb = Persistence.getByID(DoSDashboard.ID, DoSDashboard.class); + } + + /** + * @return the county for this report. + */ + public County county() { + return my_county; + } + + /** + * @return the timestamp for this report. + */ + public Instant timestamp() { + return my_timestamp; + } + + /** + * @return the CVRs imprinted IDs to audit by round map for this report. + */ + public Map> cvrsToAuditByRound() { + return Collections.unmodifiableMap(my_cvrs_to_audit_by_round); + } + + /** + * @return the driving contest results for this report. + */ + public List drivingContestResults() { + return Collections.unmodifiableList(my_driving_contest_results); + } + + /** + * @return the list of rounds for this report. + */ + public List rounds() { + return Collections.unmodifiableList(my_rounds); + } + + /** + * @return the county dashboard for this report. + */ + public CountyDashboard dashboard() { + return my_cdb; + } + + /** + * @return the Excel representation of this report, as a byte array. + * @exception IOException if the report cannot be generated. + */ + public byte[] generateExcel() throws IOException { + final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + final Workbook workbook = generateExcelWorkbook(); + workbook.write(baos); + baos.flush(); + baos.close(); + workbook.close(); + return baos.toByteArray(); + } + + /** + * @return the Excel workbook for this report. + */ + @SuppressWarnings({"checkstyle:magicnumber", "checkstyle:executablestatementcount", + "checkstyle:methodlength", "PMD.ExcessiveMethodLength", "PMD.NcssMethodCount", + "PMD.NPathComplexity", "PMD.AvoidLiteralsInIfCondition"}) + public Workbook generateExcelWorkbook() { + final Workbook workbook = new XSSFWorkbook(); + + // data format + final DataFormat format = workbook.createDataFormat(); + + // bold font for titles and such + final Font bold_font = workbook.createFont(); + bold_font.setFontHeightInPoints(FONT_SIZE); + bold_font.setBold(true); + final CellStyle bold_style = workbook.createCellStyle(); + bold_style.setFont(bold_font); + final CellStyle bold_right_style = workbook.createCellStyle(); + bold_right_style.setFont(bold_font); + bold_right_style.setAlignment(HorizontalAlignment.RIGHT); + + // regular font for other fields + final Font standard_font = workbook.createFont(); + standard_font.setFontHeightInPoints(FONT_SIZE); + final CellStyle standard_style = workbook.createCellStyle(); + standard_style.setFont(standard_font); + standard_style.setDataFormat(format.getFormat("@")); + final CellStyle wrapped_style = workbook.createCellStyle(); + wrapped_style.setFont(standard_font); + wrapped_style.setDataFormat(format.getFormat("@")); + wrapped_style.setWrapText(true); + final CellStyle standard_right_style = workbook.createCellStyle(); + standard_right_style.setFont(standard_font); + standard_right_style.setAlignment(HorizontalAlignment.RIGHT); + standard_right_style.setDataFormat(format.getFormat("@")); + final CellStyle integer_style = workbook.createCellStyle(); + integer_style.setFont(standard_font); + integer_style.setDataFormat(format.getFormat("0")); + final CellStyle decimal_style = workbook.createCellStyle(); + decimal_style.setFont(standard_font); + decimal_style.setDataFormat(format.getFormat("0.000#####")); + final CellStyle box_style = workbook.createCellStyle(); + box_style.setBorderBottom(BorderStyle.THICK); + box_style.setBorderTop(BorderStyle.THICK); + box_style.setBorderLeft(BorderStyle.THICK); + box_style.setBorderRight(BorderStyle.THICK); + + // the summary sheet + final Sheet summary_sheet = workbook.createSheet("Summary"); + int row_number = 0; + Row row = summary_sheet.createRow(row_number++); + int cell_number = 0; + int max_cell_number = 0; + + Cell cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue(my_county.name() + " County Audit Report"); + + row = summary_sheet.createRow(row_number++); + cell_number = 0; + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Generated " + + DATE_TIME_FORMATTER. + format(LocalDateTime.ofInstant(my_timestamp, + TimeZone.getDefault().toZoneId()))); + + row = summary_sheet.createRow(row_number++); + cell_number = 0; + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + if (my_dosdb.auditInfo().electionType() == null && + my_dosdb.auditInfo().electionDate() == null) { + cell.setCellValue("ELECTION TYPE/DATE NOT SET"); + } else { + cell.setCellValue(my_dosdb.auditInfo().capitalizedElectionType() + " Election - " + + DATE_FORMATTER. + format(LocalDateTime.ofInstant(my_dosdb.auditInfo().electionDate(), + ZoneOffset.UTC))); + } + + row_number++; + row = summary_sheet.createRow(row_number++); + cell_number = 0; + + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Audit Random Seed"); + + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(standard_right_style); + cell.setCellValue(my_dosdb.auditInfo().seed()); + + row = summary_sheet.createRow(row_number++); + cell_number = 0; + + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Audit Risk Limit"); + + cell = row.createCell(cell_number++); + cell.setCellType(CellType.NUMERIC); + cell.setCellStyle(decimal_style); + cell.setCellValue(my_dosdb.auditInfo().riskLimit().doubleValue()); + + row_number++; + row = summary_sheet.createRow(row_number++); + cell_number = 0; + + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Total Ballot Cards In Manifest"); + + cell = row.createCell(cell_number++); + cell.setCellType(CellType.NUMERIC); + cell.setCellStyle(integer_style); + cell.setCellValue(my_cdb.ballotsInManifest()); + + row = summary_sheet.createRow(row_number++); + cell_number = 0; + + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Total CVRs in CVR Export File"); + + cell = row.createCell(cell_number++); + cell.setCellType(CellType.NUMERIC); + cell.setCellStyle(integer_style); + cell.setCellValue(my_cdb.cvrsImported()); + + row = summary_sheet.createRow(row_number++); + cell_number = 0; + + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Total Ballot Cards Audited"); + + cell = row.createCell(cell_number++); + cell.setCellType(CellType.NUMERIC); + cell.setCellStyle(integer_style); + cell.setCellValue(my_cdb.ballotsAudited()); + + row = summary_sheet.createRow(row_number++); + cell_number = 0; + + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Number of Audit Rounds"); + + cell = row.createCell(cell_number++); + cell.setCellType(CellType.NUMERIC); + cell.setCellStyle(integer_style); + cell.setCellValue(my_rounds.size()); + + if (!my_rounds.isEmpty()) { + row_number++; + row = summary_sheet.createRow(row_number++); + cell_number = 0; + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Round Summary"); + for (final Round round : my_rounds) { + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue(round.number()); + } + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_right_style); + cell.setCellValue("Total"); + + row = summary_sheet.createRow(row_number++); + cell_number = 0; + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Ballot Cards Audited"); + int accumulator = 0; + for (final Round round : my_rounds) { + cell = row.createCell(cell_number++); + cell.setCellStyle(integer_style); + cell.setCellType(CellType.NUMERIC); + cell.setCellValue(round.actualCount()); + accumulator = accumulator + round.actualCount(); + } + cell = row.createCell(cell_number++); + cell.setCellStyle(integer_style); + cell.setCellType(CellType.NUMERIC); + cell.setCellValue(accumulator); + + row = summary_sheet.createRow(row_number++); + cell_number = 0; + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Discrepancies (Audited Contests)"); + accumulator = 0; + for (final Round round : my_rounds) { + cell = row.createCell(cell_number++); + cell.setCellStyle(integer_style); + cell.setCellType(CellType.NUMERIC); + final int discrepancies; + if (round.discrepancies().containsKey(AuditSelection.AUDITED_CONTEST)) { + discrepancies = round.discrepancies().get(AuditSelection.AUDITED_CONTEST); + } else { + discrepancies = 0; + } + cell.setCellValue(discrepancies); + accumulator = accumulator + discrepancies; + } + cell = row.createCell(cell_number++); + cell.setCellStyle(integer_style); + cell.setCellType(CellType.NUMERIC); + cell.setCellValue(accumulator); + + row = summary_sheet.createRow(row_number++); + cell_number = 0; + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Discrepancies (Non-Audited Contests)"); + accumulator = 0; + for (final Round round : my_rounds) { + cell = row.createCell(cell_number++); + cell.setCellStyle(integer_style); + cell.setCellType(CellType.NUMERIC); + final int discrepancies; + if (round.discrepancies().containsKey(AuditSelection.UNAUDITED_CONTEST)) { + discrepancies = round.discrepancies().get(AuditSelection.UNAUDITED_CONTEST); + } else { + discrepancies = 0; + } + cell.setCellValue(discrepancies); + accumulator = accumulator + discrepancies; + } + cell = row.createCell(cell_number++); + cell.setCellStyle(integer_style); + cell.setCellType(CellType.NUMERIC); + cell.setCellValue(accumulator); + + row = summary_sheet.createRow(row_number++); + cell_number = 0; + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Disagreements (Audited Contests)"); + accumulator = 0; + for (final Round round : my_rounds) { + cell = row.createCell(cell_number++); + cell.setCellStyle(integer_style); + cell.setCellType(CellType.NUMERIC); + final int disagreements; + if (round.disagreements().containsKey(AuditSelection.AUDITED_CONTEST)) { + disagreements = round.disagreements().get(AuditSelection.AUDITED_CONTEST); + } else { + disagreements = 0; + } + cell.setCellValue(disagreements); + accumulator = accumulator + disagreements; + } + cell = row.createCell(cell_number++); + cell.setCellStyle(integer_style); + cell.setCellType(CellType.NUMERIC); + cell.setCellValue(accumulator); + + row = summary_sheet.createRow(row_number++); + cell_number = 0; + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Disagreements (Non-Audited Contests)"); + accumulator = 0; + for (final Round round : my_rounds) { + cell = row.createCell(cell_number++); + cell.setCellStyle(integer_style); + cell.setCellType(CellType.NUMERIC); + final int disagreements; + if (round.disagreements().containsKey(AuditSelection.UNAUDITED_CONTEST)) { + disagreements = round.disagreements().get(AuditSelection.UNAUDITED_CONTEST); + } else { + disagreements = 0; + } + cell.setCellValue(disagreements); + accumulator = accumulator + disagreements; + } + cell = row.createCell(cell_number++); + cell.setCellStyle(integer_style); + cell.setCellType(CellType.NUMERIC); + cell.setCellValue(accumulator); + } + + row_number++; + row = summary_sheet.createRow(row_number++); + cell_number = 0; + cell = row.createCell(cell_number++); + cell.setCellStyle(bold_style); + if (my_driving_contest_results.isEmpty()) { + cell.setCellValue("No Audited Contests"); + } else { + cell.setCellValue("Audited Contests"); + } + + max_cell_number = Math.max(max_cell_number, cell_number); + row_number = row_number - 1; // don't skip a line for the first contest + for (final CountyContestResult ccr : my_driving_contest_results) { + row_number++; + row = summary_sheet.createRow(row_number++); + cell_number = 0; + cell = row.createCell(cell_number++); + cell.setCellStyle(bold_style); + cell.setCellValue(ccr.contest().name() + " - Vote For " + ccr.contest().votesAllowed()); + + cell = row.createCell(cell_number++); + cell.setCellStyle(bold_style); + cell.setCellValue("Choice"); + + cell = row.createCell(cell_number++); + cell.setCellStyle(bold_right_style); + cell.setCellValue("W/L"); + + cell = row.createCell(cell_number++); + cell.setCellStyle(bold_right_style); + cell.setCellValue("Votes"); + + cell = row.createCell(cell_number++); + cell.setCellStyle(bold_right_style); + cell.setCellValue("Margin"); + + cell = row.createCell(cell_number++); + cell.setCellStyle(bold_right_style); + cell.setCellValue("Diluted Margin %"); + + for (final String choice : ccr.rankedChoices()) { + row = summary_sheet.createRow(row_number++); + max_cell_number = Math.max(max_cell_number, cell_number); + cell_number = 1; + cell = row.createCell(cell_number++); + cell.setCellStyle(standard_style); + cell.setCellValue(choice); + + cell = row.createCell(cell_number++); + cell.setCellStyle(standard_right_style); + if ((ccr.winners().stream().anyMatch(w -> w.equalsIgnoreCase(choice)))) { + cell.setCellValue("W"); + } else { + cell.setCellValue("L"); + } + + cell = row.createCell(cell_number++); + cell.setCellStyle(integer_style); + cell.setCellType(CellType.NUMERIC); + cell.setCellValue(ccr.voteTotals().get(choice)); + + if (ccr.winners().contains(choice)) { + cell = row.createCell(cell_number++); + cell.setCellStyle(integer_style); + cell.setCellType(CellType.NUMERIC); + final OptionalInt margin = ccr.marginToNearestLoser(choice); + if (margin.isPresent()) { + cell.setCellValue(margin.getAsInt()); + } + + cell = row.createCell(cell_number++); + cell.setCellStyle(decimal_style); + cell.setCellType(CellType.NUMERIC); + final BigDecimal diluted_margin = ccr.countyDilutedMarginToNearestLoser(choice); + if (diluted_margin != null) { + cell.setCellValue(diluted_margin.doubleValue() * 100); + } + } + } + } + + row_number++; + + summary_sheet.getPrintSetup().setLandscape(true); + summary_sheet.getPrintSetup().setFitWidth((short) 1); + for (int i = 0; i < max_cell_number; i++) { + summary_sheet.autoSizeColumn(i); + } + + // round sheets + + for (final Round round : my_rounds) { + final Sheet round_sheet = workbook.createSheet("Round " + round.number()); + row_number = 0; + row = round_sheet.createRow(row_number++); + cell_number = 0; + max_cell_number = 0; + + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Round " + round.number()); + + row_number++; + row = round_sheet.createRow(row_number++); + max_cell_number = Math.max(max_cell_number, cell_number); + cell_number = 0; + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Ballot Cards Audited"); + + cell = row.createCell(cell_number++); + cell.setCellType(CellType.NUMERIC); + cell.setCellStyle(integer_style); + cell.setCellValue(round.actualCount()); + + row = round_sheet.createRow(row_number++); + + row = round_sheet.createRow(row_number++); + cell_number = 1; // these are headers for audit reasons + final List listed_selections = new ArrayList<>(); + final Map discrepancies = round.discrepancies(); + final Map disagreements = round.disagreements(); + + for (final AuditSelection s : AuditSelection.values()) { + if (discrepancies.containsKey(s) && discrepancies.get(s) >= 0 || + disagreements.containsKey(s) && disagreements.get(s) >= 0) { + listed_selections.add(s); + } + } + + Collections.sort(listed_selections); + + for (final AuditSelection s : listed_selections) { + cell = row.createCell(cell_number++); + cell.setCellStyle(bold_right_style); + cell.setCellValue(s.prettyString()); + } + + row = round_sheet.createRow(row_number++); + max_cell_number = Math.max(max_cell_number, cell_number); + cell_number = 0; + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + if (discrepancies.isEmpty()) { + cell.setCellValue("No Discrepancies Recorded"); + } else { + cell.setCellValue("Discrepancies Recorded"); + for (final AuditSelection s : listed_selections) { + cell = row.createCell(cell_number++); + cell.setCellType(CellType.NUMERIC); + cell.setCellStyle(integer_style); + final int cell_value; + if (discrepancies.containsKey(s)) { + cell_value = discrepancies.get(s); + } else { + cell_value = 0; + } + cell.setCellValue(cell_value); + } + } + + row = round_sheet.createRow(row_number++); + max_cell_number = Math.max(max_cell_number, cell_number); + cell_number = 0; + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + if (disagreements.isEmpty()) { + cell.setCellValue("No Disagreements Recorded"); + } else { + cell.setCellValue("Disagreements Recorded"); + for (final AuditSelection s : listed_selections) { + cell = row.createCell(cell_number++); + cell.setCellType(CellType.NUMERIC); + cell.setCellStyle(integer_style); + final int cell_value; + if (disagreements.containsKey(s)) { + cell_value = disagreements.get(s); + } else { + cell_value = 0; + } + cell.setCellValue(cell_value); + } + } + + row_number++; + row = round_sheet.createRow(row_number++); + max_cell_number = Math.max(max_cell_number, cell_number); + cell_number = 0; + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Ballot Cards Selected"); + + row = round_sheet.createRow(row_number++); + cell_number = 0; + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Imprinted ID"); + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_right_style); + cell.setCellValue("Audited"); + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_right_style); + cell.setCellValue("Discrepancy"); + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_right_style); + cell.setCellValue("Disagreement"); + + max_cell_number = Math.max(max_cell_number, cell_number); + for (final CVRAuditInfo audit_info : + my_cvrs_to_audit_by_round.get(round.number())) { + row = round_sheet.createRow(row_number++); + cell_number = 0; + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(standard_style); + cell.setCellValue(audit_info.cvr().imprintedID()); + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(standard_right_style); + if (audit_info.acvr() == null) { + cell.setCellValue(booleanYesNo(false)); + } else { + cell.setCellValue(booleanYesNo(audit_info.acvr().recordType() == + RecordType.AUDITOR_ENTERED)); + } + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(standard_right_style); + cell.setCellValue(booleanYesNo(!audit_info.discrepancy().isEmpty())); + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(standard_right_style); + cell.setCellValue(booleanYesNo(!audit_info.disagreement().isEmpty())); + } + + round_sheet.getPrintSetup().setFitWidth((short) 1); + for (int i = 0; i < max_cell_number; i++) { + round_sheet.autoSizeColumn(i); + } + } + + // affirmation sheet + final Sheet affirmation_sheet = workbook.createSheet("Affirmation"); + final float affirmationRowHeight = affirmation_sheet.getDefaultRowHeightInPoints(); + row_number = 0; + row = affirmation_sheet.createRow(row_number++); + cell_number = 0; + max_cell_number = 0; + + CellUtil.createCell(row, cell_number++, "Affirmation", bold_style); + + cell_number = 0; + row_number++; + row = affirmation_sheet.createRow(row_number++); + // Two lines to accommodate current affirmation text + row.setHeightInPoints(affirmationRowHeight * 2); + CellUtil.createCell(row, cell_number++, AFFIRMATION_STATEMENT, wrapped_style); + + // Merge the affirmation row columns A-G (0-6) + affirmation_sheet.addMergedRegion( + new CellRangeAddress( + row_number - 1, + row_number - 1, + 0, + 6 + ) + ); + + for (int roundIndex = 0; roundIndex < my_rounds.size(); roundIndex++) { + final Round round = my_rounds.get(roundIndex); + final Map> signatories = round.signatories(); + final List boardIndices = new ArrayList(signatories.keySet()); + Collections.sort(boardIndices); + + cell_number = 0; + row_number++; + row = affirmation_sheet.createRow(row_number++); + CellUtil.createCell( + row, + cell_number++, + String.format("Round %d", round.number()), + bold_style + ); + + for (final Integer boardIndex : boardIndices) { + cell_number = 0; + row = affirmation_sheet.createRow(row_number++); + CellUtil.createCell( + row, + cell_number++, + String.format("Audit board %d:", boardIndex + 1), + bold_style + ); + + final List boardSignatories = signatories.get(boardIndex); + for (final Elector signatory : boardSignatories) { + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(standard_style); + cell.setCellValue(String.format("%s %s", + signatory.firstName(), + signatory.lastName())); + } + } + } + + cell_number = 0; + row_number++; + row = affirmation_sheet.createRow(row_number++); + CellUtil.createCell(row, cell_number++, "County Clerk", bold_style); + + cell_number = 0; + row = affirmation_sheet.createRow(row_number++); + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + + // Clerk signature row columns 0-2 + final CellRangeAddress clerkSignatureRange = new CellRangeAddress( + row_number - 1, + row_number - 1, + 0, + 2 + ); + + // Merge the clerk signature row + affirmation_sheet.addMergedRegion(clerkSignatureRange); + + RegionUtil.setBorderTop( + BorderStyle.THICK, + clerkSignatureRange, + affirmation_sheet + ); + RegionUtil.setBorderRight( + BorderStyle.THICK, + clerkSignatureRange, + affirmation_sheet + ); + RegionUtil.setBorderBottom( + BorderStyle.THICK, + clerkSignatureRange, + affirmation_sheet + ); + RegionUtil.setBorderLeft( + BorderStyle.THICK, + clerkSignatureRange, + affirmation_sheet + ); + + row.setHeight((short) 800); + + for (int i = 0; i <= 2; i++) { + affirmation_sheet.autoSizeColumn(i); + } + + return workbook; + } + + /** + * @return the PDF representation of this report, as a byte array. + */ + public byte[] generatePDF() { + return new byte[0]; + } + + /** + * @return the filename for the Excel version of this report. + */ + public String filenameExcel() { + // the file name should be constructed from the county name, election + // type and date, and report generation time + final LocalDateTime election_datetime = + LocalDateTime.ofInstant(my_dosdb.auditInfo().electionDate(), ZoneOffset.UTC); + final LocalDateTime report_datetime = + LocalDateTime.ofInstant(my_timestamp, TimeZone.getDefault().toZoneId()). + truncatedTo(ChronoUnit.SECONDS); + final StringBuilder sb = new StringBuilder(32); + + sb.append(my_county.name().toLowerCase(Locale.getDefault()).replace(" ", "_")); + sb.append('-'); + sb.append(my_dosdb.auditInfo().electionType(). + toLowerCase(Locale.getDefault()).replace(" ", "_")); + sb.append('-'); + sb.append(DATE_FORMATTER.format(election_datetime).replace("/", "-")); + sb.append("-report-"); + sb.append(DATE_TIME_FORMATTER.format(report_datetime).replace("/", "-").replace(":", "_")); + sb.append(".xlsx"); + + return sb.toString(); + } + + /** + * @return the filename for the PDF version of this report. + */ + public String filenamePDF() { + return filenameExcel().replaceAll(".xlsx$", ".pdf"); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/report/ReportRows.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/report/ReportRows.java new file mode 100644 index 00000000..24aaf40e --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/report/ReportRows.java @@ -0,0 +1,507 @@ +package us.freeandfair.corla.report; + + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.OptionalInt; + +import java.math.BigDecimal; + +import org.apache.commons.lang3.ArrayUtils; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.TimeZone; + +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import us.freeandfair.corla.controller.ContestCounter; +import us.freeandfair.corla.math.Audit; +import us.freeandfair.corla.model.CastVoteRecord; +import us.freeandfair.corla.model.CVRAuditInfo; +import us.freeandfair.corla.model.CVRContestInfo; +import us.freeandfair.corla.model.ComparisonAudit; +import us.freeandfair.corla.model.Tribute; +import us.freeandfair.corla.persistence.Persistence; +import us.freeandfair.corla.query.CastVoteRecordQueries; +import us.freeandfair.corla.query.ComparisonAuditQueries; +import us.freeandfair.corla.query.CountyQueries; +import us.freeandfair.corla.query.TributeQueries; + +import us.freeandfair.corla.util.SuppressFBWarnings; + +/** + * Contains the query-ing and processing of two report types: + * activity and results + **/ +// fb and pmd conflict about public static nested constants +@SuppressFBWarnings({"MS_PKGPROTECT"}) +public class ReportRows { + + /** + * Class-wide logger + */ + public static final Logger LOGGER = + LogManager.getLogger(ReportRows.class); + + /** the union set of used by activity and results reports **/ + public static final String[] ALL_HEADERS = { + "county", + "imprinted id", + "scanner id", + "batch id", + "record id", + "db id", + "round", + "audit board", + "record type", + "discrepancy", + "consensus", + "comment", + "random number", + "random number sequence position", + "multiplicity", + "revision", + "re-audit ballot comment", + "time of submission" + }; + + /** US local date time **/ + private static final DateTimeFormatter MMDDYYYY = + DateTimeFormatter.ofPattern("MM/dd/yyyy hh:mm:ss a"); + + /** cache of county id to county name **/ + private final static Map countyNames = new HashMap(); + + /** an empty error response **/ + @SuppressWarnings({"PMD.NonStaticInitializer"}) + private final static List NOT_FOUND_ROW = new ArrayList() {{ + add("audit has not started or contest name not found"); + }}; + + /** no instantiation **/ + private ReportRows() {}; + + /** + * One array to be part of an array of arrays, ie: a table or csv or xlsx. + * It keeps the headers and fields in order. + **/ + public static class Row { + + /** composition rather than inheritance **/ + private final Map map = new HashMap(); + + /** hold the headers **/ + private final String[] headers; + + /** new row with haeders **/ + public Row(final String ...headers) { + this.headers = headers; + } + + /** get the value for the given header **/ + public String get(final String key) { + return this.map.get(key); + } + + /** put the value for the given header **/ + public void put(final String key, final String value) { + this.map.put(key, value); + } + + + /** loop over headers, spit out values, keeping them in sync **/ + public List toArray() { + final List a = new ArrayList(); + for (final String h: this.headers) { + a.add(this.get(h)); + } + return a; + } + } + + /** + * query for the associated CVRAuditInfo object to access the disc data on the + * Comparison object. If the acvr has been reaudited, recompute the value + * because that process loses the association. + **/ + public static Integer findDiscrepancy(final ComparisonAudit audit, final CastVoteRecord acvr) { + if (CastVoteRecord.RecordType.REAUDITED == acvr.recordType()) { + // we recompute here because we don't have cai.acvr_id = acvr.id + final CastVoteRecord cvr = Persistence.getByID(acvr.getCvrId(), CastVoteRecord.class); + final OptionalInt disc = audit.computeDiscrepancy(cvr, acvr); + if (disc.isPresent()) { + return disc.getAsInt(); + } else { + return null; + } + } else { + // not a revised/overwritten submission + final CVRAuditInfo cai = Persistence.getByID(acvr.getCvrId(), CVRAuditInfo.class); + return audit.getDiscrepancy(cai); + } + } + + /** get the county name for the given county id **/ + public static String findCountyName(final Long countyId) { + String name = countyNames.get(countyId); + if (null == name) { + name = CountyQueries.getName(countyId); + countyNames.put(countyId, name); + return name; + } else { + return name; + } + } + + /** render helper **/ + public static String toString(final Object o) { + if (null == o) { + return null; + } else { + return o.toString(); + } + } + + /** render helper **/ + public static String renderAuditBoard(final Integer auditBoardIndex) { + if (null == auditBoardIndex) { + return null; + } else { + final Integer i = auditBoardIndex.intValue() + 1; + return i.toString(); + } + } + + /** + * render helper + * Prepend a plus sign on positive integers to make it clear that it is positive. + * Negative numbers will have the negative sign. + * These don't need to be integers because they are counted, not summed. + **/ + public static String renderDiscrepancy(final Integer discrepancy) { + if (discrepancy > 0) { + return String.format("+%d", discrepancy); + } else { + return discrepancy.toString(); + } + } + + /** render helper US local date time **/ + public static String renderTimestamp(final Instant timestamp) { + return MMDDYYYY.format(LocalDateTime + .ofInstant(timestamp, + TimeZone.getDefault().toZoneId())); + } + + /** render consensus to yesNo **/ + public static String renderConsensus(final CVRContestInfo.ConsensusValue consensus) { + // consensus can be null if not sent in the request, so there was a + // consensus, unless they said no. + return yesNo(CVRContestInfo.ConsensusValue.NO != consensus); + } + + /** add fields common to both activity and results reports **/ + public static Row addBaseFields(final Row row, final ComparisonAudit audit, final CastVoteRecord acvr) { + final Integer discrepancy = findDiscrepancy(audit, acvr); + final Optional infoMaybe = acvr.contestInfoForContestResult(audit.contestResult()); + + if (infoMaybe.isPresent()) { + final CVRContestInfo info = infoMaybe.get(); + row.put("consensus", renderConsensus(info.consensus())); + row.put("comment", info.comment()); + } + + if (null == discrepancy || 0 == discrepancy) { + row.put("discrepancy", null); + } else { + row.put("discrepancy", renderDiscrepancy(discrepancy)); + } + row.put("db id", acvr.getCvrId().toString()); + row.put("record type", acvr.recordType().toString()); + row.put("county", findCountyName(acvr.countyID())); + row.put("audit board", renderAuditBoard(acvr.getAuditBoardIndex())); + row.put("round", toString(acvr.getRoundNumber())); + row.put("imprinted id", acvr.imprintedID()); + row.put("scanner id", toString(acvr.scannerID())); + row.put("batch id", acvr.batchID()); + row.put("record id", toString(acvr.recordID())); + row.put("time of submission", renderTimestamp(acvr.timestamp())); + return row; + } + + /** add fields unique to activity report **/ + public static Row addActivityFields(final Row row, final CastVoteRecord acvr) { + row.put("revision", toString(acvr.getRevision())); + row.put("re-audit ballot comment", acvr.getComment()); + return row; + } + + /** add fields unique to results report **/ + public static Row addResultsFields(final Row row, final Tribute tribute, final Integer multiplicity) { + row.put("multiplicity", toString(multiplicity)); + return addResultsFields(row, tribute); + } + + /** add fields unique to activity report, if the multiplicity is unknown **/ + public static Row addResultsFields(final Row row, final Tribute tribute) { + row.put("random number", toString(tribute.rand)); + row.put("random number sequence position", toString(tribute.randSequencePosition)); + return row; + } + + /** tie the headers to a row **/ + public static class ActivityReport { + /** a selection of headers **/ + public static final String[] HEADERS = + ArrayUtils.removeElements(ArrayUtils.clone(ALL_HEADERS), + "random number sequence position", + "random number", + "multiplicity"); + + /** no instantiation **/ + private ActivityReport() {}; + + /** new row **/ + public static final Row newRow() { + return new Row(HEADERS); + } + } + + /** tie the headers to a row **/ + public static class ResultsReport { + /** a selection of headers **/ + public static final String[] HEADERS = + ArrayUtils.removeElements(ArrayUtils.clone(ALL_HEADERS), + "revision", + "re-audit ballot comment"); + + /** no instantiation **/ + private ResultsReport() {}; + + /** new row **/ + public static final Row newRow() { + return new Row(HEADERS); + } + } + + /** tie the headers to a row **/ + public static class SummaryReport { + + /** a selection of headers **/ + public static final String[] HEADERS = { + "Contest", + "targeted", + "Winner", + + "Risk Limit met?", + "Risk measurement %", + "Audit Risk Limit %", + "diluted margin %", + "disc +2", + "disc +1", + "disc -1", + "disc -2", + "gamma", + "audited sample count", + + "ballot count", + "min margin", + "votes for winner", + "votes for runner up", + "total votes", + "disagreement count (included in +2 and +1)" + }; + + /** no instantiation **/ + private SummaryReport() {}; + + /** new Row **/ + public static final Row newRow() { + return new Row(HEADERS); + } + } + + /** risk limit achieved according to math.Audit **/ + public static BigDecimal riskMeasurement(final ComparisonAudit ca) { + if (ca.getAuditedSampleCount() > 0 + && ca.getDilutedMargin().compareTo(BigDecimal.ZERO) > 0) { + final BigDecimal result = Audit.pValueApproximation(ca.getAuditedSampleCount(), + ca.getDilutedMargin(), + ca.getGamma(), + ca.discrepancyCount(-1), + ca.discrepancyCount(-2), + ca.discrepancyCount(1), + ca.discrepancyCount(2)); + return result.setScale(3, BigDecimal.ROUND_HALF_UP); + } else { + // full risk (100%) when nothing is known + return BigDecimal.ONE; + } + } + + /** compare risk sought vs measured **/ + public static boolean riskLimitMet(final BigDecimal sought, final BigDecimal measured) { + return sought.compareTo(measured) > 0; + } + + /** yes/no instead of true/false **/ + public static String yesNo(final Boolean bool) { + if (bool) { + return "Yes"; + } else { + return "No"; + } + } + + /** significant figures **/ + public static BigDecimal sigFig(final BigDecimal num, final int digits) { + return num.setScale(digits, BigDecimal.ROUND_HALF_UP); + } + + /** * 100 **/ + public static BigDecimal percentage(final BigDecimal num) { + return BigDecimal.valueOf(100).multiply(num); + } + + /** + * for each contest(per row), show all the variables that are interesting or + * needed to perform the risk limit calculation + **/ + public static List> genSumResultsReport() { + final List> rows = new ArrayList(); + + rows.add(Arrays.asList(SummaryReport.HEADERS)); + for (final ComparisonAudit ca: ComparisonAuditQueries.sortedList()) { + final Row row = SummaryReport.newRow(); + + final BigDecimal riskMsmnt = riskMeasurement(ca); + + // general info + row.put("Contest", ca.contestResult().getContestName()); + row.put("targeted", yesNo(ca.isTargeted())); + + if (ca.contestResult().getWinners() == null || ca.contestResult().getWinners().isEmpty()) { + LOGGER.info("no winner!!! " + ca); + } + row.put("Winner", toString(ca.contestResult().getWinners().iterator().next())); + row.put("Risk Limit met?", yesNo(riskLimitMet(ca.getRiskLimit(), riskMsmnt))); + row.put("Risk measurement %", sigFig(percentage(riskMsmnt), 1).toString()); + row.put("Audit Risk Limit %", sigFig(percentage(ca.getRiskLimit()),1).toString()); + row.put("diluted margin %", percentage(ca.getDilutedMargin()).toString()); + row.put("disc +2", toString(ca.discrepancyCount(2))); + row.put("disc +1", toString(ca.discrepancyCount(1))); + row.put("disc -1", toString(ca.discrepancyCount(-1))); + row.put("disc -2", toString(ca.discrepancyCount(-2))); + row.put("gamma", toString(ca.getGamma())); + row.put("audited sample count", toString(ca.getAuditedSampleCount())); + + // very detailed extra info + row.put("ballot count", toString(ca.contestResult().getBallotCount())); + row.put("min margin", toString(ca.contestResult().getMinMargin())); + + final List> rankedTotals = + ContestCounter.rankTotals(ca.contestResult().getVoteTotals()); + + try { + row.put("votes for winner", toString(rankedTotals.get(0).getValue())); + } catch (IndexOutOfBoundsException e) { + row.put("votes for winner", ""); + } + + try { + row.put("votes for runner up", toString(rankedTotals.get(1).getValue())); + } catch (IndexOutOfBoundsException e) { + row.put("votes for runner up", ""); + } + + row.put("total votes", toString(ca.contestResult().totalVotes())); + row.put("disagreement count (included in +2 and +1)", toString(ca.disagreementCount())); + + rows.add(row.toArray()); + } + return rows; + } + + /** build a list of rows for a contest based on acvrs **/ + public static List> getContestActivity(final String contestName) { + final List> rows = new ArrayList(); + + final ComparisonAudit audit = ComparisonAuditQueries.matching(contestName); + if (null == audit) { + // return something in a response to explain the situation + rows.add(NOT_FOUND_ROW); + return rows; + } + + rows.add(Arrays.asList(ActivityReport.HEADERS)); + final List contestCVRIds = audit.getContestCVRIds(); + if (contestCVRIds.isEmpty()) { + // Something has gone wrong, it seems, because all targeted contests should + // have contestCVRIds by the time the reports button can be clicked - at + // least that is the intention. + return rows; + } + + // now we can see if there is any activity + final List acvrs = CastVoteRecordQueries.activityReport(contestCVRIds); + acvrs.sort(Comparator.comparing(CastVoteRecord::timestamp)); + + acvrs.forEach(acvr -> { + final Row row = ActivityReport.newRow(); + rows.add(addActivityFields(addBaseFields(row, audit, acvr), acvr).toArray()); + }); + + return rows; + } + + /** build a list of rows for a contest based on tributes **/ + public static List> getResultsReport(final String contestName) { + final List> rows = new ArrayList(); + + final List tributes = TributeQueries.forContest(contestName); + tributes.sort(Comparator.comparing(t -> t.randSequencePosition)); + + final ComparisonAudit audit = ComparisonAuditQueries.matching(contestName); + if (null == audit) { + rows.add(NOT_FOUND_ROW); + return rows; + } + + final List contestCVRIds = audit.getContestCVRIds(); + + final List acvrs = CastVoteRecordQueries.resultsReport(contestCVRIds); + + rows.add(Arrays.asList(ResultsReport.HEADERS)); + + for (final Tribute tribute: tributes) { + final Row row = ResultsReport.newRow(); + // get the acvr that was submitted for this tribute + final String uri = tribute.getUri(); + final String aUri = uri.replaceFirst("^cvr", "acvr"); + final Optional acvr = acvrs.stream() + .filter(c -> c.getUri().equals(aUri)) + .findFirst(); + + if (acvr.isPresent()) { + final Integer multiplicity = audit.multiplicity(acvr.get().getCvrId()); + rows.add(addResultsFields(addBaseFields(row, audit, acvr.get()), tribute, multiplicity).toArray()); + } else { + // not yet audited, and we don't know the multiplicity + rows.add(addResultsFields(row, tribute).toArray()); + } + } + + return rows; + } + + +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/report/StateReport.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/report/StateReport.java new file mode 100644 index 00000000..982ec2f7 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/report/StateReport.java @@ -0,0 +1,781 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 30, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.report; + +import static us.freeandfair.corla.util.PrettyPrinter.booleanYesNo; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Map.Entry; +import java.util.OptionalInt; +import java.util.SortedMap; +import java.util.TimeZone; +import java.util.TreeMap; + +import org.apache.poi.ss.usermodel.BorderStyle; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.CellStyle; +import org.apache.poi.ss.usermodel.CellType; +import org.apache.poi.ss.usermodel.DataFormat; +import org.apache.poi.ss.usermodel.Font; +import org.apache.poi.ss.usermodel.HorizontalAlignment; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; + +import us.freeandfair.corla.model.AuditSelection; +import us.freeandfair.corla.model.CVRAuditInfo; +import us.freeandfair.corla.model.CastVoteRecord.RecordType; +import us.freeandfair.corla.model.County; +import us.freeandfair.corla.model.County.NameComparator; +import us.freeandfair.corla.model.CountyContestResult; +import us.freeandfair.corla.model.DoSDashboard; +import us.freeandfair.corla.model.Round; +import us.freeandfair.corla.persistence.Persistence; + +/** + * All the data required for a state audit report. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings({"PMD.CyclomaticComplexity", "PMD.StdCyclomaticComplexity", + "PMD.ModifiedCyclomaticComplexity", "PMD.ExcessiveImports", "PMD.GodClass"}) +public class StateReport { + /** + * The font size for Excel. + */ + // POI interop requires a short here + @SuppressWarnings("PMD.AvoidUsingShortType") + public static final short FONT_SIZE = 12; + + /** + * The date formatter. + */ + private static final DateTimeFormatter DATE_FORMATTER = + DateTimeFormatter.ofPattern("MM/dd/yyyy"); + + /** + * The date/time formatter. + */ + private static final DateTimeFormatter DATE_TIME_FORMATTER = + DateTimeFormatter.ofPattern("MM/dd/yyyy hh:mm a"); + + /** + * The date and time this report was generated. + */ + private final Instant my_timestamp; + + /** + * The county audit reports. + */ + private final SortedMap my_county_reports; + + /** + * The DoS dashboard. + */ + private final DoSDashboard my_dosdb; + + /** + * Initialize a state report object, timestamped at the current time. + */ + public StateReport() { + this(Instant.now()); + } + + /** + * Initialize a state report object with the specified timestamp. All + * of the individual county reports will have the same timestamp. + * + * @param the_timestamp The timestamp. + */ + public StateReport(final Instant the_timestamp) { + my_county_reports = new TreeMap<>(new NameComparator()); + my_timestamp = the_timestamp; + for (final County c : Persistence.getAll(County.class)) { + my_county_reports.put(c, new CountyReport(c, my_timestamp)); + } + my_dosdb = Persistence.getByID(DoSDashboard.ID, DoSDashboard.class); + } + + /** + * @return the timestamp of this report. + */ + public Instant timestamp() { + return my_timestamp; + } + + /** + * @return the county reports comprising this report. + */ + public Map countyReports() { + return Collections.unmodifiableMap(my_county_reports); + } + + + /** + * @return the Excel representation of this report, as a byte array. + * @exception IOException if the report cannot be generated. + */ + public byte[] generateExcel() throws IOException { + final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + final Workbook workbook = generateExcelWorkbook(); + workbook.write(baos); + baos.flush(); + baos.close(); + workbook.close(); + return baos.toByteArray(); + } + + /** + * @return the Excel workbook for this report. + */ + @SuppressWarnings({"checkstyle:magicnumber", "checkstyle:executablestatementcount", + "checkstyle:methodlength", "PMD.ExcessiveMethodLength", "PMD.NcssMethodCount", + "PMD.NPathComplexity", "PMD.AvoidLiteralsInIfCondition"}) + public Workbook generateExcelWorkbook() { + final Workbook workbook = new XSSFWorkbook(); + + // data format + final DataFormat format = workbook.createDataFormat(); + + // bold font for titles and such + final Font bold_font = workbook.createFont(); + bold_font.setFontHeightInPoints(FONT_SIZE); + bold_font.setBold(true); + final CellStyle bold_style = workbook.createCellStyle(); + bold_style.setFont(bold_font); + final CellStyle bold_right_style = workbook.createCellStyle(); + bold_right_style.setFont(bold_font); + bold_right_style.setAlignment(HorizontalAlignment.RIGHT); + + // regular font for other fields + final Font standard_font = workbook.createFont(); + standard_font.setFontHeightInPoints(FONT_SIZE); + final CellStyle standard_style = workbook.createCellStyle(); + standard_style.setFont(standard_font); + standard_style.setDataFormat(format.getFormat("@")); + final CellStyle standard_right_style = workbook.createCellStyle(); + standard_right_style.setFont(standard_font); + standard_right_style.setAlignment(HorizontalAlignment.RIGHT); + standard_right_style.setDataFormat(format.getFormat("@")); + final CellStyle integer_style = workbook.createCellStyle(); + integer_style.setFont(standard_font); + integer_style.setDataFormat(format.getFormat("0")); + final CellStyle decimal_style = workbook.createCellStyle(); + decimal_style.setFont(standard_font); + decimal_style.setDataFormat(format.getFormat("0.000#####")); + final CellStyle box_style = workbook.createCellStyle(); + box_style.setBorderBottom(BorderStyle.THICK); + box_style.setBorderTop(BorderStyle.THICK); + box_style.setBorderLeft(BorderStyle.THICK); + box_style.setBorderRight(BorderStyle.THICK); + + // the summary sheet + final Sheet summary_sheet = workbook.createSheet("Summary"); + int row_number = 0; + Row row = summary_sheet.createRow(row_number++); + int cell_number = 0; + int max_cell_number = 0; + + Cell cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("State Audit Report"); + + row = summary_sheet.createRow(row_number++); + cell_number = 0; + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Generated " + + DATE_TIME_FORMATTER. + format(LocalDateTime.ofInstant(my_timestamp, + TimeZone.getDefault().toZoneId()))); + + row = summary_sheet.createRow(row_number++); + cell_number = 0; + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + if (my_dosdb.auditInfo().electionType() == null && + my_dosdb.auditInfo().electionDate() == null) { + cell.setCellValue("ELECTION TYPE/DATE NOT SET"); + } else { + cell.setCellValue(my_dosdb.auditInfo().capitalizedElectionType() + " Election - " + + DATE_FORMATTER. + format(LocalDateTime.ofInstant(my_dosdb.auditInfo().electionDate(), + ZoneOffset.UTC))); + } + + row_number++; + row = summary_sheet.createRow(row_number++); + cell_number = 0; + + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Audit Random Seed"); + + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(standard_right_style); + cell.setCellValue(my_dosdb.auditInfo().seed()); + + row = summary_sheet.createRow(row_number++); + cell_number = 0; + + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Audit Risk Limit"); + + cell = row.createCell(cell_number++); + cell.setCellType(CellType.NUMERIC); + cell.setCellStyle(decimal_style); + cell.setCellValue(my_dosdb.auditInfo().riskLimit().doubleValue()); + + row_number++; + row = summary_sheet.createRow(row_number++); + cell_number = 0; + + int ballots_in_manifests = 0; + int cvrs_in_export_files = 0; + for (final CountyReport cr : my_county_reports.values()) { + ballots_in_manifests += cr.dashboard().ballotsInManifest(); + cvrs_in_export_files += cr.dashboard().cvrsImported(); + } + + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Total Ballot Cards In Manifests"); + + cell = row.createCell(cell_number++); + cell.setCellType(CellType.NUMERIC); + cell.setCellStyle(integer_style); + + cell.setCellValue(ballots_in_manifests); + + row = summary_sheet.createRow(row_number++); + cell_number = 0; + + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Total CVRs in CVR Export Files"); + + cell = row.createCell(cell_number++); + cell.setCellType(CellType.NUMERIC); + cell.setCellStyle(integer_style); + cell.setCellValue(cvrs_in_export_files); + + row = summary_sheet.createRow(row_number++); + cell_number = 0; + + int ballots_audited = 0; + int audit_rounds = 0; + for (final CountyReport cr : my_county_reports.values()) { + ballots_audited = ballots_audited + cr.dashboard().ballotsAudited(); + audit_rounds = Math.max(audit_rounds, cr.dashboard().rounds().size()); + } + + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Total Ballot Cards Audited"); + + cell = row.createCell(cell_number++); + cell.setCellType(CellType.NUMERIC); + cell.setCellStyle(integer_style); + cell.setCellValue(ballots_audited); + + row = summary_sheet.createRow(row_number++); + cell_number = 0; + + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Number of Audit Rounds"); + + cell = row.createCell(cell_number++); + cell.setCellType(CellType.NUMERIC); + cell.setCellStyle(integer_style); + cell.setCellValue(audit_rounds); + + max_cell_number = Math.max(max_cell_number, cell_number); + + for (final Entry e : my_county_reports.entrySet()) { + row_number++; + row = summary_sheet.createRow(row_number++); + cell_number = 0; + cell = row.createCell(cell_number++); + cell.setCellStyle(bold_style); + cell.setCellType(CellType.STRING); + cell.setCellValue(e.getKey().name() + " County"); + + if (e.getValue().drivingContestResults().isEmpty()) { + cell.setCellValue(cell.getStringCellValue() + " - No Contests Audited"); + } else { + if (!e.getValue().rounds().isEmpty()) { + cell.setCellValue(cell.getStringCellValue() + " - Round Summary"); + for (final Round round : e.getValue().rounds()) { + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue(round.number()); + } + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_right_style); + cell.setCellValue("Total"); + + row = summary_sheet.createRow(row_number++); + cell_number = 0; + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Ballot Cards Audited"); + int accumulator = 0; + for (final Round round : e.getValue().rounds()) { + cell = row.createCell(cell_number++); + cell.setCellStyle(integer_style); + cell.setCellType(CellType.NUMERIC); + cell.setCellValue(round.actualCount()); + accumulator = accumulator + round.actualCount(); + } + cell = row.createCell(cell_number++); + cell.setCellStyle(integer_style); + cell.setCellType(CellType.NUMERIC); + cell.setCellValue(accumulator); + + row = summary_sheet.createRow(row_number++); + cell_number = 0; + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Discrepancies (Audited Contests)"); + accumulator = 0; + for (final Round round : e.getValue().rounds()) { + cell = row.createCell(cell_number++); + cell.setCellStyle(integer_style); + cell.setCellType(CellType.NUMERIC); + final int discrepancies; + if (round.discrepancies().containsKey(AuditSelection.AUDITED_CONTEST)) { + discrepancies = round.discrepancies().get(AuditSelection.AUDITED_CONTEST); + } else { + discrepancies = 0; + } + cell.setCellValue(discrepancies); + accumulator = accumulator + discrepancies; + } + cell = row.createCell(cell_number++); + cell.setCellStyle(integer_style); + cell.setCellType(CellType.NUMERIC); + cell.setCellValue(accumulator); + + row = summary_sheet.createRow(row_number++); + cell_number = 0; + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Discrepancies (Non-Audited Contests)"); + accumulator = 0; + for (final Round round : e.getValue().rounds()) { + cell = row.createCell(cell_number++); + cell.setCellStyle(integer_style); + cell.setCellType(CellType.NUMERIC); + final int discrepancies; + if (round.discrepancies().containsKey(AuditSelection.UNAUDITED_CONTEST)) { + discrepancies = round.discrepancies().get(AuditSelection.UNAUDITED_CONTEST); + } else { + discrepancies = 0; + } + cell.setCellValue(discrepancies); + accumulator = accumulator + discrepancies; + } + cell = row.createCell(cell_number++); + cell.setCellStyle(integer_style); + cell.setCellType(CellType.NUMERIC); + cell.setCellValue(accumulator); + + row = summary_sheet.createRow(row_number++); + cell_number = 0; + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Disagreements (Audited Contests)"); + accumulator = 0; + for (final Round round : e.getValue().rounds()) { + cell = row.createCell(cell_number++); + cell.setCellStyle(integer_style); + cell.setCellType(CellType.NUMERIC); + final int disagreements; + if (round.disagreements().containsKey(AuditSelection.AUDITED_CONTEST)) { + disagreements = round.disagreements().get(AuditSelection.AUDITED_CONTEST); + } else { + disagreements = 0; + } + cell.setCellValue(disagreements); + accumulator = accumulator + disagreements; + } + cell = row.createCell(cell_number++); + cell.setCellStyle(integer_style); + cell.setCellType(CellType.NUMERIC); + cell.setCellValue(accumulator); + + row = summary_sheet.createRow(row_number++); + cell_number = 0; + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Disagreements (Non-Audited Contests)"); + accumulator = 0; + for (final Round round : e.getValue().rounds()) { + cell = row.createCell(cell_number++); + cell.setCellStyle(integer_style); + cell.setCellType(CellType.NUMERIC); + final int disagreements; + if (round.disagreements().containsKey(AuditSelection.UNAUDITED_CONTEST)) { + disagreements = round.disagreements().get(AuditSelection.UNAUDITED_CONTEST); + } else { + disagreements = 0; + } + cell.setCellValue(disagreements); + accumulator = accumulator + disagreements; + } + cell = row.createCell(cell_number++); + cell.setCellStyle(integer_style); + cell.setCellType(CellType.NUMERIC); + cell.setCellValue(accumulator); + } + + row_number++; + row = summary_sheet.createRow(row_number++); + cell_number = 0; + cell = row.createCell(cell_number++); + cell.setCellStyle(bold_style); + cell.setCellValue(e.getKey().name() + " County - Audited Contests"); + + row_number = row_number - 1; // don't skip a line before first contest + + for (final CountyContestResult ccr : e.getValue().drivingContestResults()) { + row_number++; + row = summary_sheet.createRow(row_number++); + cell_number = 0; + + cell = row.createCell(cell_number++); + cell.setCellStyle(bold_style); + cell.setCellValue(ccr.contest().name() + " - Vote For " + + ccr.contest().votesAllowed()); + + cell = row.createCell(cell_number++); + cell.setCellStyle(bold_style); + cell.setCellValue("Choice"); + + cell = row.createCell(cell_number++); + cell.setCellStyle(bold_right_style); + cell.setCellValue("W/L"); + + cell = row.createCell(cell_number++); + cell.setCellStyle(bold_right_style); + cell.setCellValue("Votes"); + + cell = row.createCell(cell_number++); + cell.setCellStyle(bold_right_style); + cell.setCellValue("Margin"); + + cell = row.createCell(cell_number++); + cell.setCellStyle(bold_right_style); + cell.setCellValue("Diluted Margin %"); + + for (final String choice : ccr.rankedChoices()) { + row = summary_sheet.createRow(row_number++); + max_cell_number = Math.max(max_cell_number, cell_number); + cell_number = 1; + cell = row.createCell(cell_number++); + cell.setCellStyle(standard_style); + cell.setCellValue(choice); + + cell = row.createCell(cell_number++); + cell.setCellStyle(standard_right_style); + if ((ccr.winners().stream().anyMatch(w -> w.equalsIgnoreCase(choice)))) { + cell.setCellValue("W"); + } else { + cell.setCellValue("L"); + } + + cell = row.createCell(cell_number++); + cell.setCellStyle(integer_style); + cell.setCellType(CellType.NUMERIC); + cell.setCellValue(ccr.voteTotals().get(choice)); + + if (ccr.winners().contains(choice)) { + cell = row.createCell(cell_number++); + cell.setCellStyle(integer_style); + cell.setCellType(CellType.NUMERIC); + final OptionalInt margin = ccr.marginToNearestLoser(choice); + if (margin.isPresent()) { + cell.setCellValue(margin.getAsInt()); + } + + cell = row.createCell(cell_number++); + cell.setCellStyle(decimal_style); + cell.setCellType(CellType.NUMERIC); + final BigDecimal diluted_margin = ccr.countyDilutedMarginToNearestLoser(choice); + if (diluted_margin != null) { + cell.setCellValue(diluted_margin.doubleValue() * 100); + } + } + } + } + } + } + + for (int i = 0; i < max_cell_number; i++) { + summary_sheet.autoSizeColumn(i); + } + + // county sheets + + for (final Entry e : my_county_reports.entrySet()) { + if (e.getValue().drivingContestResults().isEmpty()) { + // don't generate empty sheets + continue; + } + final Sheet county_sheet = workbook.createSheet(e.getKey().name() + " County"); + row_number = 0; + row = county_sheet.createRow(row_number++); + cell_number = 0; + max_cell_number = 0; + + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue(e.getKey().name() + " County Summary Report"); + for (final Round round : e.getValue().rounds()) { + row_number++; + row = county_sheet.createRow(row_number++); + cell_number = 0; + max_cell_number = 0; + + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Round " + round.number()); + + row_number++; + row = county_sheet.createRow(row_number++); + max_cell_number = Math.max(max_cell_number, cell_number); + cell_number = 0; + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Number of Ballot Cards Audited"); + + cell = row.createCell(cell_number++); + cell.setCellType(CellType.NUMERIC); + cell.setCellStyle(integer_style); + cell.setCellValue(round.actualCount()); + + row = county_sheet.createRow(row_number++); + cell_number = 1; // these are headers for audit selections + final List listed_selections = new ArrayList<>(); + final Map discrepancies = round.discrepancies(); + final Map disagreements = round.disagreements(); + + for (final AuditSelection r : AuditSelection.values()) { + if (discrepancies.containsKey(r) && discrepancies.get(r) >= 0 || + disagreements.containsKey(r) && disagreements.get(r) >= 0) { + listed_selections.add(r); + } + } + + Collections.sort(listed_selections); + + for (final AuditSelection s : listed_selections) { + cell = row.createCell(cell_number++); + cell.setCellStyle(bold_right_style); + cell.setCellValue(s.prettyString()); + } + + row = county_sheet.createRow(row_number++); + max_cell_number = Math.max(max_cell_number, cell_number); + cell_number = 0; + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + if (discrepancies.isEmpty()) { + cell.setCellValue("No Discrepancies Recorded"); + } else { + cell.setCellValue("Discrepancies Recorded"); + for (final AuditSelection s : listed_selections) { + cell = row.createCell(cell_number++); + cell.setCellType(CellType.NUMERIC); + cell.setCellStyle(integer_style); + final int cell_value; + if (discrepancies.containsKey(s)) { + cell_value = discrepancies.get(s); + } else { + cell_value = 0; + } + cell.setCellValue(cell_value); + } + } + + row = county_sheet.createRow(row_number++); + max_cell_number = Math.max(max_cell_number, cell_number); + cell_number = 0; + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + if (disagreements.isEmpty()) { + cell.setCellValue("No Disagreements Recorded"); + } else { + cell.setCellValue("Disagreements Recorded"); + for (final AuditSelection s : listed_selections) { + cell = row.createCell(cell_number++); + cell.setCellType(CellType.NUMERIC); + cell.setCellStyle(integer_style); + final int cell_value; + if (disagreements.containsKey(s)) { + cell_value = disagreements.get(s); + } else { + cell_value = 0; + } + cell.setCellValue(cell_value); + } + } + row_number++; + row = county_sheet.createRow(row_number++); + max_cell_number = Math.max(max_cell_number, cell_number); + cell_number = 0; + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Ballot Cards Selected"); + + row = county_sheet.createRow(row_number++); + cell_number = 0; + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_style); + cell.setCellValue("Imprinted ID"); + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_right_style); + cell.setCellValue("Audited"); + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_right_style); + cell.setCellValue("Discrepancy"); + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_right_style); + cell.setCellValue("Disagreement"); + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(bold_right_style); + cell.setCellValue("Ballot Type"); + + max_cell_number = Math.max(max_cell_number, cell_number); + for (final CVRAuditInfo audit_info : + e.getValue().cvrsToAuditByRound().get(round.number())) { + row = county_sheet.createRow(row_number++); + cell_number = 0; + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(standard_style); + cell.setCellValue(audit_info.cvr().imprintedID()); + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(standard_right_style); + if (audit_info.acvr() == null) { + cell.setCellValue(booleanYesNo(false)); + } else { + cell.setCellValue(booleanYesNo(audit_info.acvr().recordType() == + RecordType.AUDITOR_ENTERED)); + } + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(standard_right_style); + cell.setCellValue(booleanYesNo(!audit_info.discrepancy().isEmpty())); + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(standard_right_style); + cell.setCellValue(booleanYesNo(!audit_info.disagreement().isEmpty())); + cell = row.createCell(cell_number++); + cell.setCellType(CellType.STRING); + cell.setCellStyle(standard_right_style); + cell.setCellValue(audit_info.cvr().ballotType()); + + } + } + for (int i = 0; i < max_cell_number; i++) { + county_sheet.autoSizeColumn(i); + } + } + + return workbook; + } + + /** + * @return the PDF representation of this report, as a byte array. + */ + public byte[] generatePDF() { + return new byte[0]; + } + + /** + * @return the filename for the Excel version of this report. + */ + public String filenameExcel() { + // the file name should be constructed from the county name, election + // type and date, and report generation time + final LocalDateTime election_datetime = + LocalDateTime.ofInstant(my_dosdb.auditInfo().electionDate(), ZoneOffset.UTC); + final LocalDateTime report_datetime = + LocalDateTime.ofInstant(my_timestamp, TimeZone.getDefault().toZoneId()). + truncatedTo(ChronoUnit.SECONDS); + final StringBuilder sb = new StringBuilder(32); + + sb.append("state-"); + sb.append(my_dosdb.auditInfo().electionType(). + toLowerCase(Locale.getDefault()).replace(" ", "_")); + sb.append('-'); + sb.append(DATE_FORMATTER.format(election_datetime).replace("/", "-")); + sb.append("-report-"); + sb.append(DATE_TIME_FORMATTER.format(report_datetime).replace("/", "-").replace(":", "_")); + sb.append(".xlsx"); + + return sb.toString(); + } + + /** + * @return the filename for the PDF version of this report. + */ + public String filenamePDF() { + return filenameExcel().replaceAll(".xlsx$", ".pdf"); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/report/WorkbookWriter.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/report/WorkbookWriter.java new file mode 100644 index 00000000..b5f72e89 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/report/WorkbookWriter.java @@ -0,0 +1,116 @@ +/* + * Colorado RLA System + */ + +package us.freeandfair.corla.report; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import java.util.List; + +import org.apache.poi.ss.usermodel.Font; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.CellStyle; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; + +import org.apache.poi.xssf.usermodel.XSSFWorkbook; + +import us.freeandfair.corla.Main; + +/** + * Generate a POI Excel workbook containing audit report sheets. + * + * Sheets include: + * + * - Summary + */ +public class WorkbookWriter { + /** + * Font size to use for all cells. + */ + private static final Short FONT_SIZE = 12; + + /** + * Internal output stream required for writing out the workbook. + */ + private final ByteArrayOutputStream baos; + + + /** + * one workbook to hold multiple sheets, another way to say Excel file, xlsx + **/ + private final Workbook workbook; + + /** regular style **/ + private final CellStyle regStyle; + + /** bold style for headers **/ + private final CellStyle boldStyle; + + /** + * Initializes the AuditReport + */ + public WorkbookWriter() { + this.baos = new ByteArrayOutputStream(); + this.workbook = new XSSFWorkbook(); + this.boldStyle = this.workbook.createCellStyle(); + this.regStyle = this.workbook.createCellStyle(); + + setStyles(); + } + + private void setStyles() { + final Font boldFont = workbook.createFont(); + final Font regFont = workbook.createFont(); + + regFont.setFontHeightInPoints(FONT_SIZE); + this.regStyle.setFont(regFont); + + boldFont.setFontHeightInPoints(FONT_SIZE); + boldFont.setBold(true); + this.boldStyle.setFont(boldFont); + } + + /** + * Given some raw data, generate the actual POI workbook. + * + * @param rows the raw data + * @return the POI workbook ready for output + */ + public void addSheet(final String sheetname, final List> rows) { + Sheet sheet = null; + sheet = workbook.createSheet(sheetname); + + for (int i = 0; i < rows.size(); i++) { + final Row poiRow = sheet.createRow(i); + final List dataRow = rows.get(i); + + for (int j = 0; j < dataRow.size(); j++) { + final Cell cell = poiRow.createCell(j); + cell.setCellValue(dataRow.get(j)); + // Embolden header rows + if (i == 0) { + cell.setCellStyle(boldStyle); + } else { + cell.setCellStyle(regStyle); + } + } + } + } + + /** + * Generate the byte-array representation of this POI workbook. + * + * @return the Excel representation of this report + * @exception IOException if the report cannot be generated. + */ + public byte[] write() throws IOException { + this.workbook.write(this.baos); + this.workbook.close(); + + return this.baos.toByteArray(); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/BallotAssignment.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/BallotAssignment.java new file mode 100644 index 00000000..1544c692 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/BallotAssignment.java @@ -0,0 +1,68 @@ +/* + * Colorado RLA System + * + * @title ColoradoRLA + * @copyright 2018 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.util; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Utilities for assigning ballots to audit boards. + * + * @author Democracy Works, Inc. + */ +public final class BallotAssignment { + /** + * Prevent public construction + */ + private BallotAssignment() { + } + + /** + * Assign a given number of ballots to a given number of boards. + * + * Any "extra" ballots that do not divide evenly into the number of boards + * will be randomly assigned to a board. + * + * @param ballots number of ballots to assign + * @param boards desired number of boards + * @return a list with each element representing a board containing the number + * of ballots to assign that board. + */ + public static List assignToBoards(final int ballots, + final int boards) + throws IllegalArgumentException { + if (ballots < 0) { + throw new IllegalArgumentException("Number of ballots cannot be < 0."); + } + + if (boards <= 0) { + throw new IllegalArgumentException("Number of boards cannot be <= 0."); + } + + // Integer division + final int ballotsPerBoard = ballots / boards; + final int leftoverBallots = ballots % boards; + + // Assign all boards the even number of ballots + final List result = + new ArrayList(Collections.nCopies(boards, ballotsPerBoard)); + + // Assign the leftovers + for (int i = 0; i < leftoverBallots; i++) { + result.set(i, result.get(i) + 1); + } + + // Shuffle the results, for fairness! + Collections.shuffle(result); + + return result; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/BallotSequencer.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/BallotSequencer.java new file mode 100644 index 00000000..5342395a --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/BallotSequencer.java @@ -0,0 +1,78 @@ +/* + * Colorado RLA System + * + * @title ColoradoRLA + * @copyright 2018 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.util; + +import java.util.List; +import java.util.Map; + +import java.util.function.Function; + +import java.util.stream.Collectors; + +import us.freeandfair.corla.controller.BallotSelection; + +import us.freeandfair.corla.json.CVRToAuditResponse; + +import us.freeandfair.corla.model.CastVoteRecord; + +/** + * Ballot sequencing functionality, such as converting a list of CVRs into + * a sorted, deduplicated list of CVRs. + * + * @author Democracy Works, Inc. + */ +public final class BallotSequencer { + /** + * Prevent public construction + */ + private BallotSequencer() { + } + + /** + * Returns a sorted, deduplicated list of CVRs given a list of CVRs. + * + * The sort order must match the order of the ballots in the "pull list" that + * counties use to fetch ballots. By storing the sorted, deduplicated list of + * ballots (CVRs) to audit consistently, we can avoid having to sort them + * again and reap other benefits like easier partitioning to support multiple + * audit boards. + * + * @param cvrs input CVRs + * @return sorted, deduplicated list of CVRs + */ + public static List + sortAndDeduplicateCVRs(final List cvrs) { + // Deduplicate CVRs, creating a mapping for use later on. + final Map cvrIdToCvrs = + cvrs.stream() + .distinct() + .collect(Collectors.toMap( + cvr -> cvr.id(), + Function.identity(), + (a, b) -> b)); + + // Join with ballot manifest for the purposes of sorting by location, then + // sort it. + // + // TOOD: Abusing the CVRToAuditResponse class for sorting is wrong; we + // should reify the "joined CVR / Ballot Manifest" concept. + final List sortedAuditResponses = + BallotSelection.toResponseList( + cvrIdToCvrs.entrySet().stream() + .map(entry -> entry.getValue()) + .collect(Collectors.toList())); + sortedAuditResponses.sort(null); + + // Walk the now-sorted list, pulling CVRs back out of the map. + return sortedAuditResponses.stream() + .map(cvrar -> cvrIdToCvrs.get(cvrar.dbID())) + .collect(Collectors.toList()); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/DBExceptionUtil.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/DBExceptionUtil.java new file mode 100644 index 00000000..386a663f --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/DBExceptionUtil.java @@ -0,0 +1,91 @@ +/* + * Free & Fair Colorado RLA System + * + * @title corla-server + * + * @created Sep 10, 2020 + * + * @copyright 2020 Free & Fair + * + * @license GNU General Public License 3.0 + * + * @creator name + * + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.util; + +import java.sql.BatchUpdateException; +import java.sql.SQLException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.persistence.PersistenceException; + +import org.postgresql.util.PSQLException; + +/** + * @description + * @explanation + * @bon OPTIONAL_BON_TYPENAME + */ +public class DBExceptionUtil { + + private static final int MAX_DEPTH = 10; + + /** + * + * Returns the first PSQL Exception found in the chain. If none found, return + * null. + * + * @param PersistenceException Persistence Exception Data + */ + + private static PSQLException getFirstPSQLException(PersistenceException pe) { + SQLException nextException = null; + Throwable innerException = null; + if (pe != null) { + innerException = pe.getCause(); + } + int i = 0; + while (innerException != null && i < MAX_DEPTH) { + if (innerException instanceof PSQLException) { + return (PSQLException) innerException; + } + i++; + innerException = innerException.getCause(); + } + return null; + } + + /** + * + * Gets the reason why set contest names fails. Currently only exception + * supported is a duplication such as a contest that's already mapped. The + * error message is PSQLException in the chain. + * + * ERROR: duplicate key value violates unique constraint "XysisConstraintName" + * Detail: Key (name, county_id, description, votes_allowed)=(NameOfContest, + * 1, , 1) already exists + * + * @param PersistenceException will be used to display a meaningful error + * message to user + * + */ + public static String getConstraintFailureReason(PersistenceException pe) { + + PSQLException firstPSQLException = getFirstPSQLException(pe); + if (firstPSQLException != null) { + + Pattern pattern = Pattern.compile("Detail: Key .*=([(].*?)[.]"); + Matcher matcher = pattern.matcher(firstPSQLException.getMessage()); + if (matcher.find()) { + return matcher.group(1) ; + } + return pe.toString(); + } + return pe.toString(); + } + +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/EqualsHashcodeHelper.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/EqualsHashcodeHelper.java new file mode 100644 index 00000000..f38d0e42 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/EqualsHashcodeHelper.java @@ -0,0 +1,62 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 1, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.util; + +/** + * A utility class with useful methods for building equals and hashCode + * methods. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public final class EqualsHashcodeHelper { + /** + * Private constructor to prevent instantiation. + */ + private EqualsHashcodeHelper() { + // empty + } + + /** + * Compares two objects, which can be null, for equivalence. The + * correctness of this method depends on the correctness of the + * objects' equals() methods. + * + * @param the_first The first object. + * @param the_second The second object. + * @return true if the objects (or nulls) are equivalent, false otherwise. + */ + public static boolean nullableEquals(final Object the_first, + final Object the_second) { + if (the_first == null) { + return the_second == null; + } else { + return the_first.equals(the_second); + } + } + + /** + * Computes a hash code for an object, which can be null. The + * correctness of this method depends on the correctness of the + * object's hashCode() method. + * + * @param the_object The object. + * @return the hash code. + */ + public static int nullableHashCode(final Object the_object) { + if (the_object == null) { + return 0; + } else { + return the_object.hashCode(); + } + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/ExponentialBackoffHelper.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/ExponentialBackoffHelper.java new file mode 100644 index 00000000..a275d359 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/ExponentialBackoffHelper.java @@ -0,0 +1,56 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 4, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.util; + +import java.util.Random; + +/** + * A class of helper methods for dealing with files. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public final class ExponentialBackoffHelper { + /** + * The random number generator. + */ + private static final Random RANDOM = new Random(); + + /** + * Private constructor to prevent instantiation. + */ + private ExponentialBackoffHelper() { + // empty + } + + /** + * Calculates a delay, in milliseconds, to sleep before retrying a transaction. + * This is done using a relatively standard exponential backoff and the specified + * unit delay. + * + * @param the_retries The number of retries so far. + */ + public static long exponentialBackoff(final int the_retries, final long the_unit_delay) { + final double exponentiated = Math.pow(2, the_retries); + long multiplier = 1; + + if (Double.isNaN(exponentiated)) { + multiplier = 1; + } else { + final int max_delay_factor = + (int) Math.max(1, Math.min(Integer.MAX_VALUE, Math.round(exponentiated))); + multiplier = RANDOM.nextInt(max_delay_factor) + 1; + } + + return multiplier * the_unit_delay; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/FileHelper.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/FileHelper.java new file mode 100644 index 00000000..bf977518 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/FileHelper.java @@ -0,0 +1,57 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 4, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.util; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/** + * A class of helper methods for dealing with files. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public final class FileHelper { + /** + * Private constructor to prevent instantiation. + */ + private FileHelper() { + // empty + } + + /** + * Perform a buffered copy from the input stream to the output stream, up + * to a maximum number of bytes. + * + * @param the_input_stream The input stream. + * @param the_output_stream The output stream. + * @param the_buffer_size The buffer size. + * @param the_max_bytes The maximum number of bytes to copy. + */ + public static int bufferedCopy(final InputStream the_input_stream, + final OutputStream the_output_stream, + final int the_buffer_size, final int the_max_bytes) + throws IOException { + final byte[] buffer = new byte[the_buffer_size]; + int length = 1; + int total = 0; + while (total < the_max_bytes && length > 0) { + length = the_input_stream.read(buffer); + if (length > 0) { + the_output_stream.write(buffer, 0, length); + total = total + length; + } + } + return total; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/NaturalOrderComparator.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/NaturalOrderComparator.java new file mode 100644 index 00000000..1cb83366 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/NaturalOrderComparator.java @@ -0,0 +1,125 @@ +package us.freeandfair.corla.util; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A Singleton implementation of Comparator that treats an + * entire subsequence of my_digits as a (Long) number, ordering strings + * in a human friendly, natural order. + */ +@SuppressWarnings("PMD.CyclomaticComplexity") +public final class NaturalOrderComparator implements Comparator, Serializable { + /** + * The instance of this comparator + */ + public static final NaturalOrderComparator INSTANCE = new NaturalOrderComparator(); + + /** + * The serialVersionUID. + */ + private static final long serialVersionUID = 1; + + /** + * A pattern for groups of either my_digits or non-my_digits + */ + private final Pattern my_chunks = Pattern.compile("(\\d+|\\D+)"); + + /** + * A pattern for just a group of my_digits + */ + private final Pattern my_digits = Pattern.compile("(\\d+)"); + + private NaturalOrderComparator() { + super(); + } + + /** + * Splits a string into the longest sequences of my_digits or letters + * @param an_s A String to split into my_chunks + * @return List A list of digit/letter strings + */ + private List split(final String an_s) { + final List xs = new ArrayList(); + final Matcher m = my_chunks.matcher(an_s); + + while (m.find()) { + xs.add(m.group(1)); + } + return xs; + } + + /** + * @param an_s An String that might or might not be made of my_digits + * @return boolean + */ + private boolean isDigit(final String an_s) { + final Matcher m = my_digits.matcher(an_s); + return m.matches(); + } + + /** + * Compare two strings as Longs + * @param an_a The left string to compare + * @param an_b The right string to compare + * @return int The result of comparing an_a and an_b as Longs + */ + private int compareAsDigits(final String an_a, final String an_b) { + int result; + + try { + result = Long.compare(Long.parseLong(an_a), Long.parseLong(an_b)); + } catch (final NumberFormatException e) { + result = String.CASE_INSENSITIVE_ORDER.compare(an_a, an_b); + } + + return result; + } + + /** + * Compare two strings in a human friendly way. + * @param an_a The left string to compare + * @param an_b The right string to compare + * @return int + */ + public int compare(final String an_a, final String an_b) { + int result = 0; + + final Iterator as = split(an_a).iterator(); + final Iterator bs = split(an_b).iterator(); + + while (as.hasNext() && bs.hasNext() && result == 0) { + final String x = as.next(); + final String y = bs.next(); + + if (isDigit(x) && isDigit(y)) { + result = compareAsDigits(x, y); + } else { + result = String.CASE_INSENSITIVE_ORDER.compare(x, y); + + // take the opposite because we want the lower char number A to come + // before the larger char number B. Also, not greater than one for consistency + if (result > 0) { + result = -1; + } else if (result < 0) { + result = 1; + }// else result is zero, and that is fine + } + } + + if (result == 0) { + if (!as.hasNext() && bs.hasNext()) { + result = -1; + } else if (as.hasNext() && !bs.hasNext()) { + result = 1; + } + } + + return result; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/Pair.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/Pair.java new file mode 100644 index 00000000..9cec0a6d --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/Pair.java @@ -0,0 +1,94 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 25, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.util; + +import static us.freeandfair.corla.util.EqualsHashcodeHelper.*; + +/** + * A pair of objects, potentially of different types. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public class Pair { + /** + * The first object. + */ + private final A my_first; + + /** + * The second object. + */ + private final B my_second; + + /** + * Constructs a new pair from two objects. + * + * @param the_first The first object. + * @param the_second The second object. + */ + public Pair(final A the_first, final B the_second) { + my_first = the_first; + my_second = the_second; + } + + /** + * Statically constructs a new pair from two objects. + * + * @param the_first The first object. + * @param the_second The second object. + */ + public static Pair make(final A the_first, final B the_second) { + return new Pair(the_first, the_second); + } + + /** + * @return the first object in this pair. + */ + public A first() { + return my_first; + } + + /** + * @return the second object in this pair. + */ + public B second() { + return my_second; + } + + /** + * Compares this pair with another for equivalence. + * + * @param the_other The other pair. + * @return true if the two pairs are equivalent, false otherwise. + */ + public boolean equals(final Object the_other) { + final boolean result; + + if (the_other instanceof Pair) { + final Pair other_pair = (Pair) the_other; + result = nullableEquals(other_pair.first(), first()) && + nullableEquals(other_pair.second(), second()); + } else { + result = false; + } + + return result; + } + + /** + * @return a hash code for this pair. + */ + public int hashCode() { + return nullableHashCode(first()) + 7 * nullableHashCode(second()); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/PhantomBallots.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/PhantomBallots.java new file mode 100644 index 00000000..50466e15 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/PhantomBallots.java @@ -0,0 +1,122 @@ +/* + * Colorado RLA System + */ + +package us.freeandfair.corla.util; + +import java.time.Instant; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import java.util.stream.Collectors; + +import us.freeandfair.corla.controller.ComparisonAuditController; + +import us.freeandfair.corla.model.CVRAuditInfo; +import us.freeandfair.corla.model.CVRContestInfo; +import us.freeandfair.corla.model.CastVoteRecord; +import us.freeandfair.corla.model.Contest; +import us.freeandfair.corla.model.CountyDashboard; + +import us.freeandfair.corla.persistence.Persistence; + +import us.freeandfair.corla.query.ContestQueries; + +/** + * Phantom ballot handling. + */ +public final class PhantomBallots { + /** + * Prevent public construction + */ + private PhantomBallots() { + } + + /** + * Audit phantom records as if by an audit board. + */ + public static List auditPhantomRecords( + final CountyDashboard cdb, + final List cvrs) { + return cvrs.stream() + .map(cvr -> { + return isPhantomRecord(cvr) + ? auditPhantomRecord(cdb, cvr) + : cvr; + }) + .collect(Collectors.toList()); + } + + /** + * Returns a list of CastVoteRecords with phantom records removed. + */ + public static List removePhantomRecords( + final List cvrs) { + return cvrs.stream() + .filter(cvr -> !isPhantomRecord(cvr)) + .collect(Collectors.toList()); + } + + /** + * Tests if the CVR is a phantom record. + */ + public static boolean isPhantomRecord(final CastVoteRecord cvr) { + return cvr.recordType() == CastVoteRecord.RecordType.PHANTOM_RECORD; + } + + /** + * Audit a phantom record as if by an audit board. + */ + private static CastVoteRecord auditPhantomRecord(final CountyDashboard cdb, + final CastVoteRecord cvr) { + CVRAuditInfo cvrAuditInfo = + Persistence.getByID(cvr.id(), CVRAuditInfo.class); + + if (null != cvrAuditInfo && null != cvrAuditInfo.acvr()) { + // CVR has already been audited. + return cvr; + } + + // we need to create a discrepancy for every contest that COULD have + // appeared on the ballot, which we take to mean all the contests that occur + // in the county + final Set contests = ContestQueries.forCounty(cdb.county()); + + final List phantomContestInfos = contests.stream() + .map(c -> { + return new CVRContestInfo(c, + "PHANTOM_RECORD - CVR not found", + null, + new ArrayList()); + }) + .collect(Collectors.toList()); + + cvr.setContestInfo(phantomContestInfos); + Persistence.saveOrUpdate(cvr); + + if (null == cvrAuditInfo) { + cvrAuditInfo = new CVRAuditInfo(cvr); + Persistence.save(cvrAuditInfo); + } + + final CastVoteRecord acvr = new CastVoteRecord( + CastVoteRecord.RecordType.PHANTOM_RECORD_ACVR, + Instant.now(), + cvr.countyID(), + cvr.cvrNumber(), + null, + cvr.scannerID(), + cvr.batchID(), + cvr.recordID(), + cvr.imprintedID(), + cvr.ballotType(), + phantomContestInfos); + Persistence.save(acvr); + + ComparisonAuditController.submitAuditCVR(cdb, cvr, acvr); + + return cvr; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/PrettyPrinter.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/PrettyPrinter.java new file mode 100644 index 00000000..70b71c24 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/PrettyPrinter.java @@ -0,0 +1,45 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Oct 2, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.util; + +/** + * A pretty-printer for various data types, for use in reporting. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public final class PrettyPrinter { + /** + * Private constructor to prevent instantiation. + */ + private PrettyPrinter() { + // do nothing + } + + /** + * Pretty-prints a Boolean as "Yes" or "No". + * + * @param the_boolean The Boolean. + * @return "Yes" if the_boolean is true, "No" if the_boolean is false. + */ + public static String booleanYesNo(final boolean the_boolean) { + final String result; + + if (the_boolean) { + result = "Yes"; + } else { + result = "No"; + } + + return result; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/SetCreator.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/SetCreator.java new file mode 100644 index 00000000..0dacce4c --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/SetCreator.java @@ -0,0 +1,42 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Sep 1, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.util; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +/** + * Utility class that creates a set from a sequence. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public final class SetCreator { + /** + * Private constructor to prevent instantiation. + */ + private SetCreator() { + // do nothing + } + + /** + * Constructs a set from the specified sequence of values. + * + * @param the_values The values. + * @return a set containing the specified values. + */ + @SafeVarargs + public static Set setOf(final T... the_values) { + return new HashSet(Arrays.asList(the_values)); + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/SparkHelper.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/SparkHelper.java new file mode 100644 index 00000000..7cba8912 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/SparkHelper.java @@ -0,0 +1,85 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 4, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.util; + +import java.io.IOException; + +import javax.servlet.ServletRequest; +import javax.servlet.ServletRequestWrapper; +import javax.servlet.ServletResponse; +import javax.servlet.ServletResponseWrapper; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import spark.Request; +import spark.Response; + +/** + * A class of helper methods for use with Spark. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +public final class SparkHelper { + /** + * Private constructor to prevent instantiation. + */ + private SparkHelper() { + // empty + } + + /** + * Gets the unwrapped raw request from a Spark request, if available; + * otherwise, gets the output of the_request.raw(). + * This is used primarily to circumvent Spark's caching mechanism + * to handle large file uploads. + * + * @param the_request The request. + * @return the raw request, unwrapped if possible. + */ + public static HttpServletRequest getRaw(final Request the_request) + throws IOException { + HttpServletRequest raw = the_request.raw(); + + if (raw instanceof ServletRequestWrapper) { + final ServletRequest sr = ((ServletRequestWrapper) raw).getRequest(); + if (sr instanceof HttpServletRequest) { + raw = (HttpServletRequest) sr; + } + } + + return raw; + } + + /** + * Gets the unwrapped raw response from a Spark response, if available; + * otherwise, gets the output of the_response.raw(). + * This is used primarily to circumvent Spark's caching mechanism + * to handle large file downloads. + * + * @param the_response The response. + * @return the raw response, unwrapped if possible. + */ + public static HttpServletResponse getRaw(final Response the_response) + throws IOException { + HttpServletResponse raw = the_response.raw(); + + if (raw instanceof ServletResponseWrapper) { + final ServletResponse sr = ((ServletResponseWrapper) raw).getResponse(); + if (sr instanceof HttpServletResponse) { + raw = (HttpServletResponse) sr; + } + } + + return raw; + } +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/SuppressFBWarnings.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/SuppressFBWarnings.java new file mode 100644 index 00000000..11e5ecf8 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/SuppressFBWarnings.java @@ -0,0 +1,35 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Jul 28, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.util; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +/** + * Annotation to suppress FindBugs warnings. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@Retention(RetentionPolicy.CLASS) +public @interface SuppressFBWarnings { + /** + * The set of FindBugs warnings that are to be suppressed in annotated + * element. The value can be a bug category, kind or pattern. + */ + String[] value() default {}; + + /** + * Optional documentation of the reason why the warning is suppressed + */ + String justification() default ""; +} diff --git a/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/UploadedFileStreamer.java b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/UploadedFileStreamer.java new file mode 100644 index 00000000..8a1f3571 --- /dev/null +++ b/server/eclipse-project/src/main/java/us/us/freeandfair/corla/util/UploadedFileStreamer.java @@ -0,0 +1,126 @@ +/* + * Free & Fair Colorado RLA System + * + * @title ColoradoRLA + * @created Aug 30, 2017 + * @copyright 2017 Colorado Department of State + * @license SPDX-License-Identifier: AGPL-3.0-or-later + * @creator Daniel M. Zimmerman + * @description A system to assist in conducting statewide risk-limiting audits. + */ + +package us.freeandfair.corla.util; + +import java.io.IOException; +import java.io.InputStream; +import java.sql.SQLException; + +import javax.persistence.PersistenceException; + +import us.freeandfair.corla.json.UploadedFileDTO; +import us.freeandfair.corla.model.UploadedFile; +import us.freeandfair.corla.persistence.Persistence; + +/** + * A Runnable class that provides streaming read access to an UploadedFile. + * + * @author Daniel M. Zimmerman + * @version 1.0.0 + */ +@SuppressWarnings("PMD.DoNotUseThreads") +public class UploadedFileStreamer implements Runnable { + /** + * The uploaded file. + */ + private UploadedFile my_file; + + + private UploadedFileDTO uploadedFileDTO; + + /** + * The input stream from the blob. + */ + private InputStream my_stream; + + /** + * The running flag. + */ + @SuppressWarnings("PMD.AvoidUsingVolatile") + private volatile boolean my_running; + + /** + * Constructs a new streamer for the specified file. + * + * @param the_file The file. + */ + public UploadedFileStreamer(final UploadedFile the_file) { + my_file = the_file; + } + + public UploadedFileStreamer(final UploadedFileDTO uploadedFileDTO) { + this.uploadedFileDTO = uploadedFileDTO; + } + + /** + * The run method. This opens up a new persistence session and database + * transaction, and sets up the stream for reading. This method should + * only be called as a result of Thread.start(), in a fresh thread; any + * other use may have unpredictable consequences due to the persistence + * subsystem's handling of threads. + * + * @exception PersistenceException if there is a problem during the + * execution. + */ + @Override + public synchronized void run() throws PersistenceException { + my_running = true; + Persistence.beginTransaction(); + // get a session-local reference to the file + my_file = Persistence.getByID(this.uploadedFileDTO.getFileId(), UploadedFile.class); + // get the blob stream + try { + my_stream = my_file.file().getBinaryStream(); + notifyAll(); + } catch (final SQLException e) { + throw new PersistenceException(e); + } + while (my_running) { + try { + wait(); + } catch (final InterruptedException e) { + // ignored, since we don't care if we were interrupted + } + } + try { + my_stream.close(); + } catch (final IOException e) { + // ignored, since we're already done with it + } + Persistence.rollbackTransaction(); + } + + /** + * Stops this thread, closing the persistence session and the transaction. + * The transactions is rolled back, so as to not interfere with any other + * transactions, since we have already read all the data we needed. + */ + public synchronized void stop() { + my_running = false; + notifyAll(); + } + + /** + * @return the open binary stream. This stream can only be used once; to + * read the same uploaded file again, a new UploadedFileStreamer is required. + */ + public synchronized InputStream inputStream() { + while (my_stream == null) { + try { + wait(); + } catch (final InterruptedException e) { + // ignored, since we don't care if we're interrupted + } + } + return my_stream; + } +}