diff --git a/dev/articles/scoring-rules.html b/dev/articles/scoring-rules.html index 822d3894..6c02133d 100644 --- a/dev/articles/scoring-rules.html +++ b/dev/articles/scoring-rules.html @@ -54,7 +54,7 @@

Scoring rules in `scoringutils`

Nikos Bosse

-

2024-05-19

+

2024-05-20

Source: vignettes/scoring-rules.Rmd
scoring-rules.Rmd
diff --git a/dev/articles/scoringutils.html b/dev/articles/scoringutils.html index 7f9233ea..1a64e2d0 100644 --- a/dev/articles/scoringutils.html +++ b/dev/articles/scoringutils.html @@ -54,7 +54,7 @@

Getting started

Nikos Bosse

-

2024-05-19

+

2024-05-20

Source: vignettes/scoringutils.Rmd
scoringutils.Rmd
diff --git a/dev/pkgdown.yml b/dev/pkgdown.yml index 4b6f882f..15ccd43c 100644 --- a/dev/pkgdown.yml +++ b/dev/pkgdown.yml @@ -1,12 +1,12 @@ pandoc: 3.1.11 pkgdown: 2.0.9.9000 -pkgdown_sha: 5761658b7582743d0962136cd0c1e6e8d42b56d5 +pkgdown_sha: f0f28244ce8aa6ea4fffc08c7f9e688b4a8daeb9 articles: Deprecated-functions: Deprecated-functions.html Deprecated-visualisations: Deprecated-visualisations.html scoring-rules: scoring-rules.html scoringutils: scoringutils.html -last_built: 2024-05-19T13:58Z +last_built: 2024-05-20T21:00Z urls: reference: https://epiforecasts.io/scoringutils/reference article: https://epiforecasts.io/scoringutils/articles diff --git a/dev/reference/as_forecast.html b/dev/reference/as_forecast.html index 592116a7..885f709a 100644 --- a/dev/reference/as_forecast.html +++ b/dev/reference/as_forecast.html @@ -162,7 +162,7 @@

Forecast unit + + + + + diff --git a/dev/reference/assert_forecast.forecast_binary.html b/dev/reference/assert_forecast.forecast_binary.html new file mode 100644 index 00000000..42a6ddd3 --- /dev/null +++ b/dev/reference/assert_forecast.forecast_binary.html @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/dev/reference/assert_forecast.forecast_point.html b/dev/reference/assert_forecast.forecast_point.html new file mode 100644 index 00000000..42a6ddd3 --- /dev/null +++ b/dev/reference/assert_forecast.forecast_point.html @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/dev/reference/assert_forecast.html b/dev/reference/assert_forecast.html index 3c62bd1c..955a0914 100644 --- a/dev/reference/assert_forecast.html +++ b/dev/reference/assert_forecast.html @@ -41,6 +41,15 @@

Assert that input is a forecast object and passes validations

Usage

assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...)
 
+# Default S3 method
+assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...)
+
+# S3 method for class 'forecast_binary'
+assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...)
+
+# S3 method for class 'forecast_point'
+assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...)
+
 # S3 method for class 'forecast_quantile'
 assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...)
 
diff --git a/dev/reference/forecast_types.html b/dev/reference/forecast_types.html
index 6d4c1136..cc70c385 100644
--- a/dev/reference/forecast_types.html
+++ b/dev/reference/forecast_types.html
@@ -68,7 +68,7 @@ 
Forecast unitForecast unitExamples#>     brierscore <- (observed - predicted)^2
 #>     return(brierscore)
 #> }
-#> <bytecode: 0x55e1b02b9338>
+#> <bytecode: 0x5610aa8e4008>
 #> <environment: namespace:scoringutils>
 #> 
 #> $log_score
@@ -83,7 +83,7 @@ 
Examples#>     logs <- -log(1 - abs(observed - predicted))
 #>     return(logs)
 #> }
-#> <bytecode: 0x55e1b26f1b40>
+#> <bytecode: 0x5610aa8e31d0>
 #> <environment: namespace:scoringutils>
 #> 
 metrics_binary(select = "brier_score")
@@ -95,7 +95,7 @@ 
Examples#>     brierscore <- (observed - predicted)^2
 #>     return(brierscore)
 #> }
-#> <bytecode: 0x55e1b02b9338>
+#> <bytecode: 0x5610aa8e4008>
 #> <environment: namespace:scoringutils>
 #> 
 metrics_binary(exclude = "log_score")
@@ -107,7 +107,7 @@ 
Examples#>     brierscore <- (observed - predicted)^2
 #>     return(brierscore)
 #> }
-#> <bytecode: 0x55e1b02b9338>
+#> <bytecode: 0x5610aa8e4008>
 #> <environment: namespace:scoringutils>
 #>
diff --git a/dev/reference/metrics_point.html b/dev/reference/metrics_point.html index 03c249f0..d79b619e 100644 --- a/dev/reference/metrics_point.html +++ b/dev/reference/metrics_point.html @@ -72,7 +72,7 @@

Examples#> { #> return(abs(actual - predicted)) #> } -#> <bytecode: 0x55e1b4024fb8> +#> <bytecode: 0x5610aaf30fa0> #> <environment: namespace:Metrics> #> #> $se_point @@ -80,7 +80,7 @@

Examples#> { #> return((actual - predicted)^2) #> } -#> <bytecode: 0x55e1b4214f30> +#> <bytecode: 0x5610ab94e828> #> <environment: namespace:Metrics> #> #> $ape @@ -88,7 +88,7 @@

Examples#> { #> return(ae(actual, predicted)/abs(actual)) #> } -#> <bytecode: 0x55e1b4214590> +#> <bytecode: 0x5610ab94de88> #> <environment: namespace:Metrics> #> metrics_point(select = "ape") @@ -97,7 +97,7 @@

Examples#> { #> return(ae(actual, predicted)/abs(actual)) #> } -#> <bytecode: 0x55e1b4214590> +#> <bytecode: 0x5610ab94de88> #> <environment: namespace:Metrics> #> diff --git a/dev/reference/metrics_quantile.html b/dev/reference/metrics_quantile.html index 3e19180a..571cd43e 100644 --- a/dev/reference/metrics_quantile.html +++ b/dev/reference/metrics_quantile.html @@ -148,7 +148,7 @@

Examples#> return(reformatted$wis) #> } #> } -#> <bytecode: 0x55e1af3dfa78> +#> <bytecode: 0x5610a9cd1c08> #> <environment: namespace:scoringutils> #> #> $overprediction @@ -161,7 +161,7 @@

Examples#> args)) #> return(out$overprediction) #> } -#> <bytecode: 0x55e1af3e41e8> +#> <bytecode: 0x5610a9cd6378> #> <environment: namespace:scoringutils> #> #> $underprediction @@ -174,7 +174,7 @@

Examples#> args)) #> return(out$underprediction) #> } -#> <bytecode: 0x55e1af3e2d90> +#> <bytecode: 0x5610a9cdac70> #> <environment: namespace:scoringutils> #> #> $dispersion @@ -187,7 +187,7 @@

Examples#> args)) #> return(out$dispersion) #> } -#> <bytecode: 0x55e1af3e5768> +#> <bytecode: 0x5610a9cd9818> #> <environment: namespace:scoringutils> #> #> $bias @@ -200,7 +200,7 @@

Examples#> dim(predicted) <- c(n, N) #> } #> if (!(0.5 %in% quantile_level)) { -#> cli_inform(c(i = "Median not available, computing bias as mean of the two\n innermost quantiles in order to compute bias.")) +#> cli_inform(c(i = "Median not available, interpolating median from the two\n innermost quantiles in order to compute bias.")) #> } #> bias <- sapply(1:n, function(i) { #> bias_quantile_single_vector(observed[i], predicted[i, @@ -208,7 +208,7 @@

Examples#> }) #> return(bias) #> } -#> <bytecode: 0x55e1af21f1c8> +#> <bytecode: 0x5610a9bfeb98> #> <environment: namespace:scoringutils> #> #> $interval_coverage_50 @@ -231,7 +231,7 @@

Examples#> (observed <= upper))] #> return(reformatted$interval_coverage) #> } -#> <bytecode: 0x55e1af38b2c0> +#> <bytecode: 0x5610a9c830c0> #> <environment: namespace:scoringutils> #> #> $interval_coverage_90 @@ -239,8 +239,8 @@

Examples#> { #> do.call(metric, c(list(...), dots)) #> } -#> <bytecode: 0x55e1af285de0> -#> <environment: 0x55e1b3d6a198> +#> <bytecode: 0x5610a9b71160> +#> <environment: 0x5610ae4bd2e0> #> #> $interval_coverage_deviation #> function (observed, predicted, quantile_level) @@ -266,7 +266,7 @@

Examples#> by = "forecast_id"] #> return(out$interval_coverage_deviation) #> } -#> <bytecode: 0x55e1af3e6d58> +#> <bytecode: 0x5610a9cdebc8> #> <environment: namespace:scoringutils> #> #> $ae_median @@ -285,7 +285,7 @@

Examples#> abs_error_median <- abs(observed - predicted) #> return(abs_error_median) #> } -#> <bytecode: 0x55e1b1901ee8> +#> <bytecode: 0x5610ac293370> #> <environment: namespace:scoringutils> #> metrics_quantile(select = "wis") @@ -327,7 +327,7 @@

Examples#> return(reformatted$wis) #> } #> } -#> <bytecode: 0x55e1af3dfa78> +#> <bytecode: 0x5610a9cd1c08> #> <environment: namespace:scoringutils> #> diff --git a/dev/reference/metrics_sample.html b/dev/reference/metrics_sample.html index c6a53689..c21c3dad 100644 --- a/dev/reference/metrics_sample.html +++ b/dev/reference/metrics_sample.html @@ -96,7 +96,7 @@

Examples#> return(res) #> } #> } -#> <bytecode: 0x55e1b0b3e748> +#> <bytecode: 0x5610ab4dd1b8> #> <environment: namespace:scoringutils> #> #> $dss @@ -105,7 +105,7 @@

Examples#> assert_input_sample(observed, predicted) #> scoringRules::dss_sample(y = observed, dat = predicted, ...) #> } -#> <bytecode: 0x55e1b086bc18> +#> <bytecode: 0x5610ab502460> #> <environment: namespace:scoringutils> #> #> $crps @@ -115,7 +115,7 @@

Examples#> scoringRules::crps_sample(y = observed, dat = predicted, #> ...) #> } -#> <bytecode: 0x55e1b037fbb8> +#> <bytecode: 0x5610aac1d510> #> <environment: namespace:scoringutils> #> #> $log_score @@ -125,7 +125,7 @@

Examples#> scoringRules::logs_sample(y = observed, dat = predicted, #> ...) #> } -#> <bytecode: 0x55e1b0d96ec8> +#> <bytecode: 0x5610abd9dba0> #> <environment: namespace:scoringutils> #> #> $mad @@ -135,7 +135,7 @@

Examples#> sharpness <- apply(predicted, MARGIN = 1, mad, ...) #> return(sharpness) #> } -#> <bytecode: 0x55e1aff679b8> +#> <bytecode: 0x5610ac3de708> #> <environment: namespace:scoringutils> #> #> $ae_median @@ -147,7 +147,7 @@

Examples#> ae_median <- abs(observed - median_predictions) #> return(ae_median) #> } -#> <bytecode: 0x55e1b000bfc8> +#> <bytecode: 0x5610aa990ea8> #> <environment: namespace:scoringutils> #> #> $se_mean @@ -158,7 +158,7 @@

Examples#> se_mean <- (observed - mean_predictions)^2 #> return(se_mean) #> } -#> <bytecode: 0x55e1ac5bfb98> +#> <bytecode: 0x5610aa6b0b60> #> <environment: namespace:scoringutils> #> metrics_sample(select = "mad") @@ -169,7 +169,7 @@

Examples#> sharpness <- apply(predicted, MARGIN = 1, mad, ...) #> return(sharpness) #> } -#> <bytecode: 0x55e1aff679b8> +#> <bytecode: 0x5610ac3de708> #> <environment: namespace:scoringutils> #> diff --git a/dev/reference/score.html b/dev/reference/score.html index d590e7d1..dfcc337c 100644 --- a/dev/reference/score.html +++ b/dev/reference/score.html @@ -131,7 +131,7 @@

Forecast unitExamples#> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } -#> <bytecode: 0x55e1b02b9338> +#> <bytecode: 0x5610aa8e4008> #> <environment: namespace:scoringutils> #> select_metrics( @@ -82,7 +82,7 @@

Examples#> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } -#> <bytecode: 0x55e1b02b9338> +#> <bytecode: 0x5610aa8e4008> #> <environment: namespace:scoringutils> #> diff --git a/dev/search.json b/dev/search.json index 1ed85fd5..9340da63 100644 --- a/dev/search.json +++ b/dev/search.json @@ -1 +1 @@ -[{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"our-pledge","dir":"","previous_headings":"","what":"Our Pledge","title":"Contributor Covenant Code of Conduct","text":"members, contributors, leaders pledge make participation community harassment-free experience everyone, regardless age, body size, visible invisible disability, ethnicity, sex characteristics, gender identity expression, level experience, education, socio-economic status, nationality, personal appearance, race, religion, sexual identity orientation. pledge act interact ways contribute open, welcoming, diverse, inclusive, healthy community.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"our-standards","dir":"","previous_headings":"","what":"Our Standards","title":"Contributor Covenant Code of Conduct","text":"Examples behavior contributes positive environment community include: Demonstrating empathy kindness toward people respectful differing opinions, viewpoints, experiences Giving gracefully accepting constructive feedback Accepting responsibility apologizing affected mistakes, learning experience Focusing best just us individuals, overall community Examples unacceptable behavior include: use sexualized language imagery, sexual attention advances kind Trolling, insulting derogatory comments, personal political attacks Public private harassment Publishing others’ private information, physical email address, without explicit permission conduct reasonably considered inappropriate professional setting","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"enforcement-responsibilities","dir":"","previous_headings":"","what":"Enforcement Responsibilities","title":"Contributor Covenant Code of Conduct","text":"Community leaders responsible clarifying enforcing standards acceptable behavior take appropriate fair corrective action response behavior deem inappropriate, threatening, offensive, harmful. Community leaders right responsibility remove, edit, reject comments, commits, code, wiki edits, issues, contributions aligned Code Conduct, communicate reasons moderation decisions appropriate.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"scope","dir":"","previous_headings":"","what":"Scope","title":"Contributor Covenant Code of Conduct","text":"Code Conduct applies within community spaces, also applies individual officially representing community public spaces. Examples representing community include using official e-mail address, posting via official social media account, acting appointed representative online offline event.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"enforcement","dir":"","previous_headings":"","what":"Enforcement","title":"Contributor Covenant Code of Conduct","text":"Instances abusive, harassing, otherwise unacceptable behavior may reported community leaders responsible enforcement Sebastian.Funk@lshtm.ac.uk. complaints reviewed investigated promptly fairly. community leaders obligated respect privacy security reporter incident.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"enforcement-guidelines","dir":"","previous_headings":"","what":"Enforcement Guidelines","title":"Contributor Covenant Code of Conduct","text":"Community leaders follow Community Impact Guidelines determining consequences action deem violation Code Conduct:","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"id_1-correction","dir":"","previous_headings":"Enforcement Guidelines","what":"1. Correction","title":"Contributor Covenant Code of Conduct","text":"Community Impact: Use inappropriate language behavior deemed unprofessional unwelcome community. Consequence: private, written warning community leaders, providing clarity around nature violation explanation behavior inappropriate. public apology may requested.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"id_2-warning","dir":"","previous_headings":"Enforcement Guidelines","what":"2. Warning","title":"Contributor Covenant Code of Conduct","text":"Community Impact: violation single incident series actions. Consequence: warning consequences continued behavior. interaction people involved, including unsolicited interaction enforcing Code Conduct, specified period time. includes avoiding interactions community spaces well external channels like social media. Violating terms may lead temporary permanent ban.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"id_3-temporary-ban","dir":"","previous_headings":"Enforcement Guidelines","what":"3. Temporary Ban","title":"Contributor Covenant Code of Conduct","text":"Community Impact: serious violation community standards, including sustained inappropriate behavior. Consequence: temporary ban sort interaction public communication community specified period time. public private interaction people involved, including unsolicited interaction enforcing Code Conduct, allowed period. Violating terms may lead permanent ban.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"id_4-permanent-ban","dir":"","previous_headings":"Enforcement Guidelines","what":"4. Permanent Ban","title":"Contributor Covenant Code of Conduct","text":"Community Impact: Demonstrating pattern violation community standards, including sustained inappropriate behavior, harassment individual, aggression toward disparagement classes individuals. Consequence: permanent ban sort public interaction within community.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"attribution","dir":"","previous_headings":"","what":"Attribution","title":"Contributor Covenant Code of Conduct","text":"Code Conduct adapted Contributor Covenant, version 2.0, available https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. Community Impact Guidelines inspired Mozilla’s code conduct enforcement ladder. answers common questions code conduct, see FAQ https://www.contributor-covenant.org/faq. Translations available https://www.contributor-covenant.org/translations.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":null,"dir":"","previous_headings":"","what":"Contributing to scoringutils","title":"Contributing to scoringutils","text":"outlines propose change scoringutils.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"fixing-typos","dir":"","previous_headings":"","what":"Fixing typos","title":"Contributing to scoringutils","text":"can fix typos, spelling mistakes, grammatical errors documentation directly using GitHub web interface, long changes made source file. generally means ’ll need edit roxygen2 comments .R, .Rd file. can find .R file generates .Rd reading comment first line.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"bigger-changes","dir":"","previous_headings":"","what":"Bigger changes","title":"Contributing to scoringutils","text":"want make bigger change, ’s good idea first file issue make sure someone team agrees ’s needed. ’ve found bug, please file issue illustrates bug minimal reprex (also help write unit test, needed).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"pull-request-process","dir":"","previous_headings":"Bigger changes","what":"Pull request process","title":"Contributing to scoringutils","text":"Fork package clone onto computer. haven’t done , recommend using usethis::create_from_github(\"epiforecasts/scoringutils\", fork = TRUE). Install development dependences devtools::install_dev_deps(), make sure package passes R CMD check running devtools::check(). R CMD check doesn’t pass cleanly, ’s good idea ask help continuing. Create Git branch pull request (PR). recommend using usethis::pr_init(\"brief-description--change\"). Make changes, commit git, create PR running usethis::pr_push(), following prompts browser. title PR briefly describe change. body PR contain Fixes #issue-number. user-facing changes, add bullet top NEWS.md (.e. just first header). Follow style described https://style.tidyverse.org/news.html.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"code-style","dir":"","previous_headings":"Bigger changes","what":"Code style","title":"Contributing to scoringutils","text":"New code follow tidyverse style guide. can use styler package apply styles, please don’t restyle code nothing PR. use roxygen2, Markdown syntax, documentation. use testthat unit tests. Contributions test cases included easier accept.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"code-of-conduct","dir":"","previous_headings":"","what":"Code of Conduct","title":"Contributing to scoringutils","text":"Please note scoringutils project released Contributor Code Conduct. contributing project agree abide terms.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/LICENSE.html","id":null,"dir":"","previous_headings":"","what":"MIT License","title":"MIT License","text":"Copyright (c) 2020 EpiForecasts Permission hereby granted, free charge, person obtaining copy software associated documentation files (“Software”), deal Software without restriction, including without limitation rights use, copy, modify, merge, publish, distribute, sublicense, /sell copies Software, permit persons Software furnished , subject following conditions: copyright notice permission notice shall included copies substantial portions Software. SOFTWARE PROVIDED “”, WITHOUT WARRANTY KIND, EXPRESS IMPLIED, INCLUDING LIMITED WARRANTIES MERCHANTABILITY, FITNESS PARTICULAR PURPOSE NONINFRINGEMENT. EVENT SHALL AUTHORS COPYRIGHT HOLDERS LIABLE CLAIM, DAMAGES LIABILITY, WHETHER ACTION CONTRACT, TORT OTHERWISE, ARISING , CONNECTION SOFTWARE USE DEALINGS SOFTWARE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/PULL_REQUEST_TEMPLATE.html","id":"description","dir":"","previous_headings":"","what":"Description","title":"NA","text":"PR closes #. [Describe changes made pull request.]","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/PULL_REQUEST_TEMPLATE.html","id":"checklist","dir":"","previous_headings":"","what":"Checklist","title":"NA","text":"PR based package issue explicitly linked . included target issue issues PR title follows: issue-number: PR title tested changes locally. added updated unit tests necessary. updated documentation required. built package locally run rebuilt docs using roxygen2. code follows established coding standards run lintr::lint_package() check style issues introduced changes. added news item linked PR. reviewed CI checks PR addressed far able.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/Deprecated-functions.html","id":"merge_pred_and_obs","dir":"Articles","previous_headings":"","what":"merge_pred_and_obs()","title":"Deprecated functions","text":"scoringutils requires forecasts observations provided single data frame. forecasts observations two different data frames, merge_pred_and_obs() may help merge two. function mostly wrapper around merge(), additional work deal duplicated column names.","code":"#' @title Merge forecast data and observations #' #' @description #' #' The function more or less provides a wrapper around `merge` that #' aims to handle the merging well if additional columns are present #' in one or both data sets. If in doubt, you should probably merge the #' data sets manually. #' #' @param forecasts A data.frame with the forecast data (as can be passed to #' [score()]). #' @param observations A data.frame with the observations. #' @param join Character, one of `c(\"left\", \"full\", \"right\")`. Determines the #' type of the join. Usually, a left join is appropriate, but sometimes you #' may want to do a full join to keep dates for which there is a forecast, but #' no ground truth data. #' @param by Character vector that denotes the columns by which to merge. Any #' value that is not a column in observations will be removed. #' @return a data.table with forecasts and observations #' @importFrom checkmate assert_subset #' @importFrom data.table as.data.table #' @keywords data-handling #' @export merge_pred_and_obs <- function(forecasts, observations, join = c(\"left\", \"full\", \"right\"), by = NULL) { forecasts <- as.data.table(forecasts) observations <- as.data.table(observations) join <- match.arg(join) assert_subset(by, intersect(names(forecasts), names(observations))) if (is.null(by)) { protected_columns <- c( \"predicted\", \"observed\", \"sample_id\", \"quantile_level\", \"interval_range\", \"boundary\" ) by <- setdiff(colnames(forecasts), protected_columns) } obs_cols <- colnames(observations) by <- intersect(by, obs_cols) join <- match.arg(join) if (join == \"left\") { # do a left_join, where all data in the observations are kept. combined <- merge(observations, forecasts, by = by, all.x = TRUE) } else if (join == \"full\") { # do a full, where all data is kept. combined <- merge(observations, forecasts, by = by, all = TRUE) } else { combined <- merge(observations, forecasts, by = by, all.y = TRUE) } # get colnames that are the same for x and y colnames <- colnames(combined) colnames_x <- colnames[endsWith(colnames, \".x\")] colnames_y <- colnames[endsWith(colnames, \".y\")] # extract basenames basenames_x <- sub(\".x$\", \"\", colnames_x) basenames_y <- sub(\".y$\", \"\", colnames_y) # see whether the column name as well as the content is the same content_x <- as.list(combined[, ..colnames_x]) content_y <- as.list(combined[, ..colnames_y]) overlapping <- (content_x %in% content_y) & (basenames_x == basenames_y) overlap_names <- colnames_x[overlapping] basenames_overlap <- sub(\".x$\", \"\", overlap_names) # delete overlapping columns if (length(basenames_overlap) > 0) { combined[, paste0(basenames_overlap, \".x\") := NULL] combined[, paste0(basenames_overlap, \".y\") := NULL] } return(combined[]) }"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/Deprecated-visualisations.html","id":"functions-plot_predictions-and-make_na","dir":"Articles","previous_headings":"","what":"Functions plot_predictions() and make_na()","title":"Deprecated Visualisations","text":"previous versions scoringutils, forecasts observed values visualised using function plot_predictions() make_na() helper function. following shows function code first example. plot_predictions() actual work producing plot. argument needed user can facet plot correctly user needs specify columns relevant facetting. make_NA() represents form filtering, instead filtering entire rows, relevant entries columns “predicted” “observed” made NA. allows user filter observations forecasts independently. following examples using two functions create plot using scoringutils example data. Visualising median forecasts example data. truth data restricted period 2021-05-01 2021-07-22. forecast data forecast model “EuroCOVIDhub-ensemble” made “2021-06-07”. data set NA, effectively removing plot. plot, variety prediction intervals shown, instead just median. similar plot, time based continuous forecasts. predictions automatically converted quantile-based forecasts plotting. Displaying two forecasts time additional colours:","code":"#\" @title Plot Predictions vs True Values #\" #\" @description #\" Make a plot of observed and predicted values #\" #\" @param data a data.frame that follows the same specifications outlined in #\" [score()]. To customise your plotting, you can filter your data using the #\" function [make_NA()]. #\" @param by character vector with column names that denote categories by which #\" the plot should be stratified. If for example you want to have a facetted #\" plot, this should be a character vector with the columns used in facetting #\" (note that the facetting still needs to be done outside of the function call) #\" @param x character vector of length one that denotes the name of the variable #\" @param interval_range numeric vector indicating the interval ranges to plot. #\" If 0 is included in `interval_range`, the median prediction will be shown. #\" @return ggplot object with a plot of true vs predicted values #\" @importFrom ggplot2 ggplot scale_colour_manual scale_fill_manual theme_light #\" @importFrom ggplot2 facet_wrap facet_grid aes geom_line .data geom_point #\" @importFrom data.table dcast #\" @importFrom ggdist geom_lineribbon #\" @export #\" @examples #\" library(ggplot2) #\" library(magrittr) #\" #\" example_sample_continuous %>% #\" make_NA ( #\" what = \"truth\", #\" target_end_date >= \"2021-07-22\", #\" target_end_date < \"2021-05-01\" #\" ) %>% #\" make_NA ( #\" what = \"forecast\", #\" model != \"EuroCOVIDhub-ensemble\", #\" forecast_date != \"2021-06-07\" #\" ) %>% #\" plot_predictions ( #\" x = \"target_end_date\", #\" by = c(\"target_type\", \"location\"), #\" interval_range = c(0, 50, 90, 95) #\" ) + #\" facet_wrap(~ location + target_type, scales = \"free_y\") + #\" aes(fill = model, color = model) #\" #\" example_sample_continuous %>% #\" make_NA ( #\" what = \"truth\", #\" target_end_date >= \"2021-07-22\", #\" target_end_date < \"2021-05-01\" #\" ) %>% #\" make_NA ( #\" what = \"forecast\", #\" forecast_date != \"2021-06-07\" #\" ) %>% #\" plot_predictions ( #\" x = \"target_end_date\", #\" by = c(\"target_type\", \"location\"), #\" interval_range = 0 #\" ) + #\" facet_wrap(~ location + target_type, scales = \"free_y\") + #\" aes(fill = model, color = model) plot_predictions <- function(data, by = NULL, x = \"date\", interval_range = c(0, 50, 90)) { # split truth data and forecasts in order to apply different filtering truth_data <- data.table::as.data.table(data)[!is.na(observed)] forecasts <- data.table::as.data.table(data)[!is.na(predicted)] del_cols <- colnames(truth_data)[!(colnames(truth_data) %in% c(by, \"observed\", x))] truth_data <- unique(suppressWarnings(truth_data[, eval(del_cols) := NULL])) # find out what type of predictions we have. convert sample based to # interval range data if (get_forecast_type(data) == \"quantile\") { forecasts <- scoringutils:::quantile_to_interval( forecasts, keep_quantile_col = FALSE ) } else if (get_forecast_type(data) == \"sample\") { # using a scoringutils internal function forecasts <- scoringutils:::sample_to_interval_long( as_forecast(forecasts), interval_range = interval_range, keep_quantile_col = FALSE ) } # select appropriate boundaries and pivot wider select <- forecasts$interval_range %in% setdiff(interval_range, 0) intervals <- forecasts[select, ] # delete quantile column in intervals if present. This is important for # pivoting if (\"quantile_level\" %in% names(intervals)) { intervals[, quantile_level := NULL] } plot <- ggplot(data = data, aes(x = .data[[x]])) + theme_scoringutils() + ylab(\"True and predicted values\") if (nrow(intervals) != 0) { # pivot wider and convert range to a factor intervals <- data.table::dcast(intervals, ... ~ boundary, value.var = \"predicted\") # only plot interval ranges if there are interval ranges to plot plot <- plot + ggdist::geom_lineribbon( data = intervals, aes( ymin = lower, ymax = upper, # We use the fill_ramp aesthetic for this instead of the default fill # because we want to keep fill to be able to use it for other # variables fill_ramp = factor( interval_range, levels = sort(unique(interval_range), decreasing = TRUE) ) ), lwd = 0.4 ) + ggdist::scale_fill_ramp_discrete( name = \"interval_range\", # range argument was added to make sure that the line for the median # and the ribbon don\"t have the same opacity, making the line # invisible range = c(0.15, 0.75) ) } # We could treat this step as part of ggdist::geom_lineribbon() but we treat # it separately here to deal with the case when only the median is provided # (in which case ggdist::geom_lineribbon() will fail) if (0 %in% interval_range) { select_median <- forecasts$interval_range == 0 & forecasts$boundary == \"lower\" median <- forecasts[select_median] if (nrow(median) > 0) { plot <- plot + geom_line( data = median, mapping = aes(y = predicted), lwd = 0.4 ) } } # add observed values if (nrow(truth_data) > 0) { plot <- plot + geom_point( data = truth_data, show.legend = FALSE, inherit.aes = FALSE, aes(x = .data[[x]], y = observed), color = \"black\", size = 0.5 ) + geom_line( data = truth_data, inherit.aes = FALSE, show.legend = FALSE, aes(x = .data[[x]], y = observed), linetype = 1, color = \"grey40\", lwd = 0.2 ) } return(plot) } #\" @title Make Rows NA in Data for Plotting #\" #\" @description #\" Filters the data and turns values into `NA` before the data gets passed to #\" [plot_predictions()]. The reason to do this is to this is that it allows to #\" \"filter\" prediction and truth data separately. Any value that is NA will then #\" be removed in the subsequent call to [plot_predictions()]. #\" #\" @inheritParams score #\" @param what character vector that determines which values should be turned #\" into `NA`. If `what = \"truth\"`, values in the column \"observed\" will be #\" turned into `NA`. If `what = \"forecast\"`, values in the column \"prediction\" #\" will be turned into `NA`. If `what = \"both\"`, values in both column will be #\" turned into `NA`. #\" @param ... logical statements used to filter the data #\" @return A data.table #\" @importFrom rlang enexprs #\" @keywords plotting #\" @export #\" #\" @examples #\" make_NA ( #\" example_sample_continuous, #\" what = \"truth\", #\" target_end_date >= \"2021-07-22\", #\" target_end_date < \"2021-05-01\" #\" ) make_NA <- function(data = NULL, what = c(\"truth\", \"forecast\", \"both\"), ...) { stopifnot(is.data.frame(data)) data <- as.data.table(data) what <- match.arg(what) # turn ... arguments into expressions args <- enexprs(...) vars <- NULL if (what %in% c(\"forecast\", \"both\")) { vars <- c(vars, \"predicted\") } if (what %in% c(\"truth\", \"both\")) { vars <- c(vars, \"observed\") } for (expr in args) { data <- data[eval(expr), eval(vars) := NA_real_] } return(data[]) } median_forecasts <- example_quantile[quantile_level == 0.5] median_forecasts %>% make_NA(what = \"truth\", target_end_date <= \"2021-05-01\", target_end_date > \"2021-07-22\") %>% make_NA(what = \"forecast\", model != \"EuroCOVIDhub-ensemble\", forecast_date != \"2021-06-07\") %>% plot_predictions( by = c(\"location\", \"target_type\"), x = \"target_end_date\" ) + facet_wrap(location ~ target_type, scales = \"free_y\") example_quantile %>% make_NA(what = \"truth\", target_end_date <= \"2021-05-01\", target_end_date > \"2021-07-22\") %>% make_NA(what = \"forecast\", model != \"EuroCOVIDhub-ensemble\", forecast_date != \"2021-06-07\") %>% plot_predictions( by = c(\"location\", \"target_type\"), x = \"target_end_date\", interval_range = c(0, 10, 20, 30, 40, 50, 60) ) + facet_wrap(location ~ target_type, scales = \"free_y\") example_sample_continuous %>% make_NA(what = \"truth\", target_end_date <= \"2021-05-01\", target_end_date > \"2021-07-22\") %>% make_NA(what = \"forecast\", model != \"EuroCOVIDhub-ensemble\", forecast_date != \"2021-06-07\") %>% plot_predictions( by = c(\"location\", \"target_type\"), x = \"target_end_date\", interval_range = c(0, 50, 90, 95) ) + facet_wrap(location ~ target_type, scales = \"free_y\") example_quantile %>% make_NA(what = \"truth\", target_end_date > \"2021-07-15\", target_end_date <= \"2021-05-22\") %>% make_NA(what = \"forecast\", !(model %in% c(\"EuroCOVIDhub-ensemble\", \"EuroCOVIDhub-baseline\")), forecast_date != \"2021-06-28\") %>% plot_predictions(x = \"target_end_date\", by = c(\"target_type\", \"location\")) + aes(colour = model, fill = model) + facet_wrap(target_type ~ location, ncol = 4, scales = \"free_y\") + labs(x = \"Target end date\")"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/Deprecated-visualisations.html","id":"function-plot_interval_ranges-formerly-plot_ranges","dir":"Articles","previous_headings":"","what":"Function plot_interval_ranges() (formerly plot_ranges())","title":"Deprecated Visualisations","text":"functionality currently relies hack. previous versions scoringutils, scores computed per interval range/per quantile. Now, scoringutils returns one score per forecast, per interval range/quantile. therefore need add range column, using internal function get_range_from_quantile(). column interpreted one defines unit single forecast scoringutils. also means get warning different number quantile levels different forecasts (0% prediction interval one median forecast, prediction intervals two (lower upper bound)). Plotting dispersion instead WIS:","code":"#\" @title Plot Metrics by Range of the Prediction Interval #\" #\" @description #\" Visualise the metrics by range, e.g. if you are interested how different #\" interval ranges contribute to the overall interval score, or how #\" sharpness / dispersion changes by range. #\" #\" @param scores A data.frame of scores based on quantile forecasts as #\" produced by [score()] or [summarise_scores()]. Note that \"range\" must be included #\" in the `by` argument when running [summarise_scores()] #\" @param y The variable from the scores you want to show on the y-Axis. #\" This could be something like \"wis\" (the default) or \"dispersion\" #\" @param x The variable from the scores you want to show on the x-Axis. #\" Usually this will be \"model\" #\" @param colour Character vector of length one used to determine a variable #\" for colouring dots. The Default is \"range\". #\" @return A ggplot2 object showing a contributions from the three components of #\" the weighted interval score #\" @importFrom ggplot2 ggplot aes aes geom_point geom_line #\" expand_limits theme theme_light element_text scale_color_continuous labs #\" @export #\" @examples #\" library(ggplot2) #\" ex <- data.table::copy(example_quantile) #\" ex$range <- scoringutils:::get_range_from_quantile(ex$quantile) #\" scores <- suppressWarnings(score(as_forecast(ex), metrics = list(\"wis\" = wis))) #\" summarised <- summarise_scores( #\" scores, #\" by = c(\"model\", \"target_type\", \"range\") #\" ) #\" plot_interval_ranges(summarised, x = \"model\") + #\" facet_wrap(~target_type, scales = \"free\") plot_interval_ranges <- function(scores, y = \"wis\", x = \"model\", colour = \"range\") { plot <- ggplot( scores, aes( x = .data[[x]], y = .data[[y]], colour = .data[[colour]] ) ) + geom_point(size = 2) + geom_line(aes(group = range), colour = \"black\", linewidth = 0.01 ) + scale_color_continuous(low = \"steelblue\", high = \"salmon\") + theme_scoringutils() + expand_limits(y = 0) + theme( legend.position = \"right\", axis.text.x = element_text( angle = 90, vjust = 1, hjust = 1 ) ) return(plot) } range_example <- copy(example_quantile) %>% na.omit() %>% .[, range := scoringutils:::get_range_from_quantile(quantile_level)] sum_scores <- range_example %>% as_forecast() %>% score(metrics = list(wis = wis, dispersion = dispersion)) %>% summarise_scores(by = c(\"model\", \"target_type\", \"range\")) %>% suppressWarnings() plot_interval_ranges(sum_scores, x = \"model\") + facet_wrap(~target_type, scales = \"free\") plot_interval_ranges(sum_scores, y = \"dispersion\", x = \"model\") + facet_wrap(~target_type, scales = \"free_y\")"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/Deprecated-visualisations.html","id":"function-plot_score_table","dir":"Articles","previous_headings":"","what":"Function plot_score_table()","title":"Deprecated Visualisations","text":"function allowed users turn table (summarised) scores coloured table. function hard-coded information colour scale pick metric. scoringutils 2.0.0, allowed users assign names metrics use custom scoring functions. stick default names provided scoringutils, function still work. However, functionality easily generalisable, decided deprecate function. main functionality old function provided, scaling scores order obtain reasonable colour shades. per metric, one also pass additional grouping variables argument. allowed users achieve faceting table (note course scores also needed summarised according grouping). function also allowed users combine different facets one, creating combined y-variable. done passing vector column names y argument.","code":"#' @title Plot Coloured Score Table #' #' @description #' Plots a coloured table of summarised scores obtained using #' [score()]. #' #' @param y the variable to be shown on the y-axis. Instead of a single character string, #' you can also specify a vector with column names, e.g. #' `y = c(\"model\", \"location\")`. These column names will be concatenated #' to create a unique row identifier (e.g. \"model1_location1\"). #' @param by A character vector that determines how the colour shading for the #' plot gets computed. By default (`NULL`), shading will be determined per #' metric, but you can provide additional column names (see examples). #' @param metrics A character vector with the metrics to show. If set to #' `NULL` (default), all metrics present in `scores` will be shown. #' #' @return A ggplot object with a coloured table of summarised scores #' @inheritParams get_pairwise_comparisons #' @importFrom ggplot2 ggplot aes element_blank element_text labs coord_cartesian coord_flip #' @importFrom data.table setDT melt #' @importFrom stats sd #' @export #' #' @examples #' library(ggplot2) #' library(magrittr) # pipe operator #' \\dontshow{ #' data.table::setDTthreads(2) # restricts number of cores used on CRAN #' } #' #' scores <- score(as_forecast(example_quantile)) %>% #' summarise_scores(by = c(\"model\", \"target_type\")) %>% #' summarise_scores(by = c(\"model\", \"target_type\"), fun = signif, digits = 2) #' #' plot_score_table(scores, y = \"model\", by = \"target_type\") + #' facet_wrap(~target_type, ncol = 1) #' #' # can also put target description on the y-axis #' plot_score_table(scores, #' y = c(\"model\", \"target_type\"), #' by = \"target_type\") plot_score_table <- function(scores, y = \"model\", by = NULL, metrics = NULL) { # identify metrics ----------------------------------------------------------- id_vars <- get_forecast_unit(scores) metrics <- get_metrics(scores) cols_to_delete <- names(scores)[!(names(scores) %in% c(metrics, id_vars))] suppressWarnings(scores[, eval(cols_to_delete) := NULL]) # compute scaled values ------------------------------------------------------ # scaling is done in order to colour the different scores # for most metrics larger is worse, but others like bias are better if they # are close to zero and deviations in both directions are bad # define which metrics are scaled using min (larger is worse) and # which not (metrics like bias where deviations in both directions are bad) metrics_zero_good <- c(\"bias\", \"interval_coverage_deviation\") metrics_no_color <- \"coverage\" metrics_min_good <- setdiff(metrics, c( metrics_zero_good, metrics_no_color )) # write scale functions that can be used in data.table scale <- function(x) { scaled <- x / sd(x, na.rm = TRUE) return(scaled) } scale_min_good <- function(x) { scaled <- (x - min(x)) / sd(x, na.rm = TRUE) return(scaled) } # pivot longer and add scaled values df <- data.table::melt(scores, value.vars = metrics, id.vars = id_vars, variable.name = \"metric\" ) df[metric %in% metrics_min_good, value_scaled := scale_min_good(value), by = c(\"metric\", by) ] df[metric %in% metrics_zero_good, value_scaled := scale(value), by = c(\"metric\", by) ] df[metric %in% metrics_no_color, value_scaled := 0, by = c(\"metric\", by) ] # create identifier column for plot ------------------------------------------ # if there is only one column, leave column as is. Reason to do that is that # users can then pass in a factor and keep the ordering of that column intact if (length(y) > 1) { df[, identifCol := do.call(paste, c(.SD, sep = \"_\")), .SDcols = y[y %in% names(df)]] } else { setnames(df, old = eval(y), new = \"identifCol\") } # plot ----------------------------------------------------------------------- # make plot with all metrics that are not NA plot <- ggplot( df[!is.na(value), ], aes(y = identifCol, x = metric) ) + geom_tile(aes(fill = value_scaled), colour = \"white\", show.legend = FALSE) + geom_text(aes(y = identifCol, label = value)) + scale_fill_gradient2(low = \"steelblue\", high = \"salmon\") + theme_scoringutils() + theme( legend.title = element_blank(), legend.position = \"none\", axis.text.x = element_text( angle = 90, vjust = 1, hjust = 1 ) ) + labs(x = \"\", y = \"\") + coord_cartesian(expand = FALSE) return(plot) } scores <- score(as_forecast(example_quantile)) %>% summarise_scores(by = c(\"model\", \"target_type\")) %>% summarise_scores(by = c(\"model\", \"target_type\"), fun = signif, digits = 2) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_score_table(scores, y = \"model\", by = \"target_type\") + facet_wrap(~target_type, ncol = 1) # can also put target description on the y-axis plot_score_table(scores, y = c(\"model\", \"target_type\"), by = \"target_type\")"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"introduction","dir":"Articles","previous_headings":"","what":"Introduction","title":"Scoring rules in `scoringutils`","text":"vignette gives overview default scoring rules made available scoringutils package. can, course, also use scoring rules, provided follow format. want obtain detailed information pacakge works, look revised version scoringutils paper. can distinguish two types forecasts: point forecasts probabilistic forecasts. point forecast single number representing single outcome. probabilistic forecast full predictive probability distribution multiple possible outcomes. contrast point forecasts, probabilistic forecasts incorporate uncertainty different possible outcomes. Scoring rules functions take forecast observation input return single numeric value. point forecasts, take form \\(S(\\hat{y}, y)\\), \\(\\hat{y}\\) forecast \\(y\\) observation. probabilistic forecasts, usually take form \\(S(F, y)\\), \\(F\\) cumulative density function (CDF) predictive distribution \\(y\\) observation. convention, scoring rules usually negatively oriented, meaning smaller values better (best possible score usually zero). sense, score can understood penalty. Many scoring rules probabilistic forecasts -called (strictly) proper scoring rules. Essentially, means “cheated”: forecaster evaluated strictly proper scoring rule always incentivised report honest best belief future , expectation, improve score reporting something else. formal definition following: Let \\(G\\) true, unobserved data-generating distribution. scoring rule said proper, \\(G\\) ideal forecast \\(F = G\\), forecast \\(F' \\neq F\\) expectation receives better score \\(F\\). scoring rule considered strictly proper , \\(G\\), forecast \\(F'\\) expectation receives score better \\(F\\).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"metrics-for-point-forecasts","dir":"Articles","previous_headings":"","what":"Metrics for point forecasts","title":"Scoring rules in `scoringutils`","text":"See list default metrics point forecasts calling ?metrics_point(). overview input output formats point forecasts: Input output formats: metrics point.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"a-note-of-caution","dir":"Articles","previous_headings":"Metrics for point forecasts","what":"A note of caution","title":"Scoring rules in `scoringutils`","text":"Scoring point forecasts can tricky business. Depending choice scoring rule, forecaster clearly worse another, might consistently receive better scores (see Gneiting (2011) illustrative example). Every scoring rule point forecast implicitly minimised specific aspect predictive distribution. mean squared error, example, meaningful scoring rule forecaster actually reported mean predictive distribution point forecast. forecaster reported median, mean absolute error appropriate scoring rule. scoring rule predictive task align, misleading results ensue. Consider following example:","code":"set.seed(123) n <- 1000 observed <- rnorm(n, 5, 4)^2 predicted_mu <- mean(observed) predicted_not_mu <- predicted_mu - rnorm(n, 10, 2) mean(Metrics::ae(observed, predicted_mu)) #> [1] 34.45981 mean(Metrics::ae(observed, predicted_not_mu)) #> [1] 32.54821 mean(Metrics::se(observed, predicted_mu)) #> [1] 2171.089 mean(Metrics::se(observed, predicted_not_mu)) #> [1] 2290.155"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"absolute-error","dir":"Articles","previous_headings":"Metrics for point forecasts","what":"Absolute error","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), real number Forecast: \\(\\hat{y}\\), real number, median forecaster’s predictive distribution. absolute error absolute difference predicted observed values. See ?Metrics::ae. \\[\\text{ae} = |y - \\hat{y}|\\] absolute error appropriate rule \\(\\hat{y}\\) corresponds median forecaster’s predictive distribution. Otherwise, results misleading (see Gneiting (2011)).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"squared-error","dir":"Articles","previous_headings":"Metrics for point forecasts","what":"Squared error","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), real number Forecast: \\(\\hat{y}\\), real number, mean forecaster’s predictive distribution. squared error squared difference predicted observed values. See ?Metrics::se. \\[\\text{se} = (y - \\hat{y})^2\\] squared error appropriate rule \\(\\hat{y}\\) corresponds mean forecaster’s predictive distribution. Otherwise, results misleading (see Gneiting (2011)).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"absolute-percentage-error","dir":"Articles","previous_headings":"Metrics for point forecasts","what":"Absolute percentage error","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), real number Forecast: \\(\\hat{y}\\), real number absolute percentage error absolute percent difference predicted observed values. See ?Metrics::ape. \\[\\text{ape} = \\frac{|y - \\hat{y}|}{|y|}\\] absolute percentage error appropriate rule \\(\\hat{y}\\) corresponds \\(\\beta\\)-median forecaster’s predictive distribution \\(\\beta = -1\\). \\(\\beta\\)-median, \\(\\text{med}^{(\\beta)}(F)\\), median random variable whose density proportional \\(y^\\beta f(y)\\). specific \\(\\beta\\)-median corresponds absolute percentage error \\(\\text{med}^{(-1)}(F)\\). Otherwise, results misleading (see Gneiting (2011)).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"binary-forecasts","dir":"Articles","previous_headings":"","what":"Binary forecasts","title":"Scoring rules in `scoringutils`","text":"See list default metrics point forecasts calling ?metrics_binary(). overview input output formats point forecasts: Input output formats: metrics binary forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"brier-score","dir":"Articles","previous_headings":"Binary forecasts","what":"Brier score","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), either 0 1 Forecast: \\(p\\), probability observed outcome 1. Brier score strictly proper scoring rule. computed mean squared error probabilistic prediction observed outcome. \\[\\begin{equation} \\text{BS}(p, y) = (p - y)^2 = \\begin{cases} p^2, & \\text{} y = 1\\\\ (1 - p)^2, & \\text{} y = 0 \\end{cases} \\end{equation}\\] Brier score logarithmic score (see ) differ penalise - underconfidence (see Machete (2012)). Brier score penalises overconfidence underconfidence probability space . Consider following example: See ?brier_score() information.","code":"n <- 1e6 p_true <- 0.7 observed <- factor(rbinom(n = n, size = 1, prob = p_true), levels = c(0, 1)) p_over <- p_true + 0.15 p_under <- p_true - 0.15 abs(mean(brier_score(observed, p_true)) - mean(brier_score(observed, p_over))) #> [1] 0.0223866 abs(mean(brier_score(observed, p_true)) - mean(brier_score(observed, p_under))) #> [1] 0.0226134"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"logarithmic-score","dir":"Articles","previous_headings":"Binary forecasts","what":"Logarithmic score","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), either 0 1 Forecast: \\(p\\), probability observed outcome 1. logarithmic score (log score) strictly proper scoring rule. computed negative logarithm probability assigned observed outcome. \\[\\begin{equation} \\text{Log score}(p, y) = - \\log(1 - |y - p|) = \\begin{cases} -\\log (p), & \\text{} y = 1\\\\ -\\log (1 - p), & \\text{} y = 0 \\end{cases} \\end{equation}\\] log score penalises overconfidence strongly underconfidence (probability space). Consider following example: See ?logs_binary() information.","code":"abs(mean(logs_binary(observed, p_true)) - mean(logs_binary(observed, p_over))) #> [1] 0.07169954 abs(mean(logs_binary(observed, p_true)) - mean(logs_binary(observed, p_under))) #> [1] 0.04741833"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"sample-based-forecasts","dir":"Articles","previous_headings":"","what":"Sample-based forecasts","title":"Scoring rules in `scoringutils`","text":"See list default metrics sample-based forecasts calling ?metrics_sample(). overview input output formats quantile forecasts: Input output formats: metrics sample-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"crps","dir":"Articles","previous_headings":"Sample-based forecasts","what":"CRPS","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), real number (discrete number). Forecast: continuous (\\(F\\)) discrete (\\(P\\)) forecast. continuous ranked probability score (CRPS) popular fields meteorology epidemiology. CRPS defined \\[\\text{CRPS}(F, y) = \\int_{-\\infty}^\\infty \\left( F(x) - 1(x \\geq y) \\right)^2 dx,\\] \\(y\\) observed value \\(F\\) CDF predictive distribution. discrete forecasts, example count data, ranked probability score (RPS) can used instead commonly defined : \\[ \\text{RPS}(P, y) = \\sum_{x = 0}^\\infty (P(x) - 1(x \\geq y))^2, \\] \\(P\\) cumulative probability mass function (PMF) predictive distribution. CRPS can understood generalisation absolute error predictive distributions (Gneiting Raftery 2007). can also understood integral Brier score binary probability forecasts implied CDF possible observed values. CRPS also related Cramér-distance two distributions equals special case one distributions concentrated single point (see e.g. Ziel (2021)). CRPS global scoring rule, meaning entire predictive distribution taken account determining quality forecast. scoringutils re-exports crps_sample() function scoringRules package, assumes forecast represented set samples predictive distribution. See ?crps_sample() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"log-score","dir":"Articles","previous_headings":"Sample-based forecasts","what":"Log score","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), real number (discrete number). Forecast: continuous (\\(F\\)) discrete (\\(P\\)) forecast. logarithmic scoring rule simply negative logarithm density predictive distribution evaluated observed value: \\[ \\text{log score}(F, y) = -\\log f(y), \\] \\(f\\) predictive probability density function (PDF) corresponding Forecast \\(F\\) \\(y\\) observed value. discrete forecasts, log score can computed \\[ \\text{log score}(F, y) = -\\log p_y, \\] \\(p_y\\) probability assigned observed outcome \\(y\\) forecast \\(F\\). logarithmic scoring rule can produce large penalties observed value takes values \\(f(y)\\) (\\(p_y\\)) close zero. therefore considered sensitive outlier forecasts. may desirable applications, also means scores can easily dominated extreme values. logarithmic scoring rule local scoring rule, meaning score depends probability assigned actual outcome. often regarded desirable property example context Bayesian inference . implies example, ranking forecasters invariant monotone transformations predictive distribution target. scoringutils re-exports logs_sample() function scoringRules package, assumes forecast represented set samples predictive distribution. One implications currently advisable use log score discrete forecasts. reason scoringRules::logs_sample() estimates predictive density samples, can problematic discrete forecasts. See ?logs_sample() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"dawid-sebastiani-score","dir":"Articles","previous_headings":"Sample-based forecasts","what":"Dawid-Sebastiani score","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), real number (discrete number). Forecast: \\(F\\). predictive distribution mean \\(\\mu\\) standard deviation \\(\\sigma\\). Dawid-Sebastiani score proper scoring rule relies first moments predictive distribution therefore easy compute. given \\[\\text{dss}(F, y) = \\left( \\frac{y - \\mu}{\\sigma} \\right)^2 + 2 \\cdot \\log \\sigma.\\] scoringutils re-exports implementation DSS scoringRules package. assumes forecast represented set samples drawn predictive distribution. See ?dss_sample() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"dispersion---median-absolute-deviation-mad","dir":"Articles","previous_headings":"Sample-based forecasts","what":"Dispersion - Median Absolute Deviation (MAD)","title":"Scoring rules in `scoringutils`","text":"Observation: required. Forecast: \\(F\\), predictive distribution. Dispersion (also called sharpness) ability produce narrow forecasts. feature forecasts depend observations. Dispersion therefore interest conditional calibration: precise forecast useful clearly wrong. One way measure sharpness (suggested Funk et al. (2019)) normalised median absolute deviation median (MADN) ). computed \\[ S(F) = \\frac{1}{0.675} \\cdot \\text{median}(|F - \\text{median(F)}|). \\] forecast \\(F\\) follows normal distribution, sharpness equal standard deviation \\(F\\).details, see ?mad_sample().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"bias","dir":"Articles","previous_headings":"Sample-based forecasts","what":"Bias","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), real number (discrete number). Forecast: continuous (\\(F\\)) discrete (\\(P\\)) forecast. Bias measure tendency forecaster - underpredict. continuous forecasts, scoringutils implementation calculates bias \\[B(F, y) = 1 - 2 \\cdot F (y), \\] \\(F(y)\\) cumulative distribution function forecast. discrete forecasts, calculate bias \\[B(P, y) = 1 - (P(y) + P(y + 1)). \\] \\(P(y)\\) cumulative probability assigned outcomes smaller equal \\(y\\), .e. cumulative probability mass function. Bias bound -1 1 represents tendency forecasts biased rather absolute amount - underprediction (e.g. case weighted interval score (WIS, see ).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"absolute-error-of-the-median","dir":"Articles","previous_headings":"Sample-based forecasts","what":"Absolute error of the median","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), real number (discrete number). Forecast: forecast \\(F\\) \\[\\text{ae}_{\\text{median}}(F, y) = |\\text{median} (F) - y|.\\] See section note caution Gneiting (2011) discussion correspondence absolute error median.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"squared-error-of-the-mean","dir":"Articles","previous_headings":"Sample-based forecasts","what":"Squared error of the mean","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), real number (discrete number). Forecast: forecast \\(F\\) \\[\\text{se}_{\\text{medn}}(F, y) = (\\text{mean} (F) - y)^2.\\] See section note caution Gneiting (2011) discussion correspondence squared error mean.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"quantile-based-forecasts","dir":"Articles","previous_headings":"","what":"Quantile-based forecasts","title":"Scoring rules in `scoringutils`","text":"See list default metrics quantile-based forecasts calling ?metrics_quantile(). overview input output formats quantile forecasts: Input output formats: metrics quantile-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"weighted-interval-score-wis","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Weighted interval score (WIS)","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), real number Forecast: \\(F\\). CDF predictive distribution represented set quantiles. quantiles form lower (\\(l\\)) upper (\\(u\\)) bounds central prediction intervals. weighted interval score (WIS) strictly proper scoring rule can understood approximation CRPS forecasts quantile format (turn represents generalisation absolute error). Quantiles assumed lower upper bounds prediction intervals symmetric around median. single interval, interval score \\[IS_\\alpha(F,y) = \\underbrace{(u-l)}_\\text{dispersion} + \\underbrace{\\frac{2}{\\alpha} \\cdot (l-y) \\cdot \\mathbf{1}(y \\leq l)}_{\\text{overprediction}} + \\underbrace{\\frac{2}{\\alpha} \\cdot (y-u) \\cdot \\mathbf{1}(y \\geq u)}_{\\text{underprediction}}, \\] \\(\\mathbf{1}()\\) indicator function, \\(l\\) \\(u\\) \\(\\frac{\\alpha}{2}\\) \\(1 - \\frac{\\alpha}{2}\\) quantiles predictive distribution \\(F\\). \\(l\\) \\(u\\) together form prediction interval. interval score can understood sum three components: dispersion, overprediction underprediction. set \\(K\\) prediction intervals median \\(m\\), score given weighted sum individual interval scores, .e. \\[WIS = \\frac{1}{K + 0.5} \\cdot \\left(w_0 \\cdot |y - m| + \\sum_{k = 1}^{K} w_k \\cdot IS_{\\alpha_{k}}(F, y)\\right),\\] \\(m\\) median forecast \\(w_k\\) weight assigned every interval. weights set \\(w_k = \\frac{\\alpha_k}{2}\\) \\(w_0 = 0.5\\), WIS converges CRPS increasing number equally spaced quantiles. See ?wis() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"overprediction-underprediction-and-dispersion","dir":"Articles","previous_headings":"Quantile-based forecasts > Weighted interval score (WIS)","what":"Overprediction, underprediction and dispersion","title":"Scoring rules in `scoringutils`","text":"individual components WIS. See ?overprediction(), ?underprediction() ?dispersion() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"bias-1","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Bias","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), real number Forecast: \\(F\\). CDF predictive distribution represented set quantiles, \\(Q\\). Bias can measured \\[\\begin{equation} \\text{B}(F, y) = \\begin{cases} (1 - 2 \\cdot \\max \\{\\alpha | q_\\alpha \\Q \\land q_\\alpha \\leq y\\}), & \\text{} y < q_{0.5} \\quad \\text{(overprediction)}\\\\ (1 - 2 \\cdot \\min \\{\\alpha | q_\\alpha \\Q_t \\land q_\\alpha \\geq y\\}, & \\text{} y > q_{0.5} \\quad \\text{(underprediction)}\\\\ 0, & \\text{} y = q_{0.5}, \\\\ \\end{cases} \\end{equation}\\] \\(q_\\alpha\\) \\(\\alpha\\)-quantile predictive distribution. consistency, define \\(Q\\) (set quantiles form predictive distribution \\(F\\)) always includes element \\(q_0 = -\\infty\\) \\(q_1 = \\infty\\). clearer terms, bias : \\(1 - (2 \\times\\) maximum percentile rank corresponding quantile still observed value), observed value smaller median predictive distribution. \\(1 - (2 \\times\\) minimum percentile rank corresponding quantile still larger observed value) observed value larger median predictive distribution.. \\(0\\) observed value exactly median. Bias can assume values -1 (underprediction) 1 (overpredictin) 0 ideally (.e. unbiased). increasing number quantiles, percentile rank equal proportion predictive samples observed value, bias metric coincides one continuous forecasts (see ). See ?bias_quantile() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"interval-coverage","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Interval coverage","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), real number Forecast: \\(F\\). CDF predictive distribution represented set quantiles. quantiles form central prediction intervals. Interval coverage given interval range defined proportion observations fall within corresponding central prediction intervals. Central prediction intervals symmetric around median formed two quantiles denote lower upper bound. example, 50% central prediction interval interval 0.25 0.75 quantiles predictive distribution.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"interval-coverage-deviation","dir":"Articles","previous_headings":"Quantile-based forecasts > Interval coverage","what":"Interval coverage deviation","title":"Scoring rules in `scoringutils`","text":"interval coverage deviation difference observed interval coverage nominal interval coverage. example, observed interval coverage 50% central prediction interval 0.6, interval coverage deviation \\(0.6 = - 0.5 = 0.1.\\) \\[\\text{interval coverage deviation} = \\text{observed interval coverage} - \\text{nominal interval coverage}\\]","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"absolute-error-of-the-median-1","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Absolute error of the median","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), real number Forecast: \\(F\\). CDF predictive distribution represented set quantiles. absolute error median absolute difference median predictive distribution observed value. \\[\\text{ae}_\\text{median} = |\\text{median}(F) - y|\\] See section note caution Gneiting (2011) discussion correspondence absolute error median.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"quantile-score","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Quantile score","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), real number Forecast: \\(F\\). CDF predictive distribution represented set quantiles. quantile score, also called pinball loss, single quantile level \\(\\tau\\) defined \\[\\begin{equation} \\text{QS}_\\tau(F, y) = 2 \\cdot \\{ \\mathbf{1}(y \\leq q_\\tau) - \\tau\\} \\cdot (q_\\tau − y) = \\begin{cases} 2 \\cdot (1 - \\tau) * q_\\tau - y, & \\text{} y \\leq q_\\tau\\\\ 2 \\cdot \\tau * |q_\\tau - y|, & \\text{} y > q_\\tau, \\end{cases} \\end{equation}\\] \\(q_\\tau\\) \\(\\tau\\)-quantile predictive distribution \\(F\\), \\(\\mathbf{1}(\\cdot)\\) indicator function. (unweighted) interval score (see ) \\(1 - \\alpha\\) prediction interval can computed quantile scores levels \\(\\alpha/2\\) \\(1 - \\alpha/2\\) \\[\\text{}_\\alpha(F, y) = \\frac{\\text{QS}_{\\alpha/2}(F, y) + \\text{QS}_{1 - \\alpha/2}(F, y)}{\\alpha}\\]. weighted interval score can obtained simple average quantile scores: \\[\\text{WIS}_\\alpha(F, y) = \\frac{\\text{QS}_{\\alpha/2}(F, y) + \\text{QS}_{1 - \\alpha/2}(F, y)}{2}\\]. See ?quantile_score Bracher et al. (2021) details.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"quantile-coverage","dir":"Articles","previous_headings":"Additional metrics","what":"Quantile coverage","title":"Scoring rules in `scoringutils`","text":"Quantile coverage given quantile level defined proportion observed values smaller corresponding predictive quantiles. example, 0.5 quantile coverage proportion observed values smaller 0.5-quantiles predictive distribution.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoringutils.html","id":"input-formats","dir":"Articles","previous_headings":"","what":"Input formats","title":"Getting started","text":"time, score() function able entire evaluation . need pass data.frame appropriate columns. columns required depends format forecasts come . forecast format can either based quantiles (see example_quantile expected format), based predictive samples (see example_sample_continuous example_sample_discrete expected format case) binary format. following table gives overview (pairwise comparisons explained detail ): Additional columns may present indicate grouping forecasts. example, forecasts made different models various locations different time points, several weeks future. scoringutils automatically tries determine unit single forecast, .e. combination existing columns able uniquely identify single forecast. uses existing columns , can sometimes lead issues. therefore recommend using function set_forecast_unit() determine forecast unit manually. function simply drops unneeded columns, making sure necessary, ‘protected columns’ like “predicted” “observed” retained.","code":"colnames(example_quantile) #> [1] \"location\" \"target_end_date\" \"target_type\" \"observed\" #> [5] \"location_name\" \"forecast_date\" \"quantile_level\" \"predicted\" #> [9] \"model\" \"horizon\" set_forecast_unit( example_quantile, c(\"location\", \"target_end_date\", \"target_type\", \"horizon\", \"model\") ) %>% colnames() #> [1] \"observed\" \"quantile_level\" \"predicted\" \"location\" #> [5] \"target_end_date\" \"target_type\" \"horizon\" \"model\""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoringutils.html","id":"constructing-a-forecast-object","dir":"Articles","previous_headings":"","what":"Constructing a forecast object","title":"Getting started","text":"function as_forecast() used construct forecast object check input data. determines forecast type, creates object appropriate class (forecast_binary, forecast_quantile forecast_sample), validates input data. Objects class forecast_* print method provides additional information. unsure input data look like, look example_quantile, example_sample_discrete, example_sample_continuous example_binary data sets provided package. output as_forecast() can later directly used input score(). recommended workflow (however, score() run as_forecast() internally hasn’t happened ). Note example columns contain duplicated information regards forecast unit, e.g. “location” “location_name”, can dropped. drop essential information, example, “target_type” column, ’ll get error informing us forecasts aren’t uniquely identified . function get_duplicate_forecasts() may help investigate issue. filtering single quantile EuroCOVIDhub-ensemble, can see indeed two forecasts every date, location horizon.","code":"head(example_quantile) #> Key: #> location target_end_date target_type observed location_name forecast_date #> #> 1: DE 2021-01-02 Cases 127300 Germany #> 2: DE 2021-01-02 Deaths 4534 Germany #> 3: DE 2021-01-09 Cases 154922 Germany #> 4: DE 2021-01-09 Deaths 6117 Germany #> 5: DE 2021-01-16 Cases 110183 Germany #> 6: DE 2021-01-16 Deaths 5867 Germany #> quantile_level predicted model horizon #> #> 1: NA NA NA #> 2: NA NA NA #> 3: NA NA NA #> 4: NA NA NA #> 5: NA NA NA #> 6: NA NA NA forecast_quantile <- example_quantile %>% set_forecast_unit( c(\"model\", \"location\", \"target_end_date\", \"forecast_date\", \"target_type\", \"horizon\") ) %>% as_forecast() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. forecast_quantile #> Forecast type: quantile #> Forecast unit: #> model, location, target_end_date, forecast_date, target_type, and horizon #> #> Key: #> observed quantile_level predicted model location #> #> 1: 127300 NA NA DE #> 2: 4534 NA NA DE #> 3: 154922 NA NA DE #> 4: 6117 NA NA DE #> 5: 110183 NA NA DE #> --- #> 20541: 78 0.850 352 epiforecasts-EpiNow2 IT #> 20542: 78 0.900 397 epiforecasts-EpiNow2 IT #> 20543: 78 0.950 499 epiforecasts-EpiNow2 IT #> 20544: 78 0.975 611 epiforecasts-EpiNow2 IT #> 20545: 78 0.990 719 epiforecasts-EpiNow2 IT #> target_end_date forecast_date target_type horizon #> #> 1: 2021-01-02 Cases NA #> 2: 2021-01-02 Deaths NA #> 3: 2021-01-09 Cases NA #> 4: 2021-01-09 Deaths NA #> 5: 2021-01-16 Cases NA #> --- #> 20541: 2021-07-24 2021-07-12 Deaths 2 #> 20542: 2021-07-24 2021-07-12 Deaths 2 #> 20543: 2021-07-24 2021-07-12 Deaths 2 #> 20544: 2021-07-24 2021-07-12 Deaths 2 #> 20545: 2021-07-24 2021-07-12 Deaths 2 example_quantile %>% set_forecast_unit( c(\"location\", \"target_end_date\", \"forecast_date\", \"model\", \"horizon\") ) %>% as_forecast() #> Error: Assertion on 'data' failed: There are instances with more than one forecast for the same target. This can't be right and needs to be resolved. Maybe you need to check the unit of a single forecast and add missing columns? Use the function get_duplicate_forecasts() to identify duplicate rows. duplicates <- example_quantile %>% set_forecast_unit( c(\"location\", \"target_end_date\", \"forecast_date\", \"model\", \"horizon\") ) %>% get_duplicate_forecasts() duplicates[quantile_level == 0.5 & model == \"EuroCOVIDhub-ensemble\", ] %>% head() #> Key: #> observed quantile_level predicted location target_end_date forecast_date #> #> 1: 106987 0.5 119258 DE 2021-05-08 2021-05-03 #> 2: 1582 0.5 1568 DE 2021-05-08 2021-05-03 #> 3: 64985 0.5 110716 DE 2021-05-15 2021-05-03 #> 4: 64985 0.5 92649 DE 2021-05-15 2021-05-10 #> 5: 1311 0.5 1521 DE 2021-05-15 2021-05-03 #> 6: 1311 0.5 1422 DE 2021-05-15 2021-05-10 #> model horizon #> #> 1: EuroCOVIDhub-ensemble 1 #> 2: EuroCOVIDhub-ensemble 1 #> 3: EuroCOVIDhub-ensemble 2 #> 4: EuroCOVIDhub-ensemble 1 #> 5: EuroCOVIDhub-ensemble 2 #> 6: EuroCOVIDhub-ensemble 1"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoringutils.html","id":"showing-available-forecasts","dir":"Articles","previous_headings":"","what":"Showing available forecasts","title":"Getting started","text":"function get_forecast_counts() may also helpful determine forecasts available. Using argument can specify level summary. example, see many forecasts per model target_type, can run see ‘epiforecasts-EpiNow2’ missing forecasts deaths forecast target UMass-MechBayes case forecasts. information can also visualised using plot():","code":"get_forecast_counts(forecast_quantile, by = c(\"model\", \"target_type\")) #> Key: #> model target_type count #> #> 1: EuroCOVIDhub-baseline Cases 128 #> 2: EuroCOVIDhub-baseline Deaths 128 #> 3: EuroCOVIDhub-ensemble Cases 128 #> 4: EuroCOVIDhub-ensemble Deaths 128 #> 5: UMass-MechBayes Cases 0 #> 6: UMass-MechBayes Deaths 128 #> 7: epiforecasts-EpiNow2 Cases 128 #> 8: epiforecasts-EpiNow2 Deaths 119 forecast_quantile %>% get_forecast_counts(by = c(\"model\", \"forecast_date\", \"target_type\")) %>% plot_forecast_counts(x = \"forecast_date\") + facet_wrap(~ target_type)"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoringutils.html","id":"scoring-and-summarising-forecasts","dir":"Articles","previous_headings":"","what":"Scoring and summarising forecasts","title":"Getting started","text":"Forecasts can easily scored using score() function. clarity, suggest setting forecast unit explicitly calling as_forecast() explicitly. function score() returns unsumarised scores, cases user wants. returns single score per forecast (determined forecast unit). second function, summarise_scores() takes care summarising scores level specified user. argument can used define level summary. default, = model one score per model returned. argument can specify unit summary want. can also call sumarise_scores() multiple tines, e.g round outputs specifying e.g. signif() summary function.","code":"scores <- forecast_quantile %>% score() print(scores, 2) #> model location target_end_date forecast_date target_type #> #> 1: EuroCOVIDhub-ensemble DE 2021-05-08 2021-05-03 Cases #> 2: EuroCOVIDhub-baseline DE 2021-05-08 2021-05-03 Cases #> --- #> 886: epiforecasts-EpiNow2 IT 2021-07-24 2021-07-05 Deaths #> 887: epiforecasts-EpiNow2 IT 2021-07-24 2021-07-12 Deaths #> horizon wis overprediction underprediction dispersion bias #> #> 1: 1 7990.85478 2549.869565 0 5440.98522 0.50 #> 2: 1 16925.04696 15275.826087 0 1649.22087 0.95 #> --- #> 886: 3 19.76261 5.478261 0 14.28435 0.50 #> 887: 2 66.16174 40.608696 0 25.55304 0.90 #> interval_coverage_50 interval_coverage_90 interval_coverage_deviation #> #> 1: TRUE TRUE 0.05181818 #> 2: FALSE FALSE -0.40272727 #> --- #> 886: TRUE TRUE 0.05181818 #> 887: FALSE TRUE -0.31181818 #> ae_median #> #> 1: 12271 #> 2: 25620 #> --- #> 886: 26 #> 887: 108 scores %>% summarise_scores(by = c(\"model\", \"target_type\")) %>% summarise_scores(fun = signif, digits = 2) %>% kable()"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoringutils.html","id":"scoring-point-forecasts","dir":"Articles","previous_headings":"Scoring and summarising forecasts","what":"Scoring point forecasts","title":"Getting started","text":"Point forecasts can scored separate format.","code":"suppressMessages(score(as_forecast(example_point))) %>% summarise_scores(by = \"model\", na.rm = TRUE) #> model ae_point se_point ape #> #> 1: EuroCOVIDhub-ensemble 12077.10156 1.945118e+09 0.2996696 #> 2: EuroCOVIDhub-baseline 19353.42969 2.883446e+09 0.7383936 #> 3: epiforecasts-EpiNow2 14521.10526 2.680928e+09 0.3715552 #> 4: UMass-MechBayes 78.47656 1.170976e+04 0.2823206"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoringutils.html","id":"adding-empirical-coverage","dir":"Articles","previous_headings":"Scoring and summarising forecasts","what":"Adding empirical coverage","title":"Getting started","text":"quantile-based forecasts often interested specific coverage-levels, example, percentage true values fell 50% 90% prediction intervals. can add information using function get_coverage(). function also requires argument defines level grouping percentage true values covered certain prediction intervals computed.","code":"score(as_forecast(example_quantile)) %>% summarise_scores(by = c(\"model\", \"target_type\")) %>% summarise_scores(fun = signif, digits = 2) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> model wis overprediction underprediction dispersion #> #> 1: EuroCOVIDhub-ensemble 18000 10000.0 4200.0 3700 #> 2: EuroCOVIDhub-ensemble 41 7.1 4.1 30 #> 3: EuroCOVIDhub-baseline 28000 14000.0 10000.0 4100 #> 4: EuroCOVIDhub-baseline 160 66.0 2.1 91 #> 5: epiforecasts-EpiNow2 21000 12000.0 3300.0 5700 #> 6: epiforecasts-EpiNow2 67 19.0 16.0 32 #> 7: UMass-MechBayes 53 9.0 17.0 27 #> bias interval_coverage_50 interval_coverage_90 #> #> 1: -0.0560 0.39 0.80 #> 2: 0.0730 0.88 1.00 #> 3: 0.0980 0.33 0.82 #> 4: 0.3400 0.66 1.00 #> 5: -0.0790 0.47 0.79 #> 6: -0.0051 0.42 0.91 #> 7: -0.0220 0.46 0.88 #> interval_coverage_deviation ae_median #> #> 1: -0.100 24000 #> 2: 0.200 53 #> 3: -0.120 38000 #> 4: 0.120 230 #> 5: -0.070 28000 #> 6: -0.045 100 #> 7: -0.025 78"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoringutils.html","id":"adding-relative-scores","dir":"Articles","previous_headings":"Scoring and summarising forecasts","what":"Adding relative scores","title":"Getting started","text":"order better compare models can use relative scores computed based pairwise comparisons (see details ). Relative scores can added evaluation using function summarise_scores(). requires column called ‘model’ present. Pairwise comparisons computed according grouping specified : essentially, data.frame scores gets split different data.frames according values specified relative scores computed every individual group separately. baseline argument allows us specify baseline can used scale relative scores (scores divided baseline relative score). example, obtain relative scores separately different forecast targets, can run","code":"score(as_forecast(example_quantile)) %>% add_relative_skill( by = c(\"model\", \"target_type\"), baseline = \"EuroCOVIDhub-ensemble\" ) %>% summarise_scores(by = c(\"model\", \"target_type\")) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Key: #> model target_type wis overprediction underprediction #> #> 1: EuroCOVIDhub-baseline Cases 28483.57465 14096.100883 10284.972826 #> 2: EuroCOVIDhub-baseline Deaths 159.40387 65.899117 2.098505 #> 3: EuroCOVIDhub-ensemble Cases 17943.82383 10043.121943 4237.177310 #> 4: EuroCOVIDhub-ensemble Deaths 41.42249 7.138247 4.103261 #> 5: UMass-MechBayes Deaths 52.65195 8.978601 16.800951 #> 6: epiforecasts-EpiNow2 Cases 20831.55662 11906.823030 3260.355639 #> 7: epiforecasts-EpiNow2 Deaths 66.64282 18.892583 15.893314 #> dispersion bias interval_coverage_50 interval_coverage_90 #> #> 1: 4102.50094 0.09796875 0.3281250 0.8203125 #> 2: 91.40625 0.33906250 0.6640625 1.0000000 #> 3: 3663.52458 -0.05640625 0.3906250 0.8046875 #> 4: 30.18099 0.07265625 0.8750000 1.0000000 #> 5: 26.87239 -0.02234375 0.4609375 0.8750000 #> 6: 5664.37795 -0.07890625 0.4687500 0.7890625 #> 7: 31.85692 -0.00512605 0.4201681 0.9075630 #> interval_coverage_deviation ae_median wis_relative_skill #> #> 1: -0.11721591 38473.60156 1.2947445 #> 2: 0.12142045 233.25781 2.2958723 #> 3: -0.10230114 24101.07031 0.8156514 #> 4: 0.20380682 53.13281 0.5966310 #> 5: -0.02488636 78.47656 0.7475873 #> 6: -0.06963068 27923.81250 0.9469157 #> 7: -0.04520244 104.74790 0.9765276 #> wis_scaled_relative_skill #> #> 1: 1.587375 #> 2: 3.848060 #> 3: 1.000000 #> 4: 1.000000 #> 5: 1.253014 #> 6: 1.160932 #> 7: 1.636736"},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoringutils.html","id":"score-heatmap","dir":"Articles","previous_headings":"Visualising scores","what":"Score heatmap","title":"Getting started","text":"can also summarise one particular metric across different categories using simple heatmap:","code":"score(as_forecast(example_sample_continuous)) %>% summarise_scores(by = c(\"model\", \"location\", \"target_type\")) %>% plot_heatmap(x = \"location\", metric = \"bias\") + facet_wrap(~ target_type) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected."},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoringutils.html","id":"weighted-interval-score-components","dir":"Articles","previous_headings":"Visualising scores","what":"Weighted interval score components","title":"Getting started","text":"weighted interval score can split three components: -prediction, -prediction dispersion. can visualised separately following way:","code":"score(as_forecast(example_quantile)) %>% summarise_scores(by = c(\"target_type\", \"model\")) %>% plot_wis() + facet_wrap(~ target_type, scales = \"free\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected."},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoringutils.html","id":"calibration","dir":"Articles","previous_headings":"","what":"Calibration","title":"Getting started","text":"Calibration measure statistical consistency forecasts observed values. common way assessing calibration (precisely: probabilistic calibration) PIT histograms. probability integral transform (PIT) equal cumulative distribution function forecast evaluated observed value. Ideally, pit values uniformly distributed transformation. can compute pit values : visualise results : Similarly quantile-based forecasts:","code":"example_sample_continuous %>% as_forecast() %>% get_pit(by = \"model\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> model pit_value #> #> 1: EuroCOVIDhub-baseline 0.025 #> 2: EuroCOVIDhub-baseline 0.525 #> 3: EuroCOVIDhub-baseline 0.000 #> 4: EuroCOVIDhub-baseline 0.000 #> 5: EuroCOVIDhub-baseline 0.200 #> --- #> 883: UMass-MechBayes 0.950 #> 884: UMass-MechBayes 0.500 #> 885: UMass-MechBayes 0.100 #> 886: UMass-MechBayes 0.450 #> 887: UMass-MechBayes 0.100 example_sample_continuous %>% as_forecast() %>% get_pit(by = c(\"model\", \"target_type\")) %>% plot_pit() + facet_grid(model ~ target_type) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. example_quantile[quantile_level %in% seq(0.1, 0.9, 0.1), ] %>% as_forecast() %>% get_pit(by = c(\"model\", \"target_type\")) %>% plot_pit() + facet_grid(model ~ target_type)"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoringutils.html","id":"pairwise-comparisons","dir":"Articles","previous_headings":"","what":"Pairwise comparisons","title":"Getting started","text":"Relative scores different models can computed using pairwise comparisons, sort pairwise tournament combinations two models compared based overlapping set available forecasts common models. Internally, ratio mean scores models computed. relative score model geometric mean mean score ratios involve model. baseline provided, baseline excluded relative scores individual models (therefore differ slightly relative scores without baseline) relative scores scaled (.e. divided ) relative score baseline model. scoringutils, pairwise comparisons can made two ways: standalone function get_pairwise_comparisons() within summarise_scores() simply adds relative scores existing set scores. using get_pairwise_comparisons() function, can also visualise pairwise comparisons showing mean score ratios models. default, smaller values better model care showing y axis left, model compared shown x-axis bottom. example , EuroCOVIDhub-ensemble performs best (values smaller 1), EuroCOVIDhub-baseline performs worst (values larger 1). cases, UMass-MechBayes model course excluded case forecasts available therefore set overlapping forecasts empty.","code":"forecast_quantile %>% score() %>% get_pairwise_comparisons(by = \"model\", baseline = \"EuroCOVIDhub-baseline\") #> model compare_against mean_scores_ratio pval #> #> 1: EuroCOVIDhub-baseline epiforecasts-EpiNow2 1.3703452 9.164893e-18 #> 2: EuroCOVIDhub-baseline UMass-MechBayes 3.0275019 2.627464e-20 #> 3: EuroCOVIDhub-baseline EuroCOVIDhub-ensemble 1.5925819 2.608666e-32 #> 4: EuroCOVIDhub-baseline EuroCOVIDhub-baseline 1.0000000 1.000000e+00 #> 5: EuroCOVIDhub-ensemble EuroCOVIDhub-baseline 0.6279112 2.608666e-32 #> 6: EuroCOVIDhub-ensemble UMass-MechBayes 0.7867229 1.244731e-04 #> 7: EuroCOVIDhub-ensemble epiforecasts-EpiNow2 0.8606607 1.881520e-02 #> 8: EuroCOVIDhub-ensemble EuroCOVIDhub-ensemble 1.0000000 1.000000e+00 #> 9: UMass-MechBayes EuroCOVIDhub-ensemble 1.2710955 1.244731e-04 #> 10: UMass-MechBayes epiforecasts-EpiNow2 0.7439673 7.253878e-03 #> 11: UMass-MechBayes EuroCOVIDhub-baseline 0.3303053 2.627464e-20 #> 12: UMass-MechBayes UMass-MechBayes 1.0000000 1.000000e+00 #> 13: epiforecasts-EpiNow2 UMass-MechBayes 1.3441452 7.253878e-03 #> 14: epiforecasts-EpiNow2 EuroCOVIDhub-ensemble 1.1618981 1.881520e-02 #> 15: epiforecasts-EpiNow2 EuroCOVIDhub-baseline 0.7297431 9.164893e-18 #> 16: epiforecasts-EpiNow2 epiforecasts-EpiNow2 1.0000000 1.000000e+00 #> adj_pval wis_relative_skill wis_scaled_relative_skill #> #> 1: 3.665957e-17 1.6032604 1.0000000 #> 2: 1.313732e-19 1.6032604 1.0000000 #> 3: 1.565200e-31 1.6032604 1.0000000 #> 4: 1.000000e+00 1.6032604 1.0000000 #> 5: 1.565200e-31 0.8074916 0.5036559 #> 6: 3.734192e-04 0.8074916 0.5036559 #> 7: 1.881520e-02 0.8074916 0.5036559 #> 8: 1.000000e+00 0.8074916 0.5036559 #> 9: 3.734192e-04 0.7475873 0.4662919 #> 10: 1.450776e-02 0.7475873 0.4662919 #> 11: 1.313732e-19 0.7475873 0.4662919 #> 12: 1.000000e+00 0.7475873 0.4662919 #> 13: 1.450776e-02 1.0332277 0.6444541 #> 14: 1.881520e-02 1.0332277 0.6444541 #> 15: 3.665957e-17 1.0332277 0.6444541 #> 16: 1.000000e+00 1.0332277 0.6444541 forecast_quantile %>% score() %>% summarise_scores(by = \"model\") %>% add_relative_skill(baseline = \"EuroCOVIDhub-baseline\") #> Warning: ! `by` is set to 'model', which is also the unit of a single forecast. This #> doesn't look right. #> ℹ All relative skill scores will be equal to 1. #> Key: #> model wis overprediction underprediction dispersion #> #> 1: EuroCOVIDhub-baseline 14321.48926 7081.000000 5143.53567 2096.95360 #> 2: EuroCOVIDhub-ensemble 8992.62316 5025.130095 2120.64029 1846.85278 #> 3: UMass-MechBayes 52.65195 8.978601 16.80095 26.87239 #> 4: epiforecasts-EpiNow2 10827.40786 6179.439535 1697.23411 2950.73422 #> bias interval_coverage_50 interval_coverage_90 #> #> 1: 0.21851562 0.4960938 0.9101562 #> 2: 0.00812500 0.6328125 0.9023438 #> 3: -0.02234375 0.4609375 0.8750000 #> 4: -0.04336032 0.4453441 0.8461538 #> interval_coverage_deviation ae_median wis_relative_skill #> #> 1: 0.002102273 19353.42969 1 #> 2: 0.050752841 12077.10156 1 #> 3: -0.024886364 78.47656 1 #> 4: -0.057861612 14521.10526 1 #> wis_scaled_relative_skill #> #> 1: 1 #> 2: 1 #> 3: 1 #> 4: 1 forecast_quantile %>% score() %>% get_pairwise_comparisons(by = c(\"model\", \"target_type\")) %>% plot_pairwise_comparisons() + facet_wrap(~ target_type)"},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoringutils.html","id":"correlation-between-scores","dir":"Articles","previous_headings":"Additional analyses and visualisations","what":"Correlation between scores","title":"Getting started","text":"may sometimes interesting see different scores correlate . can examine using function correlation(). dealing quantile-based forecasts, important call summarise_scorees() get_correlations() summarise quantiles computing correlations. Visualising correlations:","code":"forecast_quantile %>% score() %>% summarise_scores() %>% get_correlations() #> wis overprediction underprediction dispersion bias #> #> 1: 1.0000000 0.9917884 0.8821058 0.86564717 0.61587946 #> 2: 0.9917884 1.0000000 0.8158534 0.91900505 0.51068021 #> 3: 0.8821058 0.8158534 1.0000000 0.52982259 0.91280769 #> 4: 0.8656472 0.9190050 0.5298226 1.00000000 0.14860643 #> 5: 0.6158795 0.5106802 0.9128077 0.14860643 1.00000000 #> 6: 0.1565567 0.1740382 0.1630289 0.04407105 0.06639491 #> 7: 0.2818348 0.1968480 0.6141229 -0.17841559 0.74137210 #> 8: 0.1364221 0.1123720 0.2954741 -0.12711127 0.30908292 #> 9: 0.9999780 0.9909177 0.8851559 0.86247754 0.62106954 #> interval_coverage_50 interval_coverage_90 interval_coverage_deviation #> #> 1: 0.15655670 0.2818348 0.1364221 #> 2: 0.17403818 0.1968480 0.1123720 #> 3: 0.16302890 0.6141229 0.2954741 #> 4: 0.04407105 -0.1784156 -0.1271113 #> 5: 0.06639491 0.7413721 0.3090829 #> 6: 1.00000000 0.6417542 0.9466440 #> 7: 0.64175425 1.0000000 0.8451292 #> 8: 0.94664396 0.8451292 1.0000000 #> 9: 0.15565388 0.2861731 0.1376834 #> ae_median metric #> #> 1: 0.9999780 wis #> 2: 0.9909177 overprediction #> 3: 0.8851559 underprediction #> 4: 0.8624775 dispersion #> 5: 0.6210695 bias #> 6: 0.1556539 interval_coverage_50 #> 7: 0.2861731 interval_coverage_90 #> 8: 0.1376834 interval_coverage_deviation #> 9: 1.0000000 ae_median forecast_quantile %>% score() %>% summarise_scores() %>% get_correlations(digits = 2) %>% plot_correlations()"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoringutils.html","id":"converting-to-quantile-based-forecasts","dir":"Articles","previous_headings":"","what":"Converting to quantile-based forecasts","title":"Getting started","text":"Different metrics available different forecasting formats. cases, may example forecasts sample-based format, wish make use functionality available quantile-based forecasts. example, may want use decomposition weighted interval score, may like compute interval coverage values. can convert sample-based forecasts quantile-based format using function sample_to_quantile(). , however, one caveat: Quantiles calculated based predictive samples, may introduce bias number available samples small.","code":"example_sample_discrete %>% as_forecast() %>% sample_to_quantile( quantile_level = c(0.01, 0.025, seq(0.05, 0.95, 0.05), 0.975, 0.99) ) %>% as_forecast() %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> location location_name target_end_date target_type forecast_date #> #> 1: DE Germany 2021-05-08 Cases 2021-05-03 #> 2: DE Germany 2021-05-08 Cases 2021-05-03 #> 3: DE Germany 2021-05-08 Cases 2021-05-03 #> 4: DE Germany 2021-05-08 Deaths 2021-05-03 #> 5: DE Germany 2021-05-08 Deaths 2021-05-03 #> --- #> 883: IT Italy 2021-07-24 Deaths 2021-07-12 #> 884: IT Italy 2021-07-24 Deaths 2021-07-05 #> 885: IT Italy 2021-07-24 Deaths 2021-07-12 #> 886: IT Italy 2021-07-24 Deaths 2021-07-05 #> 887: IT Italy 2021-07-24 Deaths 2021-07-12 #> model horizon wis overprediction underprediction #> #> 1: EuroCOVIDhub-ensemble 1 6671.68207 2.110043e+03 0.000000 #> 2: EuroCOVIDhub-baseline 1 18441.13061 1.446498e+04 0.000000 #> 3: epiforecasts-EpiNow2 1 22042.77309 1.548071e+04 0.000000 #> 4: EuroCOVIDhub-ensemble 1 59.46028 9.682609e+00 0.000000 #> 5: EuroCOVIDhub-baseline 1 75.93215 0.000000e+00 1.195652 #> --- #> 883: EuroCOVIDhub-baseline 2 60.71862 1.323913e+01 0.000000 #> 884: UMass-MechBayes 3 5.79650 3.695652e-01 0.000000 #> 885: UMass-MechBayes 2 26.18112 1.891304e+01 0.000000 #> 886: epiforecasts-EpiNow2 3 25.13032 3.478261e-01 0.000000 #> 887: epiforecasts-EpiNow2 2 44.07931 2.410435e+01 0.000000 #> dispersion bias interval_coverage_50 interval_coverage_90 #> #> 1: 4561.638591 0.60 FALSE TRUE #> 2: 3976.145828 0.98 FALSE FALSE #> 3: 6562.060048 0.90 FALSE TRUE #> 4: 49.777674 0.30 TRUE TRUE #> 5: 74.736502 -0.10 TRUE TRUE #> --- #> 883: 47.479491 0.50 TRUE TRUE #> 884: 5.426935 0.20 TRUE TRUE #> 885: 7.268076 0.90 FALSE TRUE #> 886: 24.782489 0.10 TRUE TRUE #> 887: 19.974959 0.70 FALSE TRUE #> interval_coverage_deviation ae_median #> #> 1: -0.03909091 9414.0 #> 2: -0.49363636 29379.0 #> 3: -0.31181818 36513.0 #> 4: 0.23363636 77.0 #> 5: 0.41545455 27.5 #> --- #> 883: 0.05181818 58.5 #> 884: 0.32454545 5.0 #> 885: -0.31181818 42.5 #> 886: 0.41545455 8.0 #> 887: -0.13000000 73.5"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoringutils.html","id":"lower-level-functions","dir":"Articles","previous_headings":"","what":"Lower level functions","title":"Getting started","text":"metrics available lower level functions extensively documented. look help files understand better.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/authors.html","id":null,"dir":"","previous_headings":"","what":"Authors","title":"Authors and Citation","text":"Nikos Bosse. Author, maintainer. Sam Abbott. Author. Hugo Gruson. Author. Johannes Bracher. Contributor. Toshiaki Asakura. Contributor. James Mba Azam. Contributor. Sebastian Funk. Author.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/authors.html","id":"citation","dir":"","previous_headings":"","what":"Citation","title":"Authors and Citation","text":"Nikos . Bosse, Hugo Gruson, Sebastian Funk, Anne Cori, Edwin van Leeuwen, Sam Abbott (2022). Evaluating Forecasts scoringutils R, arXiv. DOI: 10.48550/ARXIV.2205.07090 Alexander Jordan, Fabian Krueger, Sebastian Lerch (2019). Evaluating Probabilistic Forecasts scoringRules. Journal Statistical Software, 90(12), 1-37. DOI 10.18637/jss.v090.i12","code":"@Article{, title = {Evaluating Forecasts with scoringutils in R}, author = {Nikos I. Bosse and Hugo Gruson and Anne Cori and Edwin {van Leeuwen} and Sebastian Funk and Sam Abbott}, journal = {arXiv}, url = {https://arxiv.org/abs/2205.07090}, year = {2022}, doi = {10.48550/ARXIV.2205.07090}, } @Article{, title = {Evaluating Probabilistic Forecasts with {scoringRules}}, author = {Alexander Jordan and Fabian Kr\\\"uger and Sebastian Lerch}, journal = {Journal of Statistical Software}, year = {2019}, volume = {90}, number = {12}, pages = {1--37}, doi = {10.18637/jss.v090.i12}, }"},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"scoringutils-utilities-for-scoring-and-assessing-predictions","dir":"","previous_headings":"","what":"Utilities for Scoring and Assessing Predictions","title":"Utilities for Scoring and Assessing Predictions","text":"Note: documentation refers development version scoringutils. can also view documentation stable version. scoringutils package facilitates process evaluating forecasts R, using convenient flexible data.table-based framework. provides broad functionality check input data diagnose issues, visualise forecasts missing data, transform data scoring, handle missing forecasts, aggregate scores, visualise results evaluation. package easily extendable, meaning users can supply scoring rules extend existing classes handle new types forecasts. package underwent major re-write. comprehensive documentation updated package revised version original scoringutils paper. Another good starting point vignettes Getting started, Details metrics implemented Scoring forecasts directly. details specific issue transforming forecasts scoring see: Nikos . Bosse, Sam Abbott, Anne Cori, Edwin van Leeuwen, Johannes Bracher* Sebastian Funk* (*: equal contribution) (2023). Scoring epidemiological forecasts transformed scales, PLoS Comput Biol 19(8): e1011393 https://doi.org/10.1371/journal.pcbi.1011393","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"installation","dir":"","previous_headings":"","what":"Installation","title":"Utilities for Scoring and Assessing Predictions","text":"Install CRAN version package using Install unstable development version GitHub using","code":"install.packages(\"scoringutils\") remotes::install_github(\"epiforecasts/scoringutils\", dependencies = TRUE)"},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"forecast-types","dir":"","previous_headings":"Quick start","what":"Forecast types","title":"Utilities for Scoring and Assessing Predictions","text":"scoringutils currently supports scoring following forecast types: - binary: probability binary (yes/) outcome variable. - point: forecast continuous discrete outcome variable represented single number. - quantile: probabilistic forecast continuous discrete outcome variable, forecast distribution represented set predictive quantiles. - sample: probabilistic forecast continuous discrete outcome variable, forecast represented finite set samples drawn predictive distribution.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"input-formats-and-input-validation","dir":"","previous_headings":"Quick start","what":"Input formats and input validation","title":"Utilities for Scoring and Assessing Predictions","text":"expected input format generally data.frame (similar) required columns observed, predicted, model holds forecasts observed values. Exact requirements depend forecast type. information, look paper, call ?as_forecast(), look example data provided package (example_binary, example_point, example_quantile, example_sample_continuous, example_sample_discrete). scoring, input data needs validated transformed forecast object using function as_forecast().","code":"forecast_quantile <- example_quantile |> as_forecast( forecast_unit = c( \"location\", \"forecast_date\", \"target_end_date\", \"target_type\", \"model\", \"horizon\" ), forecast_type = \"quantile\" ) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. print(forecast_quantile, 2) #> Forecast type: quantile #> Forecast unit: #> location, forecast_date, target_end_date, target_type, model, and horizon #> #> Key: #> observed quantile_level predicted location forecast_date target_end_date #> #> 1: 127300 NA NA DE 2021-01-02 #> 2: 4534 NA NA DE 2021-01-02 #> --- #> 20544: 78 0.975 611 IT 2021-07-12 2021-07-24 #> 20545: 78 0.990 719 IT 2021-07-12 2021-07-24 #> target_type model horizon #> #> 1: Cases NA #> 2: Deaths NA #> --- #> 20544: Deaths epiforecasts-EpiNow2 2 #> 20545: Deaths epiforecasts-EpiNow2 2"},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"the-forecast-unit","dir":"","previous_headings":"Quick start","what":"The forecast unit","title":"Utilities for Scoring and Assessing Predictions","text":"quantile-based sample-based forecasts, single prediction represented set several quantiles (samples) predictive distribution, .e. several rows input data. scoringutils therefore needs group rows together form single forecast. scoringutils uses existing columns input data achieve - values columns uniquely identify single forecast. Additional columns unrelated forecast unit can mess . forecast_unit argument as_forecast() makes sure columns retained relevant defining unit single forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"scoring-forecasts","dir":"","previous_headings":"Quick start","what":"Scoring forecasts","title":"Utilities for Scoring and Assessing Predictions","text":"Forecasts can scored calling score() validated forecast object. score() takes additional argument, metrics, list scoring rules. Every forecast type default list metrics. can easily add scoring functions, long conform format forecast type. See paper information. can summarise scores using function summarise_scores(). argument used specify desired level summary. fun let’s specify summary function, although recommended stick mean primary summary function, functions can lead improper scores.","code":"scores <- forecast_quantile |> score() scores |> summarise_scores(by = c(\"model\", \"target_type\")) |> summarise_scores(by = c(\"model\", \"target_type\"), fun = signif, digits = 3) #> model target_type wis overprediction underprediction #> #> 1: EuroCOVIDhub-ensemble Cases 17900.0 10000.00 4240.0 #> 2: EuroCOVIDhub-baseline Cases 28500.0 14100.00 10300.0 #> 3: epiforecasts-EpiNow2 Cases 20800.0 11900.00 3260.0 #> 4: EuroCOVIDhub-ensemble Deaths 41.4 7.14 4.1 #> 5: EuroCOVIDhub-baseline Deaths 159.0 65.90 2.1 #> 6: UMass-MechBayes Deaths 52.7 8.98 16.8 #> 7: epiforecasts-EpiNow2 Deaths 66.6 18.90 15.9 #> dispersion bias interval_coverage_50 interval_coverage_90 #> #> 1: 3660.0 -0.05640 0.391 0.805 #> 2: 4100.0 0.09800 0.328 0.820 #> 3: 5660.0 -0.07890 0.469 0.789 #> 4: 30.2 0.07270 0.875 1.000 #> 5: 91.4 0.33900 0.664 1.000 #> 6: 26.9 -0.02230 0.461 0.875 #> 7: 31.9 -0.00513 0.420 0.908 #> interval_coverage_deviation ae_median #> #> 1: -0.1020 24100.0 #> 2: -0.1170 38500.0 #> 3: -0.0696 27900.0 #> 4: 0.2040 53.1 #> 5: 0.1210 233.0 #> 6: -0.0249 78.5 #> 7: -0.0452 105.0"},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"package-workflow","dir":"","previous_headings":"","what":"Package workflow","title":"Utilities for Scoring and Assessing Predictions","text":"following depicts suggested workflow evaluating forecasts scoringutils (sections refer paper). Please find information paper, function documentation vignettes.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"citation","dir":"","previous_headings":"","what":"Citation","title":"Utilities for Scoring and Assessing Predictions","text":"using scoringutils work please consider citing using output citation(\"scoringutils\") (print(citation(\"scoringutils\"), bibtex = TRUE)):","code":"#> To cite scoringutils in publications use the following. If you use the #> CRPS, DSS, or Log Score, please also cite scoringRules. #> #> Nikos I. Bosse, Hugo Gruson, Sebastian Funk, Anne Cori, Edwin van #> Leeuwen, and Sam Abbott (2022). Evaluating Forecasts with #> scoringutils in R, arXiv. DOI: 10.48550/ARXIV.2205.07090 #> #> To cite scoringRules in publications use: #> #> Alexander Jordan, Fabian Krueger, Sebastian Lerch (2019). Evaluating #> Probabilistic Forecasts with scoringRules. Journal of Statistical #> Software, 90(12), 1-37. DOI 10.18637/jss.v090.i12 #> #> To see these entries in BibTeX format, use 'print(, #> bibtex=TRUE)', 'toBibtex(.)', or set #> 'options(citation.bibtex.max=999)'."},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"how-to-make-a-bug-report-or-feature-request","dir":"","previous_headings":"","what":"How to make a bug report or feature request","title":"Utilities for Scoring and Assessing Predictions","text":"Please briefly describe problem output expect issue. question, please don’t open issue. Instead, ask Q page.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"contributing","dir":"","previous_headings":"","what":"Contributing","title":"Utilities for Scoring and Assessing Predictions","text":"welcome contributions new contributors! particularly appreciate help priority problems issues. Please check add issues, /add pull request.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"code-of-conduct","dir":"","previous_headings":"","what":"Code of Conduct","title":"Utilities for Scoring and Assessing Predictions","text":"Please note scoringutils project released Contributor Code Conduct. contributing project, agree abide terms.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/add_relative_skill.html","id":null,"dir":"Reference","previous_headings":"","what":"Add relative skill scores based on pairwise comparisons — add_relative_skill","title":"Add relative skill scores based on pairwise comparisons — add_relative_skill","text":"Adds columns relative skills computed running pairwise comparisons scores. information computation relative skill, see get_pairwise_comparisons(). Relative skill calculated aggregation level specified .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/add_relative_skill.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Add relative skill scores based on pairwise comparisons — add_relative_skill","text":"","code":"add_relative_skill( scores, by = \"model\", metric = intersect(c(\"wis\", \"crps\", \"brier_score\"), names(scores)), baseline = NULL )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/add_relative_skill.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Add relative skill scores based on pairwise comparisons — add_relative_skill","text":"scores object class scores (data.table scores additional attribute metrics produced score()). Character vector column names define grouping level pairwise comparisons. default (model), one relative skill score per model. , example, = c(\"model\", \"location\"). get separate relative skill score every model every location. Internally, data.table scores split according (removing \"model\" splitting) pairwise comparisons computed separately split data.tables. metric string name metric relative skill shall computed. default either \"crps\", \"wis\" \"brier_score\" available. baseline string name model. baseline given, scaled relative skill respect baseline returned. default (NULL), relative skill scaled respect baseline model.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Absolute error of the median (quantile-based version) — ae_median_quantile","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"Compute absolute error median calculated $$ \\textrm{abs}(\\textrm{observed} - \\textrm{median prediction}) $$ median prediction predicted value quantile_level == 0.5, function therefore requires 0.5 among quantile levels quantile_level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"","code":"ae_median_quantile(observed, predicted, quantile_level)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"Numeric vector length N absolute error median.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"","code":"observed <- rnorm(30, mean = 1:30) predicted_values <- replicate(3, rnorm(30, mean = 1:30)) ae_median_quantile( observed, predicted_values, quantile_level = c(0.2, 0.5, 0.8) ) #> [1] 2.47438940 0.92040530 3.55121603 0.24032512 1.79911603 2.12426222 #> [7] 2.88687498 0.37899594 0.73282842 1.41674512 0.91703692 0.34483170 #> [13] 0.72770448 1.86768569 0.80586643 2.38692128 1.12876056 0.05733376 #> [19] 0.37081463 0.82374754 1.45618892 0.93544150 2.05333481 0.18155199 #> [25] 2.43676219 1.20798000 1.67648698 0.13974346 1.26067874 1.13044854"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Absolute error of the median (sample-based version) — ae_median_sample","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"Absolute error median calculated $$% \\textrm{abs}(\\textrm{observevd} - \\textrm{median\\_prediction}) $$","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"","code":"ae_median_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"vector scoring values","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"","code":"observed <- rnorm(30, mean = 1:30) predicted_values <- matrix(rnorm(30, mean = 1:30)) ae_median_sample(observed, predicted_values) #> [1] 1.61022189 0.32735036 2.52982645 0.98458168 0.94495454 0.65538891 #> [7] 0.56511146 0.09373061 1.31110818 0.61226219 0.75386115 0.08959962 #> [13] 0.39077113 1.56818369 0.84567980 1.24260044 0.27781917 0.65054779 #> [19] 1.18084954 0.45036469 0.05976767 0.14675942 0.60583332 0.19442459 #> [25] 0.21123533 0.28585022 0.64582375 1.78993469 1.20347916 0.67902801"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/apply_metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Apply a list of functions to a data table of forecasts — apply_metrics","title":"Apply a list of functions to a data table of forecasts — apply_metrics","text":"helper function applies scoring rules (stored list functions) data table forecasts. apply_metrics used within score() apply scoring rules data. Scoring rules wrapped run_safely() catch errors make sure arguments passed scoring rule actually accepted .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/apply_metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Apply a list of functions to a data table of forecasts — apply_metrics","text":"","code":"apply_metrics(forecast, metrics, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/apply_metrics.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Apply a list of functions to a data table of forecasts — apply_metrics","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()) metrics named list scoring functions. Names used column names output. See metrics_point(), metrics_binary(), metrics_quantile(), metrics_sample() information default metrics used. Note want pass arguments given metric, function customise_metric() pass updated list functions custom metric metrics argument score(). ... Additional arguments passed scoring rules. Note currently used, calls apply_scores currently avoid passing arguments via ... instead expect metrics directly modified using customise_metric().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/apply_metrics.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Apply a list of functions to a data table of forecasts — apply_metrics","text":"data table forecasts calculated metrics.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a forecast object — as_forecast","title":"Create a forecast object — as_forecast","text":"Process validate data.frame (similar) similar forecasts observations. input passes input checks, converted forecast object. class object depends forecast type input. See details section information expected input formats. as_forecast() gives users control data parsed. Using arguments observed, predicted, model, users can rename existing columns input data match required columns forecast object. Using argument forecast_unit, users can specify columns uniquely identify single forecast (remove others, see set_forecast_unit() details).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a forecast object — as_forecast","text":"","code":"as_forecast(data, ...) # Default S3 method as_forecast( data, forecast_unit = NULL, forecast_type = NULL, observed = NULL, predicted = NULL, model = NULL, quantile_level = NULL, sample_id = NULL, ... )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a forecast object — as_forecast","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. ... Additional arguments forecast_unit (optional) Name columns data (renaming columns done as_forecast()) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. forecast_type (optional) forecast type expect forecasts . forecast type determined scoringutils based input match , error thrown. NULL (default), forecast type inferred data. observed (optional) Name column data contains observed values. column renamed \"observed\". predicted (optional) Name column data contains predicted values. column renamed \"predicted\". model (optional) Name column data contains names models/forecasters generated predicted values. column renamed \"model\". quantile_level (optional) Name column data contains quantile level predicted values. column renamed \"quantile_level\". applicable quantile-based forecasts. sample_id (optional) Name column data contains sample id. column renamed \"sample_id\". applicable sample-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Create a forecast object — as_forecast","text":"Depending forecast type, object following class returned: forecast_binary binary forecasts forecast_point point forecasts forecast_sample sample-based forecasts forecast_quantile quantile-based forecasts","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"forecast-types-and-input-formats","dir":"Reference","previous_headings":"","what":"Forecast types and input formats","title":"Create a forecast object — as_forecast","text":"Various different forecast types / forecast formats supported. moment, : point forecasts binary forecasts (\"soft binary classification\") Probabilistic forecasts quantile-based format (forecast represented set predictive quantiles) Probabilistic forecasts sample-based format (forecast represented set predictive samples) Forecast types determined based columns present input data. overview required format forecast type: forecast types require data.frame similar columns observed predicted, model. Point forecasts require column observed type numeric column predicted type numeric. Binary forecasts require column observed type factor exactly two levels column predicted type numeric probabilities, corresponding probability observed equal second factor level. See details information. Quantile-based forecasts require column observed type numeric, column predicted type numeric, column quantile_level type numeric quantile-levels (0 1). Sample-based forecasts require column observed type numeric, column predicted type numeric, column sample_id type numeric sample indices. information see vignettes example data (example_quantile, example_sample_continuous, example_sample_discrete, example_point(), example_binary).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"forecast-unit","dir":"Reference","previous_headings":"","what":"Forecast unit","title":"Create a forecast object — as_forecast","text":"order score forecasts, scoringutils needs know rows data belong together jointly form single forecasts. easy e.g. point forecast, one row per forecast. quantile sample-based forecasts, however, multiple rows belong single forecast. forecast unit unit single forecast described combination columns uniquely identify single forecast. example, forecasts made different models various locations different time points, several weeks future. forecast unit described forecast_unit = c(\"model\", \"location\", \"forecast_date\", \"forecast_horizon\"). scoringutils automatically tries determine unit single forecast. uses existing columns , means columns must present unrelated forecast unit. simplistic example, additional row, \"even\", one row number even zero otherwise, mess scoring scoringutils thinks column relevant defining forecast unit. order avoid issues, recommend setting forecast unit explicitly, usually forecast_unit argument as_forecast(). drop unneeded columns, making sure necessary, 'protected columns' like \"predicted\" \"observed\" retained.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Create a forecast object — as_forecast","text":"","code":"as_forecast(example_binary) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Forecast type: binary #> Forecast unit: #> location, location_name, target_end_date, target_type, forecast_date, model, #> and horizon #> #> location location_name target_end_date target_type forecast_date #> #> 1: DE Germany 2021-01-02 Cases #> 2: DE Germany 2021-01-02 Deaths #> 3: DE Germany 2021-01-09 Cases #> 4: DE Germany 2021-01-09 Deaths #> 5: DE Germany 2021-01-16 Cases #> --- #> 1027: IT Italy 2021-07-24 Deaths 2021-07-12 #> 1028: IT Italy 2021-07-24 Deaths 2021-07-05 #> 1029: IT Italy 2021-07-24 Deaths 2021-07-12 #> 1030: IT Italy 2021-07-24 Deaths 2021-07-05 #> 1031: IT Italy 2021-07-24 Deaths 2021-07-12 #> model horizon predicted observed #> #> 1: NA NA #> 2: NA NA #> 3: NA NA #> 4: NA NA #> 5: NA NA #> --- #> 1027: EuroCOVIDhub-baseline 2 0.250 0 #> 1028: UMass-MechBayes 3 0.475 0 #> 1029: UMass-MechBayes 2 0.450 0 #> 1030: epiforecasts-EpiNow2 3 0.375 0 #> 1031: epiforecasts-EpiNow2 2 0.300 0 as_forecast( example_quantile, forecast_unit = c(\"model\", \"target_type\", \"target_end_date\", \"horizon\", \"location\") ) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Forecast type: quantile #> Forecast unit: #> model, target_type, target_end_date, horizon, and location #> #> Key: #> observed quantile_level predicted model target_type #> #> 1: 127300 NA NA Cases #> 2: 4534 NA NA Deaths #> 3: 154922 NA NA Cases #> 4: 6117 NA NA Deaths #> 5: 110183 NA NA Cases #> --- #> 20541: 78 0.850 352 epiforecasts-EpiNow2 Deaths #> 20542: 78 0.900 397 epiforecasts-EpiNow2 Deaths #> 20543: 78 0.950 499 epiforecasts-EpiNow2 Deaths #> 20544: 78 0.975 611 epiforecasts-EpiNow2 Deaths #> 20545: 78 0.990 719 epiforecasts-EpiNow2 Deaths #> target_end_date horizon location #> #> 1: 2021-01-02 NA DE #> 2: 2021-01-02 NA DE #> 3: 2021-01-09 NA DE #> 4: 2021-01-09 NA DE #> 5: 2021-01-16 NA DE #> --- #> 20541: 2021-07-24 2 IT #> 20542: 2021-07-24 2 IT #> 20543: 2021-07-24 2 IT #> 20544: 2021-07-24 2 IT #> 20545: 2021-07-24 2 IT"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Create an object of class scores from data — as_scores","title":"Create an object of class scores from data — as_scores","text":"convenience function wraps new_scores() validates scores object.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create an object of class scores from data — as_scores","text":"","code":"as_scores(scores, metrics)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create an object of class scores from data — as_scores","text":"scores data.table similar scores produced score(). metrics character vector names scores (.e. names scoring rules used scoring).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_scores.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Create an object of class scores from data — as_scores","text":"object class scores","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_dims_ok_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","title":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","text":"Function assesses whether input dimensions match. following, n number observations / forecasts. Scalar values may repeated match length input. Allowed options therefore: observed vector length 1 length n predicted : vector length 1 length n matrix n rows 1 column","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_dims_ok_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","text":"","code":"assert_dims_ok_point(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_dims_ok_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","text":"observed Input checked. factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_dims_ok_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that input is a forecast object and passes validations — assert_forecast","title":"Assert that input is a forecast object and passes validations — assert_forecast","text":"Methods different classes run assert_forecast_generic(), performs checks forecast types perform specific checks specific forecast type.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that input is a forecast object and passes validations — assert_forecast","text":"","code":"assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) # S3 method for class 'forecast_quantile' assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) # S3 method for class 'forecast_sample' assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that input is a forecast object and passes validations — assert_forecast","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()) forecast_type (optional) forecast type expect forecasts . forecast type determined scoringutils based input match , error thrown. NULL (default), forecast type inferred data. verbose Logical. FALSE (default TRUE), messages warnings created. ... Additional arguments","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that input is a forecast object and passes validations — assert_forecast","text":"Depending forecast type, object class forecast_binary, forecast_point, forecast_sample forecast_quantile.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"forecast-types-and-input-formats","dir":"Reference","previous_headings":"","what":"Forecast types and input formats","title":"Assert that input is a forecast object and passes validations — assert_forecast","text":"Various different forecast types / forecast formats supported. moment, : point forecasts binary forecasts (\"soft binary classification\") Probabilistic forecasts quantile-based format (forecast represented set predictive quantiles) Probabilistic forecasts sample-based format (forecast represented set predictive samples) Forecast types determined based columns present input data. overview required format forecast type: forecast types require data.frame similar columns observed predicted, model. Point forecasts require column observed type numeric column predicted type numeric. Binary forecasts require column observed type factor exactly two levels column predicted type numeric probabilities, corresponding probability observed equal second factor level. See details information. Quantile-based forecasts require column observed type numeric, column predicted type numeric, column quantile_level type numeric quantile-levels (0 1). Sample-based forecasts require column observed type numeric, column predicted type numeric, column sample_id type numeric sample indices. information see vignettes example data (example_quantile, example_sample_continuous, example_sample_discrete, example_point(), example_binary).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Assert that input is a forecast object and passes validations — assert_forecast","text":"","code":"forecast <- as_forecast(example_binary) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. assert_forecast(forecast) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected."},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_generic.html","id":null,"dir":"Reference","previous_headings":"","what":"Validation common to all forecast types — assert_forecast_generic","title":"Validation common to all forecast types — assert_forecast_generic","text":"function runs input checks apply input data, regardless forecast type. function asserts forecast data.table columns observed predicted, well column called model. checks forecast type forecast unit checks duplicate forecasts appropriate, checks number samples / quantiles forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_generic.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Validation common to all forecast types — assert_forecast_generic","text":"","code":"assert_forecast_generic(data, verbose = TRUE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_generic.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Validation common to all forecast types — assert_forecast_generic","text":"data data.table forecasts observed values validated. verbose Logical. FALSE (default TRUE), messages warnings created.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_generic.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Validation common to all forecast types — assert_forecast_generic","text":"returns input","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_type.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that forecast type is as expected — assert_forecast_type","title":"Assert that forecast type is as expected — assert_forecast_type","text":"Assert forecast type expected","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_type.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that forecast type is as expected — assert_forecast_type","text":"","code":"assert_forecast_type(data, actual = get_forecast_type(data), desired = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_type.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that forecast type is as expected — assert_forecast_type","text":"data forecast object produced as_forecast(). actual actual forecast type data desired desired forecast type data","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_type.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that forecast type is as expected — assert_forecast_type","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for binary forecast — assert_input_binary","title":"Assert that inputs are correct for binary forecast — assert_input_binary","text":"Function assesses whether inputs correspond requirements scoring binary forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for binary forecast — assert_input_binary","text":"","code":"assert_input_binary(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for binary forecast — assert_input_binary","text":"observed Input checked. factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for binary forecast — assert_input_binary","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_interval.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for interval-based forecast — assert_input_interval","title":"Assert that inputs are correct for interval-based forecast — assert_input_interval","text":"Function assesses whether inputs correspond requirements scoring interval-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_interval.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for interval-based forecast — assert_input_interval","text":"","code":"assert_input_interval(observed, lower, upper, interval_range)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_interval.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for interval-based forecast — assert_input_interval","text":"observed Input checked. numeric vector observed values size n. lower Input checked. numeric vector size n holds predicted value lower bounds prediction intervals. upper Input checked. numeric vector size n holds predicted value upper bounds prediction intervals. interval_range Input checked. vector size n denotes interval range percent. E.g. value 50 denotes (25%, 75%) prediction interval.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_interval.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for interval-based forecast — assert_input_interval","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for point forecast — assert_input_point","title":"Assert that inputs are correct for point forecast — assert_input_point","text":"Function assesses whether inputs correspond requirements scoring point forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for point forecast — assert_input_point","text":"","code":"assert_input_point(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for point forecast — assert_input_point","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. numeric vector predicted values size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for point forecast — assert_input_point","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","title":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","text":"Function assesses whether inputs correspond requirements scoring quantile-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","text":"","code":"assert_input_quantile( observed, predicted, quantile_level, unique_quantile_levels = TRUE )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. nxN matrix predictive quantiles, n (number rows) number data points N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Input checked. vector size N denotes quantile levels corresponding columns prediction matrix. unique_quantile_levels Whether quantile levels required unique (TRUE, default) (FALSE).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for sample-based forecast — assert_input_sample","title":"Assert that inputs are correct for sample-based forecast — assert_input_sample","text":"Function assesses whether inputs correspond requirements scoring sample-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for sample-based forecast — assert_input_sample","text":"","code":"assert_input_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for sample-based forecast — assert_input_sample","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. numeric nxN matrix predictive samples, n (number rows) number data points N (number columns) number samples per forecast. observed just single number, predicted values can just vector size N.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for sample-based forecast — assert_input_sample","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Determines bias of quantile forecasts — bias_quantile","title":"Determines bias of quantile forecasts — bias_quantile","text":"Determines bias quantile forecasts. increasing number quantiles measure converges sample based bias version integer continuous forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Determines bias of quantile forecasts — bias_quantile","text":"","code":"bias_quantile(observed, predicted, quantile_level, na.rm = TRUE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Determines bias of quantile forecasts — bias_quantile","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made. Note contain median (0.5) median imputed mean two innermost quantiles. na.rm Logical. missing values removed?","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Determines bias of quantile forecasts — bias_quantile","text":"scalar quantile bias single quantile prediction","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Determines bias of quantile forecasts — bias_quantile","text":"quantile forecasts, bias measured $$ B_t = (1 - 2 \\cdot \\max \\{| q_{t,} \\Q_t \\land q_{t,} \\leq x_t\\}) \\mathbf{1}( x_t \\leq q_{t, 0.5}) \\\\ + (1 - 2 \\cdot \\min \\{| q_{t,} \\Q_t \\land q_{t,} \\geq x_t\\}) 1( x_t \\geq q_{t, 0.5}),$$ \\(Q_t\\) set quantiles form predictive distribution time \\(t\\) \\(x_t\\) observed value. consistency, define \\(Q_t\\) always includes element \\(q_{t, 0} = - \\infty\\) \\(q_{t,1} = \\infty\\). \\(1()\\) indicator function \\(1\\) condition satisfied \\(0\\) otherwise. clearer terms, bias \\(B_t\\) : \\(1 - 2 \\cdot\\) maximum percentile rank corresponding quantile still smaller equal observed value, observed value smaller median predictive distribution. \\(1 - 2 \\cdot\\) minimum percentile rank corresponding quantile still larger equal observed value observed value larger median predictive distribution.. \\(0\\) observed value exactly median (terms cancel ) Bias can assume values -1 1 0 ideally (.e. unbiased). Note given quantiles contain median, median imputed mean two innermost quantiles. large enough number quantiles, percentile rank equal proportion predictive samples observed value, bias metric coincides one continuous forecasts (see bias_sample()).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Determines bias of quantile forecasts — bias_quantile","text":"","code":"predicted <- matrix(c(1.5:23.5, 3.3:25.3), nrow = 2, byrow = TRUE) quantile_level <- c(0.01, 0.025, seq(0.05, 0.95, 0.05), 0.975, 0.99) observed <- c(15, 12.4) bias_quantile(observed, predicted, quantile_level) #> [1] -0.3 0.2"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile_single_vector.html","id":null,"dir":"Reference","previous_headings":"","what":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","title":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","text":"Internal function compute bias single observed value, vector predicted values vector quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile_single_vector.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","text":"","code":"bias_quantile_single_vector(observed, predicted, quantile_level, na.rm)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile_single_vector.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","text":"observed Scalar observed value. predicted Vector length N (corresponding number quantiles) holds predictions. quantile_level Vector size N quantile levels predictions made. Note contain median (0.5) median imputed mean two innermost quantiles. na.rm Logical. missing values removed?","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile_single_vector.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","text":"scalar quantile bias single quantile prediction","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Determine bias of forecasts — bias_sample","title":"Determine bias of forecasts — bias_sample","text":"Determines bias predictive Monte-Carlo samples. function automatically recognises, whether forecasts continuous integer valued adapts Bias function accordingly.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Determine bias of forecasts — bias_sample","text":"","code":"bias_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Determine bias of forecasts — bias_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Determine bias of forecasts — bias_sample","text":"Numeric vector length n biases predictive samples respect observed values.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Determine bias of forecasts — bias_sample","text":"continuous forecasts, Bias measured $$ B_t (P_t, x_t) = 1 - 2 * (P_t (x_t)) $$ \\(P_t\\) empirical cumulative distribution function prediction observed value \\(x_t\\). Computationally, \\(P_t (x_t)\\) just calculated fraction predictive samples \\(x_t\\) smaller \\(x_t\\). integer valued forecasts, Bias measured $$ B_t (P_t, x_t) = 1 - (P_t (x_t) + P_t (x_t + 1)) $$ adjust integer nature forecasts. cases, Bias can assume values -1 1 0 ideally.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Determine bias of forecasts — bias_sample","text":"integer valued Bias function discussed Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15 Funk S, Camacho , Kucharski AJ, Lowe R, Eggo RM, et al. (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15. PLOS Computational Biology 15(2): e1006785. doi:10.1371/journal.pcbi.1006785","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Determine bias of forecasts — bias_sample","text":"","code":"## integer valued forecasts observed <- rpois(30, lambda = 1:30) predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) bias_sample(observed, predicted) #> [1] 0.660 -0.025 0.710 -0.930 0.870 0.435 -0.885 -0.940 -0.515 -0.790 #> [11] 0.975 -0.975 -0.620 -0.740 -0.640 0.395 0.695 -0.765 -0.935 -0.680 #> [21] -0.725 -0.320 0.355 0.730 0.250 0.995 -0.650 0.235 0.250 0.850 ## continuous forecasts observed <- rnorm(30, mean = 1:30) predicted <- replicate(200, rnorm(30, mean = 1:30)) bias_sample(observed, predicted) #> [1] -0.46 0.02 0.02 0.12 -0.18 -0.07 0.96 -0.60 0.16 -0.31 0.79 0.49 #> [13] -0.74 -0.48 0.26 -0.56 0.82 0.89 0.41 -0.31 0.19 0.47 -0.85 0.32 #> [25] 0.15 -0.16 0.34 -0.30 0.80 0.27"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_columns_present.html","id":null,"dir":"Reference","previous_headings":"","what":"Check column names are present in a data.frame — check_columns_present","title":"Check column names are present in a data.frame — check_columns_present","text":"functions loops column names checks whether present. issue encountered, function immediately stops returns message first issue encountered.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_columns_present.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check column names are present in a data.frame — check_columns_present","text":"","code":"check_columns_present(data, columns)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_columns_present.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check column names are present in a data.frame — check_columns_present","text":"data data.frame similar checked columns character vector column names check","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_columns_present.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check column names are present in a data.frame — check_columns_present","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_dims_ok_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Check Inputs Have Matching Dimensions — check_dims_ok_point","title":"Check Inputs Have Matching Dimensions — check_dims_ok_point","text":"Function assesses whether input dimensions match. following, n number observations / forecasts. Scalar values may repeated match length input. Allowed options therefore: observed vector length 1 length n predicted : vector length 1 length n matrix n rows 1 column","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_dims_ok_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check Inputs Have Matching Dimensions — check_dims_ok_point","text":"","code":"check_dims_ok_point(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_dims_ok_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check Inputs Have Matching Dimensions — check_dims_ok_point","text":"observed Input checked. factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_dims_ok_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check Inputs Have Matching Dimensions — check_dims_ok_point","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_duplicates.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that there are no duplicate forecasts — check_duplicates","title":"Check that there are no duplicate forecasts — check_duplicates","text":"Runs get_duplicate_forecasts() returns message issue encountered","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_duplicates.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that there are no duplicate forecasts — check_duplicates","text":"","code":"check_duplicates(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_duplicates.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that there are no duplicate forecasts — check_duplicates","text":"data data.frame used score()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_duplicates.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that there are no duplicate forecasts — check_duplicates","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for binary forecast — check_input_binary","title":"Check that inputs are correct for binary forecast — check_input_binary","text":"Function assesses whether inputs correspond requirements scoring binary forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for binary forecast — check_input_binary","text":"","code":"check_input_binary(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for binary forecast — check_input_binary","text":"observed Input checked. factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for binary forecast — check_input_binary","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_interval.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for interval-based forecast — check_input_interval","title":"Check that inputs are correct for interval-based forecast — check_input_interval","text":"Function assesses whether inputs correspond requirements scoring interval-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_interval.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for interval-based forecast — check_input_interval","text":"","code":"check_input_interval(observed, lower, upper, interval_range)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_interval.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for interval-based forecast — check_input_interval","text":"observed Input checked. numeric vector observed values size n. lower Input checked. numeric vector size n holds predicted value lower bounds prediction intervals. upper Input checked. numeric vector size n holds predicted value upper bounds prediction intervals. interval_range Input checked. vector size n denotes interval range percent. E.g. value 50 denotes (25%, 75%) prediction interval.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_interval.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for interval-based forecast — check_input_interval","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for point forecast — check_input_point","title":"Check that inputs are correct for point forecast — check_input_point","text":"Function assesses whether inputs correspond requirements scoring point forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for point forecast — check_input_point","text":"","code":"check_input_point(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for point forecast — check_input_point","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. numeric vector predicted values size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for point forecast — check_input_point","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for quantile-based forecast — check_input_quantile","title":"Check that inputs are correct for quantile-based forecast — check_input_quantile","text":"Function assesses whether inputs correspond requirements scoring quantile-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for quantile-based forecast — check_input_quantile","text":"","code":"check_input_quantile(observed, predicted, quantile_level)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for quantile-based forecast — check_input_quantile","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. nxN matrix predictive quantiles, n (number rows) number data points N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Input checked. vector size N denotes quantile levels corresponding columns prediction matrix.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for quantile-based forecast — check_input_quantile","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for sample-based forecast — check_input_sample","title":"Check that inputs are correct for sample-based forecast — check_input_sample","text":"Function assesses whether inputs correspond requirements scoring sample-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for sample-based forecast — check_input_sample","text":"","code":"check_input_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for sample-based forecast — check_input_sample","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. numeric nxN matrix predictive samples, n (number rows) number data points N (number columns) number samples per forecast. observed just single number, predicted values can just vector size N.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for sample-based forecast — check_input_sample","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_number_per_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that all forecasts have the same number of quantiles or samples — check_number_per_forecast","title":"Check that all forecasts have the same number of quantiles or samples — check_number_per_forecast","text":"Function checks number quantiles samples per forecast. number quantiles samples forecasts, returns TRUE string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_number_per_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that all forecasts have the same number of quantiles or samples — check_number_per_forecast","text":"","code":"check_number_per_forecast(data, forecast_unit)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_number_per_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that all forecasts have the same number of quantiles or samples — check_number_per_forecast","text":"data data.frame similar checked forecast_unit Character vector denoting unit single forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_number_per_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that all forecasts have the same number of quantiles or samples — check_number_per_forecast","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_numeric_vector.html","id":null,"dir":"Reference","previous_headings":"","what":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","title":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","text":"Helper function check whether input numeric vector.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_numeric_vector.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","text":"","code":"check_numeric_vector(x, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_numeric_vector.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","text":"x input check ... Arguments passed checkmate::check_numeric lower [numeric(1)] Lower value elements x must greater equal . upper [numeric(1)] Upper value elements x must lower equal . finite [logical(1)] Check finite values? Default FALSE. .missing [logical(1)] vectors missing values allowed? Default TRUE. .missing [logical(1)] vectors non-missing values allowed? Default TRUE. Note empty vectors non-missing values. len [integer(1)] Exact expected length x. min.len [integer(1)] Minimal length x. max.len [integer(1)] Maximal length x. unique [logical(1)] Must values unique? Default FALSE. sorted [logical(1)] Elements must sorted ascending order. Missing values ignored. names [character(1)] Check names. See checkNamed possible values. Default “” performs check . Note can use checkSubset check specific set names. typed.missing [logical(1)] set FALSE (default), types missing values (NA, NA_integer_, NA_real_, NA_character_ NA_character_) well empty vectors allowed type-checking atomic input. Set TRUE enable strict type checking. null.ok [logical(1)] set TRUE, x may also NULL. case type check x performed, additional checks disabled.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_numeric_vector.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_try.html","id":null,"dir":"Reference","previous_headings":"","what":"Helper function to convert assert statements into checks — check_try","title":"Helper function to convert assert statements into checks — check_try","text":"Tries execute expression. Internally, used see whether assertions fail checking inputs (.e. convert assert_*() statement check). expression fails, error message returned. expression succeeds, TRUE returned.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_try.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Helper function to convert assert statements into checks — check_try","text":"","code":"check_try(expr)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_try.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Helper function to convert assert statements into checks — check_try","text":"expr expression evaluated","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_try.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Helper function to convert assert statements into checks — check_try","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/clean_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Clean forecast object — clean_forecast","title":"Clean forecast object — clean_forecast","text":"function makes possible silently validate object. addition, can return copy data remove rows missing values.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/clean_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Clean forecast object — clean_forecast","text":"","code":"clean_forecast(forecast, copy = FALSE, na.omit = FALSE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/clean_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Clean forecast object — clean_forecast","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()) copy Logical, default FALSE. TRUE, copy input data created. na.omit Logical, default FALSE. TRUE, rows missing values removed.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_two_models.html","id":null,"dir":"Reference","previous_headings":"","what":"Compare two models based on subset of common forecasts — compare_two_models","title":"Compare two models based on subset of common forecasts — compare_two_models","text":"function compares two models based subset forecasts models made prediction. gets called pairwise_comparison_one_group(), handles comparison multiple models single set forecasts (subsets forecasts distinguished). pairwise_comparison_one_group() turn gets called get_pairwise_comparisons() can handle pairwise comparisons set forecasts multiple subsets, e.g. pairwise comparisons one set forecasts, done separately two different forecast targets.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_two_models.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Compare two models based on subset of common forecasts — compare_two_models","text":"","code":"compare_two_models( scores, name_model1, name_model2, metric, one_sided = FALSE, test_type = c(\"non_parametric\", \"permutation\"), n_permutations = 999 )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_two_models.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Compare two models based on subset of common forecasts — compare_two_models","text":"scores object class scores (data.table scores additional attribute metrics produced score()). name_model1 Character, name first model name_model2 Character, name model compare metric string name metric relative skill shall computed. default either \"crps\", \"wis\" \"brier_score\" available. one_sided Boolean, default FALSE, whether two conduct one-sided instead two-sided test determine significance pairwise comparison. test_type Character, either \"non_parametric\" (default) \"permutation\". determines kind test shall conducted determine p-values. n_permutations Numeric, number permutations permutation test. Default 999.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_two_models.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Compare two models based on subset of common forecasts — compare_two_models","text":"list mean score ratios p-values comparison two models","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_two_models.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Compare two models based on subset of common forecasts — compare_two_models","text":"Johannes Bracher, johannes.bracher@kit.edu Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"(Continuous) ranked probability score — crps_sample","title":"(Continuous) ranked probability score — crps_sample","text":"Wrapper around crps_sample() function scoringRules package. Can used continuous well integer valued forecasts","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"(Continuous) ranked probability score — crps_sample","text":"","code":"crps_sample(observed, predicted, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"(Continuous) ranked probability score — crps_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. ... Additional arguments passed crps_sample() scoringRules package.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"(Continuous) ranked probability score — crps_sample","text":"Vector scores.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"(Continuous) ranked probability score — crps_sample","text":"Alexander Jordan, Fabian Krüger, Sebastian Lerch, Evaluating Probabilistic Forecasts scoringRules, https://www.jstatsoft.org/article/view/v090i12","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"(Continuous) ranked probability score — crps_sample","text":"","code":"observed <- rpois(30, lambda = 1:30) predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) crps_sample(observed, predicted) #> [1] 0.231225 0.329850 1.060100 0.412625 0.991550 0.793850 2.847025 1.126200 #> [9] 0.690675 0.992875 1.118300 4.911975 4.135800 1.411050 0.967825 3.425125 #> [17] 0.839600 2.404700 1.376075 1.665525 2.771375 1.168550 1.244700 2.377250 #> [25] 2.847875 2.019175 1.827975 2.074850 1.868775 3.171950"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/customise_metric.html","id":null,"dir":"Reference","previous_headings":"","what":"Customises a metric function with additional arguments. — customise_metric","title":"Customises a metric function with additional arguments. — customise_metric","text":"function takes metric function additional arguments, returns new function includes additional arguments calling original metric function. expected way pass additional arguments metric evaluating forecast using score(): evaluate forecast using metric additional argument, need create custom version scoring function argument included. need create updated version list scoring functions includes customised metric pass score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/customise_metric.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Customises a metric function with additional arguments. — customise_metric","text":"","code":"customise_metric(metric, ...) customize_metric(metric, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/customise_metric.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Customises a metric function with additional arguments. — customise_metric","text":"metric metric function customised. ... Additional arguments included calling metric function.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/customise_metric.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Customises a metric function with additional arguments. — customise_metric","text":"customised metric function.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/customise_metric.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Customises a metric function with additional arguments. — customise_metric","text":"","code":"# Create a customised metric function custom_metric <- customise_metric(mean, na.rm = TRUE) # Use the customised metric function values <- c(1, 2, NA, 4, 5) custom_metric(values) #> [1] 3 # Updating metrics list to calculate 70% coverage in `score()` interval_coverage_70 <- customise_metric( interval_coverage, interval_range = 70 ) updated_metrics <- c( metrics_quantile(), \"interval_coverage_70\" = interval_coverage_70 ) score( as_forecast(example_quantile), metrics = updated_metrics ) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> location target_end_date target_type location_name forecast_date #> #> 1: DE 2021-05-08 Cases Germany 2021-05-03 #> 2: DE 2021-05-08 Cases Germany 2021-05-03 #> 3: DE 2021-05-08 Cases Germany 2021-05-03 #> 4: DE 2021-05-08 Deaths Germany 2021-05-03 #> 5: DE 2021-05-08 Deaths Germany 2021-05-03 #> --- #> 883: IT 2021-07-24 Deaths Italy 2021-07-12 #> 884: IT 2021-07-24 Deaths Italy 2021-07-05 #> 885: IT 2021-07-24 Deaths Italy 2021-07-12 #> 886: IT 2021-07-24 Deaths Italy 2021-07-05 #> 887: IT 2021-07-24 Deaths Italy 2021-07-12 #> model horizon wis overprediction underprediction #> #> 1: EuroCOVIDhub-ensemble 1 7990.854783 2.549870e+03 0.0000000 #> 2: EuroCOVIDhub-baseline 1 16925.046957 1.527583e+04 0.0000000 #> 3: epiforecasts-EpiNow2 1 25395.960870 1.722226e+04 0.0000000 #> 4: EuroCOVIDhub-ensemble 1 53.880000 0.000000e+00 0.6086957 #> 5: EuroCOVIDhub-baseline 1 46.793043 2.130435e+00 0.0000000 #> --- #> 883: EuroCOVIDhub-baseline 2 80.336957 3.608696e+00 0.0000000 #> 884: UMass-MechBayes 3 4.881739 4.347826e-02 0.0000000 #> 885: UMass-MechBayes 2 25.581739 1.782609e+01 0.0000000 #> 886: epiforecasts-EpiNow2 3 19.762609 5.478261e+00 0.0000000 #> 887: epiforecasts-EpiNow2 2 66.161739 4.060870e+01 0.0000000 #> dispersion bias interval_coverage_50 interval_coverage_90 #> #> 1: 5440.985217 0.50 TRUE TRUE #> 2: 1649.220870 0.95 FALSE FALSE #> 3: 8173.700000 0.90 FALSE TRUE #> 4: 53.271304 -0.10 TRUE TRUE #> 5: 44.662609 0.30 TRUE TRUE #> --- #> 883: 76.728261 0.20 TRUE TRUE #> 884: 4.838261 0.10 TRUE TRUE #> 885: 7.755652 0.80 FALSE TRUE #> 886: 14.284348 0.50 TRUE TRUE #> 887: 25.553043 0.90 FALSE TRUE #> interval_coverage_deviation ae_median interval_coverage_70 #> #> 1: 0.05181818 12271 TRUE #> 2: -0.40272727 25620 FALSE #> 3: -0.31181818 44192 FALSE #> 4: 0.41545455 14 TRUE #> 5: 0.23363636 15 TRUE #> --- #> 883: 0.32454545 53 TRUE #> 884: 0.41545455 1 TRUE #> 885: -0.22090909 46 FALSE #> 886: 0.05181818 26 TRUE #> 887: -0.31181818 108 FALSE"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_assert_functions.html","id":null,"dir":"Reference","previous_headings":"","what":"Documentation template for assert functions — document_assert_functions","title":"Documentation template for assert functions — document_assert_functions","text":"Documentation template assert functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_assert_functions.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Documentation template for assert functions — document_assert_functions","text":"observed Input checked. numeric vector observed values size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_assert_functions.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Documentation template for assert functions — document_assert_functions","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_check_functions.html","id":null,"dir":"Reference","previous_headings":"","what":"Documentation template for check functions — document_check_functions","title":"Documentation template for check functions — document_check_functions","text":"Documentation template check functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_check_functions.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Documentation template for check functions — document_check_functions","text":"data data.frame similar checked observed Input checked. numeric vector observed values size n. columns character vector column names check","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_check_functions.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Documentation template for check functions — document_check_functions","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_test_functions.html","id":null,"dir":"Reference","previous_headings":"","what":"Documentation template for test functions — document_test_functions","title":"Documentation template for test functions — document_test_functions","text":"Documentation template test functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_test_functions.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Documentation template for test functions — document_test_functions","text":"Returns TRUE check successful FALSE otherwise","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Dawid-Sebastiani score — dss_sample","title":"Dawid-Sebastiani score — dss_sample","text":"Wrapper around dss_sample() function scoringRules package.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Dawid-Sebastiani score — dss_sample","text":"","code":"dss_sample(observed, predicted, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Dawid-Sebastiani score — dss_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. ... Additional arguments passed dss_sample() scoringRules package.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Dawid-Sebastiani score — dss_sample","text":"Vector scores.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Dawid-Sebastiani score — dss_sample","text":"Alexander Jordan, Fabian Krüger, Sebastian Lerch, Evaluating Probabilistic Forecasts scoringRules, https://www.jstatsoft.org/article/view/v090i12","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Dawid-Sebastiani score — dss_sample","text":"","code":"observed <- rpois(30, lambda = 1:30) predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) dss_sample(observed, predicted) #> [1] -0.06179111 2.81530809 2.46742176 1.79360855 1.61740613 2.39081466 #> [7] 1.92865409 15.58374079 3.88246189 4.97070116 2.24551342 2.71528477 #> [13] 2.79485162 2.62249405 2.80770087 4.30015607 3.23096351 3.16329747 #> [19] 3.23001192 7.67628290 2.92052357 5.37555852 4.78857945 3.12865561 #> [25] 3.61999703 4.32028466 3.38593521 6.87116360 4.71995903 4.99107110"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_data.table.html","id":null,"dir":"Reference","previous_headings":"","what":"Ensure that an object is a data.table — ensure_data.table","title":"Ensure that an object is a data.table — ensure_data.table","text":"function ensures object data table. object data table, converted one. object data table, copy object returned.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_data.table.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Ensure that an object is a data.table — ensure_data.table","text":"","code":"ensure_data.table(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_data.table.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Ensure that an object is a data.table — ensure_data.table","text":"data object ensure data table.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_data.table.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Ensure that an object is a data.table — ensure_data.table","text":"data.table/copy existing data.table.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_model_column.html","id":null,"dir":"Reference","previous_headings":"","what":"Assure that data has a model column — ensure_model_column","title":"Assure that data has a model column — ensure_model_column","text":"Check whether data.table column called model. , column called model added value Unspecified model.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_model_column.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assure that data has a model column — ensure_model_column","text":"","code":"ensure_model_column(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_model_column.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assure that data has a model column — ensure_model_column","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_model_column.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assure that data has a model column — ensure_model_column","text":"data.table column called model","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Binary forecast example data — example_binary","title":"Binary forecast example data — example_binary","text":"data set binary predictions COVID-19 cases deaths constructed data submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Binary forecast example data — example_binary","text":"","code":"example_binary"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Binary forecast example data — example_binary","text":"data frame following columns: location country prediction made location_name name country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed factor observed values forecast_date date prediction made model name model generated forecasts horizon forecast horizon weeks predicted predicted value","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Binary forecast example data — example_binary","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Binary forecast example data — example_binary","text":"Predictions data set constructed based continuous example data looking number samples mean prediction. outcome constructed whether actually observed value mean prediction. understood sound statistical practice, rather practical way create example data set. data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Point forecast example data — example_point","title":"Point forecast example data — example_point","text":"data set predictions COVID-19 cases deaths submitted European Forecast Hub. data set like quantile example data, median replaced point forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Point forecast example data — example_point","text":"","code":"example_point"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Point forecast example data — example_point","text":"data frame following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed observed values location_name name country prediction made forecast_date date prediction made predicted predicted value model name model generated forecasts horizon forecast horizon weeks","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Point forecast example data — example_point","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Point forecast example data — example_point","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Quantile example data — example_quantile","title":"Quantile example data — example_quantile","text":"data set predictions COVID-19 cases deaths submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Quantile example data — example_quantile","text":"","code":"example_quantile"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Quantile example data — example_quantile","text":"data frame following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed Numeric: observed values location_name name country prediction made forecast_date date prediction made quantile_level quantile level corresponding prediction predicted predicted value model name model generated forecasts horizon forecast horizon weeks","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Quantile example data — example_quantile","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Quantile example data — example_quantile","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":null,"dir":"Reference","previous_headings":"","what":"Continuous forecast example data — example_sample_continuous","title":"Continuous forecast example data — example_sample_continuous","text":"data set continuous predictions COVID-19 cases deaths constructed data submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Continuous forecast example data — example_sample_continuous","text":"","code":"example_sample_continuous"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Continuous forecast example data — example_sample_continuous","text":"data frame following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed observed values location_name name country prediction made forecast_date date prediction made model name model generated forecasts horizon forecast horizon weeks predicted predicted value sample_id id corresponding sample","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Continuous forecast example data — example_sample_continuous","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Continuous forecast example data — example_sample_continuous","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":null,"dir":"Reference","previous_headings":"","what":"Discrete forecast example data — example_sample_discrete","title":"Discrete forecast example data — example_sample_discrete","text":"data set integer predictions COVID-19 cases deaths constructed data submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Discrete forecast example data — example_sample_discrete","text":"","code":"example_sample_discrete"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Discrete forecast example data — example_sample_discrete","text":"data frame following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed observed values location_name name country prediction made forecast_date date prediction made model name model generated forecasts horizon forecast horizon weeks predicted predicted value sample_id id corresponding sample","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Discrete forecast example data — example_sample_discrete","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Discrete forecast example data — example_sample_discrete","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/forecast_types.html","id":null,"dir":"Reference","previous_headings":"","what":"Documentation template for forecast types — forecast_types","title":"Documentation template for forecast types — forecast_types","text":"Documentation template forecast types","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/forecast_types.html","id":"forecast-types-and-input-formats","dir":"Reference","previous_headings":"","what":"Forecast types and input formats","title":"Documentation template for forecast types — forecast_types","text":"Various different forecast types / forecast formats supported. moment, : point forecasts binary forecasts (\"soft binary classification\") Probabilistic forecasts quantile-based format (forecast represented set predictive quantiles) Probabilistic forecasts sample-based format (forecast represented set predictive samples) Forecast types determined based columns present input data. overview required format forecast type: forecast types require data.frame similar columns observed predicted, model. Point forecasts require column observed type numeric column predicted type numeric. Binary forecasts require column observed type factor exactly two levels column predicted type numeric probabilities, corresponding probability observed equal second factor level. See details information. Quantile-based forecasts require column observed type numeric, column predicted type numeric, column quantile_level type numeric quantile-levels (0 1). Sample-based forecasts require column observed type numeric, column predicted type numeric, column sample_id type numeric sample indices. information see vignettes example data (example_quantile, example_sample_continuous, example_sample_discrete, example_point(), example_binary).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/forecast_types.html","id":"forecast-unit","dir":"Reference","previous_headings":"","what":"Forecast unit","title":"Documentation template for forecast types — forecast_types","text":"order score forecasts, scoringutils needs know rows data belong together jointly form single forecasts. easy e.g. point forecast, one row per forecast. quantile sample-based forecasts, however, multiple rows belong single forecast. forecast unit unit single forecast described combination columns uniquely identify single forecast. example, forecasts made different models various locations different time points, several weeks future. forecast unit described forecast_unit = c(\"model\", \"location\", \"forecast_date\", \"forecast_horizon\"). scoringutils automatically tries determine unit single forecast. uses existing columns , means columns must present unrelated forecast unit. simplistic example, additional row, \"even\", one row number even zero otherwise, mess scoring scoringutils thinks column relevant defining forecast unit. order avoid issues, recommend setting forecast unit explicitly, usually forecast_unit argument as_forecast(). drop unneeded columns, making sure necessary, 'protected columns' like \"predicted\" \"observed\" retained.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":null,"dir":"Reference","previous_headings":"","what":"Calculate geometric mean — geometric_mean","title":"Calculate geometric mean — geometric_mean","text":"Calculate geometric mean","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Calculate geometric mean — geometric_mean","text":"","code":"geometric_mean(x)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Calculate geometric mean — geometric_mean","text":"x Numeric vector values calculate geometric mean.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Calculate geometric mean — geometric_mean","text":"geometric mean values x. NA values ignored.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Calculate geometric mean — geometric_mean","text":"Used get_pairwise_comparisons().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":null,"dir":"Reference","previous_headings":"","what":"Calculate correlation between metrics — get_correlations","title":"Calculate correlation between metrics — get_correlations","text":"Calculate correlation different metrics data.frame scores produced score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Calculate correlation between metrics — get_correlations","text":"","code":"get_correlations(scores, metrics = get_metrics(scores), digits = NULL, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Calculate correlation between metrics — get_correlations","text":"scores object class scores (data.table scores additional attribute metrics produced score()). metrics character vector metrics show. set NULL (default), metrics present scores shown. digits number indicating many decimal places result rounded . default (digits = NULL) rounding takes place. ... Additional arguments pass cor().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Calculate correlation between metrics — get_correlations","text":"object class scores (data.table additional attribute metrics holding names scores) correlations different metrics","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Calculate correlation between metrics — get_correlations","text":"","code":"scores <- score(as_forecast(example_quantile)) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. get_correlations(scores, digits = 2) #> wis overprediction underprediction dispersion bias interval_coverage_50 #> #> 1: 1.00 0.94 0.28 0.46 0.11 -0.21 #> 2: 0.94 1.00 -0.03 0.32 0.22 -0.15 #> 3: 0.28 -0.03 1.00 0.15 -0.35 -0.21 #> 4: 0.46 0.32 0.15 1.00 0.11 -0.09 #> 5: 0.11 0.22 -0.35 0.11 1.00 0.01 #> 6: -0.21 -0.15 -0.21 -0.09 0.01 1.00 #> 7: -0.41 -0.32 -0.36 -0.09 0.10 0.37 #> 8: -0.34 -0.25 -0.33 -0.12 0.06 0.85 #> 9: 0.99 0.90 0.34 0.54 0.10 -0.25 #> interval_coverage_90 interval_coverage_deviation ae_median #> #> 1: -0.41 -0.34 0.99 #> 2: -0.32 -0.25 0.90 #> 3: -0.36 -0.33 0.34 #> 4: -0.09 -0.12 0.54 #> 5: 0.10 0.06 0.10 #> 6: 0.37 0.85 -0.25 #> 7: 1.00 0.64 -0.41 #> 8: 0.64 1.00 -0.38 #> 9: -0.41 -0.38 1.00 #> metric #> #> 1: wis #> 2: overprediction #> 3: underprediction #> 4: dispersion #> 5: bias #> 6: interval_coverage_50 #> 7: interval_coverage_90 #> 8: interval_coverage_deviation #> 9: ae_median"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"validated forecast object quantile-based format (see as_forecast() information), function computes: interval coverage central prediction intervals quantile coverage predictive quantiles deviation desired actual coverage (interval quantile coverage) Coverage values computed specific level grouping, specified argument. default, coverage values computed per model. Interval coverage Interval coverage given interval range defined proportion observations fall within corresponding central prediction intervals. Central prediction intervals symmetric around median formed two quantiles denote lower upper bound. example, 50% central prediction interval interval 0.25 0.75 quantiles predictive distribution. Quantile coverage Quantile coverage given quantile level defined proportion observed values smaller corresponding predictive quantile. example, 0.5 quantile coverage proportion observed values smaller 0.5 quantile predictive distribution. Just , single observation quantile single predictive distribution, value either TRUE FALSE. Coverage deviation coverage deviation difference desired coverage (can either interval quantile coverage) actual coverage. example, desired coverage 90% actual coverage 80%, coverage deviation -0.1.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"","code":"get_coverage(forecast, by = \"model\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()) character vector denotes level grouping coverage values computed. default (\"model\"), one coverage value per model returned.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"data.table columns specified additional columns coverage values described data.table columns \"interval_coverage\", \"interval_coverage_deviation\", \"quantile_coverage\", \"quantile_coverage_deviation\" columns specified .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"","code":"library(magrittr) # pipe operator example_quantile %>% as_forecast() %>% get_coverage(by = \"model\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> model quantile_level interval_range interval_coverage #> #> 1: EuroCOVIDhub-baseline 0.500 0 0.000000000 #> 2: EuroCOVIDhub-baseline 0.450 10 0.085937500 #> 3: EuroCOVIDhub-baseline 0.550 10 0.085937500 #> 4: EuroCOVIDhub-baseline 0.400 20 0.191406250 #> 5: EuroCOVIDhub-baseline 0.600 20 0.191406250 #> 6: EuroCOVIDhub-baseline 0.350 30 0.289062500 #> 7: EuroCOVIDhub-baseline 0.650 30 0.289062500 #> 8: EuroCOVIDhub-baseline 0.300 40 0.375000000 #> 9: EuroCOVIDhub-baseline 0.700 40 0.375000000 #> 10: EuroCOVIDhub-baseline 0.250 50 0.496093750 #> 11: EuroCOVIDhub-baseline 0.750 50 0.496093750 #> 12: EuroCOVIDhub-baseline 0.200 60 0.628906250 #> 13: EuroCOVIDhub-baseline 0.800 60 0.628906250 #> 14: EuroCOVIDhub-baseline 0.150 70 0.773437500 #> 15: EuroCOVIDhub-baseline 0.850 70 0.773437500 #> 16: EuroCOVIDhub-baseline 0.100 80 0.843750000 #> 17: EuroCOVIDhub-baseline 0.900 80 0.843750000 #> 18: EuroCOVIDhub-baseline 0.050 90 0.910156250 #> 19: EuroCOVIDhub-baseline 0.950 90 0.910156250 #> 20: EuroCOVIDhub-baseline 0.025 95 0.925781250 #> 21: EuroCOVIDhub-baseline 0.975 95 0.925781250 #> 22: EuroCOVIDhub-baseline 0.010 98 0.933593750 #> 23: EuroCOVIDhub-baseline 0.990 98 0.933593750 #> 24: EuroCOVIDhub-ensemble 0.500 0 0.003906250 #> 25: EuroCOVIDhub-ensemble 0.450 10 0.148437500 #> 26: EuroCOVIDhub-ensemble 0.550 10 0.148437500 #> 27: EuroCOVIDhub-ensemble 0.400 20 0.250000000 #> 28: EuroCOVIDhub-ensemble 0.600 20 0.250000000 #> 29: EuroCOVIDhub-ensemble 0.350 30 0.386718750 #> 30: EuroCOVIDhub-ensemble 0.650 30 0.386718750 #> 31: EuroCOVIDhub-ensemble 0.300 40 0.519531250 #> 32: EuroCOVIDhub-ensemble 0.700 40 0.519531250 #> 33: EuroCOVIDhub-ensemble 0.250 50 0.632812500 #> 34: EuroCOVIDhub-ensemble 0.750 50 0.632812500 #> 35: EuroCOVIDhub-ensemble 0.200 60 0.667968750 #> 36: EuroCOVIDhub-ensemble 0.800 60 0.667968750 #> 37: EuroCOVIDhub-ensemble 0.150 70 0.753906250 #> 38: EuroCOVIDhub-ensemble 0.850 70 0.753906250 #> 39: EuroCOVIDhub-ensemble 0.100 80 0.816406250 #> 40: EuroCOVIDhub-ensemble 0.900 80 0.816406250 #> 41: EuroCOVIDhub-ensemble 0.050 90 0.902343750 #> 42: EuroCOVIDhub-ensemble 0.950 90 0.902343750 #> 43: EuroCOVIDhub-ensemble 0.025 95 0.941406250 #> 44: EuroCOVIDhub-ensemble 0.975 95 0.941406250 #> 45: EuroCOVIDhub-ensemble 0.010 98 0.968750000 #> 46: EuroCOVIDhub-ensemble 0.990 98 0.968750000 #> 47: epiforecasts-EpiNow2 0.500 0 0.004048583 #> 48: epiforecasts-EpiNow2 0.450 10 0.093117409 #> 49: epiforecasts-EpiNow2 0.550 10 0.093117409 #> 50: epiforecasts-EpiNow2 0.400 20 0.165991903 #> 51: epiforecasts-EpiNow2 0.600 20 0.165991903 #> 52: epiforecasts-EpiNow2 0.350 30 0.230769231 #> 53: epiforecasts-EpiNow2 0.650 30 0.230769231 #> 54: epiforecasts-EpiNow2 0.300 40 0.319838057 #> 55: epiforecasts-EpiNow2 0.700 40 0.319838057 #> 56: epiforecasts-EpiNow2 0.250 50 0.445344130 #> 57: epiforecasts-EpiNow2 0.750 50 0.445344130 #> 58: epiforecasts-EpiNow2 0.200 60 0.538461538 #> 59: epiforecasts-EpiNow2 0.800 60 0.538461538 #> 60: epiforecasts-EpiNow2 0.150 70 0.635627530 #> 61: epiforecasts-EpiNow2 0.850 70 0.635627530 #> 62: epiforecasts-EpiNow2 0.100 80 0.732793522 #> 63: epiforecasts-EpiNow2 0.900 80 0.732793522 #> 64: epiforecasts-EpiNow2 0.050 90 0.846153846 #> 65: epiforecasts-EpiNow2 0.950 90 0.846153846 #> 66: epiforecasts-EpiNow2 0.025 95 0.874493927 #> 67: epiforecasts-EpiNow2 0.975 95 0.874493927 #> 68: epiforecasts-EpiNow2 0.010 98 0.910931174 #> 69: epiforecasts-EpiNow2 0.990 98 0.910931174 #> 70: UMass-MechBayes 0.500 0 0.015625000 #> 71: UMass-MechBayes 0.450 10 0.101562500 #> 72: UMass-MechBayes 0.550 10 0.101562500 #> 73: UMass-MechBayes 0.400 20 0.195312500 #> 74: UMass-MechBayes 0.600 20 0.195312500 #> 75: UMass-MechBayes 0.350 30 0.281250000 #> 76: UMass-MechBayes 0.650 30 0.281250000 #> 77: UMass-MechBayes 0.300 40 0.382812500 #> 78: UMass-MechBayes 0.700 40 0.382812500 #> 79: UMass-MechBayes 0.250 50 0.460937500 #> 80: UMass-MechBayes 0.750 50 0.460937500 #> 81: UMass-MechBayes 0.200 60 0.539062500 #> 82: UMass-MechBayes 0.800 60 0.539062500 #> 83: UMass-MechBayes 0.150 70 0.617187500 #> 84: UMass-MechBayes 0.850 70 0.617187500 #> 85: UMass-MechBayes 0.100 80 0.765625000 #> 86: UMass-MechBayes 0.900 80 0.765625000 #> 87: UMass-MechBayes 0.050 90 0.875000000 #> 88: UMass-MechBayes 0.950 90 0.875000000 #> 89: UMass-MechBayes 0.025 95 0.953125000 #> 90: UMass-MechBayes 0.975 95 0.953125000 #> 91: UMass-MechBayes 0.010 98 0.984375000 #> 92: UMass-MechBayes 0.990 98 0.984375000 #> model quantile_level interval_range interval_coverage #> interval_coverage_deviation quantile_coverage quantile_coverage_deviation #> #> 1: 0.000000000 0.69921875 0.199218750 #> 2: -0.014062500 0.65625000 0.206250000 #> 3: -0.014062500 0.74218750 0.192187500 #> 4: -0.008593750 0.58593750 0.185937500 #> 5: -0.008593750 0.77343750 0.173437500 #> 6: -0.010937500 0.52343750 0.173437500 #> 7: -0.010937500 0.80859375 0.158593750 #> 8: -0.025000000 0.46875000 0.168750000 #> 9: -0.025000000 0.84375000 0.143750000 #> 10: -0.003906250 0.36718750 0.117187500 #> 11: -0.003906250 0.86328125 0.113281250 #> 12: 0.028906250 0.25000000 0.050000000 #> 13: 0.028906250 0.87500000 0.075000000 #> 14: 0.073437500 0.13281250 -0.017187500 #> 15: 0.073437500 0.90625000 0.056250000 #> 16: 0.043750000 0.08203125 -0.017968750 #> 17: 0.043750000 0.92578125 0.025781250 #> 18: 0.010156250 0.04296875 -0.007031250 #> 19: 0.010156250 0.95312500 0.003125000 #> 20: -0.024218750 0.03125000 0.006250000 #> 21: -0.024218750 0.95703125 -0.017968750 #> 22: -0.046406250 0.03125000 0.021250000 #> 23: -0.046406250 0.96484375 -0.025156250 #> 24: 0.003906250 0.53125000 0.031250000 #> 25: 0.048437500 0.46484375 0.014843750 #> 26: 0.048437500 0.60156250 0.051562500 #> 27: 0.050000000 0.40625000 0.006250000 #> 28: 0.050000000 0.65625000 0.056250000 #> 29: 0.086718750 0.33984375 -0.010156250 #> 30: 0.086718750 0.72656250 0.076562500 #> 31: 0.119531250 0.26562500 -0.034375000 #> 32: 0.119531250 0.76562500 0.065625000 #> 33: 0.132812500 0.16406250 -0.085937500 #> 34: 0.132812500 0.79687500 0.046875000 #> 35: 0.067968750 0.14062500 -0.059375000 #> 36: 0.067968750 0.80468750 0.004687500 #> 37: 0.053906250 0.10156250 -0.048437500 #> 38: 0.053906250 0.85546875 0.005468750 #> 39: 0.016406250 0.07812500 -0.021875000 #> 40: 0.016406250 0.89453125 -0.005468750 #> 41: 0.002343750 0.04296875 -0.007031250 #> 42: 0.002343750 0.94531250 -0.004687500 #> 43: -0.008593750 0.03125000 0.006250000 #> 44: -0.008593750 0.97265625 -0.002343750 #> 45: -0.011250000 0.01562500 0.005625000 #> 46: -0.011250000 0.98437500 -0.005625000 #> 47: 0.004048583 0.49392713 -0.006072874 #> 48: -0.006882591 0.43724696 -0.012753036 #> 49: -0.006882591 0.53036437 -0.019635628 #> 50: -0.034008097 0.39676113 -0.003238866 #> 51: -0.034008097 0.55870445 -0.041295547 #> 52: -0.069230769 0.36437247 0.014372470 #> 53: -0.069230769 0.59514170 -0.054858300 #> 54: -0.080161943 0.31983806 0.019838057 #> 55: -0.080161943 0.63967611 -0.060323887 #> 56: -0.054655870 0.26315789 0.013157895 #> 57: -0.054655870 0.70445344 -0.045546559 #> 58: -0.061538462 0.20647773 0.006477733 #> 59: -0.061538462 0.74493927 -0.055060729 #> 60: -0.064372470 0.14574899 -0.004251012 #> 61: -0.064372470 0.78137652 -0.068623482 #> 62: -0.067206478 0.10121457 0.001214575 #> 63: -0.067206478 0.83400810 -0.065991903 #> 64: -0.053846154 0.06072874 0.010728745 #> 65: -0.053846154 0.90688259 -0.043117409 #> 66: -0.075506073 0.04858300 0.023582996 #> 67: -0.075506073 0.92307692 -0.051923077 #> 68: -0.069068826 0.02429150 0.014291498 #> 69: -0.069068826 0.93522267 -0.054777328 #> 70: 0.015625000 0.50000000 0.000000000 #> 71: 0.001562500 0.42187500 -0.028125000 #> 72: 0.001562500 0.52343750 -0.026562500 #> 73: -0.004687500 0.37500000 -0.025000000 #> 74: -0.004687500 0.57031250 -0.029687500 #> 75: -0.018750000 0.35156250 0.001562500 #> 76: -0.018750000 0.61718750 -0.032812500 #> 77: -0.017187500 0.28906250 -0.010937500 #> 78: -0.017187500 0.66406250 -0.035937500 #> 79: -0.039062500 0.27343750 0.023437500 #> 80: -0.039062500 0.71875000 -0.031250000 #> 81: -0.060937500 0.24218750 0.042187500 #> 82: -0.060937500 0.78125000 -0.018750000 #> 83: -0.082812500 0.20312500 0.053125000 #> 84: -0.082812500 0.82031250 -0.029687500 #> 85: -0.034375000 0.12500000 0.025000000 #> 86: -0.034375000 0.87500000 -0.025000000 #> 87: -0.025000000 0.06250000 0.012500000 #> 88: -0.025000000 0.93750000 -0.012500000 #> 89: 0.003125000 0.01562500 -0.009375000 #> 90: 0.003125000 0.96875000 -0.006250000 #> 91: 0.004375000 0.00781250 -0.002187500 #> 92: 0.004375000 0.99218750 0.002187500 #> interval_coverage_deviation quantile_coverage quantile_coverage_deviation"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":null,"dir":"Reference","previous_headings":"","what":"Find duplicate forecasts — get_duplicate_forecasts","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"Helper function identify duplicate forecasts, .e. instances one forecast prediction target.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"","code":"get_duplicate_forecasts(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"data data.frame used score()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"data.frame rows duplicate forecast found","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"","code":"example <- rbind(example_quantile, example_quantile[1000:1010]) get_duplicate_forecasts(example) #> location target_end_date target_type observed location_name forecast_date #> #> 1: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 2: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 3: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 4: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 5: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 6: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 7: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 8: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 9: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 10: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 11: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 12: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 13: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 14: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 15: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 16: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 17: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 18: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 19: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 20: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 21: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 22: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> location target_end_date target_type observed location_name forecast_date #> quantile_level predicted model horizon #> #> 1: 0.950 1464 epiforecasts-EpiNow2 1 #> 2: 0.975 1642 epiforecasts-EpiNow2 1 #> 3: 0.990 1951 epiforecasts-EpiNow2 1 #> 4: 0.010 28999 EuroCOVIDhub-ensemble 3 #> 5: 0.025 32612 EuroCOVIDhub-ensemble 3 #> 6: 0.050 36068 EuroCOVIDhub-ensemble 3 #> 7: 0.100 41484 EuroCOVIDhub-ensemble 3 #> 8: 0.150 47110 EuroCOVIDhub-ensemble 3 #> 9: 0.200 50929 EuroCOVIDhub-ensemble 3 #> 10: 0.250 54561 EuroCOVIDhub-ensemble 3 #> 11: 0.300 57739 EuroCOVIDhub-ensemble 3 #> 12: 0.950 1464 epiforecasts-EpiNow2 1 #> 13: 0.975 1642 epiforecasts-EpiNow2 1 #> 14: 0.990 1951 epiforecasts-EpiNow2 1 #> 15: 0.010 28999 EuroCOVIDhub-ensemble 3 #> 16: 0.025 32612 EuroCOVIDhub-ensemble 3 #> 17: 0.050 36068 EuroCOVIDhub-ensemble 3 #> 18: 0.100 41484 EuroCOVIDhub-ensemble 3 #> 19: 0.150 47110 EuroCOVIDhub-ensemble 3 #> 20: 0.200 50929 EuroCOVIDhub-ensemble 3 #> 21: 0.250 54561 EuroCOVIDhub-ensemble 3 #> 22: 0.300 57739 EuroCOVIDhub-ensemble 3 #> quantile_level predicted model horizon"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":null,"dir":"Reference","previous_headings":"","what":"Count number of available forecasts — get_forecast_counts","title":"Count number of available forecasts — get_forecast_counts","text":"Given data set forecasts, function counts number available forecasts. level grouping can specified using argument (e.g. count number forecasts per model, number forecasts per model location). useful determine whether missing forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Count number of available forecasts — get_forecast_counts","text":"","code":"get_forecast_counts( forecast, by = get_forecast_unit(forecast), collapse = c(\"quantile_level\", \"sample_id\") )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Count number of available forecasts — get_forecast_counts","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()) character vector NULL (default) denotes categories number forecasts counted. default unit single forecast (.e. available columns (apart \"protected\" columns 'predicted' 'observed') plus \"quantile_level\" \"sample_id\" present). collapse character vector (default: c(\"quantile_level\", \"sample_id\") names categories number rows collapsed one counting. example, single forecast usually represented set several quantiles samples collapsing one makes sure single forecast gets counted . Setting collapse = c() mean quantiles / samples counted individual forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Count number of available forecasts — get_forecast_counts","text":"data.table columns specified additional column \"count\" number forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Count number of available forecasts — get_forecast_counts","text":"","code":"# \\dontshow{ data.table::setDTthreads(2) # restricts number of cores used on CRAN # } get_forecast_counts( as_forecast(example_quantile), by = c(\"model\", \"target_type\") ) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Key: #> model target_type count #> #> 1: EuroCOVIDhub-baseline Cases 128 #> 2: EuroCOVIDhub-baseline Deaths 128 #> 3: EuroCOVIDhub-ensemble Cases 128 #> 4: EuroCOVIDhub-ensemble Deaths 128 #> 5: UMass-MechBayes Cases 0 #> 6: UMass-MechBayes Deaths 128 #> 7: epiforecasts-EpiNow2 Cases 128 #> 8: epiforecasts-EpiNow2 Deaths 119"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_type.html","id":null,"dir":"Reference","previous_headings":"","what":"Infer forecast type from data — get_forecast_type","title":"Infer forecast type from data — get_forecast_type","text":"Helper function infer forecast type based data.frame similar forecasts observed values. See details section information different forecast types.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_type.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Infer forecast type from data — get_forecast_type","text":"","code":"get_forecast_type(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_type.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Infer forecast type from data — get_forecast_type","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_type.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Infer forecast type from data — get_forecast_type","text":"Character vector length one either \"binary\", \"quantile\", \"sample\" \"point\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_type.html","id":"forecast-types-and-input-formats","dir":"Reference","previous_headings":"","what":"Forecast types and input formats","title":"Infer forecast type from data — get_forecast_type","text":"Various different forecast types / forecast formats supported. moment, : point forecasts binary forecasts (\"soft binary classification\") Probabilistic forecasts quantile-based format (forecast represented set predictive quantiles) Probabilistic forecasts sample-based format (forecast represented set predictive samples) Forecast types determined based columns present input data. overview required format forecast type: forecast types require data.frame similar columns observed predicted, model. Point forecasts require column observed type numeric column predicted type numeric. Binary forecasts require column observed type factor exactly two levels column predicted type numeric probabilities, corresponding probability observed equal second factor level. See details information. Quantile-based forecasts require column observed type numeric, column predicted type numeric, column quantile_level type numeric quantile-levels (0 1). Sample-based forecasts require column observed type numeric, column predicted type numeric, column sample_id type numeric sample indices. information see vignettes example data (example_quantile, example_sample_continuous, example_sample_discrete, example_point(), example_binary).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":null,"dir":"Reference","previous_headings":"","what":"Get unit of a single forecast — get_forecast_unit","title":"Get unit of a single forecast — get_forecast_unit","text":"Helper function get unit single forecast, .e. column names define single forecast made . just takes columns available data subtracts columns protected, .e. returned get_protected_columns() well names metrics specified scoring, .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get unit of a single forecast — get_forecast_unit","text":"","code":"get_forecast_unit(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get unit of a single forecast — get_forecast_unit","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get unit of a single forecast — get_forecast_unit","text":"character vector column names define unit single forecast","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":"forecast-unit","dir":"Reference","previous_headings":"","what":"Forecast unit","title":"Get unit of a single forecast — get_forecast_unit","text":"order score forecasts, scoringutils needs know rows data belong together jointly form single forecasts. easy e.g. point forecast, one row per forecast. quantile sample-based forecasts, however, multiple rows belong single forecast. forecast unit unit single forecast described combination columns uniquely identify single forecast. example, forecasts made different models various locations different time points, several weeks future. forecast unit described forecast_unit = c(\"model\", \"location\", \"forecast_date\", \"forecast_horizon\"). scoringutils automatically tries determine unit single forecast. uses existing columns , means columns must present unrelated forecast unit. simplistic example, additional row, \"even\", one row number even zero otherwise, mess scoring scoringutils thinks column relevant defining forecast unit. order avoid issues, recommend setting forecast unit explicitly, usually forecast_unit argument as_forecast(). drop unneeded columns, making sure necessary, 'protected columns' like \"predicted\" \"observed\" retained.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Get names of the metrics that were used for scoring — get_metrics","title":"Get names of the metrics that were used for scoring — get_metrics","text":"applying scoring rule via score(), names scoring rules become column names resulting data.table. addition, attribute metrics added output, holding names scores vector. done functions like get_forecast_unit() summarise_scores() can still identify columns part forecast unit hold score. get_metrics() accesses returns metrics attribute. attribute, function return NULL (, error = TRUE produce error instead). addition, checks column names input consistency data stored metrics attribute. Handling missing inconsistent metrics attribute: metrics attribute missing consistent column names data.table, can either run score() , specifying names scoring rules manually, add/update attribute manually using attr(scores, \"metrics\") <- c(\"names\", \"\", \"\", \"scores\") (order matter).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get names of the metrics that were used for scoring — get_metrics","text":"","code":"get_metrics(scores, error = FALSE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get names of the metrics that were used for scoring — get_metrics","text":"scores data.table attribute metrics. error Throw error attribute called metrics? Default FALSE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get names of the metrics that were used for scoring — get_metrics","text":"Character vector names scoring rules used scoring NULL scores computed previously.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":null,"dir":"Reference","previous_headings":"","what":"Obtain pairwise comparisons between models — get_pairwise_comparisons","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"Compare scores obtained different models pairwise tournament. combinations two models compared based overlapping set available forecasts common models. input scores object produced score(). Note adding additional unrelated columns can unpredictably change results, present columns taken account determining set overlapping forecasts two models. output pairwise comparisons set mean score ratios, relative skill scores p-values. following illustrates pairwise comparison process: Mean score ratios every pair two models, mean score ratio computed. simply mean score first model divided mean score second. Mean score ratios computed based set overlapping forecasts two models. means scores targets taken account models submitted forecast. (Scaled) Relative skill scores relative score model geometric mean mean score ratios involve model. baseline provided, scaled relative skill scores calculated well. Scaled relative skill scores simply relative skill score model divided relative skill score baseline model. p-values addition, function computes p-values comparison two models (based set overlapping forecasts). P-values can computed two ways: based nonparametric Wilcoxon signed-rank test (internally using wilcox.test() paired = TRUE) based permutation test. permutation test based difference mean scores two models. default null hypothesis mean score difference zero (see permutation_test()). Adjusted p-values computed calling p.adjust() raw p-values. code pairwise comparisons inspired implementation Johannes Bracher. implementation permutation test follows function permutationTest surveillance package Michael Höhle, Andrea Riebler Michaela Paul.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"","code":"get_pairwise_comparisons( scores, by = \"model\", metric = intersect(c(\"wis\", \"crps\", \"brier_score\"), names(scores)), baseline = NULL, ... )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"scores object class scores (data.table scores additional attribute metrics produced score()). Character vector column names define grouping level pairwise comparisons. default (model), one relative skill score per model. , example, = c(\"model\", \"location\"). get separate relative skill score every model every location. Internally, data.table scores split according (removing \"model\" splitting) pairwise comparisons computed separately split data.tables. metric string name metric relative skill shall computed. default either \"crps\", \"wis\" \"brier_score\" available. baseline string name model. baseline given, scaled relative skill respect baseline returned. default (NULL), relative skill scaled respect baseline model. ... Additional arguments comparison two models. See compare_two_models() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"data.table results pairwise comparisons containing mean score ratios (mean_scores_ratio), unadjusted (pval) adjusted (adj_pval) p-values, relative skill values model (..._relative_skill). baseline model given scaled relative skill reported well (..._scaled_relative_skill).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"Nikos Bosse nikosbosse@gmail.com Johannes Bracher, johannes.bracher@kit.edu","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"","code":"# \\dontshow{ data.table::setDTthreads(2) # restricts number of cores used on CRAN # } scores <- score(as_forecast(example_quantile)) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. pairwise <- get_pairwise_comparisons(scores, by = \"target_type\") pairwise2 <- get_pairwise_comparisons( scores, by = \"target_type\", baseline = \"EuroCOVIDhub-baseline\" ) library(ggplot2) plot_pairwise_comparisons(pairwise, type = \"mean_scores_ratio\") + facet_wrap(~target_type)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit.html","id":null,"dir":"Reference","previous_headings":"","what":"Probability integral transformation (data.frame version) — get_pit","title":"Probability integral transformation (data.frame version) — get_pit","text":"Compute Probability Integral Transformation (PIT) validated forecast objects.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Probability integral transformation (data.frame version) — get_pit","text":"","code":"get_pit(forecast, by, n_replicates = 100)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Probability integral transformation (data.frame version) — get_pit","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()) Character vector columns according PIT values shall grouped. e.g. columns 'model' 'location' input data want PIT histogram every model location, specify = c(\"model\", \"location\"). n_replicates number draws randomised PIT discrete predictions. ignored forecasts continuous.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Probability integral transformation (data.frame version) — get_pit","text":"data.table PIT values according grouping specified .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Probability integral transformation (data.frame version) — get_pit","text":"Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind M. Eggo, W. John Edmunds (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15, doi:10.1371/journal.pcbi.1006785","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Probability integral transformation (data.frame version) — get_pit","text":"","code":"result <- get_pit(as_forecast(example_sample_continuous), by = \"model\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_pit(result) # example with quantile data result <- get_pit(as_forecast(example_quantile), by = \"model\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_pit(result)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_protected_columns.html","id":null,"dir":"Reference","previous_headings":"","what":"Get protected columns from data — get_protected_columns","title":"Get protected columns from data — get_protected_columns","text":"Helper function get names columns data frame protected columns.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_protected_columns.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get protected columns from data — get_protected_columns","text":"","code":"get_protected_columns(data = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_protected_columns.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get protected columns from data — get_protected_columns","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_protected_columns.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get protected columns from data — get_protected_columns","text":"character vector names protected columns data. data NULL (default) returns list columns protected scoringutils.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_range_from_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Get interval range belonging to a quantile — get_range_from_quantile","title":"Get interval range belonging to a quantile — get_range_from_quantile","text":"Every quantile can thought either lower upper bound symmetric central prediction interval. helper function returns range central prediction interval quantile belongs. Due numeric instability sometimes occurred past, ranges rounded 10 decimal places. problem vast majority use cases, something aware .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_range_from_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get interval range belonging to a quantile — get_range_from_quantile","text":"","code":"get_range_from_quantile(quantile_level)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_range_from_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get interval range belonging to a quantile — get_range_from_quantile","text":"quantile_level numeric vector quantile levels size N.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_range_from_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get interval range belonging to a quantile — get_range_from_quantile","text":"numeric vector interval ranges size N","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_type.html","id":null,"dir":"Reference","previous_headings":"","what":"Get type of a vector or matrix of observed values or predictions — get_type","title":"Get type of a vector or matrix of observed values or predictions — get_type","text":"Internal helper function get type vector (usually observed predicted values). function checks whether input factor, else whether integer (can coerced integer) whether continuous.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_type.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get type of a vector or matrix of observed values or predictions — get_type","text":"","code":"get_type(x)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_type.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get type of a vector or matrix of observed values or predictions — get_type","text":"x Input type determined .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_type.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get type of a vector or matrix of observed values or predictions — get_type","text":"Character vector length one either \"classification\", \"integer\", \"continuous\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interpolate_median.html","id":null,"dir":"Reference","previous_headings":"","what":"Helper function to interpolate the median prediction if it is not available — interpolate_median","title":"Helper function to interpolate the median prediction if it is not available — interpolate_median","text":"Internal function interpolate median prediction available given quantile levels. done using linear interpolation two innermost quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interpolate_median.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Helper function to interpolate the median prediction if it is not available — interpolate_median","text":"","code":"interpolate_median(predicted, quantile_level)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interpolate_median.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Helper function to interpolate the median prediction if it is not available — interpolate_median","text":"predicted Vector length N (corresponding number quantiles) holds predictions. quantile_level Vector size N quantile levels predictions made. Note contain median (0.5) median imputed mean two innermost quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interpolate_median.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Helper function to interpolate the median prediction if it is not available — interpolate_median","text":"scalar imputed median prediction","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Interval coverage (for quantile-based forecasts) — interval_coverage","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"Check whether observed value within given central prediction interval. prediction interval defined lower upper bound formed pair predictive quantiles. example, 50% prediction interval formed 0.25 0.75 quantiles predictive distribution.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"","code":"interval_coverage(observed, predicted, quantile_level, interval_range = 50)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made. interval_range single number range prediction interval percent (e.g. 50 50% prediction interval) want compute interval coverage.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"vector length n elements either TRUE, observed value within corresponding prediction interval, FALSE otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"","code":"observed <- c(1, -15, 22) predicted <- rbind( c(-1, 0, 1, 2, 3), c(-2, 1, 2, 2, 4), c(-2, 0, 3, 3, 4) ) quantile_level <- c(0.1, 0.25, 0.5, 0.75, 0.9) interval_coverage(observed, predicted, quantile_level) #> [1] TRUE FALSE FALSE"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage_deviation.html","id":null,"dir":"Reference","previous_headings":"","what":"Interval coverage deviation (for quantile-based forecasts) — interval_coverage_deviation","title":"Interval coverage deviation (for quantile-based forecasts) — interval_coverage_deviation","text":"Check agreement desired actual interval coverage forecast. function similar interval_coverage(), takes provided prediction intervals account compares nominal interval coverage (.e. desired interval coverage) actual observed interval coverage. central symmetric prediction interval defined lower upper bound formed pair predictive quantiles. example, 50% prediction interval formed 0.25 0.75 quantiles predictive distribution. Ideally, forecaster aim cover 50% observed values 50% prediction intervals, 90% observed values 90% prediction intervals, . every prediction interval, deviation computed difference observed interval coverage nominal interval coverage single observed value single prediction interval, coverage always either 0 1 (FALSE TRUE). case single observed value multiple prediction intervals, still make much sense compare nominal (desired) coverage actual coverage single observation. sense coverage deviation really starts make sense metric averaged across multiple observations). Positive values interval coverage deviation indication underconfidence, .e. forecaster likely issued narrower forecast. Negative values indication overconfidence, .e. forecasts narrow. $$ \\textrm{interval coverage deviation} = \\mathbf{1}(\\textrm{observed value falls within interval}) - \\textrm{nominal interval coverage} $$ interval coverage deviation averaged across prediction intervals. median ignored computing coverage deviation.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage_deviation.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Interval coverage deviation (for quantile-based forecasts) — interval_coverage_deviation","text":"","code":"interval_coverage_deviation(observed, predicted, quantile_level)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage_deviation.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Interval coverage deviation (for quantile-based forecasts) — interval_coverage_deviation","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage_deviation.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Interval coverage deviation (for quantile-based forecasts) — interval_coverage_deviation","text":"numeric vector length n interval coverage deviation forecast (forecast comprising one multiple prediction intervals).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage_deviation.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Interval coverage deviation (for quantile-based forecasts) — interval_coverage_deviation","text":"","code":"observed <- c(1, -15, 22) predicted <- rbind( c(-1, 0, 1, 2, 3), c(-2, 1, 2, 2, 4), c(-2, 0, 3, 3, 4) ) quantile_level <- c(0.1, 0.25, 0.5, 0.75, 0.9) interval_coverage_deviation(observed, predicted, quantile_level) #> [1] 0.35 -0.65 -0.65"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":null,"dir":"Reference","previous_headings":"","what":"Interval score — interval_score","title":"Interval score — interval_score","text":"Proper Scoring Rule score quantile predictions, following Gneiting Raftery (2007). Smaller values better. score computed $$ \\textrm{score} = (\\textrm{upper} - \\textrm{lower}) + \\frac{2}{\\alpha}(\\textrm{lower} - \\textrm{observed}) * \\mathbf{1}(\\textrm{observed} < \\textrm{lower}) + \\frac{2}{\\alpha}(\\textrm{observed} - \\textrm{upper}) * \\mathbf{1}(\\textrm{observed} > \\textrm{upper}) $$ \\(\\mathbf{1}()\\) indicator function indicates much outside prediction interval. \\(\\alpha\\) decimal value indicates much outside prediction interval. improve usability, user asked provide interval range percentage terms, .e. interval_range = 90 (percent) 90 percent prediction interval. Correspondingly, user provide 5% 95% quantiles (corresponding alpha 0.1). specific distribution assumed, interval symmetric around median (.e use 0.1 quantile lower bound 0.7 quantile upper bound). Non-symmetric quantiles can scored using function quantile_score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Interval score — interval_score","text":"","code":"interval_score( observed, lower, upper, interval_range, weigh = TRUE, separate_results = FALSE )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Interval score — interval_score","text":"observed vector observed values size n lower Vector size n prediction lower quantile given interval range. upper Vector size n prediction upper quantile given interval range. interval_range Numeric vector (either single number vector size n) range prediction intervals. example, forecasting 0.05 0.95 quantile, interval range 90. interval range corresponds \\((100-\\alpha)/100\\), \\(\\alpha\\) decimal value indicates much outside prediction interval (see e.g. Gneiting Raftery (2007)). weigh Logical. TRUE (default), weigh score \\(\\alpha / 2\\), can averaged interval score , limit (increasing number equally spaced quantiles/prediction intervals), corresponds CRPS. \\(\\alpha\\) value corresponds (\\(\\alpha/2\\)) (\\(1 - \\alpha/2\\)), .e. decimal value represents much outside central prediction interval (E.g. 90 percent central prediction interval, alpha 0.1). separate_results Logical. TRUE (default FALSE), separate parts interval score (dispersion penalty, penalties - -prediction get returned separate elements list). want data.frame instead, simply call .data.frame() output.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Interval score — interval_score","text":"Vector scoring values, list separate entries separate_results TRUE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Interval score — interval_score","text":"Strictly Proper Scoring Rules, Prediction,Estimation, Tilmann Gneiting Adrian E. Raftery, 2007, Journal American Statistical Association, Volume 102, 2007 - Issue 477 Evaluating epidemic forecasts interval format, Johannes Bracher, Evan L. Ray, Tilmann Gneiting Nicholas G. Reich, https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1008618 # nolint","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Interval score — interval_score","text":"","code":"observed <- rnorm(30, mean = 1:30) interval_range <- rep(90, 30) alpha <- (100 - interval_range) / 100 lower <- qnorm(alpha / 2, rnorm(30, mean = 1:30)) upper <- qnorm((1 - alpha / 2), rnorm(30, mean = 11:40)) scoringutils:::interval_score( observed = observed, lower = lower, upper = upper, interval_range = interval_range ) #> [1] 0.6186832 0.6857411 0.6851103 0.6735228 0.7207120 0.6350107 0.6326772 #> [8] 0.7765693 0.9162766 0.7696477 0.5718187 0.5947846 0.6873026 0.6733580 #> [15] 0.6672086 0.7094583 0.5695223 0.6244414 0.7408444 1.4548649 0.6493377 #> [22] 0.9306896 0.6973638 0.7058552 0.8090509 0.7450232 0.5687430 0.7296127 #> [29] 0.7483414 0.6927440 # gives a warning, as the interval_range should likely be 50 instead of 0.5 scoringutils:::interval_score( observed = 4, upper = 8, lower = 2, interval_range = 0.5 ) #> Warning: ! Found interval ranges between 0 and 1. Are you sure that's right? An interval #> range of 0.5 e.g. implies a (49.75%, 50.25%) prediction interval. #> ℹ If you want to score a (25%, 75%) prediction interval, set `interval_range = #> 50`. #> This warning is displayed once per session. #> [1] 2.985 # example with missing values and separate results scoringutils:::interval_score( observed = c(observed, NA), lower = c(lower, NA), upper = c(NA, upper), separate_results = TRUE, interval_range = 90 ) #> $interval_score #> [1] NA 0.5428048 0.7656798 0.5305954 0.6509690 0.6780044 0.5082637 #> [8] 0.7404846 0.8621966 0.7375557 0.5326411 0.5665304 0.6259204 0.5696571 #> [15] 0.6555712 0.5798147 0.6144623 0.5308622 0.6850722 1.4237942 0.6161064 #> [22] 0.8500130 0.6461361 0.6502278 0.7400150 0.6942545 0.5921649 0.5858396 #> [29] 0.6975714 0.7079685 NA #> #> $dispersion #> [1] NA 0.5428048 0.7656798 0.5305954 0.6509690 0.6780044 0.5082637 #> [8] 0.7404846 0.5823934 0.7375557 0.5326411 0.5665304 0.6259204 0.5696571 #> [15] 0.6555712 0.5798147 0.6144623 0.5308622 0.6850722 0.5303743 0.6161064 #> [22] 0.5536698 0.6461361 0.6502278 0.7400150 0.6942545 0.5921649 0.5858396 #> [29] 0.6975714 0.7079685 NA #> #> $underprediction #> [1] NA 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 #> [26] 0 0 0 0 0 NA #> #> $overprediction #> [1] 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 #> [8] 0.0000000 0.2798032 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 #> [15] 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.8934199 0.0000000 #> [22] 0.2963432 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 #> [29] 0.0000000 0.0000000 NA #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Test whether an object is a forecast object — is_forecast","title":"Test whether an object is a forecast object — is_forecast","text":"Generic function test whether object class forecast_*. can also test specific forecast_* class using appropriate is_forecast.forecast_* method. example, check whether object class forecast_quantile, use scoringutils:::is_forecast.forecast_quantile().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Test whether an object is a forecast object — is_forecast","text":"","code":"is_forecast(x, ...) # Default S3 method is_forecast(x, ...) # S3 method for class 'forecast_sample' is_forecast(x, ...) # S3 method for class 'forecast_binary' is_forecast(x, ...) # S3 method for class 'forecast_point' is_forecast(x, ...) # S3 method for class 'forecast_quantile' is_forecast(x, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Test whether an object is a forecast object — is_forecast","text":"x R object. ... Additional arguments","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Test whether an object is a forecast object — is_forecast","text":"TRUE object class forecast_*, FALSE otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Test whether an object is a forecast object — is_forecast","text":"","code":"forecast_binary <- as_forecast(example_binary) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. is_forecast(forecast_binary) #> [1] TRUE"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":null,"dir":"Reference","previous_headings":"","what":"Log transformation with an additive shift — log_shift","title":"Log transformation with an additive shift — log_shift","text":"Function shifts value offset applies natural logarithm .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Log transformation with an additive shift — log_shift","text":"","code":"log_shift(x, offset = 0, base = exp(1))"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Log transformation with an additive shift — log_shift","text":"x vector input values transformed offset Number add input value taking natural logarithm. base positive number: base respect logarithms computed. Defaults e = exp(1).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Log transformation with an additive shift — log_shift","text":"numeric vector transformed values","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Log transformation with an additive shift — log_shift","text":"output computed log(x + offset)","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Log transformation with an additive shift — log_shift","text":"Transformation forecasts evaluating predictive performance epidemiological context Nikos . Bosse, Sam Abbott, Anne Cori, Edwin van Leeuwen, Johannes Bracher, Sebastian Funk medRxiv 2023.01.23.23284722 doi:10.1101/2023.01.23.23284722 https://www.medrxiv.org/content/10.1101/2023.01.23.23284722v1 # nolint","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Log transformation with an additive shift — log_shift","text":"","code":"log_shift(1:10) #> [1] 0.0000000 0.6931472 1.0986123 1.3862944 1.6094379 1.7917595 1.9459101 #> [8] 2.0794415 2.1972246 2.3025851 log_shift(0:9, offset = 1) #> [1] 0.0000000 0.6931472 1.0986123 1.3862944 1.6094379 1.7917595 1.9459101 #> [8] 2.0794415 2.1972246 2.3025851 transform_forecasts( as_forecast(example_quantile)[observed > 0, ], fun = log_shift, offset = 1 ) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Forecast type: quantile #> Forecast unit: #> location, target_end_date, target_type, location_name, forecast_date, model, #> horizon, and scale #> #> location target_end_date target_type observed location_name #> #> 1: DE 2021-01-02 Cases 1.273000e+05 Germany #> 2: DE 2021-01-02 Deaths 4.534000e+03 Germany #> 3: DE 2021-01-09 Cases 1.549220e+05 Germany #> 4: DE 2021-01-09 Deaths 6.117000e+03 Germany #> 5: DE 2021-01-16 Cases 1.101830e+05 Germany #> --- #> 40672: IT 2021-07-24 Deaths 4.369448e+00 Italy #> 40673: IT 2021-07-24 Deaths 4.369448e+00 Italy #> 40674: IT 2021-07-24 Deaths 4.369448e+00 Italy #> 40675: IT 2021-07-24 Deaths 4.369448e+00 Italy #> 40676: IT 2021-07-24 Deaths 4.369448e+00 Italy #> forecast_date quantile_level predicted model horizon #> #> 1: NA NA NA #> 2: NA NA NA #> 3: NA NA NA #> 4: NA NA NA #> 5: NA NA NA #> --- #> 40672: 2021-07-12 0.850 5.866468 epiforecasts-EpiNow2 2 #> 40673: 2021-07-12 0.900 5.986452 epiforecasts-EpiNow2 2 #> 40674: 2021-07-12 0.950 6.214608 epiforecasts-EpiNow2 2 #> 40675: 2021-07-12 0.975 6.416732 epiforecasts-EpiNow2 2 #> 40676: 2021-07-12 0.990 6.579251 epiforecasts-EpiNow2 2 #> scale #> #> 1: natural #> 2: natural #> 3: natural #> 4: natural #> 5: natural #> --- #> 40672: log #> 40673: log #> 40674: log #> 40675: log #> 40676: log"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Logarithmic score (sample-based version) — logs_sample","title":"Logarithmic score (sample-based version) — logs_sample","text":"function wrapper around logs_sample() function scoringRules package. function used score continuous predictions . Log Score theory also applicable discrete forecasts, problem lies implementation: Log score needs kernel density estimation, well defined integer-valued Monte Carlo Samples. Log score can used specific discrete probability distributions. See scoringRules package details.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Logarithmic score (sample-based version) — logs_sample","text":"","code":"logs_sample(observed, predicted, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Logarithmic score (sample-based version) — logs_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. ... Additional arguments passed logs_sample() scoringRules package.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Logarithmic score (sample-based version) — logs_sample","text":"Vector scores.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Logarithmic score (sample-based version) — logs_sample","text":"Alexander Jordan, Fabian Krüger, Sebastian Lerch, Evaluating Probabilistic Forecasts scoringRules, https://www.jstatsoft.org/article/view/v090i12","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Logarithmic score (sample-based version) — logs_sample","text":"","code":"observed <- rpois(30, lambda = 1:30) predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) logs_sample(observed, predicted) #> [1] 0.9877018 1.3179060 2.9308071 3.0967973 1.7353294 2.1375898 3.0671846 #> [8] 2.0115847 2.3128147 2.2232410 2.3023663 6.6464809 2.2126378 2.6212927 #> [15] 2.3262577 2.5675490 3.3007417 2.2614306 2.4344484 2.4467472 3.5018847 #> [22] 2.7833854 2.9574125 2.7755341 2.7756007 3.1608762 3.1106262 2.8047358 #> [29] 2.6202545 2.6531665"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Determine dispersion of a probabilistic forecast — mad_sample","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"Sharpness ability model generate predictions within narrow range dispersion lack thereof. data-independent measure, purely feature forecasts . Dispersion predictive samples corresponding one single observed value measured normalised median absolute deviation median predictive samples. details, see mad() explanations given Funk et al. (2019)","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"","code":"mad_sample(observed = NULL, predicted, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"observed Place holder, argument ignored exists consistency scoring functions. output depend observed values. predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. ... Additional arguments passed mad().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"Vector dispersion values.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"Funk S, Camacho , Kucharski AJ, Lowe R, Eggo RM, Edmunds WJ (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15. PLoS Comput Biol 15(2): e1006785. doi:10.1371/journal.pcbi.1006785","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"","code":"predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) mad_sample(predicted = predicted) #> [1] 1.4826 1.4826 1.4826 2.9652 2.9652 2.9652 2.9652 2.9652 2.9652 2.9652 #> [11] 2.9652 2.9652 4.4478 4.4478 2.9652 3.7065 4.4478 3.7065 4.4478 4.4478 #> [21] 4.4478 5.9304 4.4478 4.4478 4.4478 5.9304 4.4478 4.4478 5.9304 5.9304"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Default metrics and scoring rules for binary forecasts — metrics_binary","title":"Default metrics and scoring rules for binary forecasts — metrics_binary","text":"Helper function returns named list default scoring rules suitable binary forecasts. default scoring rules : \"brier_score\" = brier_score() \"log_score\" = logs_binary()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Default metrics and scoring rules for binary forecasts — metrics_binary","text":"","code":"metrics_binary(select = NULL, exclude = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Default metrics and scoring rules for binary forecasts — metrics_binary","text":"select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Default metrics and scoring rules for binary forecasts — metrics_binary","text":"list scoring rules.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_binary.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Default metrics and scoring rules for binary forecasts — metrics_binary","text":"","code":"metrics_binary() #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> #> #> #> $log_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> logs <- -log(1 - abs(observed - predicted)) #> return(logs) #> } #> #> #> metrics_binary(select = \"brier_score\") #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> #> #> metrics_binary(exclude = \"log_score\") #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> #> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Default metrics and scoring rules for point forecasts — metrics_point","title":"Default metrics and scoring rules for point forecasts — metrics_point","text":"Helper function returns named list default scoring rules suitable point forecasts. default scoring rules : \"ae_point\" = ae() \"se_point\" = se() \"ape\" = ape()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Default metrics and scoring rules for point forecasts — metrics_point","text":"","code":"metrics_point(select = NULL, exclude = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Default metrics and scoring rules for point forecasts — metrics_point","text":"select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Default metrics and scoring rules for point forecasts — metrics_point","text":"list scoring rules.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_point.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Default metrics and scoring rules for point forecasts — metrics_point","text":"","code":"metrics_point() #> $ae_point #> function (actual, predicted) #> { #> return(abs(actual - predicted)) #> } #> #> #> #> $se_point #> function (actual, predicted) #> { #> return((actual - predicted)^2) #> } #> #> #> #> $ape #> function (actual, predicted) #> { #> return(ae(actual, predicted)/abs(actual)) #> } #> #> #> metrics_point(select = \"ape\") #> $ape #> function (actual, predicted) #> { #> return(ae(actual, predicted)/abs(actual)) #> } #> #> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Default metrics and scoring rules for quantile-based forecasts — metrics_quantile","title":"Default metrics and scoring rules for quantile-based forecasts — metrics_quantile","text":"Helper function returns named list default scoring rules suitable forecasts quantile-based format. default scoring rules : \"wis\" = wis \"overprediction\" = overprediction() \"underprediction\" = underprediction() \"dispersion\" = dispersion() \"bias\" = bias_quantile() \"interval_coverage_50\" = interval_coverage() \"interval_coverage_90\" = customise_metric( interval_coverage, interval_range = 90 ) \"interval_coverage_deviation\" = interval_coverage_deviation(), \"ae_median\" = ae_median_quantile() Note: interval_coverage_90 scoring rule created modifying interval_coverage(), making use function customise_metric(). construct allows function deal arbitrary arguments ..., making sure interval_coverage() can accept get passed . interval_range = 90 set function definition, passing argument interval_range = 90 score() mean also get passed interval_coverage_50.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Default metrics and scoring rules for quantile-based forecasts — metrics_quantile","text":"","code":"metrics_quantile(select = NULL, exclude = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Default metrics and scoring rules for quantile-based forecasts — metrics_quantile","text":"select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Default metrics and scoring rules for quantile-based forecasts — metrics_quantile","text":"list scoring rules.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_quantile.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Default metrics and scoring rules for quantile-based forecasts — metrics_quantile","text":"","code":"metrics_quantile() #> $wis #> function (observed, predicted, quantile_level, separate_results = FALSE, #> weigh = TRUE, count_median_twice = FALSE, na.rm = TRUE) #> { #> assert_input_quantile(observed, predicted, quantile_level) #> reformatted <- quantile_to_interval(observed, predicted, #> quantile_level) #> assert_logical(separate_results, len = 1) #> assert_logical(weigh, len = 1) #> assert_logical(count_median_twice, len = 1) #> assert_logical(na.rm, len = 1) #> if (separate_results) { #> cols <- c(\"wis\", \"dispersion\", \"underprediction\", \"overprediction\") #> } #> else { #> cols <- \"wis\" #> } #> reformatted[, `:=`(eval(cols), do.call(interval_score, list(observed = observed, #> lower = lower, upper = upper, interval_range = interval_range, #> weigh = weigh, separate_results = separate_results)))] #> if (count_median_twice) { #> reformatted[, `:=`(weight, 1)] #> } #> else { #> reformatted[, `:=`(weight, ifelse(interval_range == 0, #> 0.5, 1))] #> } #> reformatted <- reformatted[, lapply(.SD, weighted.mean, na.rm = na.rm, #> w = weight), by = \"forecast_id\", .SDcols = colnames(reformatted) %like% #> paste(cols, collapse = \"|\")] #> if (separate_results) { #> return(list(wis = reformatted$wis, dispersion = reformatted$dispersion, #> underprediction = reformatted$underprediction, overprediction = reformatted$overprediction)) #> } #> else { #> return(reformatted$wis) #> } #> } #> #> #> #> $overprediction #> function (observed, predicted, quantile_level, ...) #> { #> args <- list(...) #> args$separate_results <- TRUE #> assert_input_quantile(observed, predicted, quantile_level) #> out <- do.call(wis, c(list(observed), list(predicted), list(quantile_level), #> args)) #> return(out$overprediction) #> } #> #> #> #> $underprediction #> function (observed, predicted, quantile_level, ...) #> { #> args <- list(...) #> args$separate_results <- TRUE #> assert_input_quantile(observed, predicted, quantile_level) #> out <- do.call(wis, c(list(observed), list(predicted), list(quantile_level), #> args)) #> return(out$underprediction) #> } #> #> #> #> $dispersion #> function (observed, predicted, quantile_level, ...) #> { #> args <- list(...) #> args$separate_results <- TRUE #> assert_input_quantile(observed, predicted, quantile_level) #> out <- do.call(wis, c(list(observed), list(predicted), list(quantile_level), #> args)) #> return(out$dispersion) #> } #> #> #> #> $bias #> function (observed, predicted, quantile_level, na.rm = TRUE) #> { #> assert_input_quantile(observed, predicted, quantile_level) #> n <- length(observed) #> N <- length(quantile_level) #> if (is.null(dim(predicted))) { #> dim(predicted) <- c(n, N) #> } #> if (!(0.5 %in% quantile_level)) { #> cli_inform(c(i = \"Median not available, computing bias as mean of the two\\n innermost quantiles in order to compute bias.\")) #> } #> bias <- sapply(1:n, function(i) { #> bias_quantile_single_vector(observed[i], predicted[i, #> ], quantile_level, na.rm) #> }) #> return(bias) #> } #> #> #> #> $interval_coverage_50 #> function (observed, predicted, quantile_level, interval_range = 50) #> { #> assert_input_quantile(observed, predicted, quantile_level) #> assert_number(interval_range) #> necessary_quantiles <- c((100 - interval_range)/2, 100 - #> (100 - interval_range)/2)/100 #> if (!all(necessary_quantiles %in% quantile_level)) { #> cli_warn(c(`!` = \"To compute the interval coverage for an interval range of\\n {.val {interval_range}%}, the {.val {necessary_quantiles}} quantiles\\n are required.\", #> i = \"Returning {.val {NA}}.\")) #> return(NA) #> } #> r <- interval_range #> reformatted <- quantile_to_interval(observed, predicted, #> quantile_level) #> reformatted <- reformatted[interval_range %in% r] #> reformatted[, `:=`(interval_coverage, (observed >= lower) & #> (observed <= upper))] #> return(reformatted$interval_coverage) #> } #> #> #> #> $interval_coverage_90 #> function (...) #> { #> do.call(metric, c(list(...), dots)) #> } #> #> #> #> $interval_coverage_deviation #> function (observed, predicted, quantile_level) #> { #> assert_input_quantile(observed, predicted, quantile_level) #> available_ranges <- unique(get_range_from_quantile(quantile_level)) #> necessary_quantiles <- unique(c((100 - available_ranges)/2, #> 100 - (100 - available_ranges)/2)/100) #> if (!all(necessary_quantiles %in% quantile_level)) { #> missing <- necessary_quantiles[!necessary_quantiles %in% #> quantile_level] #> cli_warn(c(x = \"To compute interval coverage deviation, all quantiles must form\\n central symmetric prediction intervals.\", #> i = \"Missing quantiles: {.val {missing}}. Returning {.val {NA}}.\")) #> return(NA) #> } #> reformatted <- quantile_to_interval(observed, predicted, #> quantile_level)[interval_range != 0] #> reformatted[, `:=`(interval_coverage, (observed >= lower) & #> (observed <= upper))] #> reformatted[, `:=`(interval_coverage_deviation, interval_coverage - #> interval_range/100)] #> out <- reformatted[, .(interval_coverage_deviation = mean(interval_coverage_deviation)), #> by = \"forecast_id\"] #> return(out$interval_coverage_deviation) #> } #> #> #> #> $ae_median #> function (observed, predicted, quantile_level) #> { #> assert_input_quantile(observed, predicted, quantile_level) #> if (!any(quantile_level == 0.5)) { #> cli_warn(c(x = \"In order to compute the absolute error of the median,\\n {.val 0.5} must be among the quantiles given.\", #> i = \"Returning {.val NA}.\")) #> return(NA_real_) #> } #> if (is.null(dim(predicted))) { #> predicted <- matrix(predicted, nrow = 1) #> } #> predicted <- predicted[, quantile_level == 0.5] #> abs_error_median <- abs(observed - predicted) #> return(abs_error_median) #> } #> #> #> metrics_quantile(select = \"wis\") #> $wis #> function (observed, predicted, quantile_level, separate_results = FALSE, #> weigh = TRUE, count_median_twice = FALSE, na.rm = TRUE) #> { #> assert_input_quantile(observed, predicted, quantile_level) #> reformatted <- quantile_to_interval(observed, predicted, #> quantile_level) #> assert_logical(separate_results, len = 1) #> assert_logical(weigh, len = 1) #> assert_logical(count_median_twice, len = 1) #> assert_logical(na.rm, len = 1) #> if (separate_results) { #> cols <- c(\"wis\", \"dispersion\", \"underprediction\", \"overprediction\") #> } #> else { #> cols <- \"wis\" #> } #> reformatted[, `:=`(eval(cols), do.call(interval_score, list(observed = observed, #> lower = lower, upper = upper, interval_range = interval_range, #> weigh = weigh, separate_results = separate_results)))] #> if (count_median_twice) { #> reformatted[, `:=`(weight, 1)] #> } #> else { #> reformatted[, `:=`(weight, ifelse(interval_range == 0, #> 0.5, 1))] #> } #> reformatted <- reformatted[, lapply(.SD, weighted.mean, na.rm = na.rm, #> w = weight), by = \"forecast_id\", .SDcols = colnames(reformatted) %like% #> paste(cols, collapse = \"|\")] #> if (separate_results) { #> return(list(wis = reformatted$wis, dispersion = reformatted$dispersion, #> underprediction = reformatted$underprediction, overprediction = reformatted$overprediction)) #> } #> else { #> return(reformatted$wis) #> } #> } #> #> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Default metrics and scoring rules sample-based forecasts — metrics_sample","title":"Default metrics and scoring rules sample-based forecasts — metrics_sample","text":"Helper function returns named list default scoring rules suitable forecasts sample-based format. default scoring rules : \"crps\" = crps_sample() \"log_score\" = logs_sample() \"dss\" = dss_sample() \"mad\" = mad_sample() \"bias\" = bias_sample() \"ae_median\" = ae_median_sample() \"se_mean\" = se_mean_sample()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Default metrics and scoring rules sample-based forecasts — metrics_sample","text":"","code":"metrics_sample(select = NULL, exclude = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Default metrics and scoring rules sample-based forecasts — metrics_sample","text":"select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Default metrics and scoring rules sample-based forecasts — metrics_sample","text":"list scoring rules.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Default metrics and scoring rules sample-based forecasts — metrics_sample","text":"","code":"metrics_sample() #> $bias #> function (observed, predicted) #> { #> assert_input_sample(observed, predicted) #> prediction_type <- get_type(predicted) #> n_pred <- ncol(predicted) #> p_x <- rowSums(predicted <= observed)/n_pred #> if (prediction_type == \"continuous\") { #> res <- 1 - 2 * p_x #> return(res) #> } #> else { #> p_xm1 <- rowSums(predicted <= (observed - 1))/n_pred #> res <- 1 - (p_x + p_xm1) #> return(res) #> } #> } #> #> #> #> $dss #> function (observed, predicted, ...) #> { #> assert_input_sample(observed, predicted) #> scoringRules::dss_sample(y = observed, dat = predicted, ...) #> } #> #> #> #> $crps #> function (observed, predicted, ...) #> { #> assert_input_sample(observed, predicted) #> scoringRules::crps_sample(y = observed, dat = predicted, #> ...) #> } #> #> #> #> $log_score #> function (observed, predicted, ...) #> { #> assert_input_sample(observed, predicted) #> scoringRules::logs_sample(y = observed, dat = predicted, #> ...) #> } #> #> #> #> $mad #> function (observed = NULL, predicted, ...) #> { #> assert_input_sample(rep(NA_real_, nrow(predicted)), predicted) #> sharpness <- apply(predicted, MARGIN = 1, mad, ...) #> return(sharpness) #> } #> #> #> #> $ae_median #> function (observed, predicted) #> { #> assert_input_sample(observed, predicted) #> median_predictions <- apply(as.matrix(predicted), MARGIN = 1, #> FUN = median) #> ae_median <- abs(observed - median_predictions) #> return(ae_median) #> } #> #> #> #> $se_mean #> function (observed, predicted) #> { #> assert_input_sample(observed, predicted) #> mean_predictions <- rowMeans(as.matrix(predicted)) #> se_mean <- (observed - mean_predictions)^2 #> return(se_mean) #> } #> #> #> metrics_sample(select = \"mad\") #> $mad #> function (observed = NULL, predicted, ...) #> { #> assert_input_sample(rep(NA_real_, nrow(predicted)), predicted) #> sharpness <- apply(predicted, MARGIN = 1, mad, ...) #> return(sharpness) #> } #> #> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Class constructor for forecast objects — new_forecast","title":"Class constructor for forecast objects — new_forecast","text":"Construct class based data.frame similar. constructor coerces data data.table makes sure column called model exists creates one assigns class","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Class constructor for forecast objects — new_forecast","text":"","code":"new_forecast(data, classname)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Class constructor for forecast objects — new_forecast","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. classname name class created","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Class constructor for forecast objects — new_forecast","text":"object class indicated classname","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Construct an object of class scores — new_scores","title":"Construct an object of class scores — new_scores","text":"function creates object class scores based data.table similar.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Construct an object of class scores — new_scores","text":"","code":"new_scores(scores, metrics, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Construct an object of class scores — new_scores","text":"scores data.table similar scores produced score(). metrics character vector names scores (.e. names scoring rules used scoring). ... Additional arguments .data.table()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Construct an object of class scores — new_scores","text":"object class scores","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Construct an object of class scores — new_scores","text":"","code":"if (FALSE) { df <- data.frame( model = \"A\", wis = \"0.1\" ) new_scores(df, \"wis\") }"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pairwise_comparison_one_group.html","id":null,"dir":"Reference","previous_headings":"","what":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","title":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","text":"function pairwise comparison one set forecasts, multiple models involved. gets called get_pairwise_comparisons(). get_pairwise_comparisons() splits data arbitrary subgroups specified user (e.g. pairwise comparison done separately different forecast targets) actual pairwise comparison subgroup managed pairwise_comparison_one_group(). order actually comparison two models subset common forecasts calls compare_two_models().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pairwise_comparison_one_group.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","text":"","code":"pairwise_comparison_one_group(scores, metric, baseline, by, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pairwise_comparison_one_group.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","text":"scores object class scores (data.table scores additional attribute metrics produced score()). metric string name metric relative skill shall computed. default either \"crps\", \"wis\" \"brier_score\" available. baseline string name model. baseline given, scaled relative skill respect baseline returned. default (NULL), relative skill scaled respect baseline model. Character vector column names define grouping level pairwise comparisons. default (model), one relative skill score per model. , example, = c(\"model\", \"location\"). get separate relative skill score every model every location. Internally, data.table scores split according (removing \"model\" splitting) pairwise comparisons computed separately split data.tables. ... Additional arguments comparison two models. See compare_two_models() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pairwise_comparison_one_group.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","text":"data.table results pairwise comparisons containing mean score ratios (mean_scores_ratio), unadjusted (pval) adjusted (adj_pval) p-values, relative skill values model (..._relative_skill). baseline model given scaled relative skill reported well (..._scaled_relative_skill).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/permutation_test.html","id":null,"dir":"Reference","previous_headings":"","what":"Simple permutation test — permutation_test","title":"Simple permutation test — permutation_test","text":"implementation permutation test follows function permutationTest surveillance package Michael Höhle, Andrea Riebler Michaela Paul. function compares two vectors scores. computes mean vector independently takes either difference ratio two. observed difference ratio compared test statistic based permutations original data. Used get_pairwise_comparisons().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/permutation_test.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Simple permutation test — permutation_test","text":"","code":"permutation_test( scores1, scores2, n_permutation = 999, one_sided = FALSE, comparison_mode = c(\"difference\", \"ratio\") )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/permutation_test.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Simple permutation test — permutation_test","text":"scores1 Vector scores compare another vector scores. scores2 second vector scores compare first n_permutation number replications use permutation test. replications yield exact results, require computation. one_sided Whether compute one-sided test. Default FALSE. comparison_mode compute test statistic comparison two scores. either \"difference\" \"ratio\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/permutation_test.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Simple permutation test — permutation_test","text":"p-value permutation test","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Probability integral transformation (sample-based version) — pit_sample","title":"Probability integral transformation (sample-based version) — pit_sample","text":"Uses Probability integral transformation (PIT) (randomised PIT integer forecasts) assess calibration predictive Monte Carlo samples.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Probability integral transformation (sample-based version) — pit_sample","text":"","code":"pit_sample(observed, predicted, n_replicates = 100)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Probability integral transformation (sample-based version) — pit_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. n_replicates number draws randomised PIT discrete predictions. ignored forecasts continuous.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Probability integral transformation (sample-based version) — pit_sample","text":"vector PIT-values. continuous forecasts, vector correspond length observed. integer forecasts, randomised PIT returned length length(observed) * n_replicates.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Probability integral transformation (sample-based version) — pit_sample","text":"Calibration reliability forecasts ability model correctly identify uncertainty making predictions. model perfect calibration, observed data time point look came predictive probability distribution time. Equivalently, one can inspect probability integral transform predictive distribution time t, $$ u_t = F_t (x_t) $$ \\(x_t\\) observed data point time \\(t \\textrm{ } t_1, …, t_n\\), n number forecasts, \\(F_t\\) (continuous) predictive cumulative probability distribution time t. true probability distribution outcomes time t \\(G_t\\) forecasts \\(F_t\\) said ideal \\(F_t = G_t\\) times t. case, probabilities \\(u_t\\) distributed uniformly. case discrete outcomes incidence counts, PIT longer uniform even forecasts ideal. case randomised PIT can used instead: $$ u_t = P_t(k_t) + v * (P_t(k_t) - P_t(k_t - 1) ) $$ \\(k_t\\) observed count, \\(P_t(x)\\) predictive cumulative probability observing incidence k time t, \\(P_t (-1) = 0\\) definition v standard uniform independent k. \\(P_t\\) true cumulative probability distribution, \\(u_t\\) standard uniform. function checks whether integer continuous forecasts provided. applies (randomised) probability integral tests values \\(u_t\\) uniformity using Anderson-Darling test. rule thumb, evidence suggest forecasting model miscalibrated p-value found greater threshold p >= 0.1, evidence miscalibrated 0.01 < p < 0.1, good evidence miscalibrated p <= 0.01. However, AD-p-values may overly strict actual usefulness may questionable. context noted, though, uniformity PIT necessary sufficient condition calibration.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Probability integral transformation (sample-based version) — pit_sample","text":"Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind M. Eggo, W. John Edmunds (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15, doi:10.1371/journal.pcbi.1006785","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Probability integral transformation (sample-based version) — pit_sample","text":"","code":"# \\dontshow{ data.table::setDTthreads(2) # restricts number of cores used on CRAN # } ## continuous predictions observed <- rnorm(20, mean = 1:20) predicted <- replicate(100, rnorm(n = 20, mean = 1:20)) pit <- pit_sample(observed, predicted) plot_pit(pit) ## integer predictions observed <- rpois(20, lambda = 1:20) predicted <- replicate(100, rpois(n = 20, lambda = 1:20)) pit <- pit_sample(observed, predicted, n_replicates = 30) plot_pit(pit)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot correlation between metrics — plot_correlations","title":"Plot correlation between metrics — plot_correlations","text":"Plots heatmap correlations different metrics.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot correlation between metrics — plot_correlations","text":"","code":"plot_correlations(correlations)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot correlation between metrics — plot_correlations","text":"correlations data.table correlations scores produced get_correlations().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot correlation between metrics — plot_correlations","text":"ggplot object showing coloured matrix correlations metrics. ggplot object visualisation correlations metrics","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot correlation between metrics — plot_correlations","text":"","code":"scores <- score(as_forecast(example_quantile)) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. correlations <- get_correlations( summarise_scores(scores), digits = 2 ) plot_correlations(correlations)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":null,"dir":"Reference","previous_headings":"","what":"Visualise the number of available forecasts — plot_forecast_counts","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"Visualise Forecasts Available.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"","code":"plot_forecast_counts( forecast_counts, x, y = \"model\", x_as_factor = TRUE, show_counts = TRUE )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"forecast_counts data.table (similar) column count holding forecast counts, produced get_forecast_counts(). x Character vector length one denotes name column appear x-axis plot. y Character vector length one denotes name column appear y-axis plot. Default \"model\". x_as_factor Logical (default TRUE). Whether convert variable x-axis factor. effect e.g. dates shown x-axis. show_counts Logical (default TRUE) indicates whether show actual count numbers plot.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"ggplot object plot forecast counts","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"","code":"library(ggplot2) forecast_counts <- get_forecast_counts( as_forecast(example_quantile), by = c(\"model\", \"target_type\", \"target_end_date\") ) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_forecast_counts( forecast_counts, x = \"target_end_date\", show_counts = FALSE ) + facet_wrap(\"target_type\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a heatmap of a scoring metric — plot_heatmap","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"function can used create heatmap one metric across different groups, e.g. interval score obtained several forecasting models different locations.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"","code":"plot_heatmap(scores, y = \"model\", x, metric)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"scores data.frame scores based quantile forecasts produced score(). y variable scores want show y-Axis. default \"model\" x variable scores want show x-Axis. something like \"horizon\", \"location\" metric String, metric determines value colour shown tiles heatmap.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"ggplot object showing heatmap desired metric","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"","code":"scores <- score(as_forecast(example_quantile)) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. scores <- summarise_scores(scores, by = c(\"model\", \"target_type\")) scores <- summarise_scores( scores, by = c(\"model\", \"target_type\"), fun = signif, digits = 2 ) plot_heatmap(scores, x = \"target_type\", metric = \"bias\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot interval coverage — plot_interval_coverage","title":"Plot interval coverage — plot_interval_coverage","text":"Plot interval coverage values (see get_coverage() information).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot interval coverage — plot_interval_coverage","text":"","code":"plot_interval_coverage(coverage, colour = \"model\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot interval coverage — plot_interval_coverage","text":"coverage data frame coverage values produced get_coverage(). colour According variable shall graphs coloured? Default \"model\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot interval coverage — plot_interval_coverage","text":"ggplot object plot interval coverage","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot interval coverage — plot_interval_coverage","text":"","code":"# \\dontshow{ data.table::setDTthreads(2) # restricts number of cores used on CRAN # } coverage <- get_coverage(as_forecast(example_quantile), by = \"model\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_interval_coverage(coverage)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"Creates heatmap ratios pvalues pairwise comparison models.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"","code":"plot_pairwise_comparisons( comparison_result, type = c(\"mean_scores_ratio\", \"pval\") )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"comparison_result data.frame produced get_pairwise_comparisons(). type Character vector length one either \"mean_scores_ratio\" \"pval\". denotes whether visualise ratio p-value pairwise comparison. Default \"mean_scores_ratio\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"ggplot object heatmap mean score ratios pairwise comparisons.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"","code":"library(ggplot2) scores <- score(as_forecast(example_quantile)) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. pairwise <- get_pairwise_comparisons(scores, by = \"target_type\") plot_pairwise_comparisons(pairwise, type = \"mean_scores_ratio\") + facet_wrap(~target_type)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pit.html","id":null,"dir":"Reference","previous_headings":"","what":"PIT histogram — plot_pit","title":"PIT histogram — plot_pit","text":"Make simple histogram probability integral transformed values visually check whether uniform distribution seems likely.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pit.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"PIT histogram — plot_pit","text":"","code":"plot_pit(pit, num_bins = \"auto\", breaks = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pit.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"PIT histogram — plot_pit","text":"pit Either vector PIT values, data.table produced get_pit(). num_bins number bins PIT histogram, default \"auto\". num_bins == \"auto\", plot_pit() either display 10 bins, display bin available quantile case passed data quantile-based format. can control number bins supplying number. fine sample-based pit histograms, may fail quantile-based formats. case preferred supply explicit breaks points using breaks argument. breaks Numeric vector break points bins PIT histogram. preferred creating PIT histogram based quantile-based data. Default NULL breaks determined num_bins. breaks used, num_bins ignored.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pit.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"PIT histogram — plot_pit","text":"ggplot object histogram PIT values","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pit.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"PIT histogram — plot_pit","text":"","code":"# \\dontshow{ data.table::setDTthreads(2) # restricts number of cores used on CRAN # } # PIT histogram in vector based format observed <- rnorm(30, mean = 1:30) predicted <- replicate(200, rnorm(n = 30, mean = 1:30)) pit <- pit_sample(observed, predicted) plot_pit(pit) # quantile-based pit pit <- get_pit(as_forecast(example_quantile), by = \"model\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_pit(pit, breaks = seq(0.1, 1, 0.1)) # sample-based pit pit <- get_pit(as_forecast(example_sample_discrete), by = \"model\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_pit(pit)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot quantile coverage — plot_quantile_coverage","title":"Plot quantile coverage — plot_quantile_coverage","text":"Plot quantile coverage values (see get_coverage() information).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot quantile coverage — plot_quantile_coverage","text":"","code":"plot_quantile_coverage(coverage, colour = \"model\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot quantile coverage — plot_quantile_coverage","text":"coverage data frame coverage values produced get_coverage(). colour String, according variable shall graphs coloured? Default \"model\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot quantile coverage — plot_quantile_coverage","text":"ggplot object plot interval coverage","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot quantile coverage — plot_quantile_coverage","text":"","code":"coverage <- get_coverage(as_forecast(example_quantile), by = \"model\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_quantile_coverage(coverage)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot contributions to the weighted interval score — plot_wis","title":"Plot contributions to the weighted interval score — plot_wis","text":"Visualise components weighted interval score: penalties -prediction, -prediction high dispersion (lack sharpness).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot contributions to the weighted interval score — plot_wis","text":"","code":"plot_wis(scores, x = \"model\", relative_contributions = FALSE, flip = FALSE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot contributions to the weighted interval score — plot_wis","text":"scores data.table scores based quantile forecasts produced score() summarised using summarise_scores(). x variable scores want show x-Axis. Usually \"model\". relative_contributions Logical. Show relative contributions instead absolute contributions? Default FALSE functionality available yet. flip Boolean (default FALSE), whether flip axes.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot contributions to the weighted interval score — plot_wis","text":"ggplot object showing contributions three components weighted interval score. ggplot object visualisation WIS decomposition","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Plot contributions to the weighted interval score — plot_wis","text":"Bracher J, Ray E, Gneiting T, Reich, N (2020) Evaluating epidemic forecasts interval format. https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1008618","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot contributions to the weighted interval score — plot_wis","text":"","code":"library(ggplot2) scores <- score(as_forecast(example_quantile)) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. scores <- summarise_scores(scores, by = c(\"model\", \"target_type\")) plot_wis(scores, x = \"model\", relative_contributions = TRUE ) + facet_wrap(~target_type) plot_wis(scores, x = \"model\", relative_contributions = FALSE ) + facet_wrap(~target_type, scales = \"free_x\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Print information about a forecast object — print.forecast_binary","title":"Print information about a forecast object — print.forecast_binary","text":"function prints information forecast object, including \"Forecast type\", \"Score columns\", \"Forecast unit\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Print information about a forecast object — print.forecast_binary","text":"","code":"# S3 method for class 'forecast_binary' print(x, ...) # S3 method for class 'forecast_quantile' print(x, ...) # S3 method for class 'forecast_point' print(x, ...) # S3 method for class 'forecast_sample' print(x, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Print information about a forecast object — print.forecast_binary","text":"x forecast object (validated data.table predicted observed values, see as_forecast()). ... Additional arguments print().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast_binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Print information about a forecast object — print.forecast_binary","text":"Returns x invisibly.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast_binary.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Print information about a forecast object — print.forecast_binary","text":"","code":"dat <- as_forecast(example_quantile) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. print(dat) #> Forecast type: quantile #> Forecast unit: #> location, target_end_date, target_type, location_name, forecast_date, model, #> and horizon #> #> Key: #> location target_end_date target_type observed location_name #> #> 1: DE 2021-01-02 Cases 127300 Germany #> 2: DE 2021-01-02 Deaths 4534 Germany #> 3: DE 2021-01-09 Cases 154922 Germany #> 4: DE 2021-01-09 Deaths 6117 Germany #> 5: DE 2021-01-16 Cases 110183 Germany #> --- #> 20541: IT 2021-07-24 Deaths 78 Italy #> 20542: IT 2021-07-24 Deaths 78 Italy #> 20543: IT 2021-07-24 Deaths 78 Italy #> 20544: IT 2021-07-24 Deaths 78 Italy #> 20545: IT 2021-07-24 Deaths 78 Italy #> forecast_date quantile_level predicted model horizon #> #> 1: NA NA NA #> 2: NA NA NA #> 3: NA NA NA #> 4: NA NA NA #> 5: NA NA NA #> --- #> 20541: 2021-07-12 0.850 352 epiforecasts-EpiNow2 2 #> 20542: 2021-07-12 0.900 397 epiforecasts-EpiNow2 2 #> 20543: 2021-07-12 0.950 499 epiforecasts-EpiNow2 2 #> 20544: 2021-07-12 0.975 611 epiforecasts-EpiNow2 2 #> 20545: 2021-07-12 0.990 719 epiforecasts-EpiNow2 2"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":null,"dir":"Reference","previous_headings":"","what":"Quantile score — quantile_score","title":"Quantile score — quantile_score","text":"Proper Scoring Rule score quantile predictions. Smaller values better. quantile score closely related interval score (see wis()) quantile equivalent works single quantiles instead central prediction intervals. quantile score, also called pinball loss, single quantile level \\(\\tau\\) defined $$ \\text{QS}_\\tau(F, y) = 2 \\cdot \\{ \\mathbf{1}(y \\leq q_\\tau) - \\tau\\} \\cdot (q_\\tau − y) = \\begin{cases} 2 \\cdot (1 - \\tau) * q_\\tau - y, & \\text{} y \\leq q_\\tau\\\\ 2 \\cdot \\tau * |q_\\tau - y|, & \\text{} y > q_\\tau, \\end{cases} $$ \\(q_\\tau\\) \\(\\tau\\)-quantile predictive distribution \\(F\\), \\(\\mathbf{1}(\\cdot)\\) indicator function. weighted interval score single prediction interval can obtained average quantile scores lower upper quantile prediction interval: $$ \\text{WIS}_\\alpha(F, y) = \\frac{\\text{QS}_{\\alpha/2}(F, y) + \\text{QS}_{1 - \\alpha/2}(F, y)}{2}. $$ See SI Bracher et al. (2021) details. quantile_score() returns average quantile score across quantile levels provided. set quantile levels form pairwise central prediction intervals, quantile score equivalent interval score.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Quantile score — quantile_score","text":"","code":"quantile_score(observed, predicted, quantile_level, weigh = TRUE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Quantile score — quantile_score","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made. weigh Logical. TRUE (default), weigh score \\(\\alpha / 2\\), can averaged interval score , limit (increasing number equally spaced quantiles/prediction intervals), corresponds CRPS. \\(\\alpha\\) value corresponds (\\(\\alpha/2\\)) (\\(1 - \\alpha/2\\)), .e. decimal value represents much outside central prediction interval (E.g. 90 percent central prediction interval, alpha 0.1).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Quantile score — quantile_score","text":"Numeric vector length n quantile score. scores averaged across quantile levels multiple quantile levels provided (result calling rowMeans() matrix quantile scores computed based observed predicted values).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Quantile score — quantile_score","text":"Strictly Proper Scoring Rules, Prediction,Estimation, Tilmann Gneiting Adrian E. Raftery, 2007, Journal American Statistical Association, Volume 102, 2007 - Issue 477 Evaluating epidemic forecasts interval format, Johannes Bracher, Evan L. Ray, Tilmann Gneiting Nicholas G. Reich, 2021, https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1008618","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Quantile score — quantile_score","text":"","code":"observed <- rnorm(10, mean = 1:10) alpha <- 0.5 lower <- qnorm(alpha / 2, observed) upper <- qnorm((1 - alpha / 2), observed) qs_lower <- quantile_score(observed, predicted = matrix(lower), quantile_level = alpha / 2 ) qs_upper <- quantile_score(observed, predicted = matrix(upper), quantile_level = 1 - alpha / 2 ) interval_score <- (qs_lower + qs_upper) / 2 interval_score2 <- quantile_score( observed, predicted = cbind(lower, upper), quantile_level = c(alpha / 2, 1 - alpha / 2) ) # this is the same as the following wis( observed, predicted = cbind(lower, upper), quantile_level = c(alpha / 2, 1 - alpha / 2) ) #> [1] 0.3372449 0.3372449 0.3372449 0.3372449 0.3372449 0.3372449 0.3372449 #> [8] 0.3372449 0.3372449 0.3372449"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_to_interval.html","id":null,"dir":"Reference","previous_headings":"","what":"Transform from a quantile format to an interval format — quantile_to_interval","title":"Transform from a quantile format to an interval format — quantile_to_interval","text":"Internal helper function transform quantile format interval format (longer supported forecast format, still used internally. function mimics S3 generic, actually S3 generic, want functions internal exported.) Quantile format quantile format, prediction characterised one multiple predicted values corresponding quantile levels. example, prediction quantile format represented 0.05, 0.25, 0.5, 0.75 0.95 quantiles predictive distribution. Interval format interval format, two quantiles assumed form prediction interval. Prediction intervals need symmetric around median characterised lower upper bound. lower bound defined lower quantile upper bound defined upper quantile. 90% prediction interval, example, covers 90% probability mass defined 5% 95% quantiles. forecast therefore characterised one multiple prediction intervals, e.g. lower upper bounds 50% 90% prediction intervals (corresponding 0.25 0.75 well 0.05 0.095 quantiles).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_to_interval.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Transform from a quantile format to an interval format — quantile_to_interval","text":"","code":"quantile_to_interval(...) quantile_to_interval_dataframe( forecast, format = \"long\", keep_quantile_col = FALSE, ... ) quantile_to_interval_numeric(observed, predicted, quantile_level, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_to_interval.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Transform from a quantile format to an interval format — quantile_to_interval","text":"... Arguments forecast data.table forecasts quantile-based format (see as_forecast()). format format output. Either \"long\" \"wide\". \"long\" (default), column boundary (values either \"upper\" \"lower\" column interval_range contains range interval. \"wide\", column interval_range two columns lower upper contain lower upper bounds prediction interval, respectively. keep_quantile_col keep quantile_level column final output transformation (default FALSE). works format = \"long\". format = \"wide\", quantile_level column always dropped. observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_to_interval.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Transform from a quantile format to an interval format — quantile_to_interval","text":"data.table forecasts interval format. quantile_to_interval_dataframe: data.table interval format (either \"long\" \"wide\"), without quantile_level column. Rows reordered. quantile_to_interval.numeric: data.table wide interval format columns forecast_id, observed, lower, upper, interval_range. forecast_id column unique identifier forecast. Rows reordered according forecast_id interval_range.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/run_safely.html","id":null,"dir":"Reference","previous_headings":"","what":"Run a function safely — run_safely","title":"Run a function safely — run_safely","text":"wrapper function designed run function safely completely clear arguments passed function. named arguments ... accepted fun removed. unnamed arguments passed function. case fun errors, error converted warning run_safely returns NULL. run_safely can useful constructing functions used metrics score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/run_safely.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Run a function safely — run_safely","text":"","code":"run_safely(..., fun, metric_name)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/run_safely.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Run a function safely — run_safely","text":"... Arguments pass fun. fun function execute. metric_name character string name metric. Used provide informative warning message case fun errors.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/run_safely.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Run a function safely — run_safely","text":"result fun NULL fun errors","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/run_safely.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Run a function safely — run_safely","text":"","code":"f <- function(x) {x} scoringutils:::run_safely(2, fun = f, metric_name = \"f\") #> [1] 2 scoringutils:::run_safely(2, y = 3, fun = f, metric_name = \"f\") #> [1] 2 scoringutils:::run_safely(fun = f, metric_name = \"f\") #> Warning: ! Computation for `f` failed. Error: argument \"x\" is missing, with no default. #> NULL scoringutils:::run_safely(y = 3, fun = f, metric_name = \"f\") #> Warning: ! Computation for `f` failed. Error: argument \"x\" is missing, with no default. #> NULL"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_interval_long.html","id":null,"dir":"Reference","previous_headings":"","what":"Change data from a sample-based format to a long interval range format — sample_to_interval_long","title":"Change data from a sample-based format to a long interval range format — sample_to_interval_long","text":"Transform data format based predictive samples format based interval ranges.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_interval_long.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Change data from a sample-based format to a long interval range format — sample_to_interval_long","text":"","code":"sample_to_interval_long( data, interval_range = c(0, 50, 90), type = 7, keep_quantile_col = TRUE )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_interval_long.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Change data from a sample-based format to a long interval range format — sample_to_interval_long","text":"type Type argument passed quantile function. information, see quantile(). keep_quantile_col keep quantile_level column, default TRUE","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_interval_long.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Change data from a sample-based format to a long interval range format — sample_to_interval_long","text":"data.table long interval interval range format","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Change data from a sample based format to a quantile format — sample_to_quantile","title":"Change data from a sample based format to a quantile format — sample_to_quantile","text":"Transform data format based predictive samples format based plain quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Change data from a sample based format to a quantile format — sample_to_quantile","text":"","code":"sample_to_quantile( forecast, quantile_level = c(0.05, 0.25, 0.5, 0.75, 0.95), type = 7 )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Change data from a sample based format to a quantile format — sample_to_quantile","text":"forecast forecast object class forecast_sample (validated data.table predicted observed values, see as_forecast()). quantile_level numeric vector quantile levels quantiles computed. type Type argument passed quantile function. information, see quantile().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Change data from a sample based format to a quantile format — sample_to_quantile","text":"data.table long interval range format","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_quantile.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Change data from a sample based format to a quantile format — sample_to_quantile","text":"","code":"sample_to_quantile(as_forecast(example_sample_discrete)) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Forecast type: quantile #> Forecast unit: #> location, location_name, target_end_date, target_type, forecast_date, model, #> and horizon #> #> location location_name target_end_date target_type forecast_date #> #> 1: DE Germany 2021-01-02 Cases #> 2: DE Germany 2021-01-02 Cases #> 3: DE Germany 2021-01-02 Cases #> 4: DE Germany 2021-01-02 Cases #> 5: DE Germany 2021-01-02 Cases #> --- #> 5151: IT Italy 2021-07-24 Deaths 2021-07-12 #> 5152: IT Italy 2021-07-24 Deaths 2021-07-12 #> 5153: IT Italy 2021-07-24 Deaths 2021-07-12 #> 5154: IT Italy 2021-07-24 Deaths 2021-07-12 #> 5155: IT Italy 2021-07-24 Deaths 2021-07-12 #> model horizon observed quantile_level predicted #> #> 1: NA 127300 0.05 NA #> 2: NA 127300 0.25 NA #> 3: NA 127300 0.50 NA #> 4: NA 127300 0.75 NA #> 5: NA 127300 0.95 NA #> --- #> 5151: epiforecasts-EpiNow2 2 78 0.05 44.90 #> 5152: epiforecasts-EpiNow2 2 78 0.25 89.25 #> 5153: epiforecasts-EpiNow2 2 78 0.50 151.50 #> 5154: epiforecasts-EpiNow2 2 78 0.75 208.00 #> 5155: epiforecasts-EpiNow2 2 78 0.95 469.10"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":null,"dir":"Reference","previous_headings":"","what":"Evaluate forecasts — score","title":"Evaluate forecasts — score","text":"score() applies selection scoring metrics forecast object (data.table forecasts observations) produced as_forecast(). score() generic dispatches different methods depending class input data. See details section information forecast types input formats. additional help examples, check Getting Started Vignette well paper Evaluating Forecasts scoringutils R.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Evaluate forecasts — score","text":"","code":"score(forecast, metrics, ...) # S3 method for class 'forecast_binary' score(forecast, metrics = metrics_binary(), ...) # S3 method for class 'forecast_point' score(forecast, metrics = metrics_point(), ...) # S3 method for class 'forecast_sample' score(forecast, metrics = metrics_sample(), ...) # S3 method for class 'forecast_quantile' score(forecast, metrics = metrics_quantile(), ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Evaluate forecasts — score","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()) metrics named list scoring functions. Names used column names output. See metrics_point(), metrics_binary(), metrics_quantile(), metrics_sample() information default metrics used. Note want pass arguments given metric, function customise_metric() pass updated list functions custom metric metrics argument score(). ... Additional arguments. Currently unused allows future extensions. want pass arguments individual metrics, use customise_metric().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Evaluate forecasts — score","text":"object class scores. object data.table unsummarised scores (one score per forecast) additional attribute metrics names metrics used scoring. See summarise_scores()) information summarise scores.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"forecast-types-and-input-formats","dir":"Reference","previous_headings":"","what":"Forecast types and input formats","title":"Evaluate forecasts — score","text":"Various different forecast types / forecast formats supported. moment, : point forecasts binary forecasts (\"soft binary classification\") Probabilistic forecasts quantile-based format (forecast represented set predictive quantiles) Probabilistic forecasts sample-based format (forecast represented set predictive samples) Forecast types determined based columns present input data. overview required format forecast type: forecast types require data.frame similar columns observed predicted, model. Point forecasts require column observed type numeric column predicted type numeric. Binary forecasts require column observed type factor exactly two levels column predicted type numeric probabilities, corresponding probability observed equal second factor level. See details information. Quantile-based forecasts require column observed type numeric, column predicted type numeric, column quantile_level type numeric quantile-levels (0 1). Sample-based forecasts require column observed type numeric, column predicted type numeric, column sample_id type numeric sample indices. information see vignettes example data (example_quantile, example_sample_continuous, example_sample_discrete, example_point(), example_binary).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"forecast-unit","dir":"Reference","previous_headings":"","what":"Forecast unit","title":"Evaluate forecasts — score","text":"order score forecasts, scoringutils needs know rows data belong together jointly form single forecasts. easy e.g. point forecast, one row per forecast. quantile sample-based forecasts, however, multiple rows belong single forecast. forecast unit unit single forecast described combination columns uniquely identify single forecast. example, forecasts made different models various locations different time points, several weeks future. forecast unit described forecast_unit = c(\"model\", \"location\", \"forecast_date\", \"forecast_horizon\"). scoringutils automatically tries determine unit single forecast. uses existing columns , means columns must present unrelated forecast unit. simplistic example, additional row, \"even\", one row number even zero otherwise, mess scoring scoringutils thinks column relevant defining forecast unit. order avoid issues, recommend setting forecast unit explicitly, usually forecast_unit argument as_forecast(). drop unneeded columns, making sure necessary, 'protected columns' like \"predicted\" \"observed\" retained.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Evaluate forecasts — score","text":"Bosse NI, Gruson H, Cori , van Leeuwen E, Funk S, Abbott S (2022) Evaluating Forecasts scoringutils R. doi:10.48550/arXiv.2205.07090","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Evaluate forecasts — score","text":"Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Evaluate forecasts — score","text":"","code":"library(magrittr) # pipe operator # \\dontshow{ data.table::setDTthreads(2) # restricts number of cores used on CRAN # } validated <- as_forecast(example_quantile) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. score(validated) %>% summarise_scores(by = c(\"model\", \"target_type\")) #> model target_type wis overprediction underprediction #> #> 1: EuroCOVIDhub-ensemble Cases 17943.82383 10043.121943 4237.177310 #> 2: EuroCOVIDhub-baseline Cases 28483.57465 14096.100883 10284.972826 #> 3: epiforecasts-EpiNow2 Cases 20831.55662 11906.823030 3260.355639 #> 4: EuroCOVIDhub-ensemble Deaths 41.42249 7.138247 4.103261 #> 5: EuroCOVIDhub-baseline Deaths 159.40387 65.899117 2.098505 #> 6: UMass-MechBayes Deaths 52.65195 8.978601 16.800951 #> 7: epiforecasts-EpiNow2 Deaths 66.64282 18.892583 15.893314 #> dispersion bias interval_coverage_50 interval_coverage_90 #> #> 1: 3663.52458 -0.05640625 0.3906250 0.8046875 #> 2: 4102.50094 0.09796875 0.3281250 0.8203125 #> 3: 5664.37795 -0.07890625 0.4687500 0.7890625 #> 4: 30.18099 0.07265625 0.8750000 1.0000000 #> 5: 91.40625 0.33906250 0.6640625 1.0000000 #> 6: 26.87239 -0.02234375 0.4609375 0.8750000 #> 7: 31.85692 -0.00512605 0.4201681 0.9075630 #> interval_coverage_deviation ae_median #> #> 1: -0.10230114 24101.07031 #> 2: -0.11721591 38473.60156 #> 3: -0.06963068 27923.81250 #> 4: 0.20380682 53.13281 #> 5: 0.12142045 233.25781 #> 6: -0.02488636 78.47656 #> 7: -0.04520244 104.74790 # set forecast unit manually (to avoid issues with scoringutils trying to # determine the forecast unit automatically) example_quantile %>% as_forecast( forecast_unit = c( \"location\", \"target_end_date\", \"target_type\", \"horizon\", \"model\" ) ) %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> location target_end_date target_type horizon model #> #> 1: DE 2021-05-08 Cases 1 EuroCOVIDhub-ensemble #> 2: DE 2021-05-08 Cases 1 EuroCOVIDhub-baseline #> 3: DE 2021-05-08 Cases 1 epiforecasts-EpiNow2 #> 4: DE 2021-05-08 Deaths 1 EuroCOVIDhub-ensemble #> 5: DE 2021-05-08 Deaths 1 EuroCOVIDhub-baseline #> --- #> 883: IT 2021-07-24 Deaths 2 EuroCOVIDhub-baseline #> 884: IT 2021-07-24 Deaths 3 UMass-MechBayes #> 885: IT 2021-07-24 Deaths 2 UMass-MechBayes #> 886: IT 2021-07-24 Deaths 3 epiforecasts-EpiNow2 #> 887: IT 2021-07-24 Deaths 2 epiforecasts-EpiNow2 #> wis overprediction underprediction dispersion bias #> #> 1: 7990.854783 2.549870e+03 0.0000000 5440.985217 0.50 #> 2: 16925.046957 1.527583e+04 0.0000000 1649.220870 0.95 #> 3: 25395.960870 1.722226e+04 0.0000000 8173.700000 0.90 #> 4: 53.880000 0.000000e+00 0.6086957 53.271304 -0.10 #> 5: 46.793043 2.130435e+00 0.0000000 44.662609 0.30 #> --- #> 883: 80.336957 3.608696e+00 0.0000000 76.728261 0.20 #> 884: 4.881739 4.347826e-02 0.0000000 4.838261 0.10 #> 885: 25.581739 1.782609e+01 0.0000000 7.755652 0.80 #> 886: 19.762609 5.478261e+00 0.0000000 14.284348 0.50 #> 887: 66.161739 4.060870e+01 0.0000000 25.553043 0.90 #> interval_coverage_50 interval_coverage_90 interval_coverage_deviation #> #> 1: TRUE TRUE 0.05181818 #> 2: FALSE FALSE -0.40272727 #> 3: FALSE TRUE -0.31181818 #> 4: TRUE TRUE 0.41545455 #> 5: TRUE TRUE 0.23363636 #> --- #> 883: TRUE TRUE 0.32454545 #> 884: TRUE TRUE 0.41545455 #> 885: FALSE TRUE -0.22090909 #> 886: TRUE TRUE 0.05181818 #> 887: FALSE TRUE -0.31181818 #> ae_median #> #> 1: 12271 #> 2: 25620 #> 3: 44192 #> 4: 14 #> 5: 15 #> --- #> 883: 53 #> 884: 1 #> 885: 46 #> 886: 26 #> 887: 108 # forecast formats with different metrics if (FALSE) { score(as_forecast(example_binary)) score(as_forecast(example_quantile)) score(as_forecast(example_point)) score(as_forecast(example_sample_discrete)) score(as_forecast(example_sample_continuous)) }"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Metrics for binary outcomes — scoring-functions-binary","title":"Metrics for binary outcomes — scoring-functions-binary","text":"Brier score Brier Score mean squared error probabilistic prediction observed outcome. Brier score proper scoring rule. Small values better (best 0, worst 1). $$ \\textrm{Brier\\_Score} = (\\textrm{prediction} - \\textrm{outcome})^2, $$ \\(\\textrm{outcome} \\\\{0, 1\\}\\), \\(\\textrm{prediction} \\[0, 1]\\) represents probability outcome equal 1. Log score binary outcomes Log Score negative logarithm probability assigned observed value. proper scoring rule. Small values better (best zero, worst infinity).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Metrics for binary outcomes — scoring-functions-binary","text":"","code":"brier_score(observed, predicted) logs_binary(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Metrics for binary outcomes — scoring-functions-binary","text":"observed factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted numeric vector length n, holding probabilities. Values represent probability corresponding outcome equal highest level factor observed.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Metrics for binary outcomes — scoring-functions-binary","text":"numeric vector size n Brier scores numeric vector size n log scores","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Metrics for binary outcomes — scoring-functions-binary","text":"Input formats functions require users provide observed values factor order distinguish input input format required scoring point forecasts. Internally, however, factors converted numeric values. factor observed = factor(c(0, 1, 1, 0, 1) two levels (0 1) internally coerced numeric vector (case result numeric vector c(1, 2, 2, 1, 1)). subtracting 1, resulting vector (c(0, 1, 1, 0) case) used internal calculations. predictions assumed represent probability outcome equal highest factor level (case outcome equal 1). alternatively also provide vector like observed = factor(c(\"\", \"b\", \"b\", \"\")) (two levels, b), result exactly internal representation. Probabilities represent probability outcome equal \"b\". want predictions probabilities outcome \"\", course make observed factor levels swapped, .e. observed = factor(c(\"\", \"b\", \"b\", \"\"), levels = c(\"b\", \"\"))","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Metrics for binary outcomes — scoring-functions-binary","text":"","code":"observed <- factor(sample(c(0, 1), size = 30, replace = TRUE)) predicted <- runif(n = 30, min = 0, max = 1) brier_score(observed, predicted) #> [1] 3.697221e-01 4.333309e-01 8.019571e-01 1.560465e-01 1.214441e-02 #> [6] 4.780794e-01 2.170087e-02 9.225365e-01 4.260066e-01 1.083496e-05 #> [11] 7.643096e-01 2.958275e-02 9.868210e-01 4.245489e-01 9.405384e-01 #> [16] 2.810347e-01 6.433678e-02 7.993802e-01 9.628835e-02 7.492222e-02 #> [21] 4.786024e-03 1.914958e-01 3.273303e-02 3.325011e-01 6.258271e-02 #> [26] 8.713640e-01 5.608322e-01 1.536012e-01 1.239574e-01 4.130782e-01 logs_binary(observed, predicted) #> [1] 0.93661542 1.07375988 2.25876533 0.50257178 0.11676043 1.17581594 #> [7] 0.15936173 3.23114148 1.05754275 0.00329708 2.07344387 0.18873775 #> [13] 5.01896290 1.05432990 3.50036400 0.75529303 0.29255656 2.24507736 #> [19] 0.37150383 0.31981861 0.07169054 0.57554614 0.19957683 0.85950668 #> [25] 0.28790257 2.71008389 1.38185350 0.49744870 0.43398155 1.02921200"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoringutils-package.html","id":null,"dir":"Reference","previous_headings":"","what":"scoringutils: Utilities for Scoring and Assessing Predictions — scoringutils-package","title":"scoringutils: Utilities for Scoring and Assessing Predictions — scoringutils-package","text":"`scoringutils` facilitates evaluation forecasts convenient framework based `data.table`. allows user check forecasts diagnose issues, visualise forecasts missing data, transform data scoring, handle missing forecasts, aggregate scores, visualise results evaluation. package mostly focuses evaluation probabilistic forecasts allows evaluating several different forecast types input formats. Find information package Vignettes well accompanying paper (doi:10.48550/arXiv.2205.07090 ).","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoringutils-package.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"scoringutils: Utilities for Scoring and Assessing Predictions — scoringutils-package","text":"Maintainer: Nikos Bosse nikosbosse@gmail.com (ORCID) Authors: Sam Abbott contact@samabbott.co.uk (ORCID) Hugo Gruson hugo.gruson+R@normalesup.org (ORCID) Sebastian Funk sebastian.funk@lshtm.ac.uk contributors: Johannes Bracher johannes.bracher@kit.edu (ORCID) [contributor] Toshiaki Asakura toshiaki.asa9ra@gmail.com (ORCID) [contributor] James Mba Azam james.azam@lshtm.ac.uk (ORCID) [contributor]","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/se_mean_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Squared error of the mean (sample-based version) — se_mean_sample","title":"Squared error of the mean (sample-based version) — se_mean_sample","text":"Squared error mean calculated $$ \\textrm{mean}(\\textrm{observed} - \\textrm{mean prediction})^2 $$ mean prediction calculated mean predictive samples.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/se_mean_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Squared error of the mean (sample-based version) — se_mean_sample","text":"","code":"se_mean_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/se_mean_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Squared error of the mean (sample-based version) — se_mean_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/se_mean_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Squared error of the mean (sample-based version) — se_mean_sample","text":"","code":"observed <- rnorm(30, mean = 1:30) predicted_values <- matrix(rnorm(30, mean = 1:30)) se_mean_sample(observed, predicted_values) #> [1] 0.173184776 0.023435451 1.610298449 3.571463532 0.287637723 3.456505359 #> [7] 3.104445200 6.416765323 0.014822594 0.262562121 0.002945155 2.290331532 #> [13] 0.062723285 0.492648216 0.047323098 1.489234187 1.959968436 2.103650766 #> [19] 0.086133601 0.894739782 5.993444771 2.978954352 6.778862480 1.779654975 #> [25] 1.712346282 0.843409936 3.193731460 0.514182655 0.134914452 0.485937390"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/select_metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Select metrics from a list of functions — select_metrics","title":"Select metrics from a list of functions — select_metrics","text":"Helper function return scoring rules selected user list possible functions.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/select_metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Select metrics from a list of functions — select_metrics","text":"","code":"select_metrics(metrics, select = NULL, exclude = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/select_metrics.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Select metrics from a list of functions — select_metrics","text":"metrics list scoring functions. select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/select_metrics.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Select metrics from a list of functions — select_metrics","text":"list scoring rules.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/select_metrics.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Select metrics from a list of functions — select_metrics","text":"","code":"select_metrics( metrics = metrics_binary(), select = \"brier_score\" ) #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> #> #> select_metrics( metrics = metrics_binary(), exclude = \"log_score\" ) #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> #> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/set_forecast_unit.html","id":null,"dir":"Reference","previous_headings":"","what":"Set unit of a single forecast manually — set_forecast_unit","title":"Set unit of a single forecast manually — set_forecast_unit","text":"Helper function set unit single forecast (.e. combination columns uniquely define single forecast) manually. simple function keeps columns specified forecast_unit (plus additional protected columns, e.g. observed values, predictions quantile levels) removes duplicate rows. set_forecast_unit() mainly called as_forecast() forecast_unit argument. done explicitly, scoringutils attempts determine unit single forecast automatically simply assuming column names relevant determine forecast unit. may lead unexpected behaviour, setting forecast unit explicitly can help make code easier debug easier read.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/set_forecast_unit.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Set unit of a single forecast manually — set_forecast_unit","text":"","code":"set_forecast_unit(data, forecast_unit)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/set_forecast_unit.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Set unit of a single forecast manually — set_forecast_unit","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. forecast_unit Character vector names columns uniquely identify single forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/set_forecast_unit.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Set unit of a single forecast manually — set_forecast_unit","text":"data.table columns kept relevant scoring denote unit single forecast specified user.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/set_forecast_unit.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Set unit of a single forecast manually — set_forecast_unit","text":"","code":"set_forecast_unit( example_quantile, c(\"location\", \"target_end_date\", \"target_type\", \"horizon\", \"model\") ) #> Key: #> observed quantile_level predicted location target_end_date target_type #> #> 1: 127300 NA NA DE 2021-01-02 Cases #> 2: 4534 NA NA DE 2021-01-02 Deaths #> 3: 154922 NA NA DE 2021-01-09 Cases #> 4: 6117 NA NA DE 2021-01-09 Deaths #> 5: 110183 NA NA DE 2021-01-16 Cases #> --- #> 20541: 78 0.850 352 IT 2021-07-24 Deaths #> 20542: 78 0.900 397 IT 2021-07-24 Deaths #> 20543: 78 0.950 499 IT 2021-07-24 Deaths #> 20544: 78 0.975 611 IT 2021-07-24 Deaths #> 20545: 78 0.990 719 IT 2021-07-24 Deaths #> horizon model #> #> 1: NA #> 2: NA #> 3: NA #> 4: NA #> 5: NA #> --- #> 20541: 2 epiforecasts-EpiNow2 #> 20542: 2 epiforecasts-EpiNow2 #> 20543: 2 epiforecasts-EpiNow2 #> 20544: 2 epiforecasts-EpiNow2 #> 20545: 2 epiforecasts-EpiNow2"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/summarise_scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Summarise scores as produced by score() — summarise_scores","title":"Summarise scores as produced by score() — summarise_scores","text":"Summarise scores produced score(). summarise_scores relies way identify names scores distinguish columns denote unit single forecast. Internally, done via stored attribute, metrics stores names scores. means, however, need careful renaming scores produced score(). , also manually update attribute calling attr(scores, \"metrics\") <- new_names.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/summarise_scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Summarise scores as produced by score() — summarise_scores","text":"","code":"summarise_scores(scores, by = \"model\", across = NULL, fun = mean, ...) summarize_scores(scores, by = \"model\", across = NULL, fun = mean, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/summarise_scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Summarise scores as produced by score() — summarise_scores","text":"scores object class scores (data.table scores additional attribute metrics produced score()). Character vector column names summarise scores . Default model, meaning one score per model output. across Character vector column names summarise scores across (meaning specified columns dropped). alternative specifying directly. across set, ignored. across NULL (default), used. fun function used summarising scores. Default mean(). ... Additional parameters can passed summary function provided fun. information see documentation respective function.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/summarise_scores.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Summarise scores as produced by score() — summarise_scores","text":"data.table summarised scores. Scores summarised according names columns original data specified across using fun passed summarise_scores().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/summarise_scores.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Summarise scores as produced by score() — summarise_scores","text":"","code":"# \\dontshow{ data.table::setDTthreads(2) # restricts number of cores used on CRAN # } library(magrittr) # pipe operator scores <- score(as_forecast(example_sample_continuous)) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. # get scores by model summarise_scores(scores,by = \"model\") #> model bias dss crps log_score mad #> #> 1: EuroCOVIDhub-ensemble 0.009765625 16.40496 9876.95886 10.747811 8763.6176 #> 2: EuroCOVIDhub-baseline 0.177734375 NaN 15309.68627 Inf 9680.3792 #> 3: epiforecasts-EpiNow2 -0.024898785 26.10137 11901.43354 Inf 12999.5404 #> 4: UMass-MechBayes -0.026953125 10.08582 60.19018 5.941622 123.6211 #> ae_median se_mean #> #> 1: 12406.03100 2.103026e+09 #> 2: 18932.50196 2.885063e+09 #> 3: 14680.12285 3.152268e+09 #> 4: 79.66001 1.371418e+04 # get scores by model and target type summarise_scores(scores, by = c(\"model\", \"target_type\")) #> model target_type bias dss crps log_score #> #> 1: EuroCOVIDhub-ensemble Cases -0.04648437 22.89997 19703.05522 15.633420 #> 2: EuroCOVIDhub-baseline Cases 0.03671875 NaN 30453.58346 Inf #> 3: epiforecasts-EpiNow2 Cases -0.03867188 40.87716 22896.51608 Inf #> 4: EuroCOVIDhub-ensemble Deaths 0.06601562 9.90995 50.86249 5.862203 #> 5: EuroCOVIDhub-baseline Deaths 0.31875000 12.99360 165.78907 6.977391 #> 6: UMass-MechBayes Deaths -0.02695313 10.08582 60.19018 5.941622 #> 7: epiforecasts-EpiNow2 Deaths -0.01008403 10.20807 74.79013 6.024092 #> mad ae_median se_mean #> #> 1: 17385.2629 24749.39707 4.206042e+09 #> 2: 18982.2128 37648.01693 5.769964e+09 #> 3: 24929.3438 28233.04536 6.082863e+09 #> 4: 141.9723 62.66492 1.080233e+04 #> 5: 378.5457 216.98699 1.622417e+05 #> 6: 123.6211 79.66001 1.371418e+04 #> 7: 167.4829 102.18939 3.243111e+04 # Get scores summarised across horizon, forecast date, and target end date summarise_scores( scores, across = c(\"horizon\", \"forecast_date\", \"target_end_date\") ) #> location location_name target_type model bias #> #> 1: DE Germany Cases EuroCOVIDhub-ensemble 0.17812500 #> 2: DE Germany Cases EuroCOVIDhub-baseline 0.51093750 #> 3: DE Germany Cases epiforecasts-EpiNow2 0.15937500 #> 4: DE Germany Deaths EuroCOVIDhub-ensemble -0.04531250 #> 5: DE Germany Deaths EuroCOVIDhub-baseline 0.40312500 #> 6: DE Germany Deaths UMass-MechBayes -0.58125000 #> 7: DE Germany Deaths epiforecasts-EpiNow2 -0.34531250 #> 8: FR France Cases EuroCOVIDhub-ensemble 0.01250000 #> 9: FR France Cases EuroCOVIDhub-baseline -0.02968750 #> 10: FR France Cases epiforecasts-EpiNow2 -0.02656250 #> 11: FR France Deaths EuroCOVIDhub-ensemble 0.04687500 #> 12: FR France Deaths EuroCOVIDhub-baseline 0.34218750 #> 13: FR France Deaths UMass-MechBayes 0.05156250 #> 14: FR France Deaths epiforecasts-EpiNow2 -0.05652174 #> 15: GB United Kingdom Cases EuroCOVIDhub-ensemble -0.47656250 #> 16: GB United Kingdom Cases EuroCOVIDhub-baseline -0.61875000 #> 17: GB United Kingdom Cases epiforecasts-EpiNow2 -0.18750000 #> 18: GB United Kingdom Deaths EuroCOVIDhub-ensemble 0.10937500 #> 19: GB United Kingdom Deaths EuroCOVIDhub-baseline 0.08281250 #> 20: GB United Kingdom Deaths UMass-MechBayes 0.62812500 #> 21: GB United Kingdom Deaths epiforecasts-EpiNow2 0.43125000 #> 22: IT Italy Cases EuroCOVIDhub-ensemble 0.10000000 #> 23: IT Italy Cases EuroCOVIDhub-baseline 0.28437500 #> 24: IT Italy Cases epiforecasts-EpiNow2 -0.10000000 #> 25: IT Italy Deaths EuroCOVIDhub-ensemble 0.15312500 #> 26: IT Italy Deaths EuroCOVIDhub-baseline 0.44687500 #> 27: IT Italy Deaths UMass-MechBayes -0.20625000 #> 28: IT Italy Deaths epiforecasts-EpiNow2 -0.08281250 #> location location_name target_type model bias #> dss crps log_score mad ae_median se_mean #> #> 1: 18.913810 7936.62666 10.368232 11207.25397 11618.84123 4.494508e+08 #> 2: 21.878467 15588.64493 11.938333 13450.95537 20043.14918 9.339773e+08 #> 3: 29.470535 13453.07861 22.686464 14195.91295 18498.57965 1.286406e+09 #> 4: 10.257710 56.54768 6.104505 167.78914 67.75772 1.061735e+04 #> 5: 12.536674 155.04973 6.997312 391.89252 219.90763 1.141223e+05 #> 6: 10.620902 76.81642 6.345002 147.52889 113.14713 1.543732e+04 #> 7: 11.020430 98.55426 6.571691 222.78911 140.73457 3.401571e+04 #> 8: 31.176915 47132.62230 28.508977 20503.83242 54298.94987 1.427192e+10 #> 9: NaN 56093.15772 Inf 25117.85238 65048.92899 1.653529e+10 #> 10: 34.493853 54117.03193 32.355909 33004.99213 63535.47737 1.981982e+10 #> 11: 10.729140 67.93297 6.236167 188.67948 85.67433 1.819764e+04 #> 12: 13.289991 202.77900 7.204643 501.10175 259.38228 2.316930e+05 #> 13: 11.322153 85.42422 6.518248 186.16678 98.33634 2.700122e+04 #> 14: 11.196840 119.95348 6.553617 331.22963 160.27973 8.603595e+04 #> 15: 21.106923 17824.87005 11.748949 30723.25562 24247.60751 1.940949e+09 #> 16: 24.877031 40848.86212 13.554332 22622.32113 51882.49530 5.228815e+09 #> 17: 21.224165 20508.11818 11.483762 47677.95848 25915.83258 3.160764e+09 #> 18: 8.839111 25.48464 5.251438 81.18266 26.29047 1.515431e+03 #> 19: 13.061990 109.66110 6.625700 212.20871 110.49682 1.048123e+05 #> 20: 9.040336 40.55896 5.310473 67.03450 62.02151 8.933439e+03 #> 21: 8.859818 29.31789 5.233896 73.15706 35.78943 4.412517e+03 #> 22: 20.402224 5918.10189 11.907522 7106.70962 8832.18969 1.618505e+08 #> 23: 20.338858 9283.66908 10.903144 14737.72225 13617.49426 3.817765e+08 #> 24: 78.320096 3507.83559 Inf 4838.51176 4982.29184 6.446300e+07 #> 25: 9.813839 53.48466 5.856701 130.23793 70.93715 1.287888e+04 #> 26: 13.085745 195.66648 7.081908 408.97976 278.16121 1.983392e+05 #> 27: 9.359876 37.96112 5.592767 93.75413 45.13505 3.484721e+03 #> 28: 10.033302 64.03710 5.886091 88.80946 88.29173 2.033662e+04 #> dss crps log_score mad ae_median se_mean # get standard deviation summarise_scores(scores, by = \"model\", fun = sd) #> model bias dss crps log_score mad #> #> 1: EuroCOVIDhub-ensemble 0.5468290 14.869520 39368.24836 21.510119 19799.1620 #> 2: EuroCOVIDhub-baseline 0.5457971 NA 45020.82814 NaN 13610.4174 #> 3: epiforecasts-EpiNow2 0.6083410 108.130107 44957.07746 NaN 29616.1714 #> 4: UMass-MechBayes 0.6221914 2.248998 49.62465 1.126019 123.3465 #> ae_median se_mean #> #> 1: 42801.64123 1.564286e+10 #> 2: 49458.36446 1.760651e+10 #> 3: 51129.54601 2.209086e+10 #> 4: 76.09471 2.994664e+04 # round digits summarise_scores(scores,by = \"model\") %>% summarise_scores(fun = signif, digits = 2) #> model bias dss crps log_score mad ae_median se_mean #> #> 1: EuroCOVIDhub-ensemble 0.0098 16 9900 11.0 8800 12000 2.1e+09 #> 2: EuroCOVIDhub-baseline 0.1800 NaN 15000 Inf 9700 19000 2.9e+09 #> 3: epiforecasts-EpiNow2 -0.0250 26 12000 Inf 13000 15000 3.2e+09 #> 4: UMass-MechBayes -0.0270 10 60 5.9 120 80 1.4e+04"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_not_present.html","id":null,"dir":"Reference","previous_headings":"","what":"Test whether column names are NOT present in a data.frame — test_columns_not_present","title":"Test whether column names are NOT present in a data.frame — test_columns_not_present","text":"function checks whether column names present. none columns present, function returns TRUE. one columns present, function returns FALSE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_not_present.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Test whether column names are NOT present in a data.frame — test_columns_not_present","text":"","code":"test_columns_not_present(data, columns)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_not_present.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Test whether column names are NOT present in a data.frame — test_columns_not_present","text":"data data.frame similar checked columns character vector column names check","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_not_present.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Test whether column names are NOT present in a data.frame — test_columns_not_present","text":"Returns TRUE none columns present FALSE otherwise","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_present.html","id":null,"dir":"Reference","previous_headings":"","what":"Test whether all column names are present in a data.frame — test_columns_present","title":"Test whether all column names are present in a data.frame — test_columns_present","text":"function checks whether column names present. one columns missing, function returns FALSE. columns present, function returns TRUE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_present.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Test whether all column names are present in a data.frame — test_columns_present","text":"","code":"test_columns_present(data, columns)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_present.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Test whether all column names are present in a data.frame — test_columns_present","text":"data data.frame similar checked columns character vector column names check","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_present.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Test whether all column names are present in a data.frame — test_columns_present","text":"Returns TRUE columns present FALSE otherwise","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Test whether data could be a binary forecast. — test_forecast_type_is_binary","title":"Test whether data could be a binary forecast. — test_forecast_type_is_binary","text":"Checks type necessary columns.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Test whether data could be a binary forecast. — test_forecast_type_is_binary","text":"","code":"test_forecast_type_is_binary(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Test whether data could be a binary forecast. — test_forecast_type_is_binary","text":"data data.frame similar checked","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Test whether data could be a binary forecast. — test_forecast_type_is_binary","text":"Returns TRUE basic requirements satisfied FALSE otherwise","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Test whether data could be a point forecast. — test_forecast_type_is_point","title":"Test whether data could be a point forecast. — test_forecast_type_is_point","text":"Checks type necessary columns.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Test whether data could be a point forecast. — test_forecast_type_is_point","text":"","code":"test_forecast_type_is_point(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Test whether data could be a point forecast. — test_forecast_type_is_point","text":"data data.frame similar checked","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Test whether data could be a point forecast. — test_forecast_type_is_point","text":"Returns TRUE basic requirements satisfied FALSE otherwise","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Test whether data could be a quantile forecast. — test_forecast_type_is_quantile","title":"Test whether data could be a quantile forecast. — test_forecast_type_is_quantile","text":"Checks type necessary columns.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Test whether data could be a quantile forecast. — test_forecast_type_is_quantile","text":"","code":"test_forecast_type_is_quantile(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Test whether data could be a quantile forecast. — test_forecast_type_is_quantile","text":"data data.frame similar checked","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Test whether data could be a quantile forecast. — test_forecast_type_is_quantile","text":"Returns TRUE basic requirements satisfied FALSE otherwise","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Test whether data could be a sample-based forecast. — test_forecast_type_is_sample","title":"Test whether data could be a sample-based forecast. — test_forecast_type_is_sample","text":"Checks type necessary columns.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Test whether data could be a sample-based forecast. — test_forecast_type_is_sample","text":"","code":"test_forecast_type_is_sample(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Test whether data could be a sample-based forecast. — test_forecast_type_is_sample","text":"data data.frame similar checked","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Test whether data could be a sample-based forecast. — test_forecast_type_is_sample","text":"Returns TRUE basic requirements satisfied FALSE otherwise","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/theme_scoringutils.html","id":null,"dir":"Reference","previous_headings":"","what":"Scoringutils ggplot2 theme — theme_scoringutils","title":"Scoringutils ggplot2 theme — theme_scoringutils","text":"theme ggplot2 plots used scoringutils.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/theme_scoringutils.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Scoringutils ggplot2 theme — theme_scoringutils","text":"","code":"theme_scoringutils()"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/theme_scoringutils.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Scoringutils ggplot2 theme — theme_scoringutils","text":"ggplot2 theme","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":null,"dir":"Reference","previous_headings":"","what":"Transform forecasts and observed values — transform_forecasts","title":"Transform forecasts and observed values — transform_forecasts","text":"Function transform forecasts observed values scoring.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Transform forecasts and observed values — transform_forecasts","text":"","code":"transform_forecasts( forecast, fun = log_shift, append = TRUE, label = \"log\", ... )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Transform forecasts and observed values — transform_forecasts","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()) fun function used transform observed values predictions. default function log_shift(), custom function essentially log(), additional arguments (offset) allows add offset applying logarithm. often helpful natural log transformation defined zero. common, pragmatic solution, add small offset data applying log transformation. work often used offset 1 precise value depend application. append Logical, defaults TRUE. Whether append transformed version data currently existing data (TRUE). selected, data gets transformed appended existing data, making possible use outcome directly score(). additional column, 'scale', gets created denotes rows untransformed ('scale' value \"natural\") transformed ('scale' value passed argument label). label string newly created 'scale' column denote newly transformed values. relevant append = TRUE. ... Additional parameters pass function supplied. default option log_shift() offset argument.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Transform forecasts and observed values — transform_forecasts","text":"forecast object either transformed version data, one untransformed transformed data. includes original data well transformation original data. one additional column, `scale', present set \"natural\" untransformed forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Transform forecasts and observed values — transform_forecasts","text":"reasons, depending circumstances, might desirable (check linked reference info). epidemiology, example, may useful log-transform incidence counts evaluating forecasts using scores weighted interval score (WIS) continuous ranked probability score (CRPS). Log-transforming forecasts observations changes interpretation score measure absolute distance forecast observation score evaluates forecast exponential growth rate. Another motivation can apply variance-stabilising transformation standardise incidence counts population. Note want apply transformation, important transform forecasts observations apply score. Applying transformation score risks losing propriety proper scoring rule.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Transform forecasts and observed values — transform_forecasts","text":"Transformation forecasts evaluating predictive performance epidemiological context Nikos . Bosse, Sam Abbott, Anne Cori, Edwin van Leeuwen, Johannes Bracher, Sebastian Funk medRxiv 2023.01.23.23284722 doi:10.1101/2023.01.23.23284722 https://www.medrxiv.org/content/10.1101/2023.01.23.23284722v1","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Transform forecasts and observed values — transform_forecasts","text":"Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Transform forecasts and observed values — transform_forecasts","text":"","code":"library(magrittr) # pipe operator # transform forecasts using the natural logarithm # negative values need to be handled (here by replacing them with 0) example_quantile %>% .[, observed := ifelse(observed < 0, 0, observed)] %>% as_forecast() %>% # Here we use the default function log_shift() which is essentially the same # as log(), but has an additional arguments (offset) that allows you add an # offset before applying the logarithm. transform_forecasts(append = FALSE) %>% head() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Warning: ! Detected zeros in input values. #> ℹ Try specifying offset = 1 (or any other offset). #> Warning: ! Detected zeros in input values. #> ℹ Try specifying offset = 1 (or any other offset). #> Warning: ! Error in validating forecast object: Error in #> assert_forecast_generic(forecast, verbose) : ! After removing rows with NA #> values in the data, no forecasts are left. . #> Forecast type: quantile #> Forecast unit: #> location, target_end_date, target_type, location_name, forecast_date, model, #> and horizon #> #> Key: #> location target_end_date target_type observed location_name forecast_date #> #> 1: DE 2021-01-02 Cases 11.754302 Germany #> 2: DE 2021-01-02 Deaths 8.419360 Germany #> 3: DE 2021-01-09 Cases 11.950677 Germany #> 4: DE 2021-01-09 Deaths 8.718827 Germany #> 5: DE 2021-01-16 Cases 11.609898 Germany #> 6: DE 2021-01-16 Deaths 8.677099 Germany #> quantile_level predicted model horizon #> #> 1: NA NA NA #> 2: NA NA NA #> 3: NA NA NA #> 4: NA NA NA #> 5: NA NA NA #> 6: NA NA NA # alternatively, integrating the truncation in the transformation function: example_quantile %>% as_forecast() %>% transform_forecasts( fun = function(x) {log_shift(pmax(0, x))}, append = FALSE ) %>% head() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Warning: ! Detected zeros in input values. #> ℹ Try specifying offset = 1 (or any other offset). #> Warning: ! Detected zeros in input values. #> ℹ Try specifying offset = 1 (or any other offset). #> Warning: ! Error in validating forecast object: Error in #> assert_forecast_generic(forecast, verbose) : ! After removing rows with NA #> values in the data, no forecasts are left. . #> Forecast type: quantile #> Forecast unit: #> location, target_end_date, target_type, location_name, forecast_date, model, #> and horizon #> #> Key: #> location target_end_date target_type observed location_name forecast_date #> #> 1: DE 2021-01-02 Cases 11.754302 Germany #> 2: DE 2021-01-02 Deaths 8.419360 Germany #> 3: DE 2021-01-09 Cases 11.950677 Germany #> 4: DE 2021-01-09 Deaths 8.718827 Germany #> 5: DE 2021-01-16 Cases 11.609898 Germany #> 6: DE 2021-01-16 Deaths 8.677099 Germany #> quantile_level predicted model horizon #> #> 1: NA NA NA #> 2: NA NA NA #> 3: NA NA NA #> 4: NA NA NA #> 5: NA NA NA #> 6: NA NA NA # specifying an offset for the log transformation removes the # warning caused by zeros in the data example_quantile %>% as_forecast() %>% .[, observed := ifelse(observed < 0, 0, observed)] %>% transform_forecasts(offset = 1, append = FALSE) %>% head() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Warning: ! Error in validating forecast object: Error in #> assert_forecast_generic(forecast, verbose) : ! After removing rows with NA #> values in the data, no forecasts are left. . #> Forecast type: quantile #> Forecast unit: #> location, target_end_date, target_type, location_name, forecast_date, model, #> and horizon #> #> Key: #> location target_end_date target_type observed location_name forecast_date #> #> 1: DE 2021-01-02 Cases 11.754310 Germany #> 2: DE 2021-01-02 Deaths 8.419580 Germany #> 3: DE 2021-01-09 Cases 11.950683 Germany #> 4: DE 2021-01-09 Deaths 8.718991 Germany #> 5: DE 2021-01-16 Cases 11.609907 Germany #> 6: DE 2021-01-16 Deaths 8.677269 Germany #> quantile_level predicted model horizon #> #> 1: NA NA NA #> 2: NA NA NA #> 3: NA NA NA #> 4: NA NA NA #> 5: NA NA NA #> 6: NA NA NA # adding square root transformed forecasts to the original ones example_quantile %>% .[, observed := ifelse(observed < 0, 0, observed)] %>% as_forecast() %>% transform_forecasts(fun = sqrt, label = \"sqrt\") %>% score() %>% summarise_scores(by = c(\"model\", \"scale\")) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> model scale wis overprediction underprediction #> #> 1: EuroCOVIDhub-ensemble natural 5796.064569 1828.5715014 2120.6402853 #> 2: EuroCOVIDhub-baseline natural 11124.930667 3884.4414062 5143.5356658 #> 3: epiforecasts-EpiNow2 natural 7514.375476 2866.4071466 1697.2341137 #> 4: UMass-MechBayes natural 52.651946 8.9786005 16.8009511 #> 5: EuroCOVIDhub-ensemble sqrt 14.974344 5.5037665 5.1827454 #> 6: EuroCOVIDhub-baseline sqrt 27.742316 10.4190016 9.5936380 #> 7: epiforecasts-EpiNow2 sqrt 17.704899 6.5700431 5.7235785 #> 8: UMass-MechBayes sqrt 1.328653 0.3273746 0.4019195 #> dispersion bias interval_coverage_50 interval_coverage_90 #> #> 1: 1846.8527819 0.00812500 0.6328125 0.9023438 #> 2: 2096.9535954 0.21816406 0.4960938 0.9101562 #> 3: 2950.7342158 -0.04336032 0.4453441 0.8461538 #> 4: 26.8723947 -0.02234375 0.4609375 0.8750000 #> 5: 4.2878323 0.00812500 0.6328125 0.9023438 #> 6: 7.7296761 0.21816406 0.4960938 0.9101562 #> 7: 5.4112770 -0.04336032 0.4453441 0.8461538 #> 8: 0.5993586 -0.02234375 0.4609375 0.8750000 #> interval_coverage_deviation ae_median #> #> 1: 0.050752841 8880.542969 #> 2: 0.003522727 16156.871094 #> 3: -0.057861612 11208.072874 #> 4: -0.024886364 78.476562 #> 5: 0.050752841 22.458900 #> 6: 0.003522727 39.185406 #> 7: -0.057861612 25.585018 #> 8: -0.024886364 2.069103 # adding multiple transformations example_quantile %>% as_forecast() %>% .[, observed := ifelse(observed < 0, 0, observed)] %>% transform_forecasts(fun = log_shift, offset = 1) %>% transform_forecasts(fun = sqrt, label = \"sqrt\") %>% head() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Warning: ! Error in validating forecast object: Error in #> assert_forecast_generic(forecast, verbose) : ! After removing rows with NA #> values in the data, no forecasts are left. . #> Forecast type: quantile #> Forecast unit: #> location, target_end_date, target_type, location_name, forecast_date, model, #> horizon, and scale #> #> location target_end_date target_type observed location_name forecast_date #> #> 1: DE 2021-01-02 Cases 127300 Germany #> 2: DE 2021-01-02 Deaths 4534 Germany #> 3: DE 2021-01-09 Cases 154922 Germany #> 4: DE 2021-01-09 Deaths 6117 Germany #> 5: DE 2021-01-16 Cases 110183 Germany #> 6: DE 2021-01-16 Deaths 5867 Germany #> quantile_level predicted model horizon scale #> #> 1: NA NA NA natural #> 2: NA NA NA natural #> 3: NA NA NA natural #> 4: NA NA NA natural #> 5: NA NA NA natural #> 6: NA NA NA natural"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Re-validate an existing forecast object — validate_forecast","title":"Re-validate an existing forecast object — validate_forecast","text":"function re-validates existing forecast object. similar assert_forecast(), returns input data instead invisible NULL. See as_forecast() details expected input formats.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Re-validate an existing forecast object — validate_forecast","text":"","code":"validate_forecast(forecast, forecast_type = NULL, verbose = TRUE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Re-validate an existing forecast object — validate_forecast","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()) forecast_type (optional) forecast type expect forecasts . forecast type determined scoringutils based input match , error thrown. NULL (default), forecast type inferred data. verbose Logical. FALSE (default TRUE), messages warnings created.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Re-validate an existing forecast object — validate_forecast","text":"Depending forecast type, object class forecast_binary, forecast_point, forecast_sample forecast_quantile.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_forecast.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Re-validate an existing forecast object — validate_forecast","text":"","code":"forecast <- as_forecast(example_binary) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. assert_forecast(forecast) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected."},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Validate metrics — validate_metrics","title":"Validate metrics — validate_metrics","text":"function validates whether list metrics list valid functions. function used score() make sure metrics valid functions.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Validate metrics — validate_metrics","text":"","code":"validate_metrics(metrics)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_metrics.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Validate metrics — validate_metrics","text":"metrics named list metrics. Every element scoring function applied data.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_metrics.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Validate metrics — validate_metrics","text":"named list metrics, filtered valid functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Validate an object of class scores — validate_scores","title":"Validate an object of class scores — validate_scores","text":"function validates object class scores, checking correct class metrics attribute.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Validate an object of class scores — validate_scores","text":"","code":"validate_scores(scores)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Validate an object of class scores — validate_scores","text":"scores data.table similar scores produced score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_scores.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Validate an object of class scores — validate_scores","text":"Returns NULL invisibly","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":null,"dir":"Reference","previous_headings":"","what":"Weighted interval score (WIS) — wis","title":"Weighted interval score (WIS) — wis","text":"WIS proper scoring rule used evaluate forecasts interval- / quantile-based format. See Bracher et al. (2021). Smaller values better. name suggest score assumes forecast comes form one multiple central prediction intervals. prediction interval characterised lower upper bound formed pair predictive quantiles. example, 50% central prediction interval formed 0.25 0.75 quantiles predictive distribution. Interval score interval score () sum three components: overprediction, underprediction dispersion. single prediction interval one components non-zero. single prediction interval observed value lower bound, interval score equal absolute difference lower bound observed value (\"underprediction\"). \"Overprediction\" defined analogously. observed value falls within bounds prediction interval, interval score equal width prediction interval, .e. difference upper lower bound. single interval, therefore : $$ \\textrm{} = (\\textrm{upper} - \\textrm{lower}) + \\frac{2}{\\alpha}(\\textrm{lower} - \\textrm{observed}) * \\mathbf{1}(\\textrm{observed} < \\textrm{lower}) + \\frac{2}{\\alpha}(\\textrm{observed} - \\textrm{upper}) * \\mathbf{1}(\\textrm{observed} > \\textrm{upper}) $$ \\(\\mathbf{1}()\\) indicator function indicates much outside prediction interval. \\(\\alpha\\) decimal value indicates much outside prediction interval. 90% prediction interval, example, \\(\\alpha\\) equal 0.1. specific distribution assumed, interval formed quantiles symmetric around median (.e use 0.1 quantile lower bound 0.7 quantile upper bound). Non-symmetric quantiles can scored using function quantile_score(). Usually interval score weighted factor makes sure average score across increasing number equally spaced quantiles, converges continuous ranked probability score (CRPS). weighted score called weighted interval score (WIS). weight commonly used \\(\\alpha / 2\\). Quantile score addition interval score, also exists quantile score (QS) (see quantile_score()), equal -called pinball loss. quantile score can computed single quantile (whereas interval score requires two quantiles form interval). However, intuitive decomposition overprediction, underprediction dispersion exist quantile score. Two versions weighted interval score two ways conceptualise weighted interval score across several quantiles / prediction intervals median. one view, treat WIS average quantile scores (median 0.5-quantile) (default wis()). another view, treat WIS average several interval scores + difference observed value median forecast. effect contrast first view, median twice much weight (weighted like prediction interval, rather like single quantile). valid ways conceptualise WIS can control behaviour count_median_twice-argument. WIS components: WIS components can computed individually using functions overprediction, underprediction, dispersion.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Weighted interval score (WIS) — wis","text":"","code":"wis( observed, predicted, quantile_level, separate_results = FALSE, weigh = TRUE, count_median_twice = FALSE, na.rm = TRUE ) dispersion(observed, predicted, quantile_level, ...) overprediction(observed, predicted, quantile_level, ...) underprediction(observed, predicted, quantile_level, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Weighted interval score (WIS) — wis","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made. separate_results Logical. TRUE (default FALSE), separate parts interval score (dispersion penalty, penalties - -prediction get returned separate elements list). want data.frame instead, simply call .data.frame() output. weigh Logical. TRUE (default), weigh score \\(\\alpha / 2\\), can averaged interval score , limit (increasing number equally spaced quantiles/prediction intervals), corresponds CRPS. \\(\\alpha\\) value corresponds (\\(\\alpha/2\\)) (\\(1 - \\alpha/2\\)), .e. decimal value represents much outside central prediction interval (E.g. 90 percent central prediction interval, alpha 0.1). count_median_twice TRUE, count median twice score. na.rm TRUE, ignore NA values computing score. ... Additional arguments passed wis() functions overprediction(), underprediction() dispersion().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Weighted interval score (WIS) — wis","text":"wis(): numeric vector WIS values size n (one per observation), list separate entries separate_results TRUE. dispersion(): numeric vector dispersion values (one per observation). overprediction(): numeric vector overprediction values (one per observation). underprediction(): numeric vector underprediction values (one per observation)","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Weighted interval score (WIS) — wis","text":"","code":"observed <- c(1, -15, 22) predicted <- rbind( c(-1, 0, 1, 2, 3), c(-2, 1, 2, 2, 4), c(-2, 0, 3, 3, 4) ) quantile_level <- c(0.1, 0.25, 0.5, 0.75, 0.9) wis(observed, predicted, quantile_level) #> [1] 0.36 15.34 19.14"},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-1229000","dir":"Changelog","previous_headings":"","what":"scoringutils 1.2.2.9000","title":"scoringutils 1.2.2.9000","text":"update represents major rewrite package introduces breaking changes. want keep using older version, can download using remotes::install_github(\"epiforecasts/scoringutils@v1.2\"). update aims make package modular customisable overall cleaner easier work . particular, aimed make suggested workflows evaluating forecasts explicit easier follow (see visualisation ). , clarified input formats made consistent across functions. refactord many functions S3-methods introduced forecast objects separate classes different types forecasts. new function, as_forecast() introduced validate data convert inputs forecast object. Another major update possibility users pass scoring functions score(). updated improved function documentation added new vignettes guide users package. Internally, refactored code, improved input checks, updated notifications (now use cli package) increased test coverage. comprehensive documentation new package rewrite revised version original scoringutils paper.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"score-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"score()","title":"scoringutils 1.2.2.9000","text":"previous columns “true_value” “prediction” renamed. score() now requires columns called “observed” “predicted” “model”. column quantile renamed quantile_level sample renamed sample_id score() now generic. S3 methods classes forecast_point, forecast_binary, forecast_quantile forecast_sample, correspond different forecast types can scored scoringutils. score() now calls na.omit() data, instead removing rows missing values columns observed predicted. NA values columns can also mess e.g. grouping forecasts according unit single forecast. score() many functions now require validated forecast object. forecast objects can created using function as_forecast() (replaces previous check_forecast()). score() now returns objects class scores stored attribute metrics holds names scoring rules used. Users can call get_metrics() access names scoring rules. score() now returns one score per forecast, instead one score per sample quantile. Users can now also use scoring rules (making use metrics argument, takes named list functions). Default scoring rules can accessed using functions metrics_point(), metrics_sample(), metrics_quantile() metrics_binary(), return named list scoring rules suitable respective forecast type. Column names scores output score() correspond names scoring rules (.e. names functions list metrics). Instead supplying arguments score() manipulate individual scoring rules users now manipulate metric list supplied using customise_metric() select_metric().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"creating-a-forecast-object-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Creating a forecast object","title":"scoringutils 1.2.2.9000","text":"function as_forecast() creates forecast object validates . as_forecast() also allows users rename/specify required columns specify forecast unit single step, taking functionality set_forecast_unit() cases.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"updated-workflows-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Updated workflows","title":"scoringutils 1.2.2.9000","text":"example workflow scoring forecast now looks like : Overall, updated suggested workflows users work package. following gives overview (see updated paper details).","code":"forecast_quantile <- as_forecast( example_quantile, observed = \"observed\", predicted = \"predicted\", model = \"model\", quantile_level = \"quantile_level\", forecast_unit = c(\"model\", \"location\", \"target_end_date\", \"forecast_date\", \"target_type\") ) scores <- score(forecast_quantile)"},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"input-formats-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Input formats","title":"scoringutils 1.2.2.9000","text":"standardised input formats score() well scoring rules exported scoreingutils. following plot gives overview expected input formats different forecast types score(). Support interval format mostly dropped (see PR #525 @nikosbosse reviewed @seabbs). co-existence quantile interval format let confusing user experience many duplicated functions providing functionality. decided simplify interface focusing exclusively quantile format. function bias_range() removed (users now use bias_quantile() instead) function interval_score() made internal function rather exported users. recommend using wis() instead.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"re-validating-forecast-objects-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"(Re-)Validating forecast objects","title":"scoringutils 1.2.2.9000","text":"create validate new forecast object, users can use as_forecast(). revalidate existing forecast object users can call assert_forecast() (validates input returns invisible(NULL). assert_forecast() generic methods different forecast types. Alternatively, validate_forecast() can used (calls assert_forecast()), returns input useful pipe. Lastly, users can simply print object obtain additional information. Users can test whether object class forecast_*() using function is_forecast(). Users can also test specific forecast_* class using appropriate is_forecast.forecast_* method. example, check whether object class forecast_quantile, use use scoringutils:::is_forecast.forecast_quantile().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"pairwise-comparisons-and-relative-skill-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Pairwise comparisons and relative skill","title":"scoringutils 1.2.2.9000","text":"functionality computing pairwise comparisons now split summarise_scores(). Instead pairwise comparisons part summarising scores, new function, add_relative_skill(), introduced takes summarised scores input adds columns relative skill scores scaled relative skill scores. function pairwise_comparison() renamed get_pairwise_comparisons(), line get_-functions. Analogously, plot_pairwise_comparison() renamed plot_pairwise_comparisons(). Output columns pairwise comparisons renamed contain name metric used comparing. Replaced warnings errors get_pairwise_comparison avoid returning NULL","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"computing-coverage-values-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Computing coverage values","title":"scoringutils 1.2.2.9000","text":"add_coverage() replaced new function, get_coverage(). function comes updated workflow coverage values computed directly based original data can visualised using plot_interval_coverage() plot_quantile_coverage(). example workflow example_quantile |> as_forecast() |> get_coverage(= \"model\") |> plot_interval_coverage().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"obtaining-and-plotting-forecast-counts-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Obtaining and plotting forecast counts","title":"scoringutils 1.2.2.9000","text":"clarity, output column get_forecast_counts() renamed “Number forecasts” “count”. get_forecast_counts() now also displays combinations 0 forecasts, instead silently dropping corresponding rows. plot_avail_forecasts() renamed plot_forecast_counts() line change function name. x argument longer default value, value depend data provided user.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"renamed-functions-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Renamed functions","title":"scoringutils 1.2.2.9000","text":"function find_duplicates() renamed get_duplicate_forecasts() Renamed interval_coverage_quantile() interval_coverage_dev_quantile() interval_coverage() interval_coverage_deviation(), respectively. “range” consistently renamed “interval_range” code. “range”-format (mostly used internally) renamed “interval”-format Renamed correlation() get_correlations() plot_correlation() plot_correlations() pit() renamed get_pit().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"deleted-functions-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Deleted functions","title":"scoringutils 1.2.2.9000","text":"Removed abs_error squared_error package favour Metrics::ae Metrics::se. Deleted function plot_ranges(). want continue using functionality, can find function code Deprecated-visualisations Vignette. Removed function plot_predictions(), well helper function make_NA(), favour dedicated Vignette shows different ways visualising predictions. future reference, function code can found (Issue #659) Deprecated-visualisations Vignette. Removed function plot_score_table(). can find code Deprecated-visualisations Vignette. Removed function merge_pred_and_obs() used merge two separate data frames forecasts observations. moved contents new “Deprecated functions”-vignette. Removed interval_coverage_sample() users now expected convert quantile format first scoring.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"function-changes-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Function changes","title":"scoringutils 1.2.2.9000","text":"bias_quantile() changed way handles forecasts median missing: median now imputed linear interpolation innermost quantiles. Previously, imputed median simply taking mean innermost quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"internal-package-updates-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Internal package updates","title":"scoringutils 1.2.2.9000","text":"deprecated ..density.. replaced after_stat(density) ggplot calls. Files ending “.Rda” renamed “.rds” appropriate used together saveRDS() readRDS(). Added subsetting [ operator scores, score name attribute gets preserved subsetting. Switched using cli condition handling signalling, added tests check_*() test_*() functions. See #583 @jamesmbaazam reviewed @nikosbosse @seabbs.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"documentation-and-testing-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Documentation and testing","title":"scoringutils 1.2.2.9000","text":"Updates documentation functions made sure functions documented return statements Documentation pkgdown pages now created stable dev versions. Added unit tests many functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-122","dir":"Changelog","previous_headings":"","what":"scoringutils 1.2.2","title":"scoringutils 1.2.2","text":"CRAN release: 2023-11-29","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-2-2","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.2.2","text":"scoringutils now depends R 3.6. change made since packages testthat lifecycle, used scoringutils now require R 3.6. also updated Github action CI check work R 3.6 now. Added new PR template checklist things included PRs facilitate development review process","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"bug-fixes-1-2-2","dir":"Changelog","previous_headings":"","what":"Bug fixes","title":"scoringutils 1.2.2","text":"Fixes bug set_forecast_unit() function worked data.table, data.frame input. metrics table vignette Details metrics implemented scoringutils duplicated entries. fixed removing duplicated rows.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-2-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.2.1","text":"Gets rid preferably package _pkgdown.yml. theme toggle light dark theme didn’t work properly Updates gh pages deploy action v4 also cleans files triggered Introduces gh action automatically render Readme Readme.Rmd Removes links vignettes renamed","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-120","dir":"Changelog","previous_headings":"","what":"scoringutils 1.2.0","title":"scoringutils 1.2.0","text":"major release contains range new features bug fixes introduced minor releases since 1.1.0. important changes : Documentation updated reflect changes since version 1.1.0, including new transform workflow functions. New set_forecast_unit() function allows manual setting forecast unit. summarise_scores() gains new across argument summarizing across variables. New transform_forecasts() log_shift() functions allow forecast transformations. See documentation transform_forecasts() details example use case. Input checks test coverage improved bias functions. Bug fix get_prediction_type() integer matrix input. Links scoringutils paper citation updates. Warning added interval_score() small interval ranges. Linting updates improvements. Thanks @nikosbosse, @seabbs, @sbfnk code review contributions. Thanks @bisaloo suggestion use linting GitHub Action triggers changes, @adrian-lison suggestion add warning interval_score() interval range 0 1.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-2-0","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.2.0","text":"documentation updated reflect recent changes since scoringutils 1.1.0. particular, usage functions set_forecast_unit(), check_forecasts() transform_forecasts() now documented Vignettes. introduction functions enhances overall workflow help make code readable. functions designed used together pipe operator. example, one can now use something like following: Documentation transform_forecasts() also extended. functions allows user easily add transformations forecasts, suggested paper “Scoring epidemiological forecasts transformed scales”. epidemiological context, example, may make sense apply natural logarithm first scoring forecasts, order obtain scores reflect well models able predict exponential growth rates, rather absolute values. Users can now something like following score transformed version data addition original one: use log_shift() function apply logarithmic transformation forecasts. function introduced scoringutils 1.1.2 helper function acts just like log(), additional argument offset can add number every prediction observed value applying log transformation.","code":"example_quantile |> set_forecast_unit(c(\"model\", \"location\", \"forecast_date\", \"horizon\", \"target_type\")) |> check_forecasts() |> score() data <- example_quantile[true_value > 0, ] data |> transform_forecasts(fun = log_shift, offset = 1) |> score() |> summarise_scores(by = c(\"model\", \"scale\"))"},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-1-2-0","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.2.0","text":"Made check_forecasts() score() pipeable (see issue #290). means users can now directly use output check_forecasts() input score(). score() otherwise runs check_forecasts() internally anyway simply makes step explicit helps writing clearer code.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-117","dir":"Changelog","previous_headings":"","what":"scoringutils 1.1.7","title":"scoringutils 1.1.7","text":"Release @seabbs #305. Reviewed @nikosbosse @sbfnk.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"breaking-changes-1-1-7","dir":"Changelog","previous_headings":"","what":"Breaking changes","title":"scoringutils 1.1.7","text":"prediction_type argument get_forecast_unit() changed dropped. Instead new internal function prediction_is_quantile() used detect quantile variable present. Whilst internal function may impact users accessible via `find_duplicates().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-1-7","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.7","text":"Made imputation median bias_range() bias_quantile() obvious user may cause unexpected behaviour. Simplified bias_range() uses bias_quantile() internally. Added additional input checks bias_range(), bias_quantile(), check_predictions() make sure input valid. Improve coverage unit tests bias_range(), bias_quantile(), bias_sample(). Updated pairwise comparison unit tests use realistic data.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"bug-fixes-1-1-7","dir":"Changelog","previous_headings":"","what":"Bug fixes","title":"scoringutils 1.1.7","text":"Fixed bug get_prediction_type() led unable correctly detect integer (instead categorising continuous) forecasts input matrix. issue impacted bias_sample() also score() used integer forecasts resulting lower bias scores expected.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-1-1-6","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.1.6","text":"Added new argument, across, summarise_scores(). argument allows user summarise scores across different forecast units alternative specifying . See documentation summarise_scores() details example use case.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-1-1-5","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.1.5","text":"Added new function, set_forecast_unit() allows user set forecast unit manually. function removes columns relevant uniquely identifying single forecast. done manually, scoringutils attempts determine unit single automatically simply assuming column names relevant determine forecast unit. can lead unexpected behaviour, setting forecast unit explicitly can help make code easier debug easier read (see issue #268). used part workflow, set_forecast_unit() can directly piped check_forecasts() check everything order.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-1-4","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.4","text":"Added links scoringutils paper Evaluating Forecasts scoringutils R package. Updated citation formatting comply newer standards.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-1-3","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.3","text":"Added warning interval_score() interval range 0 1. Thanks @adrian-lison (see #277) suggestion.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-1-3-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.3","text":"Switched linting GitHub Action triggers changes. Inspired @bisaloo recent contribution epinowcast package. Updated package linters extensive. Inspired @bisaloo recent contribution epinowcast package. Resolved flagged linting issues across package.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-1-1-2","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.1.2","text":"Added new function, transform_forecasts() make easy transform forecasts scoring , suggested Bosse et al. (2023), https://www.medrxiv.org/content/10.1101/2023.01.23.23284722v1. Added function, log_shift() implements default transformation function. function allows add offset applying logarithm.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-111","dir":"Changelog","previous_headings":"","what":"scoringutils 1.1.1","title":"scoringutils 1.1.1","text":"Added small change interval_score() explicitly converts logical vector numeric one. happen implicitly anyway, now done explicitly order avoid issues may come input vector type doesn’t allow implicit conversion.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-110","dir":"Changelog","previous_headings":"","what":"scoringutils 1.1.0","title":"scoringutils 1.1.0","text":"CRAN release: 2023-01-30 minor update package bug fixes minor changes.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-1-0","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.0","text":"Removed attach message warned breaking changes 1.0.0. Renamed metric argument summarise_scores() relative_skill_metric. argument now deprecated removed future version package. Please use new argument instead. Updated documentation score() related functions make soft requirement model column input data explicit. Updated documentation score(), pairwise_comparison() summarise_scores() make clearer unit single forecast required computations Simplified function plot_pairwise_comparison() now supports plotting mean score ratios p-values removed hybrid option print time.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"bug-fixes-1-1-0","dir":"Changelog","previous_headings":"","what":"Bug fixes","title":"scoringutils 1.1.0","text":"Missing baseline forecasts pairwise_comparison() now trigger explicit informative error message. requirements table getting started vignette now correct. Added support optional sample column using quantile forecast format. Previously resulted error.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-100","dir":"Changelog","previous_headings":"","what":"scoringutils 1.0.0","title":"scoringutils 1.0.0","text":"CRAN release: 2022-05-13 Major update package package functions lots breaking changes.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-1-0-0","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.0.0","text":"New updated Readme vignette. proposed scoring workflow reworked. Functions changed can easily piped simplified arguments outputs.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"new-functions-and-function-changes-1-0-0","dir":"Changelog","previous_headings":"Feature updates","what":"New functions and function changes","title":"scoringutils 1.0.0","text":"function eval_forecasts() replaced function score() much reduced set function arguments. Functionality summarise scores add relative skill scores moved function summarise_scores() New function check_forecasts() analyse input data scoring New function correlation() compute correlations different metrics New function add_coverage() add coverage specific central prediction intervals. New function avail_forecasts() allows visualise number available forecasts. New function find_duplicates() find duplicate forecasts cause error. plotting functions renamed begin plot_. Arguments simplified. function pit() now works based data.frames. old pit function renamed pit_sample(). PIT p-values removed entirely. function plot_pit() now works directly input produced pit() Many data-handling functions removed input types score() restricted sample-based, quantile-based binary forecasts. function brier_score() now returns brier scores, rather taking mean returning output. crps(), dss() logs() renamed crps_sample(), dss_sample(), logs_sample()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"bug-fixes-1-0-0","dir":"Changelog","previous_headings":"Feature updates","what":"Bug fixes","title":"scoringutils 1.0.0","text":"Testing expanded Minor bugs fixed, example bug sample_to_quantile() function (https://github.com/epiforecasts/scoringutils/pull/223)","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-data-updated-1-0-0","dir":"Changelog","previous_headings":"Feature updates","what":"Package data updated","title":"scoringutils 1.0.0","text":"Package data now based forecasts submitted European Forecast Hub (https://covid19forecasthub.eu/). example data files renamed begin example_. new data set, summary_metrics included contains summary metrics implemented scoringutils.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"other-breaking-changes-1-0-0","dir":"Changelog","previous_headings":"","what":"Other breaking changes","title":"scoringutils 1.0.0","text":"‘sharpness’ component weighted interval score renamed dispersion. done make clear component represents maintain consistency used places.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-8","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.8","text":"Added function check_forecasts() runs basic checks input data provides feedback.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-0172","dir":"Changelog","previous_headings":"","what":"scoringutils 0.1.7.2","title":"scoringutils 0.1.7.2","text":"CRAN release: 2021-07-21","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-7-2","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.7.2","text":"Minor bug fixes (previously, ‘interval_score’ needed among selected metrics). data.tables now returned table[] rather table, don’t called twice display contents.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-017","dir":"Changelog","previous_headings":"","what":"scoringutils 0.1.7","title":"scoringutils 0.1.7","text":"CRAN release: 2021-07-14","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-7","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.7","text":"Added function, pairwise_comparison() runs pairwise comparisons models output eval_forecasts() Added functionality compute relative skill within eval_forecasts(). Added function visualise pairwise comparisons.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-7","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.7","text":"WIS definition change introduced version 0.1.5 partly corrected difference weighting introduced summarising scores different interval ranges. “sharpness” renamed ‘mad’ output [score()] sample-based forecasts.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.","text":"eval_forecasts() can now handle separate forecast truth data set input. eval_forecasts() now supports scoring point forecasts along side quantiles quantile-based format. Currently metric used absolute error.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.","text":"Many functions, especially eval_forecasts() got major rewrite. functionality unchanged, code now easier maintain data-handling functions got renamed, old names supported well now.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-5","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.5","text":"Changed default definition weighted interval score. Previously, median prediction counted twice, counted . want go back old behaviour, can call interval_score function argument count_median_twice = FALSE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-014","dir":"Changelog","previous_headings":"","what":"scoringutils 0.1.4","title":"scoringutils 0.1.4","text":"CRAN release: 2020-11-17","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-4","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.4","text":"Added basic plotting functionality visualise scores. can now easily obtain diagnostic plots based scores produced score. correlation_plot() shows correlation metrics. plot_ranges() shows contribution different prediction intervals chosen metric. plot_heatmap() visualises scores heatmap. plot_score_table() shows coloured summary table scores.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-4","dir":"Changelog","previous_headings":"","what":"package updates","title":"scoringutils 0.1.4","text":"Renamed “calibration” “coverage”. Renamed “true_values” “true_value” data.frames. Renamed “predictions” “prediction” data.frames. Renamed “is_overprediction” “overprediction”. Renamed “is_underprediction” “underprediction”.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"potentially-breaking-changes-0-1-3","dir":"Changelog","previous_headings":"","what":"(Potentially) Breaking changes","title":"scoringutils 0.1.3","text":"argument score now slightly changed meaning. now denotes lowest possible grouping unit, .e. unit one observation needs specified explicitly. default now NULL. reason change metrics need scoring observation level consistent implementation principle. pit function receives grouping now summarise_by. similar spirit, summarise_by specified explicitly e.g. doesn’t assume anymore want ‘range’ included. interval score, weigh = TRUE now default option. Renamed true_values true_value predictions prediction.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-3","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.3","text":"Updated quantile evaluation metrics score. Bias well calibration now take quantiles account. Included option summarise scores according summarise_by argument score() summary can return mean, standard deviation well arbitrary set quantiles. score() can now return pit histograms. Switched ggplot2 plotting.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"potentially-breaking-changes-0-1-2","dir":"Changelog","previous_headings":"","what":"(Potentially) Breaking changes","title":"scoringutils 0.1.2","text":"scores score consistently renamed lower case. Interval_score now interval_score, CRPS now crps etc.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-2","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.2","text":"Included support grouping scores according vector column names score(). Included support passing arguments lower-level functions score() Included support three new metrics score quantiles score(): bias, sharpness calibration","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-2","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.2","text":"Example data now horizon column illustrate use grouping. Documentation updated explain listed changes.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-1","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.1","text":"Included support long well wide input formats quantile forecasts scored score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.1","text":"Updated documentation score(). Added badges README.","code":""}] +[{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"our-pledge","dir":"","previous_headings":"","what":"Our Pledge","title":"Contributor Covenant Code of Conduct","text":"members, contributors, leaders pledge make participation community harassment-free experience everyone, regardless age, body size, visible invisible disability, ethnicity, sex characteristics, gender identity expression, level experience, education, socio-economic status, nationality, personal appearance, race, religion, sexual identity orientation. pledge act interact ways contribute open, welcoming, diverse, inclusive, healthy community.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"our-standards","dir":"","previous_headings":"","what":"Our Standards","title":"Contributor Covenant Code of Conduct","text":"Examples behavior contributes positive environment community include: Demonstrating empathy kindness toward people respectful differing opinions, viewpoints, experiences Giving gracefully accepting constructive feedback Accepting responsibility apologizing affected mistakes, learning experience Focusing best just us individuals, overall community Examples unacceptable behavior include: use sexualized language imagery, sexual attention advances kind Trolling, insulting derogatory comments, personal political attacks Public private harassment Publishing others’ private information, physical email address, without explicit permission conduct reasonably considered inappropriate professional setting","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"enforcement-responsibilities","dir":"","previous_headings":"","what":"Enforcement Responsibilities","title":"Contributor Covenant Code of Conduct","text":"Community leaders responsible clarifying enforcing standards acceptable behavior take appropriate fair corrective action response behavior deem inappropriate, threatening, offensive, harmful. Community leaders right responsibility remove, edit, reject comments, commits, code, wiki edits, issues, contributions aligned Code Conduct, communicate reasons moderation decisions appropriate.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"scope","dir":"","previous_headings":"","what":"Scope","title":"Contributor Covenant Code of Conduct","text":"Code Conduct applies within community spaces, also applies individual officially representing community public spaces. Examples representing community include using official e-mail address, posting via official social media account, acting appointed representative online offline event.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"enforcement","dir":"","previous_headings":"","what":"Enforcement","title":"Contributor Covenant Code of Conduct","text":"Instances abusive, harassing, otherwise unacceptable behavior may reported community leaders responsible enforcement Sebastian.Funk@lshtm.ac.uk. complaints reviewed investigated promptly fairly. community leaders obligated respect privacy security reporter incident.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"enforcement-guidelines","dir":"","previous_headings":"","what":"Enforcement Guidelines","title":"Contributor Covenant Code of Conduct","text":"Community leaders follow Community Impact Guidelines determining consequences action deem violation Code Conduct:","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"id_1-correction","dir":"","previous_headings":"Enforcement Guidelines","what":"1. Correction","title":"Contributor Covenant Code of Conduct","text":"Community Impact: Use inappropriate language behavior deemed unprofessional unwelcome community. Consequence: private, written warning community leaders, providing clarity around nature violation explanation behavior inappropriate. public apology may requested.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"id_2-warning","dir":"","previous_headings":"Enforcement Guidelines","what":"2. Warning","title":"Contributor Covenant Code of Conduct","text":"Community Impact: violation single incident series actions. Consequence: warning consequences continued behavior. interaction people involved, including unsolicited interaction enforcing Code Conduct, specified period time. includes avoiding interactions community spaces well external channels like social media. Violating terms may lead temporary permanent ban.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"id_3-temporary-ban","dir":"","previous_headings":"Enforcement Guidelines","what":"3. Temporary Ban","title":"Contributor Covenant Code of Conduct","text":"Community Impact: serious violation community standards, including sustained inappropriate behavior. Consequence: temporary ban sort interaction public communication community specified period time. public private interaction people involved, including unsolicited interaction enforcing Code Conduct, allowed period. Violating terms may lead permanent ban.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"id_4-permanent-ban","dir":"","previous_headings":"Enforcement Guidelines","what":"4. Permanent Ban","title":"Contributor Covenant Code of Conduct","text":"Community Impact: Demonstrating pattern violation community standards, including sustained inappropriate behavior, harassment individual, aggression toward disparagement classes individuals. Consequence: permanent ban sort public interaction within community.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"attribution","dir":"","previous_headings":"","what":"Attribution","title":"Contributor Covenant Code of Conduct","text":"Code Conduct adapted Contributor Covenant, version 2.0, available https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. Community Impact Guidelines inspired Mozilla’s code conduct enforcement ladder. answers common questions code conduct, see FAQ https://www.contributor-covenant.org/faq. Translations available https://www.contributor-covenant.org/translations.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":null,"dir":"","previous_headings":"","what":"Contributing to scoringutils","title":"Contributing to scoringutils","text":"outlines propose change scoringutils.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"fixing-typos","dir":"","previous_headings":"","what":"Fixing typos","title":"Contributing to scoringutils","text":"can fix typos, spelling mistakes, grammatical errors documentation directly using GitHub web interface, long changes made source file. generally means ’ll need edit roxygen2 comments .R, .Rd file. can find .R file generates .Rd reading comment first line.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"bigger-changes","dir":"","previous_headings":"","what":"Bigger changes","title":"Contributing to scoringutils","text":"want make bigger change, ’s good idea first file issue make sure someone team agrees ’s needed. ’ve found bug, please file issue illustrates bug minimal reprex (also help write unit test, needed).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"pull-request-process","dir":"","previous_headings":"Bigger changes","what":"Pull request process","title":"Contributing to scoringutils","text":"Fork package clone onto computer. haven’t done , recommend using usethis::create_from_github(\"epiforecasts/scoringutils\", fork = TRUE). Install development dependences devtools::install_dev_deps(), make sure package passes R CMD check running devtools::check(). R CMD check doesn’t pass cleanly, ’s good idea ask help continuing. Create Git branch pull request (PR). recommend using usethis::pr_init(\"brief-description--change\"). Make changes, commit git, create PR running usethis::pr_push(), following prompts browser. title PR briefly describe change. body PR contain Fixes #issue-number. user-facing changes, add bullet top NEWS.md (.e. just first header). Follow style described https://style.tidyverse.org/news.html.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"code-style","dir":"","previous_headings":"Bigger changes","what":"Code style","title":"Contributing to scoringutils","text":"New code follow tidyverse style guide. can use styler package apply styles, please don’t restyle code nothing PR. use roxygen2, Markdown syntax, documentation. use testthat unit tests. Contributions test cases included easier accept.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"code-of-conduct","dir":"","previous_headings":"","what":"Code of Conduct","title":"Contributing to scoringutils","text":"Please note scoringutils project released Contributor Code Conduct. contributing project agree abide terms.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/LICENSE.html","id":null,"dir":"","previous_headings":"","what":"MIT License","title":"MIT License","text":"Copyright (c) 2020 EpiForecasts Permission hereby granted, free charge, person obtaining copy software associated documentation files (“Software”), deal Software without restriction, including without limitation rights use, copy, modify, merge, publish, distribute, sublicense, /sell copies Software, permit persons Software furnished , subject following conditions: copyright notice permission notice shall included copies substantial portions Software. SOFTWARE PROVIDED “”, WITHOUT WARRANTY KIND, EXPRESS IMPLIED, INCLUDING LIMITED WARRANTIES MERCHANTABILITY, FITNESS PARTICULAR PURPOSE NONINFRINGEMENT. EVENT SHALL AUTHORS COPYRIGHT HOLDERS LIABLE CLAIM, DAMAGES LIABILITY, WHETHER ACTION CONTRACT, TORT OTHERWISE, ARISING , CONNECTION SOFTWARE USE DEALINGS SOFTWARE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/PULL_REQUEST_TEMPLATE.html","id":"description","dir":"","previous_headings":"","what":"Description","title":"NA","text":"PR closes #. [Describe changes made pull request.]","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/PULL_REQUEST_TEMPLATE.html","id":"checklist","dir":"","previous_headings":"","what":"Checklist","title":"NA","text":"PR based package issue explicitly linked . included target issue issues PR title follows: issue-number: PR title tested changes locally. added updated unit tests necessary. updated documentation required. built package locally run rebuilt docs using roxygen2. code follows established coding standards run lintr::lint_package() check style issues introduced changes. added news item linked PR. reviewed CI checks PR addressed far able.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/Deprecated-functions.html","id":"merge_pred_and_obs","dir":"Articles","previous_headings":"","what":"merge_pred_and_obs()","title":"Deprecated functions","text":"scoringutils requires forecasts observations provided single data frame. forecasts observations two different data frames, merge_pred_and_obs() may help merge two. function mostly wrapper around merge(), additional work deal duplicated column names.","code":"#' @title Merge forecast data and observations #' #' @description #' #' The function more or less provides a wrapper around `merge` that #' aims to handle the merging well if additional columns are present #' in one or both data sets. If in doubt, you should probably merge the #' data sets manually. #' #' @param forecasts A data.frame with the forecast data (as can be passed to #' [score()]). #' @param observations A data.frame with the observations. #' @param join Character, one of `c(\"left\", \"full\", \"right\")`. Determines the #' type of the join. Usually, a left join is appropriate, but sometimes you #' may want to do a full join to keep dates for which there is a forecast, but #' no ground truth data. #' @param by Character vector that denotes the columns by which to merge. Any #' value that is not a column in observations will be removed. #' @return a data.table with forecasts and observations #' @importFrom checkmate assert_subset #' @importFrom data.table as.data.table #' @keywords data-handling #' @export merge_pred_and_obs <- function(forecasts, observations, join = c(\"left\", \"full\", \"right\"), by = NULL) { forecasts <- as.data.table(forecasts) observations <- as.data.table(observations) join <- match.arg(join) assert_subset(by, intersect(names(forecasts), names(observations))) if (is.null(by)) { protected_columns <- c( \"predicted\", \"observed\", \"sample_id\", \"quantile_level\", \"interval_range\", \"boundary\" ) by <- setdiff(colnames(forecasts), protected_columns) } obs_cols <- colnames(observations) by <- intersect(by, obs_cols) join <- match.arg(join) if (join == \"left\") { # do a left_join, where all data in the observations are kept. combined <- merge(observations, forecasts, by = by, all.x = TRUE) } else if (join == \"full\") { # do a full, where all data is kept. combined <- merge(observations, forecasts, by = by, all = TRUE) } else { combined <- merge(observations, forecasts, by = by, all.y = TRUE) } # get colnames that are the same for x and y colnames <- colnames(combined) colnames_x <- colnames[endsWith(colnames, \".x\")] colnames_y <- colnames[endsWith(colnames, \".y\")] # extract basenames basenames_x <- sub(\".x$\", \"\", colnames_x) basenames_y <- sub(\".y$\", \"\", colnames_y) # see whether the column name as well as the content is the same content_x <- as.list(combined[, ..colnames_x]) content_y <- as.list(combined[, ..colnames_y]) overlapping <- (content_x %in% content_y) & (basenames_x == basenames_y) overlap_names <- colnames_x[overlapping] basenames_overlap <- sub(\".x$\", \"\", overlap_names) # delete overlapping columns if (length(basenames_overlap) > 0) { combined[, paste0(basenames_overlap, \".x\") := NULL] combined[, paste0(basenames_overlap, \".y\") := NULL] } return(combined[]) }"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/Deprecated-visualisations.html","id":"functions-plot_predictions-and-make_na","dir":"Articles","previous_headings":"","what":"Functions plot_predictions() and make_na()","title":"Deprecated Visualisations","text":"previous versions scoringutils, forecasts observed values visualised using function plot_predictions() make_na() helper function. following shows function code first example. plot_predictions() actual work producing plot. argument needed user can facet plot correctly user needs specify columns relevant facetting. make_NA() represents form filtering, instead filtering entire rows, relevant entries columns “predicted” “observed” made NA. allows user filter observations forecasts independently. following examples using two functions create plot using scoringutils example data. Visualising median forecasts example data. truth data restricted period 2021-05-01 2021-07-22. forecast data forecast model “EuroCOVIDhub-ensemble” made “2021-06-07”. data set NA, effectively removing plot. plot, variety prediction intervals shown, instead just median. similar plot, time based continuous forecasts. predictions automatically converted quantile-based forecasts plotting. Displaying two forecasts time additional colours:","code":"#\" @title Plot Predictions vs True Values #\" #\" @description #\" Make a plot of observed and predicted values #\" #\" @param data a data.frame that follows the same specifications outlined in #\" [score()]. To customise your plotting, you can filter your data using the #\" function [make_NA()]. #\" @param by character vector with column names that denote categories by which #\" the plot should be stratified. If for example you want to have a facetted #\" plot, this should be a character vector with the columns used in facetting #\" (note that the facetting still needs to be done outside of the function call) #\" @param x character vector of length one that denotes the name of the variable #\" @param interval_range numeric vector indicating the interval ranges to plot. #\" If 0 is included in `interval_range`, the median prediction will be shown. #\" @return ggplot object with a plot of true vs predicted values #\" @importFrom ggplot2 ggplot scale_colour_manual scale_fill_manual theme_light #\" @importFrom ggplot2 facet_wrap facet_grid aes geom_line .data geom_point #\" @importFrom data.table dcast #\" @importFrom ggdist geom_lineribbon #\" @export #\" @examples #\" library(ggplot2) #\" library(magrittr) #\" #\" example_sample_continuous %>% #\" make_NA ( #\" what = \"truth\", #\" target_end_date >= \"2021-07-22\", #\" target_end_date < \"2021-05-01\" #\" ) %>% #\" make_NA ( #\" what = \"forecast\", #\" model != \"EuroCOVIDhub-ensemble\", #\" forecast_date != \"2021-06-07\" #\" ) %>% #\" plot_predictions ( #\" x = \"target_end_date\", #\" by = c(\"target_type\", \"location\"), #\" interval_range = c(0, 50, 90, 95) #\" ) + #\" facet_wrap(~ location + target_type, scales = \"free_y\") + #\" aes(fill = model, color = model) #\" #\" example_sample_continuous %>% #\" make_NA ( #\" what = \"truth\", #\" target_end_date >= \"2021-07-22\", #\" target_end_date < \"2021-05-01\" #\" ) %>% #\" make_NA ( #\" what = \"forecast\", #\" forecast_date != \"2021-06-07\" #\" ) %>% #\" plot_predictions ( #\" x = \"target_end_date\", #\" by = c(\"target_type\", \"location\"), #\" interval_range = 0 #\" ) + #\" facet_wrap(~ location + target_type, scales = \"free_y\") + #\" aes(fill = model, color = model) plot_predictions <- function(data, by = NULL, x = \"date\", interval_range = c(0, 50, 90)) { # split truth data and forecasts in order to apply different filtering truth_data <- data.table::as.data.table(data)[!is.na(observed)] forecasts <- data.table::as.data.table(data)[!is.na(predicted)] del_cols <- colnames(truth_data)[!(colnames(truth_data) %in% c(by, \"observed\", x))] truth_data <- unique(suppressWarnings(truth_data[, eval(del_cols) := NULL])) # find out what type of predictions we have. convert sample based to # interval range data if (get_forecast_type(data) == \"quantile\") { forecasts <- scoringutils:::quantile_to_interval( forecasts, keep_quantile_col = FALSE ) } else if (get_forecast_type(data) == \"sample\") { # using a scoringutils internal function forecasts <- scoringutils:::sample_to_interval_long( as_forecast(forecasts), interval_range = interval_range, keep_quantile_col = FALSE ) } # select appropriate boundaries and pivot wider select <- forecasts$interval_range %in% setdiff(interval_range, 0) intervals <- forecasts[select, ] # delete quantile column in intervals if present. This is important for # pivoting if (\"quantile_level\" %in% names(intervals)) { intervals[, quantile_level := NULL] } plot <- ggplot(data = data, aes(x = .data[[x]])) + theme_scoringutils() + ylab(\"True and predicted values\") if (nrow(intervals) != 0) { # pivot wider and convert range to a factor intervals <- data.table::dcast(intervals, ... ~ boundary, value.var = \"predicted\") # only plot interval ranges if there are interval ranges to plot plot <- plot + ggdist::geom_lineribbon( data = intervals, aes( ymin = lower, ymax = upper, # We use the fill_ramp aesthetic for this instead of the default fill # because we want to keep fill to be able to use it for other # variables fill_ramp = factor( interval_range, levels = sort(unique(interval_range), decreasing = TRUE) ) ), lwd = 0.4 ) + ggdist::scale_fill_ramp_discrete( name = \"interval_range\", # range argument was added to make sure that the line for the median # and the ribbon don\"t have the same opacity, making the line # invisible range = c(0.15, 0.75) ) } # We could treat this step as part of ggdist::geom_lineribbon() but we treat # it separately here to deal with the case when only the median is provided # (in which case ggdist::geom_lineribbon() will fail) if (0 %in% interval_range) { select_median <- forecasts$interval_range == 0 & forecasts$boundary == \"lower\" median <- forecasts[select_median] if (nrow(median) > 0) { plot <- plot + geom_line( data = median, mapping = aes(y = predicted), lwd = 0.4 ) } } # add observed values if (nrow(truth_data) > 0) { plot <- plot + geom_point( data = truth_data, show.legend = FALSE, inherit.aes = FALSE, aes(x = .data[[x]], y = observed), color = \"black\", size = 0.5 ) + geom_line( data = truth_data, inherit.aes = FALSE, show.legend = FALSE, aes(x = .data[[x]], y = observed), linetype = 1, color = \"grey40\", lwd = 0.2 ) } return(plot) } #\" @title Make Rows NA in Data for Plotting #\" #\" @description #\" Filters the data and turns values into `NA` before the data gets passed to #\" [plot_predictions()]. The reason to do this is to this is that it allows to #\" \"filter\" prediction and truth data separately. Any value that is NA will then #\" be removed in the subsequent call to [plot_predictions()]. #\" #\" @inheritParams score #\" @param what character vector that determines which values should be turned #\" into `NA`. If `what = \"truth\"`, values in the column \"observed\" will be #\" turned into `NA`. If `what = \"forecast\"`, values in the column \"prediction\" #\" will be turned into `NA`. If `what = \"both\"`, values in both column will be #\" turned into `NA`. #\" @param ... logical statements used to filter the data #\" @return A data.table #\" @importFrom rlang enexprs #\" @keywords plotting #\" @export #\" #\" @examples #\" make_NA ( #\" example_sample_continuous, #\" what = \"truth\", #\" target_end_date >= \"2021-07-22\", #\" target_end_date < \"2021-05-01\" #\" ) make_NA <- function(data = NULL, what = c(\"truth\", \"forecast\", \"both\"), ...) { stopifnot(is.data.frame(data)) data <- as.data.table(data) what <- match.arg(what) # turn ... arguments into expressions args <- enexprs(...) vars <- NULL if (what %in% c(\"forecast\", \"both\")) { vars <- c(vars, \"predicted\") } if (what %in% c(\"truth\", \"both\")) { vars <- c(vars, \"observed\") } for (expr in args) { data <- data[eval(expr), eval(vars) := NA_real_] } return(data[]) } median_forecasts <- example_quantile[quantile_level == 0.5] median_forecasts %>% make_NA(what = \"truth\", target_end_date <= \"2021-05-01\", target_end_date > \"2021-07-22\") %>% make_NA(what = \"forecast\", model != \"EuroCOVIDhub-ensemble\", forecast_date != \"2021-06-07\") %>% plot_predictions( by = c(\"location\", \"target_type\"), x = \"target_end_date\" ) + facet_wrap(location ~ target_type, scales = \"free_y\") example_quantile %>% make_NA(what = \"truth\", target_end_date <= \"2021-05-01\", target_end_date > \"2021-07-22\") %>% make_NA(what = \"forecast\", model != \"EuroCOVIDhub-ensemble\", forecast_date != \"2021-06-07\") %>% plot_predictions( by = c(\"location\", \"target_type\"), x = \"target_end_date\", interval_range = c(0, 10, 20, 30, 40, 50, 60) ) + facet_wrap(location ~ target_type, scales = \"free_y\") example_sample_continuous %>% make_NA(what = \"truth\", target_end_date <= \"2021-05-01\", target_end_date > \"2021-07-22\") %>% make_NA(what = \"forecast\", model != \"EuroCOVIDhub-ensemble\", forecast_date != \"2021-06-07\") %>% plot_predictions( by = c(\"location\", \"target_type\"), x = \"target_end_date\", interval_range = c(0, 50, 90, 95) ) + facet_wrap(location ~ target_type, scales = \"free_y\") example_quantile %>% make_NA(what = \"truth\", target_end_date > \"2021-07-15\", target_end_date <= \"2021-05-22\") %>% make_NA(what = \"forecast\", !(model %in% c(\"EuroCOVIDhub-ensemble\", \"EuroCOVIDhub-baseline\")), forecast_date != \"2021-06-28\") %>% plot_predictions(x = \"target_end_date\", by = c(\"target_type\", \"location\")) + aes(colour = model, fill = model) + facet_wrap(target_type ~ location, ncol = 4, scales = \"free_y\") + labs(x = \"Target end date\")"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/Deprecated-visualisations.html","id":"function-plot_interval_ranges-formerly-plot_ranges","dir":"Articles","previous_headings":"","what":"Function plot_interval_ranges() (formerly plot_ranges())","title":"Deprecated Visualisations","text":"functionality currently relies hack. previous versions scoringutils, scores computed per interval range/per quantile. Now, scoringutils returns one score per forecast, per interval range/quantile. therefore need add range column, using internal function get_range_from_quantile(). column interpreted one defines unit single forecast scoringutils. also means get warning different number quantile levels different forecasts (0% prediction interval one median forecast, prediction intervals two (lower upper bound)). Plotting dispersion instead WIS:","code":"#\" @title Plot Metrics by Range of the Prediction Interval #\" #\" @description #\" Visualise the metrics by range, e.g. if you are interested how different #\" interval ranges contribute to the overall interval score, or how #\" sharpness / dispersion changes by range. #\" #\" @param scores A data.frame of scores based on quantile forecasts as #\" produced by [score()] or [summarise_scores()]. Note that \"range\" must be included #\" in the `by` argument when running [summarise_scores()] #\" @param y The variable from the scores you want to show on the y-Axis. #\" This could be something like \"wis\" (the default) or \"dispersion\" #\" @param x The variable from the scores you want to show on the x-Axis. #\" Usually this will be \"model\" #\" @param colour Character vector of length one used to determine a variable #\" for colouring dots. The Default is \"range\". #\" @return A ggplot2 object showing a contributions from the three components of #\" the weighted interval score #\" @importFrom ggplot2 ggplot aes aes geom_point geom_line #\" expand_limits theme theme_light element_text scale_color_continuous labs #\" @export #\" @examples #\" library(ggplot2) #\" ex <- data.table::copy(example_quantile) #\" ex$range <- scoringutils:::get_range_from_quantile(ex$quantile) #\" scores <- suppressWarnings(score(as_forecast(ex), metrics = list(\"wis\" = wis))) #\" summarised <- summarise_scores( #\" scores, #\" by = c(\"model\", \"target_type\", \"range\") #\" ) #\" plot_interval_ranges(summarised, x = \"model\") + #\" facet_wrap(~target_type, scales = \"free\") plot_interval_ranges <- function(scores, y = \"wis\", x = \"model\", colour = \"range\") { plot <- ggplot( scores, aes( x = .data[[x]], y = .data[[y]], colour = .data[[colour]] ) ) + geom_point(size = 2) + geom_line(aes(group = range), colour = \"black\", linewidth = 0.01 ) + scale_color_continuous(low = \"steelblue\", high = \"salmon\") + theme_scoringutils() + expand_limits(y = 0) + theme( legend.position = \"right\", axis.text.x = element_text( angle = 90, vjust = 1, hjust = 1 ) ) return(plot) } range_example <- copy(example_quantile) %>% na.omit() %>% .[, range := scoringutils:::get_range_from_quantile(quantile_level)] sum_scores <- range_example %>% as_forecast() %>% score(metrics = list(wis = wis, dispersion = dispersion)) %>% summarise_scores(by = c(\"model\", \"target_type\", \"range\")) %>% suppressWarnings() plot_interval_ranges(sum_scores, x = \"model\") + facet_wrap(~target_type, scales = \"free\") plot_interval_ranges(sum_scores, y = \"dispersion\", x = \"model\") + facet_wrap(~target_type, scales = \"free_y\")"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/Deprecated-visualisations.html","id":"function-plot_score_table","dir":"Articles","previous_headings":"","what":"Function plot_score_table()","title":"Deprecated Visualisations","text":"function allowed users turn table (summarised) scores coloured table. function hard-coded information colour scale pick metric. scoringutils 2.0.0, allowed users assign names metrics use custom scoring functions. stick default names provided scoringutils, function still work. However, functionality easily generalisable, decided deprecate function. main functionality old function provided, scaling scores order obtain reasonable colour shades. per metric, one also pass additional grouping variables argument. allowed users achieve faceting table (note course scores also needed summarised according grouping). function also allowed users combine different facets one, creating combined y-variable. done passing vector column names y argument.","code":"#' @title Plot Coloured Score Table #' #' @description #' Plots a coloured table of summarised scores obtained using #' [score()]. #' #' @param y the variable to be shown on the y-axis. Instead of a single character string, #' you can also specify a vector with column names, e.g. #' `y = c(\"model\", \"location\")`. These column names will be concatenated #' to create a unique row identifier (e.g. \"model1_location1\"). #' @param by A character vector that determines how the colour shading for the #' plot gets computed. By default (`NULL`), shading will be determined per #' metric, but you can provide additional column names (see examples). #' @param metrics A character vector with the metrics to show. If set to #' `NULL` (default), all metrics present in `scores` will be shown. #' #' @return A ggplot object with a coloured table of summarised scores #' @inheritParams get_pairwise_comparisons #' @importFrom ggplot2 ggplot aes element_blank element_text labs coord_cartesian coord_flip #' @importFrom data.table setDT melt #' @importFrom stats sd #' @export #' #' @examples #' library(ggplot2) #' library(magrittr) # pipe operator #' \\dontshow{ #' data.table::setDTthreads(2) # restricts number of cores used on CRAN #' } #' #' scores <- score(as_forecast(example_quantile)) %>% #' summarise_scores(by = c(\"model\", \"target_type\")) %>% #' summarise_scores(by = c(\"model\", \"target_type\"), fun = signif, digits = 2) #' #' plot_score_table(scores, y = \"model\", by = \"target_type\") + #' facet_wrap(~target_type, ncol = 1) #' #' # can also put target description on the y-axis #' plot_score_table(scores, #' y = c(\"model\", \"target_type\"), #' by = \"target_type\") plot_score_table <- function(scores, y = \"model\", by = NULL, metrics = NULL) { # identify metrics ----------------------------------------------------------- id_vars <- get_forecast_unit(scores) metrics <- get_metrics(scores) cols_to_delete <- names(scores)[!(names(scores) %in% c(metrics, id_vars))] suppressWarnings(scores[, eval(cols_to_delete) := NULL]) # compute scaled values ------------------------------------------------------ # scaling is done in order to colour the different scores # for most metrics larger is worse, but others like bias are better if they # are close to zero and deviations in both directions are bad # define which metrics are scaled using min (larger is worse) and # which not (metrics like bias where deviations in both directions are bad) metrics_zero_good <- c(\"bias\", \"interval_coverage_deviation\") metrics_no_color <- \"coverage\" metrics_min_good <- setdiff(metrics, c( metrics_zero_good, metrics_no_color )) # write scale functions that can be used in data.table scale <- function(x) { scaled <- x / sd(x, na.rm = TRUE) return(scaled) } scale_min_good <- function(x) { scaled <- (x - min(x)) / sd(x, na.rm = TRUE) return(scaled) } # pivot longer and add scaled values df <- data.table::melt(scores, value.vars = metrics, id.vars = id_vars, variable.name = \"metric\" ) df[metric %in% metrics_min_good, value_scaled := scale_min_good(value), by = c(\"metric\", by) ] df[metric %in% metrics_zero_good, value_scaled := scale(value), by = c(\"metric\", by) ] df[metric %in% metrics_no_color, value_scaled := 0, by = c(\"metric\", by) ] # create identifier column for plot ------------------------------------------ # if there is only one column, leave column as is. Reason to do that is that # users can then pass in a factor and keep the ordering of that column intact if (length(y) > 1) { df[, identifCol := do.call(paste, c(.SD, sep = \"_\")), .SDcols = y[y %in% names(df)]] } else { setnames(df, old = eval(y), new = \"identifCol\") } # plot ----------------------------------------------------------------------- # make plot with all metrics that are not NA plot <- ggplot( df[!is.na(value), ], aes(y = identifCol, x = metric) ) + geom_tile(aes(fill = value_scaled), colour = \"white\", show.legend = FALSE) + geom_text(aes(y = identifCol, label = value)) + scale_fill_gradient2(low = \"steelblue\", high = \"salmon\") + theme_scoringutils() + theme( legend.title = element_blank(), legend.position = \"none\", axis.text.x = element_text( angle = 90, vjust = 1, hjust = 1 ) ) + labs(x = \"\", y = \"\") + coord_cartesian(expand = FALSE) return(plot) } scores <- score(as_forecast(example_quantile)) %>% summarise_scores(by = c(\"model\", \"target_type\")) %>% summarise_scores(by = c(\"model\", \"target_type\"), fun = signif, digits = 2) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_score_table(scores, y = \"model\", by = \"target_type\") + facet_wrap(~target_type, ncol = 1) # can also put target description on the y-axis plot_score_table(scores, y = c(\"model\", \"target_type\"), by = \"target_type\")"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"introduction","dir":"Articles","previous_headings":"","what":"Introduction","title":"Scoring rules in `scoringutils`","text":"vignette gives overview default scoring rules made available scoringutils package. can, course, also use scoring rules, provided follow format. want obtain detailed information pacakge works, look revised version scoringutils paper. can distinguish two types forecasts: point forecasts probabilistic forecasts. point forecast single number representing single outcome. probabilistic forecast full predictive probability distribution multiple possible outcomes. contrast point forecasts, probabilistic forecasts incorporate uncertainty different possible outcomes. Scoring rules functions take forecast observation input return single numeric value. point forecasts, take form \\(S(\\hat{y}, y)\\), \\(\\hat{y}\\) forecast \\(y\\) observation. probabilistic forecasts, usually take form \\(S(F, y)\\), \\(F\\) cumulative density function (CDF) predictive distribution \\(y\\) observation. convention, scoring rules usually negatively oriented, meaning smaller values better (best possible score usually zero). sense, score can understood penalty. Many scoring rules probabilistic forecasts -called (strictly) proper scoring rules. Essentially, means “cheated”: forecaster evaluated strictly proper scoring rule always incentivised report honest best belief future , expectation, improve score reporting something else. formal definition following: Let \\(G\\) true, unobserved data-generating distribution. scoring rule said proper, \\(G\\) ideal forecast \\(F = G\\), forecast \\(F' \\neq F\\) expectation receives better score \\(F\\). scoring rule considered strictly proper , \\(G\\), forecast \\(F'\\) expectation receives score better \\(F\\).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"metrics-for-point-forecasts","dir":"Articles","previous_headings":"","what":"Metrics for point forecasts","title":"Scoring rules in `scoringutils`","text":"See list default metrics point forecasts calling ?metrics_point(). overview input output formats point forecasts: Input output formats: metrics point.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"a-note-of-caution","dir":"Articles","previous_headings":"Metrics for point forecasts","what":"A note of caution","title":"Scoring rules in `scoringutils`","text":"Scoring point forecasts can tricky business. Depending choice scoring rule, forecaster clearly worse another, might consistently receive better scores (see Gneiting (2011) illustrative example). Every scoring rule point forecast implicitly minimised specific aspect predictive distribution. mean squared error, example, meaningful scoring rule forecaster actually reported mean predictive distribution point forecast. forecaster reported median, mean absolute error appropriate scoring rule. scoring rule predictive task align, misleading results ensue. Consider following example:","code":"set.seed(123) n <- 1000 observed <- rnorm(n, 5, 4)^2 predicted_mu <- mean(observed) predicted_not_mu <- predicted_mu - rnorm(n, 10, 2) mean(Metrics::ae(observed, predicted_mu)) #> [1] 34.45981 mean(Metrics::ae(observed, predicted_not_mu)) #> [1] 32.54821 mean(Metrics::se(observed, predicted_mu)) #> [1] 2171.089 mean(Metrics::se(observed, predicted_not_mu)) #> [1] 2290.155"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"absolute-error","dir":"Articles","previous_headings":"Metrics for point forecasts","what":"Absolute error","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), real number Forecast: \\(\\hat{y}\\), real number, median forecaster’s predictive distribution. absolute error absolute difference predicted observed values. See ?Metrics::ae. \\[\\text{ae} = |y - \\hat{y}|\\] absolute error appropriate rule \\(\\hat{y}\\) corresponds median forecaster’s predictive distribution. Otherwise, results misleading (see Gneiting (2011)).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"squared-error","dir":"Articles","previous_headings":"Metrics for point forecasts","what":"Squared error","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), real number Forecast: \\(\\hat{y}\\), real number, mean forecaster’s predictive distribution. squared error squared difference predicted observed values. See ?Metrics::se. \\[\\text{se} = (y - \\hat{y})^2\\] squared error appropriate rule \\(\\hat{y}\\) corresponds mean forecaster’s predictive distribution. Otherwise, results misleading (see Gneiting (2011)).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"absolute-percentage-error","dir":"Articles","previous_headings":"Metrics for point forecasts","what":"Absolute percentage error","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), real number Forecast: \\(\\hat{y}\\), real number absolute percentage error absolute percent difference predicted observed values. See ?Metrics::ape. \\[\\text{ape} = \\frac{|y - \\hat{y}|}{|y|}\\] absolute percentage error appropriate rule \\(\\hat{y}\\) corresponds \\(\\beta\\)-median forecaster’s predictive distribution \\(\\beta = -1\\). \\(\\beta\\)-median, \\(\\text{med}^{(\\beta)}(F)\\), median random variable whose density proportional \\(y^\\beta f(y)\\). specific \\(\\beta\\)-median corresponds absolute percentage error \\(\\text{med}^{(-1)}(F)\\). Otherwise, results misleading (see Gneiting (2011)).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"binary-forecasts","dir":"Articles","previous_headings":"","what":"Binary forecasts","title":"Scoring rules in `scoringutils`","text":"See list default metrics point forecasts calling ?metrics_binary(). overview input output formats point forecasts: Input output formats: metrics binary forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"brier-score","dir":"Articles","previous_headings":"Binary forecasts","what":"Brier score","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), either 0 1 Forecast: \\(p\\), probability observed outcome 1. Brier score strictly proper scoring rule. computed mean squared error probabilistic prediction observed outcome. \\[\\begin{equation} \\text{BS}(p, y) = (p - y)^2 = \\begin{cases} p^2, & \\text{} y = 1\\\\ (1 - p)^2, & \\text{} y = 0 \\end{cases} \\end{equation}\\] Brier score logarithmic score (see ) differ penalise - underconfidence (see Machete (2012)). Brier score penalises overconfidence underconfidence probability space . Consider following example: See ?brier_score() information.","code":"n <- 1e6 p_true <- 0.7 observed <- factor(rbinom(n = n, size = 1, prob = p_true), levels = c(0, 1)) p_over <- p_true + 0.15 p_under <- p_true - 0.15 abs(mean(brier_score(observed, p_true)) - mean(brier_score(observed, p_over))) #> [1] 0.0223866 abs(mean(brier_score(observed, p_true)) - mean(brier_score(observed, p_under))) #> [1] 0.0226134"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"logarithmic-score","dir":"Articles","previous_headings":"Binary forecasts","what":"Logarithmic score","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), either 0 1 Forecast: \\(p\\), probability observed outcome 1. logarithmic score (log score) strictly proper scoring rule. computed negative logarithm probability assigned observed outcome. \\[\\begin{equation} \\text{Log score}(p, y) = - \\log(1 - |y - p|) = \\begin{cases} -\\log (p), & \\text{} y = 1\\\\ -\\log (1 - p), & \\text{} y = 0 \\end{cases} \\end{equation}\\] log score penalises overconfidence strongly underconfidence (probability space). Consider following example: See ?logs_binary() information.","code":"abs(mean(logs_binary(observed, p_true)) - mean(logs_binary(observed, p_over))) #> [1] 0.07169954 abs(mean(logs_binary(observed, p_true)) - mean(logs_binary(observed, p_under))) #> [1] 0.04741833"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"sample-based-forecasts","dir":"Articles","previous_headings":"","what":"Sample-based forecasts","title":"Scoring rules in `scoringutils`","text":"See list default metrics sample-based forecasts calling ?metrics_sample(). overview input output formats quantile forecasts: Input output formats: metrics sample-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"crps","dir":"Articles","previous_headings":"Sample-based forecasts","what":"CRPS","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), real number (discrete number). Forecast: continuous (\\(F\\)) discrete (\\(P\\)) forecast. continuous ranked probability score (CRPS) popular fields meteorology epidemiology. CRPS defined \\[\\text{CRPS}(F, y) = \\int_{-\\infty}^\\infty \\left( F(x) - 1(x \\geq y) \\right)^2 dx,\\] \\(y\\) observed value \\(F\\) CDF predictive distribution. discrete forecasts, example count data, ranked probability score (RPS) can used instead commonly defined : \\[ \\text{RPS}(P, y) = \\sum_{x = 0}^\\infty (P(x) - 1(x \\geq y))^2, \\] \\(P\\) cumulative probability mass function (PMF) predictive distribution. CRPS can understood generalisation absolute error predictive distributions (Gneiting Raftery 2007). can also understood integral Brier score binary probability forecasts implied CDF possible observed values. CRPS also related Cramér-distance two distributions equals special case one distributions concentrated single point (see e.g. Ziel (2021)). CRPS global scoring rule, meaning entire predictive distribution taken account determining quality forecast. scoringutils re-exports crps_sample() function scoringRules package, assumes forecast represented set samples predictive distribution. See ?crps_sample() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"log-score","dir":"Articles","previous_headings":"Sample-based forecasts","what":"Log score","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), real number (discrete number). Forecast: continuous (\\(F\\)) discrete (\\(P\\)) forecast. logarithmic scoring rule simply negative logarithm density predictive distribution evaluated observed value: \\[ \\text{log score}(F, y) = -\\log f(y), \\] \\(f\\) predictive probability density function (PDF) corresponding Forecast \\(F\\) \\(y\\) observed value. discrete forecasts, log score can computed \\[ \\text{log score}(F, y) = -\\log p_y, \\] \\(p_y\\) probability assigned observed outcome \\(y\\) forecast \\(F\\). logarithmic scoring rule can produce large penalties observed value takes values \\(f(y)\\) (\\(p_y\\)) close zero. therefore considered sensitive outlier forecasts. may desirable applications, also means scores can easily dominated extreme values. logarithmic scoring rule local scoring rule, meaning score depends probability assigned actual outcome. often regarded desirable property example context Bayesian inference . implies example, ranking forecasters invariant monotone transformations predictive distribution target. scoringutils re-exports logs_sample() function scoringRules package, assumes forecast represented set samples predictive distribution. One implications currently advisable use log score discrete forecasts. reason scoringRules::logs_sample() estimates predictive density samples, can problematic discrete forecasts. See ?logs_sample() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"dawid-sebastiani-score","dir":"Articles","previous_headings":"Sample-based forecasts","what":"Dawid-Sebastiani score","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), real number (discrete number). Forecast: \\(F\\). predictive distribution mean \\(\\mu\\) standard deviation \\(\\sigma\\). Dawid-Sebastiani score proper scoring rule relies first moments predictive distribution therefore easy compute. given \\[\\text{dss}(F, y) = \\left( \\frac{y - \\mu}{\\sigma} \\right)^2 + 2 \\cdot \\log \\sigma.\\] scoringutils re-exports implementation DSS scoringRules package. assumes forecast represented set samples drawn predictive distribution. See ?dss_sample() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"dispersion---median-absolute-deviation-mad","dir":"Articles","previous_headings":"Sample-based forecasts","what":"Dispersion - Median Absolute Deviation (MAD)","title":"Scoring rules in `scoringutils`","text":"Observation: required. Forecast: \\(F\\), predictive distribution. Dispersion (also called sharpness) ability produce narrow forecasts. feature forecasts depend observations. Dispersion therefore interest conditional calibration: precise forecast useful clearly wrong. One way measure sharpness (suggested Funk et al. (2019)) normalised median absolute deviation median (MADN) ). computed \\[ S(F) = \\frac{1}{0.675} \\cdot \\text{median}(|F - \\text{median(F)}|). \\] forecast \\(F\\) follows normal distribution, sharpness equal standard deviation \\(F\\).details, see ?mad_sample().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"bias","dir":"Articles","previous_headings":"Sample-based forecasts","what":"Bias","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), real number (discrete number). Forecast: continuous (\\(F\\)) discrete (\\(P\\)) forecast. Bias measure tendency forecaster - underpredict. continuous forecasts, scoringutils implementation calculates bias \\[B(F, y) = 1 - 2 \\cdot F (y), \\] \\(F(y)\\) cumulative distribution function forecast. discrete forecasts, calculate bias \\[B(P, y) = 1 - (P(y) + P(y + 1)). \\] \\(P(y)\\) cumulative probability assigned outcomes smaller equal \\(y\\), .e. cumulative probability mass function. Bias bound -1 1 represents tendency forecasts biased rather absolute amount - underprediction (e.g. case weighted interval score (WIS, see ).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"absolute-error-of-the-median","dir":"Articles","previous_headings":"Sample-based forecasts","what":"Absolute error of the median","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), real number (discrete number). Forecast: forecast \\(F\\) \\[\\text{ae}_{\\text{median}}(F, y) = |\\text{median} (F) - y|.\\] See section note caution Gneiting (2011) discussion correspondence absolute error median.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"squared-error-of-the-mean","dir":"Articles","previous_headings":"Sample-based forecasts","what":"Squared error of the mean","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), real number (discrete number). Forecast: forecast \\(F\\) \\[\\text{se}_{\\text{medn}}(F, y) = (\\text{mean} (F) - y)^2.\\] See section note caution Gneiting (2011) discussion correspondence squared error mean.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"quantile-based-forecasts","dir":"Articles","previous_headings":"","what":"Quantile-based forecasts","title":"Scoring rules in `scoringutils`","text":"See list default metrics quantile-based forecasts calling ?metrics_quantile(). overview input output formats quantile forecasts: Input output formats: metrics quantile-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"weighted-interval-score-wis","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Weighted interval score (WIS)","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), real number Forecast: \\(F\\). CDF predictive distribution represented set quantiles. quantiles form lower (\\(l\\)) upper (\\(u\\)) bounds central prediction intervals. weighted interval score (WIS) strictly proper scoring rule can understood approximation CRPS forecasts quantile format (turn represents generalisation absolute error). Quantiles assumed lower upper bounds prediction intervals symmetric around median. single interval, interval score \\[IS_\\alpha(F,y) = \\underbrace{(u-l)}_\\text{dispersion} + \\underbrace{\\frac{2}{\\alpha} \\cdot (l-y) \\cdot \\mathbf{1}(y \\leq l)}_{\\text{overprediction}} + \\underbrace{\\frac{2}{\\alpha} \\cdot (y-u) \\cdot \\mathbf{1}(y \\geq u)}_{\\text{underprediction}}, \\] \\(\\mathbf{1}()\\) indicator function, \\(l\\) \\(u\\) \\(\\frac{\\alpha}{2}\\) \\(1 - \\frac{\\alpha}{2}\\) quantiles predictive distribution \\(F\\). \\(l\\) \\(u\\) together form prediction interval. interval score can understood sum three components: dispersion, overprediction underprediction. set \\(K\\) prediction intervals median \\(m\\), score given weighted sum individual interval scores, .e. \\[WIS = \\frac{1}{K + 0.5} \\cdot \\left(w_0 \\cdot |y - m| + \\sum_{k = 1}^{K} w_k \\cdot IS_{\\alpha_{k}}(F, y)\\right),\\] \\(m\\) median forecast \\(w_k\\) weight assigned every interval. weights set \\(w_k = \\frac{\\alpha_k}{2}\\) \\(w_0 = 0.5\\), WIS converges CRPS increasing number equally spaced quantiles. See ?wis() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"overprediction-underprediction-and-dispersion","dir":"Articles","previous_headings":"Quantile-based forecasts > Weighted interval score (WIS)","what":"Overprediction, underprediction and dispersion","title":"Scoring rules in `scoringutils`","text":"individual components WIS. See ?overprediction(), ?underprediction() ?dispersion() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"bias-1","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Bias","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), real number Forecast: \\(F\\). CDF predictive distribution represented set quantiles, \\(Q\\). Bias can measured \\[\\begin{equation} \\text{B}(F, y) = \\begin{cases} (1 - 2 \\cdot \\max \\{\\alpha | q_\\alpha \\Q \\land q_\\alpha \\leq y\\}), & \\text{} y < q_{0.5} \\quad \\text{(overprediction)}\\\\ (1 - 2 \\cdot \\min \\{\\alpha | q_\\alpha \\Q_t \\land q_\\alpha \\geq y\\}, & \\text{} y > q_{0.5} \\quad \\text{(underprediction)}\\\\ 0, & \\text{} y = q_{0.5}, \\\\ \\end{cases} \\end{equation}\\] \\(q_\\alpha\\) \\(\\alpha\\)-quantile predictive distribution. consistency, define \\(Q\\) (set quantiles form predictive distribution \\(F\\)) always includes element \\(q_0 = -\\infty\\) \\(q_1 = \\infty\\). clearer terms, bias : \\(1 - (2 \\times\\) maximum percentile rank corresponding quantile still observed value), observed value smaller median predictive distribution. \\(1 - (2 \\times\\) minimum percentile rank corresponding quantile still larger observed value) observed value larger median predictive distribution.. \\(0\\) observed value exactly median. Bias can assume values -1 (underprediction) 1 (overpredictin) 0 ideally (.e. unbiased). increasing number quantiles, percentile rank equal proportion predictive samples observed value, bias metric coincides one continuous forecasts (see ). See ?bias_quantile() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"interval-coverage","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Interval coverage","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), real number Forecast: \\(F\\). CDF predictive distribution represented set quantiles. quantiles form central prediction intervals. Interval coverage given interval range defined proportion observations fall within corresponding central prediction intervals. Central prediction intervals symmetric around median formed two quantiles denote lower upper bound. example, 50% central prediction interval interval 0.25 0.75 quantiles predictive distribution.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"interval-coverage-deviation","dir":"Articles","previous_headings":"Quantile-based forecasts > Interval coverage","what":"Interval coverage deviation","title":"Scoring rules in `scoringutils`","text":"interval coverage deviation difference observed interval coverage nominal interval coverage. example, observed interval coverage 50% central prediction interval 0.6, interval coverage deviation \\(0.6 = - 0.5 = 0.1.\\) \\[\\text{interval coverage deviation} = \\text{observed interval coverage} - \\text{nominal interval coverage}\\]","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"absolute-error-of-the-median-1","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Absolute error of the median","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), real number Forecast: \\(F\\). CDF predictive distribution represented set quantiles. absolute error median absolute difference median predictive distribution observed value. \\[\\text{ae}_\\text{median} = |\\text{median}(F) - y|\\] See section note caution Gneiting (2011) discussion correspondence absolute error median.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"quantile-score","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Quantile score","title":"Scoring rules in `scoringutils`","text":"Observation: \\(y\\), real number Forecast: \\(F\\). CDF predictive distribution represented set quantiles. quantile score, also called pinball loss, single quantile level \\(\\tau\\) defined \\[\\begin{equation} \\text{QS}_\\tau(F, y) = 2 \\cdot \\{ \\mathbf{1}(y \\leq q_\\tau) - \\tau\\} \\cdot (q_\\tau − y) = \\begin{cases} 2 \\cdot (1 - \\tau) * q_\\tau - y, & \\text{} y \\leq q_\\tau\\\\ 2 \\cdot \\tau * |q_\\tau - y|, & \\text{} y > q_\\tau, \\end{cases} \\end{equation}\\] \\(q_\\tau\\) \\(\\tau\\)-quantile predictive distribution \\(F\\), \\(\\mathbf{1}(\\cdot)\\) indicator function. (unweighted) interval score (see ) \\(1 - \\alpha\\) prediction interval can computed quantile scores levels \\(\\alpha/2\\) \\(1 - \\alpha/2\\) \\[\\text{}_\\alpha(F, y) = \\frac{\\text{QS}_{\\alpha/2}(F, y) + \\text{QS}_{1 - \\alpha/2}(F, y)}{\\alpha}\\]. weighted interval score can obtained simple average quantile scores: \\[\\text{WIS}_\\alpha(F, y) = \\frac{\\text{QS}_{\\alpha/2}(F, y) + \\text{QS}_{1 - \\alpha/2}(F, y)}{2}\\]. See ?quantile_score Bracher et al. (2021) details.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"quantile-coverage","dir":"Articles","previous_headings":"Additional metrics","what":"Quantile coverage","title":"Scoring rules in `scoringutils`","text":"Quantile coverage given quantile level defined proportion observed values smaller corresponding predictive quantiles. example, 0.5 quantile coverage proportion observed values smaller 0.5-quantiles predictive distribution.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoringutils.html","id":"input-formats","dir":"Articles","previous_headings":"","what":"Input formats","title":"Getting started","text":"time, score() function able entire evaluation . need pass data.frame appropriate columns. columns required depends format forecasts come . forecast format can either based quantiles (see example_quantile expected format), based predictive samples (see example_sample_continuous example_sample_discrete expected format case) binary format. following table gives overview (pairwise comparisons explained detail ): Additional columns may present indicate grouping forecasts. example, forecasts made different models various locations different time points, several weeks future. scoringutils automatically tries determine unit single forecast, .e. combination existing columns able uniquely identify single forecast. uses existing columns , can sometimes lead issues. therefore recommend using function set_forecast_unit() determine forecast unit manually. function simply drops unneeded columns, making sure necessary, ‘protected columns’ like “predicted” “observed” retained.","code":"colnames(example_quantile) #> [1] \"location\" \"target_end_date\" \"target_type\" \"observed\" #> [5] \"location_name\" \"forecast_date\" \"quantile_level\" \"predicted\" #> [9] \"model\" \"horizon\" set_forecast_unit( example_quantile, c(\"location\", \"target_end_date\", \"target_type\", \"horizon\", \"model\") ) %>% colnames() #> [1] \"observed\" \"quantile_level\" \"predicted\" \"location\" #> [5] \"target_end_date\" \"target_type\" \"horizon\" \"model\""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoringutils.html","id":"constructing-a-forecast-object","dir":"Articles","previous_headings":"","what":"Constructing a forecast object","title":"Getting started","text":"function as_forecast() used construct forecast object check input data. determines forecast type, creates object appropriate class (forecast_binary, forecast_quantile forecast_sample), validates input data. Objects class forecast_* print method provides additional information. unsure input data look like, look example_quantile, example_sample_discrete, example_sample_continuous example_binary data sets provided package. output as_forecast() can later directly used input score(). recommended workflow (however, score() run as_forecast() internally hasn’t happened ). Note example columns contain duplicated information regards forecast unit, e.g. “location” “location_name”, can dropped. drop essential information, example, “target_type” column, ’ll get error informing us forecasts aren’t uniquely identified . function get_duplicate_forecasts() may help investigate issue. filtering single quantile EuroCOVIDhub-ensemble, can see indeed two forecasts every date, location horizon.","code":"head(example_quantile) #> Key: #> location target_end_date target_type observed location_name forecast_date #> #> 1: DE 2021-01-02 Cases 127300 Germany #> 2: DE 2021-01-02 Deaths 4534 Germany #> 3: DE 2021-01-09 Cases 154922 Germany #> 4: DE 2021-01-09 Deaths 6117 Germany #> 5: DE 2021-01-16 Cases 110183 Germany #> 6: DE 2021-01-16 Deaths 5867 Germany #> quantile_level predicted model horizon #> #> 1: NA NA NA #> 2: NA NA NA #> 3: NA NA NA #> 4: NA NA NA #> 5: NA NA NA #> 6: NA NA NA forecast_quantile <- example_quantile %>% set_forecast_unit( c(\"model\", \"location\", \"target_end_date\", \"forecast_date\", \"target_type\", \"horizon\") ) %>% as_forecast() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. forecast_quantile #> Forecast type: quantile #> Forecast unit: #> model, location, target_end_date, forecast_date, target_type, and horizon #> #> Key: #> observed quantile_level predicted model location #> #> 1: 127300 NA NA DE #> 2: 4534 NA NA DE #> 3: 154922 NA NA DE #> 4: 6117 NA NA DE #> 5: 110183 NA NA DE #> --- #> 20541: 78 0.850 352 epiforecasts-EpiNow2 IT #> 20542: 78 0.900 397 epiforecasts-EpiNow2 IT #> 20543: 78 0.950 499 epiforecasts-EpiNow2 IT #> 20544: 78 0.975 611 epiforecasts-EpiNow2 IT #> 20545: 78 0.990 719 epiforecasts-EpiNow2 IT #> target_end_date forecast_date target_type horizon #> #> 1: 2021-01-02 Cases NA #> 2: 2021-01-02 Deaths NA #> 3: 2021-01-09 Cases NA #> 4: 2021-01-09 Deaths NA #> 5: 2021-01-16 Cases NA #> --- #> 20541: 2021-07-24 2021-07-12 Deaths 2 #> 20542: 2021-07-24 2021-07-12 Deaths 2 #> 20543: 2021-07-24 2021-07-12 Deaths 2 #> 20544: 2021-07-24 2021-07-12 Deaths 2 #> 20545: 2021-07-24 2021-07-12 Deaths 2 example_quantile %>% set_forecast_unit( c(\"location\", \"target_end_date\", \"forecast_date\", \"model\", \"horizon\") ) %>% as_forecast() #> Error: Assertion on 'data' failed: There are instances with more than one forecast for the same target. This can't be right and needs to be resolved. Maybe you need to check the unit of a single forecast and add missing columns? Use the function get_duplicate_forecasts() to identify duplicate rows. duplicates <- example_quantile %>% set_forecast_unit( c(\"location\", \"target_end_date\", \"forecast_date\", \"model\", \"horizon\") ) %>% get_duplicate_forecasts() duplicates[quantile_level == 0.5 & model == \"EuroCOVIDhub-ensemble\", ] %>% head() #> Key: #> observed quantile_level predicted location target_end_date forecast_date #> #> 1: 106987 0.5 119258 DE 2021-05-08 2021-05-03 #> 2: 1582 0.5 1568 DE 2021-05-08 2021-05-03 #> 3: 64985 0.5 110716 DE 2021-05-15 2021-05-03 #> 4: 64985 0.5 92649 DE 2021-05-15 2021-05-10 #> 5: 1311 0.5 1521 DE 2021-05-15 2021-05-03 #> 6: 1311 0.5 1422 DE 2021-05-15 2021-05-10 #> model horizon #> #> 1: EuroCOVIDhub-ensemble 1 #> 2: EuroCOVIDhub-ensemble 1 #> 3: EuroCOVIDhub-ensemble 2 #> 4: EuroCOVIDhub-ensemble 1 #> 5: EuroCOVIDhub-ensemble 2 #> 6: EuroCOVIDhub-ensemble 1"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoringutils.html","id":"showing-available-forecasts","dir":"Articles","previous_headings":"","what":"Showing available forecasts","title":"Getting started","text":"function get_forecast_counts() may also helpful determine forecasts available. Using argument can specify level summary. example, see many forecasts per model target_type, can run see ‘epiforecasts-EpiNow2’ missing forecasts deaths forecast target UMass-MechBayes case forecasts. information can also visualised using plot():","code":"get_forecast_counts(forecast_quantile, by = c(\"model\", \"target_type\")) #> Key: #> model target_type count #> #> 1: EuroCOVIDhub-baseline Cases 128 #> 2: EuroCOVIDhub-baseline Deaths 128 #> 3: EuroCOVIDhub-ensemble Cases 128 #> 4: EuroCOVIDhub-ensemble Deaths 128 #> 5: UMass-MechBayes Cases 0 #> 6: UMass-MechBayes Deaths 128 #> 7: epiforecasts-EpiNow2 Cases 128 #> 8: epiforecasts-EpiNow2 Deaths 119 forecast_quantile %>% get_forecast_counts(by = c(\"model\", \"forecast_date\", \"target_type\")) %>% plot_forecast_counts(x = \"forecast_date\") + facet_wrap(~ target_type)"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoringutils.html","id":"scoring-and-summarising-forecasts","dir":"Articles","previous_headings":"","what":"Scoring and summarising forecasts","title":"Getting started","text":"Forecasts can easily scored using score() function. clarity, suggest setting forecast unit explicitly calling as_forecast() explicitly. function score() returns unsumarised scores, cases user wants. returns single score per forecast (determined forecast unit). second function, summarise_scores() takes care summarising scores level specified user. argument can used define level summary. default, = model one score per model returned. argument can specify unit summary want. can also call sumarise_scores() multiple tines, e.g round outputs specifying e.g. signif() summary function.","code":"scores <- forecast_quantile %>% score() print(scores, 2) #> model location target_end_date forecast_date target_type #> #> 1: EuroCOVIDhub-ensemble DE 2021-05-08 2021-05-03 Cases #> 2: EuroCOVIDhub-baseline DE 2021-05-08 2021-05-03 Cases #> --- #> 886: epiforecasts-EpiNow2 IT 2021-07-24 2021-07-05 Deaths #> 887: epiforecasts-EpiNow2 IT 2021-07-24 2021-07-12 Deaths #> horizon wis overprediction underprediction dispersion bias #> #> 1: 1 7990.85478 2549.869565 0 5440.98522 0.50 #> 2: 1 16925.04696 15275.826087 0 1649.22087 0.95 #> --- #> 886: 3 19.76261 5.478261 0 14.28435 0.50 #> 887: 2 66.16174 40.608696 0 25.55304 0.90 #> interval_coverage_50 interval_coverage_90 interval_coverage_deviation #> #> 1: TRUE TRUE 0.05181818 #> 2: FALSE FALSE -0.40272727 #> --- #> 886: TRUE TRUE 0.05181818 #> 887: FALSE TRUE -0.31181818 #> ae_median #> #> 1: 12271 #> 2: 25620 #> --- #> 886: 26 #> 887: 108 scores %>% summarise_scores(by = c(\"model\", \"target_type\")) %>% summarise_scores(fun = signif, digits = 2) %>% kable()"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoringutils.html","id":"scoring-point-forecasts","dir":"Articles","previous_headings":"Scoring and summarising forecasts","what":"Scoring point forecasts","title":"Getting started","text":"Point forecasts can scored separate format.","code":"suppressMessages(score(as_forecast(example_point))) %>% summarise_scores(by = \"model\", na.rm = TRUE) #> model ae_point se_point ape #> #> 1: EuroCOVIDhub-ensemble 12077.10156 1.945118e+09 0.2996696 #> 2: EuroCOVIDhub-baseline 19353.42969 2.883446e+09 0.7383936 #> 3: epiforecasts-EpiNow2 14521.10526 2.680928e+09 0.3715552 #> 4: UMass-MechBayes 78.47656 1.170976e+04 0.2823206"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoringutils.html","id":"adding-empirical-coverage","dir":"Articles","previous_headings":"Scoring and summarising forecasts","what":"Adding empirical coverage","title":"Getting started","text":"quantile-based forecasts often interested specific coverage-levels, example, percentage true values fell 50% 90% prediction intervals. can add information using function get_coverage(). function also requires argument defines level grouping percentage true values covered certain prediction intervals computed.","code":"score(as_forecast(example_quantile)) %>% summarise_scores(by = c(\"model\", \"target_type\")) %>% summarise_scores(fun = signif, digits = 2) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> model wis overprediction underprediction dispersion #> #> 1: EuroCOVIDhub-ensemble 18000 10000.0 4200.0 3700 #> 2: EuroCOVIDhub-ensemble 41 7.1 4.1 30 #> 3: EuroCOVIDhub-baseline 28000 14000.0 10000.0 4100 #> 4: EuroCOVIDhub-baseline 160 66.0 2.1 91 #> 5: epiforecasts-EpiNow2 21000 12000.0 3300.0 5700 #> 6: epiforecasts-EpiNow2 67 19.0 16.0 32 #> 7: UMass-MechBayes 53 9.0 17.0 27 #> bias interval_coverage_50 interval_coverage_90 #> #> 1: -0.0560 0.39 0.80 #> 2: 0.0730 0.88 1.00 #> 3: 0.0980 0.33 0.82 #> 4: 0.3400 0.66 1.00 #> 5: -0.0790 0.47 0.79 #> 6: -0.0051 0.42 0.91 #> 7: -0.0220 0.46 0.88 #> interval_coverage_deviation ae_median #> #> 1: -0.100 24000 #> 2: 0.200 53 #> 3: -0.120 38000 #> 4: 0.120 230 #> 5: -0.070 28000 #> 6: -0.045 100 #> 7: -0.025 78"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoringutils.html","id":"adding-relative-scores","dir":"Articles","previous_headings":"Scoring and summarising forecasts","what":"Adding relative scores","title":"Getting started","text":"order better compare models can use relative scores computed based pairwise comparisons (see details ). Relative scores can added evaluation using function summarise_scores(). requires column called ‘model’ present. Pairwise comparisons computed according grouping specified : essentially, data.frame scores gets split different data.frames according values specified relative scores computed every individual group separately. baseline argument allows us specify baseline can used scale relative scores (scores divided baseline relative score). example, obtain relative scores separately different forecast targets, can run","code":"score(as_forecast(example_quantile)) %>% add_relative_skill( by = c(\"model\", \"target_type\"), baseline = \"EuroCOVIDhub-ensemble\" ) %>% summarise_scores(by = c(\"model\", \"target_type\")) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Key: #> model target_type wis overprediction underprediction #> #> 1: EuroCOVIDhub-baseline Cases 28483.57465 14096.100883 10284.972826 #> 2: EuroCOVIDhub-baseline Deaths 159.40387 65.899117 2.098505 #> 3: EuroCOVIDhub-ensemble Cases 17943.82383 10043.121943 4237.177310 #> 4: EuroCOVIDhub-ensemble Deaths 41.42249 7.138247 4.103261 #> 5: UMass-MechBayes Deaths 52.65195 8.978601 16.800951 #> 6: epiforecasts-EpiNow2 Cases 20831.55662 11906.823030 3260.355639 #> 7: epiforecasts-EpiNow2 Deaths 66.64282 18.892583 15.893314 #> dispersion bias interval_coverage_50 interval_coverage_90 #> #> 1: 4102.50094 0.09796875 0.3281250 0.8203125 #> 2: 91.40625 0.33906250 0.6640625 1.0000000 #> 3: 3663.52458 -0.05640625 0.3906250 0.8046875 #> 4: 30.18099 0.07265625 0.8750000 1.0000000 #> 5: 26.87239 -0.02234375 0.4609375 0.8750000 #> 6: 5664.37795 -0.07890625 0.4687500 0.7890625 #> 7: 31.85692 -0.00512605 0.4201681 0.9075630 #> interval_coverage_deviation ae_median wis_relative_skill #> #> 1: -0.11721591 38473.60156 1.2947445 #> 2: 0.12142045 233.25781 2.2958723 #> 3: -0.10230114 24101.07031 0.8156514 #> 4: 0.20380682 53.13281 0.5966310 #> 5: -0.02488636 78.47656 0.7475873 #> 6: -0.06963068 27923.81250 0.9469157 #> 7: -0.04520244 104.74790 0.9765276 #> wis_scaled_relative_skill #> #> 1: 1.587375 #> 2: 3.848060 #> 3: 1.000000 #> 4: 1.000000 #> 5: 1.253014 #> 6: 1.160932 #> 7: 1.636736"},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoringutils.html","id":"score-heatmap","dir":"Articles","previous_headings":"Visualising scores","what":"Score heatmap","title":"Getting started","text":"can also summarise one particular metric across different categories using simple heatmap:","code":"score(as_forecast(example_sample_continuous)) %>% summarise_scores(by = c(\"model\", \"location\", \"target_type\")) %>% plot_heatmap(x = \"location\", metric = \"bias\") + facet_wrap(~ target_type) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected."},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoringutils.html","id":"weighted-interval-score-components","dir":"Articles","previous_headings":"Visualising scores","what":"Weighted interval score components","title":"Getting started","text":"weighted interval score can split three components: -prediction, -prediction dispersion. can visualised separately following way:","code":"score(as_forecast(example_quantile)) %>% summarise_scores(by = c(\"target_type\", \"model\")) %>% plot_wis() + facet_wrap(~ target_type, scales = \"free\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected."},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoringutils.html","id":"calibration","dir":"Articles","previous_headings":"","what":"Calibration","title":"Getting started","text":"Calibration measure statistical consistency forecasts observed values. common way assessing calibration (precisely: probabilistic calibration) PIT histograms. probability integral transform (PIT) equal cumulative distribution function forecast evaluated observed value. Ideally, pit values uniformly distributed transformation. can compute pit values : visualise results : Similarly quantile-based forecasts:","code":"example_sample_continuous %>% as_forecast() %>% get_pit(by = \"model\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> model pit_value #> #> 1: EuroCOVIDhub-baseline 0.025 #> 2: EuroCOVIDhub-baseline 0.525 #> 3: EuroCOVIDhub-baseline 0.000 #> 4: EuroCOVIDhub-baseline 0.000 #> 5: EuroCOVIDhub-baseline 0.200 #> --- #> 883: UMass-MechBayes 0.950 #> 884: UMass-MechBayes 0.500 #> 885: UMass-MechBayes 0.100 #> 886: UMass-MechBayes 0.450 #> 887: UMass-MechBayes 0.100 example_sample_continuous %>% as_forecast() %>% get_pit(by = c(\"model\", \"target_type\")) %>% plot_pit() + facet_grid(model ~ target_type) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. example_quantile[quantile_level %in% seq(0.1, 0.9, 0.1), ] %>% as_forecast() %>% get_pit(by = c(\"model\", \"target_type\")) %>% plot_pit() + facet_grid(model ~ target_type)"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoringutils.html","id":"pairwise-comparisons","dir":"Articles","previous_headings":"","what":"Pairwise comparisons","title":"Getting started","text":"Relative scores different models can computed using pairwise comparisons, sort pairwise tournament combinations two models compared based overlapping set available forecasts common models. Internally, ratio mean scores models computed. relative score model geometric mean mean score ratios involve model. baseline provided, baseline excluded relative scores individual models (therefore differ slightly relative scores without baseline) relative scores scaled (.e. divided ) relative score baseline model. scoringutils, pairwise comparisons can made two ways: standalone function get_pairwise_comparisons() within summarise_scores() simply adds relative scores existing set scores. using get_pairwise_comparisons() function, can also visualise pairwise comparisons showing mean score ratios models. default, smaller values better model care showing y axis left, model compared shown x-axis bottom. example , EuroCOVIDhub-ensemble performs best (values smaller 1), EuroCOVIDhub-baseline performs worst (values larger 1). cases, UMass-MechBayes model course excluded case forecasts available therefore set overlapping forecasts empty.","code":"forecast_quantile %>% score() %>% get_pairwise_comparisons(by = \"model\", baseline = \"EuroCOVIDhub-baseline\") #> model compare_against mean_scores_ratio pval #> #> 1: EuroCOVIDhub-baseline epiforecasts-EpiNow2 1.3703452 9.164893e-18 #> 2: EuroCOVIDhub-baseline UMass-MechBayes 3.0275019 2.627464e-20 #> 3: EuroCOVIDhub-baseline EuroCOVIDhub-ensemble 1.5925819 2.608666e-32 #> 4: EuroCOVIDhub-baseline EuroCOVIDhub-baseline 1.0000000 1.000000e+00 #> 5: EuroCOVIDhub-ensemble EuroCOVIDhub-baseline 0.6279112 2.608666e-32 #> 6: EuroCOVIDhub-ensemble UMass-MechBayes 0.7867229 1.244731e-04 #> 7: EuroCOVIDhub-ensemble epiforecasts-EpiNow2 0.8606607 1.881520e-02 #> 8: EuroCOVIDhub-ensemble EuroCOVIDhub-ensemble 1.0000000 1.000000e+00 #> 9: UMass-MechBayes EuroCOVIDhub-ensemble 1.2710955 1.244731e-04 #> 10: UMass-MechBayes epiforecasts-EpiNow2 0.7439673 7.253878e-03 #> 11: UMass-MechBayes EuroCOVIDhub-baseline 0.3303053 2.627464e-20 #> 12: UMass-MechBayes UMass-MechBayes 1.0000000 1.000000e+00 #> 13: epiforecasts-EpiNow2 UMass-MechBayes 1.3441452 7.253878e-03 #> 14: epiforecasts-EpiNow2 EuroCOVIDhub-ensemble 1.1618981 1.881520e-02 #> 15: epiforecasts-EpiNow2 EuroCOVIDhub-baseline 0.7297431 9.164893e-18 #> 16: epiforecasts-EpiNow2 epiforecasts-EpiNow2 1.0000000 1.000000e+00 #> adj_pval wis_relative_skill wis_scaled_relative_skill #> #> 1: 3.665957e-17 1.6032604 1.0000000 #> 2: 1.313732e-19 1.6032604 1.0000000 #> 3: 1.565200e-31 1.6032604 1.0000000 #> 4: 1.000000e+00 1.6032604 1.0000000 #> 5: 1.565200e-31 0.8074916 0.5036559 #> 6: 3.734192e-04 0.8074916 0.5036559 #> 7: 1.881520e-02 0.8074916 0.5036559 #> 8: 1.000000e+00 0.8074916 0.5036559 #> 9: 3.734192e-04 0.7475873 0.4662919 #> 10: 1.450776e-02 0.7475873 0.4662919 #> 11: 1.313732e-19 0.7475873 0.4662919 #> 12: 1.000000e+00 0.7475873 0.4662919 #> 13: 1.450776e-02 1.0332277 0.6444541 #> 14: 1.881520e-02 1.0332277 0.6444541 #> 15: 3.665957e-17 1.0332277 0.6444541 #> 16: 1.000000e+00 1.0332277 0.6444541 forecast_quantile %>% score() %>% summarise_scores(by = \"model\") %>% add_relative_skill(baseline = \"EuroCOVIDhub-baseline\") #> Warning: ! `by` is set to 'model', which is also the unit of a single forecast. This #> doesn't look right. #> ℹ All relative skill scores will be equal to 1. #> Key: #> model wis overprediction underprediction dispersion #> #> 1: EuroCOVIDhub-baseline 14321.48926 7081.000000 5143.53567 2096.95360 #> 2: EuroCOVIDhub-ensemble 8992.62316 5025.130095 2120.64029 1846.85278 #> 3: UMass-MechBayes 52.65195 8.978601 16.80095 26.87239 #> 4: epiforecasts-EpiNow2 10827.40786 6179.439535 1697.23411 2950.73422 #> bias interval_coverage_50 interval_coverage_90 #> #> 1: 0.21851562 0.4960938 0.9101562 #> 2: 0.00812500 0.6328125 0.9023438 #> 3: -0.02234375 0.4609375 0.8750000 #> 4: -0.04336032 0.4453441 0.8461538 #> interval_coverage_deviation ae_median wis_relative_skill #> #> 1: 0.002102273 19353.42969 1 #> 2: 0.050752841 12077.10156 1 #> 3: -0.024886364 78.47656 1 #> 4: -0.057861612 14521.10526 1 #> wis_scaled_relative_skill #> #> 1: 1 #> 2: 1 #> 3: 1 #> 4: 1 forecast_quantile %>% score() %>% get_pairwise_comparisons(by = c(\"model\", \"target_type\")) %>% plot_pairwise_comparisons() + facet_wrap(~ target_type)"},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoringutils.html","id":"correlation-between-scores","dir":"Articles","previous_headings":"Additional analyses and visualisations","what":"Correlation between scores","title":"Getting started","text":"may sometimes interesting see different scores correlate . can examine using function correlation(). dealing quantile-based forecasts, important call summarise_scorees() get_correlations() summarise quantiles computing correlations. Visualising correlations:","code":"forecast_quantile %>% score() %>% summarise_scores() %>% get_correlations() #> wis overprediction underprediction dispersion bias #> #> 1: 1.0000000 0.9917884 0.8821058 0.86564717 0.61587946 #> 2: 0.9917884 1.0000000 0.8158534 0.91900505 0.51068021 #> 3: 0.8821058 0.8158534 1.0000000 0.52982259 0.91280769 #> 4: 0.8656472 0.9190050 0.5298226 1.00000000 0.14860643 #> 5: 0.6158795 0.5106802 0.9128077 0.14860643 1.00000000 #> 6: 0.1565567 0.1740382 0.1630289 0.04407105 0.06639491 #> 7: 0.2818348 0.1968480 0.6141229 -0.17841559 0.74137210 #> 8: 0.1364221 0.1123720 0.2954741 -0.12711127 0.30908292 #> 9: 0.9999780 0.9909177 0.8851559 0.86247754 0.62106954 #> interval_coverage_50 interval_coverage_90 interval_coverage_deviation #> #> 1: 0.15655670 0.2818348 0.1364221 #> 2: 0.17403818 0.1968480 0.1123720 #> 3: 0.16302890 0.6141229 0.2954741 #> 4: 0.04407105 -0.1784156 -0.1271113 #> 5: 0.06639491 0.7413721 0.3090829 #> 6: 1.00000000 0.6417542 0.9466440 #> 7: 0.64175425 1.0000000 0.8451292 #> 8: 0.94664396 0.8451292 1.0000000 #> 9: 0.15565388 0.2861731 0.1376834 #> ae_median metric #> #> 1: 0.9999780 wis #> 2: 0.9909177 overprediction #> 3: 0.8851559 underprediction #> 4: 0.8624775 dispersion #> 5: 0.6210695 bias #> 6: 0.1556539 interval_coverage_50 #> 7: 0.2861731 interval_coverage_90 #> 8: 0.1376834 interval_coverage_deviation #> 9: 1.0000000 ae_median forecast_quantile %>% score() %>% summarise_scores() %>% get_correlations(digits = 2) %>% plot_correlations()"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoringutils.html","id":"converting-to-quantile-based-forecasts","dir":"Articles","previous_headings":"","what":"Converting to quantile-based forecasts","title":"Getting started","text":"Different metrics available different forecasting formats. cases, may example forecasts sample-based format, wish make use functionality available quantile-based forecasts. example, may want use decomposition weighted interval score, may like compute interval coverage values. can convert sample-based forecasts quantile-based format using function sample_to_quantile(). , however, one caveat: Quantiles calculated based predictive samples, may introduce bias number available samples small.","code":"example_sample_discrete %>% as_forecast() %>% sample_to_quantile( quantile_level = c(0.01, 0.025, seq(0.05, 0.95, 0.05), 0.975, 0.99) ) %>% as_forecast() %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> location location_name target_end_date target_type forecast_date #> #> 1: DE Germany 2021-05-08 Cases 2021-05-03 #> 2: DE Germany 2021-05-08 Cases 2021-05-03 #> 3: DE Germany 2021-05-08 Cases 2021-05-03 #> 4: DE Germany 2021-05-08 Deaths 2021-05-03 #> 5: DE Germany 2021-05-08 Deaths 2021-05-03 #> --- #> 883: IT Italy 2021-07-24 Deaths 2021-07-12 #> 884: IT Italy 2021-07-24 Deaths 2021-07-05 #> 885: IT Italy 2021-07-24 Deaths 2021-07-12 #> 886: IT Italy 2021-07-24 Deaths 2021-07-05 #> 887: IT Italy 2021-07-24 Deaths 2021-07-12 #> model horizon wis overprediction underprediction #> #> 1: EuroCOVIDhub-ensemble 1 6671.68207 2.110043e+03 0.000000 #> 2: EuroCOVIDhub-baseline 1 18441.13061 1.446498e+04 0.000000 #> 3: epiforecasts-EpiNow2 1 22042.77309 1.548071e+04 0.000000 #> 4: EuroCOVIDhub-ensemble 1 59.46028 9.682609e+00 0.000000 #> 5: EuroCOVIDhub-baseline 1 75.93215 0.000000e+00 1.195652 #> --- #> 883: EuroCOVIDhub-baseline 2 60.71862 1.323913e+01 0.000000 #> 884: UMass-MechBayes 3 5.79650 3.695652e-01 0.000000 #> 885: UMass-MechBayes 2 26.18112 1.891304e+01 0.000000 #> 886: epiforecasts-EpiNow2 3 25.13032 3.478261e-01 0.000000 #> 887: epiforecasts-EpiNow2 2 44.07931 2.410435e+01 0.000000 #> dispersion bias interval_coverage_50 interval_coverage_90 #> #> 1: 4561.638591 0.60 FALSE TRUE #> 2: 3976.145828 0.98 FALSE FALSE #> 3: 6562.060048 0.90 FALSE TRUE #> 4: 49.777674 0.30 TRUE TRUE #> 5: 74.736502 -0.10 TRUE TRUE #> --- #> 883: 47.479491 0.50 TRUE TRUE #> 884: 5.426935 0.20 TRUE TRUE #> 885: 7.268076 0.90 FALSE TRUE #> 886: 24.782489 0.10 TRUE TRUE #> 887: 19.974959 0.70 FALSE TRUE #> interval_coverage_deviation ae_median #> #> 1: -0.03909091 9414.0 #> 2: -0.49363636 29379.0 #> 3: -0.31181818 36513.0 #> 4: 0.23363636 77.0 #> 5: 0.41545455 27.5 #> --- #> 883: 0.05181818 58.5 #> 884: 0.32454545 5.0 #> 885: -0.31181818 42.5 #> 886: 0.41545455 8.0 #> 887: -0.13000000 73.5"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoringutils.html","id":"lower-level-functions","dir":"Articles","previous_headings":"","what":"Lower level functions","title":"Getting started","text":"metrics available lower level functions extensively documented. look help files understand better.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/authors.html","id":null,"dir":"","previous_headings":"","what":"Authors","title":"Authors and Citation","text":"Nikos Bosse. Author, maintainer. Sam Abbott. Author. Hugo Gruson. Author. Johannes Bracher. Contributor. Toshiaki Asakura. Contributor. James Mba Azam. Contributor. Sebastian Funk. Author.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/authors.html","id":"citation","dir":"","previous_headings":"","what":"Citation","title":"Authors and Citation","text":"Nikos . Bosse, Hugo Gruson, Sebastian Funk, Anne Cori, Edwin van Leeuwen, Sam Abbott (2022). Evaluating Forecasts scoringutils R, arXiv. DOI: 10.48550/ARXIV.2205.07090 Alexander Jordan, Fabian Krueger, Sebastian Lerch (2019). Evaluating Probabilistic Forecasts scoringRules. Journal Statistical Software, 90(12), 1-37. DOI 10.18637/jss.v090.i12","code":"@Article{, title = {Evaluating Forecasts with scoringutils in R}, author = {Nikos I. Bosse and Hugo Gruson and Anne Cori and Edwin {van Leeuwen} and Sebastian Funk and Sam Abbott}, journal = {arXiv}, url = {https://arxiv.org/abs/2205.07090}, year = {2022}, doi = {10.48550/ARXIV.2205.07090}, } @Article{, title = {Evaluating Probabilistic Forecasts with {scoringRules}}, author = {Alexander Jordan and Fabian Kr\\\"uger and Sebastian Lerch}, journal = {Journal of Statistical Software}, year = {2019}, volume = {90}, number = {12}, pages = {1--37}, doi = {10.18637/jss.v090.i12}, }"},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"scoringutils-utilities-for-scoring-and-assessing-predictions","dir":"","previous_headings":"","what":"Utilities for Scoring and Assessing Predictions","title":"Utilities for Scoring and Assessing Predictions","text":"Note: documentation refers development version scoringutils. can also view documentation stable version. scoringutils package facilitates process evaluating forecasts R, using convenient flexible data.table-based framework. provides broad functionality check input data diagnose issues, visualise forecasts missing data, transform data scoring, handle missing forecasts, aggregate scores, visualise results evaluation. package easily extendable, meaning users can supply scoring rules extend existing classes handle new types forecasts. package underwent major re-write. comprehensive documentation updated package revised version original scoringutils paper. Another good starting point vignettes Getting started, Details metrics implemented Scoring forecasts directly. details specific issue transforming forecasts scoring see: Nikos . Bosse, Sam Abbott, Anne Cori, Edwin van Leeuwen, Johannes Bracher* Sebastian Funk* (*: equal contribution) (2023). Scoring epidemiological forecasts transformed scales, PLoS Comput Biol 19(8): e1011393 https://doi.org/10.1371/journal.pcbi.1011393","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"installation","dir":"","previous_headings":"","what":"Installation","title":"Utilities for Scoring and Assessing Predictions","text":"Install CRAN version package using Install unstable development version GitHub using","code":"install.packages(\"scoringutils\") remotes::install_github(\"epiforecasts/scoringutils\", dependencies = TRUE)"},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"forecast-types","dir":"","previous_headings":"Quick start","what":"Forecast types","title":"Utilities for Scoring and Assessing Predictions","text":"scoringutils currently supports scoring following forecast types: - binary: probability binary (yes/) outcome variable. - point: forecast continuous discrete outcome variable represented single number. - quantile: probabilistic forecast continuous discrete outcome variable, forecast distribution represented set predictive quantiles. - sample: probabilistic forecast continuous discrete outcome variable, forecast represented finite set samples drawn predictive distribution.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"input-formats-and-input-validation","dir":"","previous_headings":"Quick start","what":"Input formats and input validation","title":"Utilities for Scoring and Assessing Predictions","text":"expected input format generally data.frame (similar) required columns observed, predicted, model holds forecasts observed values. Exact requirements depend forecast type. information, look paper, call ?as_forecast(), look example data provided package (example_binary, example_point, example_quantile, example_sample_continuous, example_sample_discrete). scoring, input data needs validated transformed forecast object using function as_forecast().","code":"forecast_quantile <- example_quantile |> as_forecast( forecast_unit = c( \"location\", \"forecast_date\", \"target_end_date\", \"target_type\", \"model\", \"horizon\" ), forecast_type = \"quantile\" ) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. print(forecast_quantile, 2) #> Forecast type: quantile #> Forecast unit: #> location, forecast_date, target_end_date, target_type, model, and horizon #> #> Key: #> observed quantile_level predicted location forecast_date target_end_date #> #> 1: 127300 NA NA DE 2021-01-02 #> 2: 4534 NA NA DE 2021-01-02 #> --- #> 20544: 78 0.975 611 IT 2021-07-12 2021-07-24 #> 20545: 78 0.990 719 IT 2021-07-12 2021-07-24 #> target_type model horizon #> #> 1: Cases NA #> 2: Deaths NA #> --- #> 20544: Deaths epiforecasts-EpiNow2 2 #> 20545: Deaths epiforecasts-EpiNow2 2"},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"the-forecast-unit","dir":"","previous_headings":"Quick start","what":"The forecast unit","title":"Utilities for Scoring and Assessing Predictions","text":"quantile-based sample-based forecasts, single prediction represented set several quantiles (samples) predictive distribution, .e. several rows input data. scoringutils therefore needs group rows together form single forecast. scoringutils uses existing columns input data achieve - values columns uniquely identify single forecast. Additional columns unrelated forecast unit can mess . forecast_unit argument as_forecast() makes sure columns retained relevant defining unit single forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"scoring-forecasts","dir":"","previous_headings":"Quick start","what":"Scoring forecasts","title":"Utilities for Scoring and Assessing Predictions","text":"Forecasts can scored calling score() validated forecast object. score() takes additional argument, metrics, list scoring rules. Every forecast type default list metrics. can easily add scoring functions, long conform format forecast type. See paper information. can summarise scores using function summarise_scores(). argument used specify desired level summary. fun let’s specify summary function, although recommended stick mean primary summary function, functions can lead improper scores.","code":"scores <- forecast_quantile |> score() scores |> summarise_scores(by = c(\"model\", \"target_type\")) |> summarise_scores(by = c(\"model\", \"target_type\"), fun = signif, digits = 3) #> model target_type wis overprediction underprediction #> #> 1: EuroCOVIDhub-ensemble Cases 17900.0 10000.00 4240.0 #> 2: EuroCOVIDhub-baseline Cases 28500.0 14100.00 10300.0 #> 3: epiforecasts-EpiNow2 Cases 20800.0 11900.00 3260.0 #> 4: EuroCOVIDhub-ensemble Deaths 41.4 7.14 4.1 #> 5: EuroCOVIDhub-baseline Deaths 159.0 65.90 2.1 #> 6: UMass-MechBayes Deaths 52.7 8.98 16.8 #> 7: epiforecasts-EpiNow2 Deaths 66.6 18.90 15.9 #> dispersion bias interval_coverage_50 interval_coverage_90 #> #> 1: 3660.0 -0.05640 0.391 0.805 #> 2: 4100.0 0.09800 0.328 0.820 #> 3: 5660.0 -0.07890 0.469 0.789 #> 4: 30.2 0.07270 0.875 1.000 #> 5: 91.4 0.33900 0.664 1.000 #> 6: 26.9 -0.02230 0.461 0.875 #> 7: 31.9 -0.00513 0.420 0.908 #> interval_coverage_deviation ae_median #> #> 1: -0.1020 24100.0 #> 2: -0.1170 38500.0 #> 3: -0.0696 27900.0 #> 4: 0.2040 53.1 #> 5: 0.1210 233.0 #> 6: -0.0249 78.5 #> 7: -0.0452 105.0"},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"package-workflow","dir":"","previous_headings":"","what":"Package workflow","title":"Utilities for Scoring and Assessing Predictions","text":"following depicts suggested workflow evaluating forecasts scoringutils (sections refer paper). Please find information paper, function documentation vignettes.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"citation","dir":"","previous_headings":"","what":"Citation","title":"Utilities for Scoring and Assessing Predictions","text":"using scoringutils work please consider citing using output citation(\"scoringutils\") (print(citation(\"scoringutils\"), bibtex = TRUE)):","code":"#> To cite scoringutils in publications use the following. If you use the #> CRPS, DSS, or Log Score, please also cite scoringRules. #> #> Nikos I. Bosse, Hugo Gruson, Sebastian Funk, Anne Cori, Edwin van #> Leeuwen, and Sam Abbott (2022). Evaluating Forecasts with #> scoringutils in R, arXiv. DOI: 10.48550/ARXIV.2205.07090 #> #> To cite scoringRules in publications use: #> #> Alexander Jordan, Fabian Krueger, Sebastian Lerch (2019). Evaluating #> Probabilistic Forecasts with scoringRules. Journal of Statistical #> Software, 90(12), 1-37. DOI 10.18637/jss.v090.i12 #> #> To see these entries in BibTeX format, use 'print(, #> bibtex=TRUE)', 'toBibtex(.)', or set #> 'options(citation.bibtex.max=999)'."},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"how-to-make-a-bug-report-or-feature-request","dir":"","previous_headings":"","what":"How to make a bug report or feature request","title":"Utilities for Scoring and Assessing Predictions","text":"Please briefly describe problem output expect issue. question, please don’t open issue. Instead, ask Q page.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"contributing","dir":"","previous_headings":"","what":"Contributing","title":"Utilities for Scoring and Assessing Predictions","text":"welcome contributions new contributors! particularly appreciate help priority problems issues. Please check add issues, /add pull request.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"code-of-conduct","dir":"","previous_headings":"","what":"Code of Conduct","title":"Utilities for Scoring and Assessing Predictions","text":"Please note scoringutils project released Contributor Code Conduct. contributing project, agree abide terms.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/add_relative_skill.html","id":null,"dir":"Reference","previous_headings":"","what":"Add relative skill scores based on pairwise comparisons — add_relative_skill","title":"Add relative skill scores based on pairwise comparisons — add_relative_skill","text":"Adds columns relative skills computed running pairwise comparisons scores. information computation relative skill, see get_pairwise_comparisons(). Relative skill calculated aggregation level specified .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/add_relative_skill.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Add relative skill scores based on pairwise comparisons — add_relative_skill","text":"","code":"add_relative_skill( scores, by = \"model\", metric = intersect(c(\"wis\", \"crps\", \"brier_score\"), names(scores)), baseline = NULL )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/add_relative_skill.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Add relative skill scores based on pairwise comparisons — add_relative_skill","text":"scores object class scores (data.table scores additional attribute metrics produced score()). Character vector column names define grouping level pairwise comparisons. default (model), one relative skill score per model. , example, = c(\"model\", \"location\"). get separate relative skill score every model every location. Internally, data.table scores split according (removing \"model\" splitting) pairwise comparisons computed separately split data.tables. metric string name metric relative skill shall computed. default either \"crps\", \"wis\" \"brier_score\" available. baseline string name model. baseline given, scaled relative skill respect baseline returned. default (NULL), relative skill scaled respect baseline model.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Absolute error of the median (quantile-based version) — ae_median_quantile","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"Compute absolute error median calculated $$ \\textrm{abs}(\\textrm{observed} - \\textrm{median prediction}) $$ median prediction predicted value quantile_level == 0.5, function therefore requires 0.5 among quantile levels quantile_level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"","code":"ae_median_quantile(observed, predicted, quantile_level)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"Numeric vector length N absolute error median.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"","code":"observed <- rnorm(30, mean = 1:30) predicted_values <- replicate(3, rnorm(30, mean = 1:30)) ae_median_quantile( observed, predicted_values, quantile_level = c(0.2, 0.5, 0.8) ) #> [1] 2.47438940 0.92040530 3.55121603 0.24032512 1.79911603 2.12426222 #> [7] 2.88687498 0.37899594 0.73282842 1.41674512 0.91703692 0.34483170 #> [13] 0.72770448 1.86768569 0.80586643 2.38692128 1.12876056 0.05733376 #> [19] 0.37081463 0.82374754 1.45618892 0.93544150 2.05333481 0.18155199 #> [25] 2.43676219 1.20798000 1.67648698 0.13974346 1.26067874 1.13044854"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Absolute error of the median (sample-based version) — ae_median_sample","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"Absolute error median calculated $$% \\textrm{abs}(\\textrm{observevd} - \\textrm{median\\_prediction}) $$","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"","code":"ae_median_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"vector scoring values","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"","code":"observed <- rnorm(30, mean = 1:30) predicted_values <- matrix(rnorm(30, mean = 1:30)) ae_median_sample(observed, predicted_values) #> [1] 1.61022189 0.32735036 2.52982645 0.98458168 0.94495454 0.65538891 #> [7] 0.56511146 0.09373061 1.31110818 0.61226219 0.75386115 0.08959962 #> [13] 0.39077113 1.56818369 0.84567980 1.24260044 0.27781917 0.65054779 #> [19] 1.18084954 0.45036469 0.05976767 0.14675942 0.60583332 0.19442459 #> [25] 0.21123533 0.28585022 0.64582375 1.78993469 1.20347916 0.67902801"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/apply_metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Apply a list of functions to a data table of forecasts — apply_metrics","title":"Apply a list of functions to a data table of forecasts — apply_metrics","text":"helper function applies scoring rules (stored list functions) data table forecasts. apply_metrics used within score() apply scoring rules data. Scoring rules wrapped run_safely() catch errors make sure arguments passed scoring rule actually accepted .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/apply_metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Apply a list of functions to a data table of forecasts — apply_metrics","text":"","code":"apply_metrics(forecast, metrics, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/apply_metrics.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Apply a list of functions to a data table of forecasts — apply_metrics","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()) metrics named list scoring functions. Names used column names output. See metrics_point(), metrics_binary(), metrics_quantile(), metrics_sample() information default metrics used. Note want pass arguments given metric, function customise_metric() pass updated list functions custom metric metrics argument score(). ... Additional arguments passed scoring rules. Note currently used, calls apply_scores currently avoid passing arguments via ... instead expect metrics directly modified using customise_metric().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/apply_metrics.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Apply a list of functions to a data table of forecasts — apply_metrics","text":"data table forecasts calculated metrics.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a forecast object — as_forecast","title":"Create a forecast object — as_forecast","text":"Process validate data.frame (similar) similar forecasts observations. input passes input checks, converted forecast object. class object depends forecast type input. See details section information expected input formats. as_forecast() gives users control data parsed. Using arguments observed, predicted, model, users can rename existing columns input data match required columns forecast object. Using argument forecast_unit, users can specify columns uniquely identify single forecast (remove others, see set_forecast_unit() details).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a forecast object — as_forecast","text":"","code":"as_forecast(data, ...) # Default S3 method as_forecast( data, forecast_unit = NULL, forecast_type = NULL, observed = NULL, predicted = NULL, model = NULL, quantile_level = NULL, sample_id = NULL, ... )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a forecast object — as_forecast","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. ... Additional arguments forecast_unit (optional) Name columns data (renaming columns done as_forecast()) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. forecast_type (optional) forecast type expect forecasts . forecast type determined scoringutils based input match , error thrown. NULL (default), forecast type inferred data. observed (optional) Name column data contains observed values. column renamed \"observed\". predicted (optional) Name column data contains predicted values. column renamed \"predicted\". model (optional) Name column data contains names models/forecasters generated predicted values. column renamed \"model\". quantile_level (optional) Name column data contains quantile level predicted values. column renamed \"quantile_level\". applicable quantile-based forecasts. sample_id (optional) Name column data contains sample id. column renamed \"sample_id\". applicable sample-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Create a forecast object — as_forecast","text":"Depending forecast type, object following class returned: forecast_binary binary forecasts forecast_point point forecasts forecast_sample sample-based forecasts forecast_quantile quantile-based forecasts","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"forecast-types-and-input-formats","dir":"Reference","previous_headings":"","what":"Forecast types and input formats","title":"Create a forecast object — as_forecast","text":"Various different forecast types / forecast formats supported. moment, : point forecasts binary forecasts (\"soft binary classification\") Probabilistic forecasts quantile-based format (forecast represented set predictive quantiles) Probabilistic forecasts sample-based format (forecast represented set predictive samples) Forecast types determined based columns present input data. overview required format forecast type: forecast types require data.frame similar columns observed predicted, model. Point forecasts require column observed type numeric column predicted type numeric. Binary forecasts require column observed type factor exactly two levels column predicted type numeric probabilities, corresponding probability observed equal second factor level. See details information. Quantile-based forecasts require column observed type numeric, column predicted type numeric, column quantile_level type numeric quantile-levels (0 1). Sample-based forecasts require column observed type numeric, column predicted type numeric, column sample_id type numeric sample indices. information see vignettes example data (example_quantile, example_sample_continuous, example_sample_discrete, example_point(), example_binary).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"forecast-unit","dir":"Reference","previous_headings":"","what":"Forecast unit","title":"Create a forecast object — as_forecast","text":"order score forecasts, scoringutils needs know rows data belong together jointly form single forecasts. easy e.g. point forecast, one row per forecast. quantile sample-based forecasts, however, multiple rows belong single forecast. forecast unit unit single forecast described combination columns uniquely identify single forecast. example, forecasts made different models various locations different time points, several weeks future. forecast unit described forecast_unit = c(\"model\", \"location\", \"forecast_date\", \"forecast_horizon\"). scoringutils automatically tries determine unit single forecast. uses existing columns , means columns must present unrelated forecast unit. simplistic example, additional row, \"even\", one row number even zero otherwise, mess scoring scoringutils thinks column relevant defining forecast unit. order avoid issues, recommend setting forecast unit explicitly, usually forecast_unit argument as_forecast(). drop unneeded columns, making sure necessary, 'protected columns' like \"predicted\" \"observed\" retained.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Create a forecast object — as_forecast","text":"","code":"as_forecast(example_binary) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Forecast type: binary #> Forecast unit: #> location, location_name, target_end_date, target_type, forecast_date, model, #> and horizon #> #> location location_name target_end_date target_type forecast_date #> #> 1: DE Germany 2021-01-02 Cases #> 2: DE Germany 2021-01-02 Deaths #> 3: DE Germany 2021-01-09 Cases #> 4: DE Germany 2021-01-09 Deaths #> 5: DE Germany 2021-01-16 Cases #> --- #> 1027: IT Italy 2021-07-24 Deaths 2021-07-12 #> 1028: IT Italy 2021-07-24 Deaths 2021-07-05 #> 1029: IT Italy 2021-07-24 Deaths 2021-07-12 #> 1030: IT Italy 2021-07-24 Deaths 2021-07-05 #> 1031: IT Italy 2021-07-24 Deaths 2021-07-12 #> model horizon predicted observed #> #> 1: NA NA #> 2: NA NA #> 3: NA NA #> 4: NA NA #> 5: NA NA #> --- #> 1027: EuroCOVIDhub-baseline 2 0.250 0 #> 1028: UMass-MechBayes 3 0.475 0 #> 1029: UMass-MechBayes 2 0.450 0 #> 1030: epiforecasts-EpiNow2 3 0.375 0 #> 1031: epiforecasts-EpiNow2 2 0.300 0 as_forecast( example_quantile, forecast_unit = c(\"model\", \"target_type\", \"target_end_date\", \"horizon\", \"location\") ) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Forecast type: quantile #> Forecast unit: #> model, target_type, target_end_date, horizon, and location #> #> Key: #> observed quantile_level predicted model target_type #> #> 1: 127300 NA NA Cases #> 2: 4534 NA NA Deaths #> 3: 154922 NA NA Cases #> 4: 6117 NA NA Deaths #> 5: 110183 NA NA Cases #> --- #> 20541: 78 0.850 352 epiforecasts-EpiNow2 Deaths #> 20542: 78 0.900 397 epiforecasts-EpiNow2 Deaths #> 20543: 78 0.950 499 epiforecasts-EpiNow2 Deaths #> 20544: 78 0.975 611 epiforecasts-EpiNow2 Deaths #> 20545: 78 0.990 719 epiforecasts-EpiNow2 Deaths #> target_end_date horizon location #> #> 1: 2021-01-02 NA DE #> 2: 2021-01-02 NA DE #> 3: 2021-01-09 NA DE #> 4: 2021-01-09 NA DE #> 5: 2021-01-16 NA DE #> --- #> 20541: 2021-07-24 2 IT #> 20542: 2021-07-24 2 IT #> 20543: 2021-07-24 2 IT #> 20544: 2021-07-24 2 IT #> 20545: 2021-07-24 2 IT"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Create an object of class scores from data — as_scores","title":"Create an object of class scores from data — as_scores","text":"convenience function wraps new_scores() validates scores object.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create an object of class scores from data — as_scores","text":"","code":"as_scores(scores, metrics)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create an object of class scores from data — as_scores","text":"scores data.table similar scores produced score(). metrics character vector names scores (.e. names scoring rules used scoring).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_scores.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Create an object of class scores from data — as_scores","text":"object class scores","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_dims_ok_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","title":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","text":"Function assesses whether input dimensions match. following, n number observations / forecasts. Scalar values may repeated match length input. Allowed options therefore: observed vector length 1 length n predicted : vector length 1 length n matrix n rows 1 column","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_dims_ok_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","text":"","code":"assert_dims_ok_point(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_dims_ok_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","text":"observed Input checked. factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_dims_ok_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that input is a forecast object and passes validations — assert_forecast","title":"Assert that input is a forecast object and passes validations — assert_forecast","text":"Methods different classes run assert_forecast_generic(), performs checks forecast types perform specific checks specific forecast type.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that input is a forecast object and passes validations — assert_forecast","text":"","code":"assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) # Default S3 method assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) # S3 method for class 'forecast_binary' assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) # S3 method for class 'forecast_point' assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) # S3 method for class 'forecast_quantile' assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) # S3 method for class 'forecast_sample' assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that input is a forecast object and passes validations — assert_forecast","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()) forecast_type (optional) forecast type expect forecasts . forecast type determined scoringutils based input match , error thrown. NULL (default), forecast type inferred data. verbose Logical. FALSE (default TRUE), messages warnings created. ... Additional arguments","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that input is a forecast object and passes validations — assert_forecast","text":"Depending forecast type, object class forecast_binary, forecast_point, forecast_sample forecast_quantile.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"forecast-types-and-input-formats","dir":"Reference","previous_headings":"","what":"Forecast types and input formats","title":"Assert that input is a forecast object and passes validations — assert_forecast","text":"Various different forecast types / forecast formats supported. moment, : point forecasts binary forecasts (\"soft binary classification\") Probabilistic forecasts quantile-based format (forecast represented set predictive quantiles) Probabilistic forecasts sample-based format (forecast represented set predictive samples) Forecast types determined based columns present input data. overview required format forecast type: forecast types require data.frame similar columns observed predicted, model. Point forecasts require column observed type numeric column predicted type numeric. Binary forecasts require column observed type factor exactly two levels column predicted type numeric probabilities, corresponding probability observed equal second factor level. See details information. Quantile-based forecasts require column observed type numeric, column predicted type numeric, column quantile_level type numeric quantile-levels (0 1). Sample-based forecasts require column observed type numeric, column predicted type numeric, column sample_id type numeric sample indices. information see vignettes example data (example_quantile, example_sample_continuous, example_sample_discrete, example_point(), example_binary).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Assert that input is a forecast object and passes validations — assert_forecast","text":"","code":"forecast <- as_forecast(example_binary) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. assert_forecast(forecast) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected."},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_generic.html","id":null,"dir":"Reference","previous_headings":"","what":"Validation common to all forecast types — assert_forecast_generic","title":"Validation common to all forecast types — assert_forecast_generic","text":"function runs input checks apply input data, regardless forecast type. function asserts forecast data.table columns observed predicted, well column called model. checks forecast type forecast unit checks duplicate forecasts appropriate, checks number samples / quantiles forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_generic.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Validation common to all forecast types — assert_forecast_generic","text":"","code":"assert_forecast_generic(data, verbose = TRUE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_generic.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Validation common to all forecast types — assert_forecast_generic","text":"data data.table forecasts observed values validated. verbose Logical. FALSE (default TRUE), messages warnings created.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_generic.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Validation common to all forecast types — assert_forecast_generic","text":"returns input","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_type.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that forecast type is as expected — assert_forecast_type","title":"Assert that forecast type is as expected — assert_forecast_type","text":"Assert forecast type expected","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_type.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that forecast type is as expected — assert_forecast_type","text":"","code":"assert_forecast_type(data, actual = get_forecast_type(data), desired = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_type.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that forecast type is as expected — assert_forecast_type","text":"data forecast object produced as_forecast(). actual actual forecast type data desired desired forecast type data","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_type.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that forecast type is as expected — assert_forecast_type","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for binary forecast — assert_input_binary","title":"Assert that inputs are correct for binary forecast — assert_input_binary","text":"Function assesses whether inputs correspond requirements scoring binary forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for binary forecast — assert_input_binary","text":"","code":"assert_input_binary(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for binary forecast — assert_input_binary","text":"observed Input checked. factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for binary forecast — assert_input_binary","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_interval.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for interval-based forecast — assert_input_interval","title":"Assert that inputs are correct for interval-based forecast — assert_input_interval","text":"Function assesses whether inputs correspond requirements scoring interval-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_interval.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for interval-based forecast — assert_input_interval","text":"","code":"assert_input_interval(observed, lower, upper, interval_range)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_interval.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for interval-based forecast — assert_input_interval","text":"observed Input checked. numeric vector observed values size n. lower Input checked. numeric vector size n holds predicted value lower bounds prediction intervals. upper Input checked. numeric vector size n holds predicted value upper bounds prediction intervals. interval_range Input checked. vector size n denotes interval range percent. E.g. value 50 denotes (25%, 75%) prediction interval.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_interval.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for interval-based forecast — assert_input_interval","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for point forecast — assert_input_point","title":"Assert that inputs are correct for point forecast — assert_input_point","text":"Function assesses whether inputs correspond requirements scoring point forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for point forecast — assert_input_point","text":"","code":"assert_input_point(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for point forecast — assert_input_point","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. numeric vector predicted values size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for point forecast — assert_input_point","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","title":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","text":"Function assesses whether inputs correspond requirements scoring quantile-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","text":"","code":"assert_input_quantile( observed, predicted, quantile_level, unique_quantile_levels = TRUE )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. nxN matrix predictive quantiles, n (number rows) number data points N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Input checked. vector size N denotes quantile levels corresponding columns prediction matrix. unique_quantile_levels Whether quantile levels required unique (TRUE, default) (FALSE).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for sample-based forecast — assert_input_sample","title":"Assert that inputs are correct for sample-based forecast — assert_input_sample","text":"Function assesses whether inputs correspond requirements scoring sample-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for sample-based forecast — assert_input_sample","text":"","code":"assert_input_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for sample-based forecast — assert_input_sample","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. numeric nxN matrix predictive samples, n (number rows) number data points N (number columns) number samples per forecast. observed just single number, predicted values can just vector size N.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for sample-based forecast — assert_input_sample","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Determines bias of quantile forecasts — bias_quantile","title":"Determines bias of quantile forecasts — bias_quantile","text":"Determines bias quantile forecasts. increasing number quantiles measure converges sample based bias version integer continuous forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Determines bias of quantile forecasts — bias_quantile","text":"","code":"bias_quantile(observed, predicted, quantile_level, na.rm = TRUE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Determines bias of quantile forecasts — bias_quantile","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made. Note contain median (0.5) median imputed mean two innermost quantiles. na.rm Logical. missing values removed?","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Determines bias of quantile forecasts — bias_quantile","text":"scalar quantile bias single quantile prediction","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Determines bias of quantile forecasts — bias_quantile","text":"quantile forecasts, bias measured $$ B_t = (1 - 2 \\cdot \\max \\{| q_{t,} \\Q_t \\land q_{t,} \\leq x_t\\}) \\mathbf{1}( x_t \\leq q_{t, 0.5}) \\\\ + (1 - 2 \\cdot \\min \\{| q_{t,} \\Q_t \\land q_{t,} \\geq x_t\\}) 1( x_t \\geq q_{t, 0.5}),$$ \\(Q_t\\) set quantiles form predictive distribution time \\(t\\) \\(x_t\\) observed value. consistency, define \\(Q_t\\) always includes element \\(q_{t, 0} = - \\infty\\) \\(q_{t,1} = \\infty\\). \\(1()\\) indicator function \\(1\\) condition satisfied \\(0\\) otherwise. clearer terms, bias \\(B_t\\) : \\(1 - 2 \\cdot\\) maximum percentile rank corresponding quantile still smaller equal observed value, observed value smaller median predictive distribution. \\(1 - 2 \\cdot\\) minimum percentile rank corresponding quantile still larger equal observed value observed value larger median predictive distribution.. \\(0\\) observed value exactly median (terms cancel ) Bias can assume values -1 1 0 ideally (.e. unbiased). Note given quantiles contain median, median imputed mean two innermost quantiles. large enough number quantiles, percentile rank equal proportion predictive samples observed value, bias metric coincides one continuous forecasts (see bias_sample()).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Determines bias of quantile forecasts — bias_quantile","text":"","code":"predicted <- matrix(c(1.5:23.5, 3.3:25.3), nrow = 2, byrow = TRUE) quantile_level <- c(0.01, 0.025, seq(0.05, 0.95, 0.05), 0.975, 0.99) observed <- c(15, 12.4) bias_quantile(observed, predicted, quantile_level) #> [1] -0.3 0.2"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile_single_vector.html","id":null,"dir":"Reference","previous_headings":"","what":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","title":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","text":"Internal function compute bias single observed value, vector predicted values vector quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile_single_vector.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","text":"","code":"bias_quantile_single_vector(observed, predicted, quantile_level, na.rm)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile_single_vector.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","text":"observed Scalar observed value. predicted Vector length N (corresponding number quantiles) holds predictions. quantile_level Vector size N quantile levels predictions made. Note contain median (0.5) median imputed mean two innermost quantiles. na.rm Logical. missing values removed?","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile_single_vector.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","text":"scalar quantile bias single quantile prediction","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Determine bias of forecasts — bias_sample","title":"Determine bias of forecasts — bias_sample","text":"Determines bias predictive Monte-Carlo samples. function automatically recognises, whether forecasts continuous integer valued adapts Bias function accordingly.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Determine bias of forecasts — bias_sample","text":"","code":"bias_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Determine bias of forecasts — bias_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Determine bias of forecasts — bias_sample","text":"Numeric vector length n biases predictive samples respect observed values.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Determine bias of forecasts — bias_sample","text":"continuous forecasts, Bias measured $$ B_t (P_t, x_t) = 1 - 2 * (P_t (x_t)) $$ \\(P_t\\) empirical cumulative distribution function prediction observed value \\(x_t\\). Computationally, \\(P_t (x_t)\\) just calculated fraction predictive samples \\(x_t\\) smaller \\(x_t\\). integer valued forecasts, Bias measured $$ B_t (P_t, x_t) = 1 - (P_t (x_t) + P_t (x_t + 1)) $$ adjust integer nature forecasts. cases, Bias can assume values -1 1 0 ideally.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Determine bias of forecasts — bias_sample","text":"integer valued Bias function discussed Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15 Funk S, Camacho , Kucharski AJ, Lowe R, Eggo RM, et al. (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15. PLOS Computational Biology 15(2): e1006785. doi:10.1371/journal.pcbi.1006785","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Determine bias of forecasts — bias_sample","text":"","code":"## integer valued forecasts observed <- rpois(30, lambda = 1:30) predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) bias_sample(observed, predicted) #> [1] 0.660 -0.025 0.710 -0.930 0.870 0.435 -0.885 -0.940 -0.515 -0.790 #> [11] 0.975 -0.975 -0.620 -0.740 -0.640 0.395 0.695 -0.765 -0.935 -0.680 #> [21] -0.725 -0.320 0.355 0.730 0.250 0.995 -0.650 0.235 0.250 0.850 ## continuous forecasts observed <- rnorm(30, mean = 1:30) predicted <- replicate(200, rnorm(30, mean = 1:30)) bias_sample(observed, predicted) #> [1] -0.46 0.02 0.02 0.12 -0.18 -0.07 0.96 -0.60 0.16 -0.31 0.79 0.49 #> [13] -0.74 -0.48 0.26 -0.56 0.82 0.89 0.41 -0.31 0.19 0.47 -0.85 0.32 #> [25] 0.15 -0.16 0.34 -0.30 0.80 0.27"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_columns_present.html","id":null,"dir":"Reference","previous_headings":"","what":"Check column names are present in a data.frame — check_columns_present","title":"Check column names are present in a data.frame — check_columns_present","text":"functions loops column names checks whether present. issue encountered, function immediately stops returns message first issue encountered.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_columns_present.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check column names are present in a data.frame — check_columns_present","text":"","code":"check_columns_present(data, columns)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_columns_present.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check column names are present in a data.frame — check_columns_present","text":"data data.frame similar checked columns character vector column names check","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_columns_present.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check column names are present in a data.frame — check_columns_present","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_dims_ok_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Check Inputs Have Matching Dimensions — check_dims_ok_point","title":"Check Inputs Have Matching Dimensions — check_dims_ok_point","text":"Function assesses whether input dimensions match. following, n number observations / forecasts. Scalar values may repeated match length input. Allowed options therefore: observed vector length 1 length n predicted : vector length 1 length n matrix n rows 1 column","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_dims_ok_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check Inputs Have Matching Dimensions — check_dims_ok_point","text":"","code":"check_dims_ok_point(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_dims_ok_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check Inputs Have Matching Dimensions — check_dims_ok_point","text":"observed Input checked. factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_dims_ok_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check Inputs Have Matching Dimensions — check_dims_ok_point","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_duplicates.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that there are no duplicate forecasts — check_duplicates","title":"Check that there are no duplicate forecasts — check_duplicates","text":"Runs get_duplicate_forecasts() returns message issue encountered","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_duplicates.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that there are no duplicate forecasts — check_duplicates","text":"","code":"check_duplicates(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_duplicates.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that there are no duplicate forecasts — check_duplicates","text":"data data.frame used score()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_duplicates.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that there are no duplicate forecasts — check_duplicates","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for binary forecast — check_input_binary","title":"Check that inputs are correct for binary forecast — check_input_binary","text":"Function assesses whether inputs correspond requirements scoring binary forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for binary forecast — check_input_binary","text":"","code":"check_input_binary(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for binary forecast — check_input_binary","text":"observed Input checked. factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for binary forecast — check_input_binary","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_interval.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for interval-based forecast — check_input_interval","title":"Check that inputs are correct for interval-based forecast — check_input_interval","text":"Function assesses whether inputs correspond requirements scoring interval-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_interval.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for interval-based forecast — check_input_interval","text":"","code":"check_input_interval(observed, lower, upper, interval_range)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_interval.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for interval-based forecast — check_input_interval","text":"observed Input checked. numeric vector observed values size n. lower Input checked. numeric vector size n holds predicted value lower bounds prediction intervals. upper Input checked. numeric vector size n holds predicted value upper bounds prediction intervals. interval_range Input checked. vector size n denotes interval range percent. E.g. value 50 denotes (25%, 75%) prediction interval.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_interval.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for interval-based forecast — check_input_interval","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for point forecast — check_input_point","title":"Check that inputs are correct for point forecast — check_input_point","text":"Function assesses whether inputs correspond requirements scoring point forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for point forecast — check_input_point","text":"","code":"check_input_point(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for point forecast — check_input_point","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. numeric vector predicted values size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for point forecast — check_input_point","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for quantile-based forecast — check_input_quantile","title":"Check that inputs are correct for quantile-based forecast — check_input_quantile","text":"Function assesses whether inputs correspond requirements scoring quantile-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for quantile-based forecast — check_input_quantile","text":"","code":"check_input_quantile(observed, predicted, quantile_level)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for quantile-based forecast — check_input_quantile","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. nxN matrix predictive quantiles, n (number rows) number data points N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Input checked. vector size N denotes quantile levels corresponding columns prediction matrix.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for quantile-based forecast — check_input_quantile","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for sample-based forecast — check_input_sample","title":"Check that inputs are correct for sample-based forecast — check_input_sample","text":"Function assesses whether inputs correspond requirements scoring sample-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for sample-based forecast — check_input_sample","text":"","code":"check_input_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for sample-based forecast — check_input_sample","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. numeric nxN matrix predictive samples, n (number rows) number data points N (number columns) number samples per forecast. observed just single number, predicted values can just vector size N.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for sample-based forecast — check_input_sample","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_number_per_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that all forecasts have the same number of quantiles or samples — check_number_per_forecast","title":"Check that all forecasts have the same number of quantiles or samples — check_number_per_forecast","text":"Function checks number quantiles samples per forecast. number quantiles samples forecasts, returns TRUE string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_number_per_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that all forecasts have the same number of quantiles or samples — check_number_per_forecast","text":"","code":"check_number_per_forecast(data, forecast_unit)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_number_per_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that all forecasts have the same number of quantiles or samples — check_number_per_forecast","text":"data data.frame similar checked forecast_unit Character vector denoting unit single forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_number_per_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that all forecasts have the same number of quantiles or samples — check_number_per_forecast","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_numeric_vector.html","id":null,"dir":"Reference","previous_headings":"","what":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","title":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","text":"Helper function check whether input numeric vector.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_numeric_vector.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","text":"","code":"check_numeric_vector(x, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_numeric_vector.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","text":"x input check ... Arguments passed checkmate::check_numeric lower [numeric(1)] Lower value elements x must greater equal . upper [numeric(1)] Upper value elements x must lower equal . finite [logical(1)] Check finite values? Default FALSE. .missing [logical(1)] vectors missing values allowed? Default TRUE. .missing [logical(1)] vectors non-missing values allowed? Default TRUE. Note empty vectors non-missing values. len [integer(1)] Exact expected length x. min.len [integer(1)] Minimal length x. max.len [integer(1)] Maximal length x. unique [logical(1)] Must values unique? Default FALSE. sorted [logical(1)] Elements must sorted ascending order. Missing values ignored. names [character(1)] Check names. See checkNamed possible values. Default “” performs check . Note can use checkSubset check specific set names. typed.missing [logical(1)] set FALSE (default), types missing values (NA, NA_integer_, NA_real_, NA_character_ NA_character_) well empty vectors allowed type-checking atomic input. Set TRUE enable strict type checking. null.ok [logical(1)] set TRUE, x may also NULL. case type check x performed, additional checks disabled.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_numeric_vector.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_try.html","id":null,"dir":"Reference","previous_headings":"","what":"Helper function to convert assert statements into checks — check_try","title":"Helper function to convert assert statements into checks — check_try","text":"Tries execute expression. Internally, used see whether assertions fail checking inputs (.e. convert assert_*() statement check). expression fails, error message returned. expression succeeds, TRUE returned.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_try.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Helper function to convert assert statements into checks — check_try","text":"","code":"check_try(expr)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_try.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Helper function to convert assert statements into checks — check_try","text":"expr expression evaluated","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_try.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Helper function to convert assert statements into checks — check_try","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/clean_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Clean forecast object — clean_forecast","title":"Clean forecast object — clean_forecast","text":"function makes possible silently validate object. addition, can return copy data remove rows missing values.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/clean_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Clean forecast object — clean_forecast","text":"","code":"clean_forecast(forecast, copy = FALSE, na.omit = FALSE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/clean_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Clean forecast object — clean_forecast","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()) copy Logical, default FALSE. TRUE, copy input data created. na.omit Logical, default FALSE. TRUE, rows missing values removed.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_two_models.html","id":null,"dir":"Reference","previous_headings":"","what":"Compare two models based on subset of common forecasts — compare_two_models","title":"Compare two models based on subset of common forecasts — compare_two_models","text":"function compares two models based subset forecasts models made prediction. gets called pairwise_comparison_one_group(), handles comparison multiple models single set forecasts (subsets forecasts distinguished). pairwise_comparison_one_group() turn gets called get_pairwise_comparisons() can handle pairwise comparisons set forecasts multiple subsets, e.g. pairwise comparisons one set forecasts, done separately two different forecast targets.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_two_models.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Compare two models based on subset of common forecasts — compare_two_models","text":"","code":"compare_two_models( scores, name_model1, name_model2, metric, one_sided = FALSE, test_type = c(\"non_parametric\", \"permutation\"), n_permutations = 999 )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_two_models.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Compare two models based on subset of common forecasts — compare_two_models","text":"scores object class scores (data.table scores additional attribute metrics produced score()). name_model1 Character, name first model name_model2 Character, name model compare metric string name metric relative skill shall computed. default either \"crps\", \"wis\" \"brier_score\" available. one_sided Boolean, default FALSE, whether two conduct one-sided instead two-sided test determine significance pairwise comparison. test_type Character, either \"non_parametric\" (default) \"permutation\". determines kind test shall conducted determine p-values. n_permutations Numeric, number permutations permutation test. Default 999.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_two_models.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Compare two models based on subset of common forecasts — compare_two_models","text":"list mean score ratios p-values comparison two models","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_two_models.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Compare two models based on subset of common forecasts — compare_two_models","text":"Johannes Bracher, johannes.bracher@kit.edu Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"(Continuous) ranked probability score — crps_sample","title":"(Continuous) ranked probability score — crps_sample","text":"Wrapper around crps_sample() function scoringRules package. Can used continuous well integer valued forecasts","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"(Continuous) ranked probability score — crps_sample","text":"","code":"crps_sample(observed, predicted, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"(Continuous) ranked probability score — crps_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. ... Additional arguments passed crps_sample() scoringRules package.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"(Continuous) ranked probability score — crps_sample","text":"Vector scores.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"(Continuous) ranked probability score — crps_sample","text":"Alexander Jordan, Fabian Krüger, Sebastian Lerch, Evaluating Probabilistic Forecasts scoringRules, https://www.jstatsoft.org/article/view/v090i12","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"(Continuous) ranked probability score — crps_sample","text":"","code":"observed <- rpois(30, lambda = 1:30) predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) crps_sample(observed, predicted) #> [1] 0.231225 0.329850 1.060100 0.412625 0.991550 0.793850 2.847025 1.126200 #> [9] 0.690675 0.992875 1.118300 4.911975 4.135800 1.411050 0.967825 3.425125 #> [17] 0.839600 2.404700 1.376075 1.665525 2.771375 1.168550 1.244700 2.377250 #> [25] 2.847875 2.019175 1.827975 2.074850 1.868775 3.171950"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/customise_metric.html","id":null,"dir":"Reference","previous_headings":"","what":"Customises a metric function with additional arguments. — customise_metric","title":"Customises a metric function with additional arguments. — customise_metric","text":"function takes metric function additional arguments, returns new function includes additional arguments calling original metric function. expected way pass additional arguments metric evaluating forecast using score(): evaluate forecast using metric additional argument, need create custom version scoring function argument included. need create updated version list scoring functions includes customised metric pass score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/customise_metric.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Customises a metric function with additional arguments. — customise_metric","text":"","code":"customise_metric(metric, ...) customize_metric(metric, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/customise_metric.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Customises a metric function with additional arguments. — customise_metric","text":"metric metric function customised. ... Additional arguments included calling metric function.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/customise_metric.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Customises a metric function with additional arguments. — customise_metric","text":"customised metric function.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/customise_metric.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Customises a metric function with additional arguments. — customise_metric","text":"","code":"# Create a customised metric function custom_metric <- customise_metric(mean, na.rm = TRUE) # Use the customised metric function values <- c(1, 2, NA, 4, 5) custom_metric(values) #> [1] 3 # Updating metrics list to calculate 70% coverage in `score()` interval_coverage_70 <- customise_metric( interval_coverage, interval_range = 70 ) updated_metrics <- c( metrics_quantile(), \"interval_coverage_70\" = interval_coverage_70 ) score( as_forecast(example_quantile), metrics = updated_metrics ) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> location target_end_date target_type location_name forecast_date #> #> 1: DE 2021-05-08 Cases Germany 2021-05-03 #> 2: DE 2021-05-08 Cases Germany 2021-05-03 #> 3: DE 2021-05-08 Cases Germany 2021-05-03 #> 4: DE 2021-05-08 Deaths Germany 2021-05-03 #> 5: DE 2021-05-08 Deaths Germany 2021-05-03 #> --- #> 883: IT 2021-07-24 Deaths Italy 2021-07-12 #> 884: IT 2021-07-24 Deaths Italy 2021-07-05 #> 885: IT 2021-07-24 Deaths Italy 2021-07-12 #> 886: IT 2021-07-24 Deaths Italy 2021-07-05 #> 887: IT 2021-07-24 Deaths Italy 2021-07-12 #> model horizon wis overprediction underprediction #> #> 1: EuroCOVIDhub-ensemble 1 7990.854783 2.549870e+03 0.0000000 #> 2: EuroCOVIDhub-baseline 1 16925.046957 1.527583e+04 0.0000000 #> 3: epiforecasts-EpiNow2 1 25395.960870 1.722226e+04 0.0000000 #> 4: EuroCOVIDhub-ensemble 1 53.880000 0.000000e+00 0.6086957 #> 5: EuroCOVIDhub-baseline 1 46.793043 2.130435e+00 0.0000000 #> --- #> 883: EuroCOVIDhub-baseline 2 80.336957 3.608696e+00 0.0000000 #> 884: UMass-MechBayes 3 4.881739 4.347826e-02 0.0000000 #> 885: UMass-MechBayes 2 25.581739 1.782609e+01 0.0000000 #> 886: epiforecasts-EpiNow2 3 19.762609 5.478261e+00 0.0000000 #> 887: epiforecasts-EpiNow2 2 66.161739 4.060870e+01 0.0000000 #> dispersion bias interval_coverage_50 interval_coverage_90 #> #> 1: 5440.985217 0.50 TRUE TRUE #> 2: 1649.220870 0.95 FALSE FALSE #> 3: 8173.700000 0.90 FALSE TRUE #> 4: 53.271304 -0.10 TRUE TRUE #> 5: 44.662609 0.30 TRUE TRUE #> --- #> 883: 76.728261 0.20 TRUE TRUE #> 884: 4.838261 0.10 TRUE TRUE #> 885: 7.755652 0.80 FALSE TRUE #> 886: 14.284348 0.50 TRUE TRUE #> 887: 25.553043 0.90 FALSE TRUE #> interval_coverage_deviation ae_median interval_coverage_70 #> #> 1: 0.05181818 12271 TRUE #> 2: -0.40272727 25620 FALSE #> 3: -0.31181818 44192 FALSE #> 4: 0.41545455 14 TRUE #> 5: 0.23363636 15 TRUE #> --- #> 883: 0.32454545 53 TRUE #> 884: 0.41545455 1 TRUE #> 885: -0.22090909 46 FALSE #> 886: 0.05181818 26 TRUE #> 887: -0.31181818 108 FALSE"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_assert_functions.html","id":null,"dir":"Reference","previous_headings":"","what":"Documentation template for assert functions — document_assert_functions","title":"Documentation template for assert functions — document_assert_functions","text":"Documentation template assert functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_assert_functions.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Documentation template for assert functions — document_assert_functions","text":"observed Input checked. numeric vector observed values size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_assert_functions.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Documentation template for assert functions — document_assert_functions","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_check_functions.html","id":null,"dir":"Reference","previous_headings":"","what":"Documentation template for check functions — document_check_functions","title":"Documentation template for check functions — document_check_functions","text":"Documentation template check functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_check_functions.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Documentation template for check functions — document_check_functions","text":"data data.frame similar checked observed Input checked. numeric vector observed values size n. columns character vector column names check","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_check_functions.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Documentation template for check functions — document_check_functions","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_test_functions.html","id":null,"dir":"Reference","previous_headings":"","what":"Documentation template for test functions — document_test_functions","title":"Documentation template for test functions — document_test_functions","text":"Documentation template test functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_test_functions.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Documentation template for test functions — document_test_functions","text":"Returns TRUE check successful FALSE otherwise","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Dawid-Sebastiani score — dss_sample","title":"Dawid-Sebastiani score — dss_sample","text":"Wrapper around dss_sample() function scoringRules package.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Dawid-Sebastiani score — dss_sample","text":"","code":"dss_sample(observed, predicted, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Dawid-Sebastiani score — dss_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. ... Additional arguments passed dss_sample() scoringRules package.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Dawid-Sebastiani score — dss_sample","text":"Vector scores.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Dawid-Sebastiani score — dss_sample","text":"Alexander Jordan, Fabian Krüger, Sebastian Lerch, Evaluating Probabilistic Forecasts scoringRules, https://www.jstatsoft.org/article/view/v090i12","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Dawid-Sebastiani score — dss_sample","text":"","code":"observed <- rpois(30, lambda = 1:30) predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) dss_sample(observed, predicted) #> [1] -0.06179111 2.81530809 2.46742176 1.79360855 1.61740613 2.39081466 #> [7] 1.92865409 15.58374079 3.88246189 4.97070116 2.24551342 2.71528477 #> [13] 2.79485162 2.62249405 2.80770087 4.30015607 3.23096351 3.16329747 #> [19] 3.23001192 7.67628290 2.92052357 5.37555852 4.78857945 3.12865561 #> [25] 3.61999703 4.32028466 3.38593521 6.87116360 4.71995903 4.99107110"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_data.table.html","id":null,"dir":"Reference","previous_headings":"","what":"Ensure that an object is a data.table — ensure_data.table","title":"Ensure that an object is a data.table — ensure_data.table","text":"function ensures object data table. object data table, converted one. object data table, copy object returned.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_data.table.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Ensure that an object is a data.table — ensure_data.table","text":"","code":"ensure_data.table(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_data.table.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Ensure that an object is a data.table — ensure_data.table","text":"data object ensure data table.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_data.table.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Ensure that an object is a data.table — ensure_data.table","text":"data.table/copy existing data.table.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_model_column.html","id":null,"dir":"Reference","previous_headings":"","what":"Assure that data has a model column — ensure_model_column","title":"Assure that data has a model column — ensure_model_column","text":"Check whether data.table column called model. , column called model added value Unspecified model.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_model_column.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assure that data has a model column — ensure_model_column","text":"","code":"ensure_model_column(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_model_column.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assure that data has a model column — ensure_model_column","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_model_column.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assure that data has a model column — ensure_model_column","text":"data.table column called model","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Binary forecast example data — example_binary","title":"Binary forecast example data — example_binary","text":"data set binary predictions COVID-19 cases deaths constructed data submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Binary forecast example data — example_binary","text":"","code":"example_binary"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Binary forecast example data — example_binary","text":"data frame following columns: location country prediction made location_name name country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed factor observed values forecast_date date prediction made model name model generated forecasts horizon forecast horizon weeks predicted predicted value","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Binary forecast example data — example_binary","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Binary forecast example data — example_binary","text":"Predictions data set constructed based continuous example data looking number samples mean prediction. outcome constructed whether actually observed value mean prediction. understood sound statistical practice, rather practical way create example data set. data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Point forecast example data — example_point","title":"Point forecast example data — example_point","text":"data set predictions COVID-19 cases deaths submitted European Forecast Hub. data set like quantile example data, median replaced point forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Point forecast example data — example_point","text":"","code":"example_point"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Point forecast example data — example_point","text":"data frame following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed observed values location_name name country prediction made forecast_date date prediction made predicted predicted value model name model generated forecasts horizon forecast horizon weeks","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Point forecast example data — example_point","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Point forecast example data — example_point","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Quantile example data — example_quantile","title":"Quantile example data — example_quantile","text":"data set predictions COVID-19 cases deaths submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Quantile example data — example_quantile","text":"","code":"example_quantile"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Quantile example data — example_quantile","text":"data frame following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed Numeric: observed values location_name name country prediction made forecast_date date prediction made quantile_level quantile level corresponding prediction predicted predicted value model name model generated forecasts horizon forecast horizon weeks","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Quantile example data — example_quantile","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Quantile example data — example_quantile","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":null,"dir":"Reference","previous_headings":"","what":"Continuous forecast example data — example_sample_continuous","title":"Continuous forecast example data — example_sample_continuous","text":"data set continuous predictions COVID-19 cases deaths constructed data submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Continuous forecast example data — example_sample_continuous","text":"","code":"example_sample_continuous"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Continuous forecast example data — example_sample_continuous","text":"data frame following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed observed values location_name name country prediction made forecast_date date prediction made model name model generated forecasts horizon forecast horizon weeks predicted predicted value sample_id id corresponding sample","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Continuous forecast example data — example_sample_continuous","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Continuous forecast example data — example_sample_continuous","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":null,"dir":"Reference","previous_headings":"","what":"Discrete forecast example data — example_sample_discrete","title":"Discrete forecast example data — example_sample_discrete","text":"data set integer predictions COVID-19 cases deaths constructed data submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Discrete forecast example data — example_sample_discrete","text":"","code":"example_sample_discrete"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Discrete forecast example data — example_sample_discrete","text":"data frame following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed observed values location_name name country prediction made forecast_date date prediction made model name model generated forecasts horizon forecast horizon weeks predicted predicted value sample_id id corresponding sample","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Discrete forecast example data — example_sample_discrete","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Discrete forecast example data — example_sample_discrete","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/forecast_types.html","id":null,"dir":"Reference","previous_headings":"","what":"Documentation template for forecast types — forecast_types","title":"Documentation template for forecast types — forecast_types","text":"Documentation template forecast types","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/forecast_types.html","id":"forecast-types-and-input-formats","dir":"Reference","previous_headings":"","what":"Forecast types and input formats","title":"Documentation template for forecast types — forecast_types","text":"Various different forecast types / forecast formats supported. moment, : point forecasts binary forecasts (\"soft binary classification\") Probabilistic forecasts quantile-based format (forecast represented set predictive quantiles) Probabilistic forecasts sample-based format (forecast represented set predictive samples) Forecast types determined based columns present input data. overview required format forecast type: forecast types require data.frame similar columns observed predicted, model. Point forecasts require column observed type numeric column predicted type numeric. Binary forecasts require column observed type factor exactly two levels column predicted type numeric probabilities, corresponding probability observed equal second factor level. See details information. Quantile-based forecasts require column observed type numeric, column predicted type numeric, column quantile_level type numeric quantile-levels (0 1). Sample-based forecasts require column observed type numeric, column predicted type numeric, column sample_id type numeric sample indices. information see vignettes example data (example_quantile, example_sample_continuous, example_sample_discrete, example_point(), example_binary).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/forecast_types.html","id":"forecast-unit","dir":"Reference","previous_headings":"","what":"Forecast unit","title":"Documentation template for forecast types — forecast_types","text":"order score forecasts, scoringutils needs know rows data belong together jointly form single forecasts. easy e.g. point forecast, one row per forecast. quantile sample-based forecasts, however, multiple rows belong single forecast. forecast unit unit single forecast described combination columns uniquely identify single forecast. example, forecasts made different models various locations different time points, several weeks future. forecast unit described forecast_unit = c(\"model\", \"location\", \"forecast_date\", \"forecast_horizon\"). scoringutils automatically tries determine unit single forecast. uses existing columns , means columns must present unrelated forecast unit. simplistic example, additional row, \"even\", one row number even zero otherwise, mess scoring scoringutils thinks column relevant defining forecast unit. order avoid issues, recommend setting forecast unit explicitly, usually forecast_unit argument as_forecast(). drop unneeded columns, making sure necessary, 'protected columns' like \"predicted\" \"observed\" retained.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":null,"dir":"Reference","previous_headings":"","what":"Calculate geometric mean — geometric_mean","title":"Calculate geometric mean — geometric_mean","text":"Calculate geometric mean","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Calculate geometric mean — geometric_mean","text":"","code":"geometric_mean(x)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Calculate geometric mean — geometric_mean","text":"x Numeric vector values calculate geometric mean.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Calculate geometric mean — geometric_mean","text":"geometric mean values x. NA values ignored.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Calculate geometric mean — geometric_mean","text":"Used get_pairwise_comparisons().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":null,"dir":"Reference","previous_headings":"","what":"Calculate correlation between metrics — get_correlations","title":"Calculate correlation between metrics — get_correlations","text":"Calculate correlation different metrics data.frame scores produced score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Calculate correlation between metrics — get_correlations","text":"","code":"get_correlations(scores, metrics = get_metrics(scores), digits = NULL, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Calculate correlation between metrics — get_correlations","text":"scores object class scores (data.table scores additional attribute metrics produced score()). metrics character vector metrics show. set NULL (default), metrics present scores shown. digits number indicating many decimal places result rounded . default (digits = NULL) rounding takes place. ... Additional arguments pass cor().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Calculate correlation between metrics — get_correlations","text":"object class scores (data.table additional attribute metrics holding names scores) correlations different metrics","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Calculate correlation between metrics — get_correlations","text":"","code":"scores <- score(as_forecast(example_quantile)) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. get_correlations(scores, digits = 2) #> wis overprediction underprediction dispersion bias interval_coverage_50 #> #> 1: 1.00 0.94 0.28 0.46 0.11 -0.21 #> 2: 0.94 1.00 -0.03 0.32 0.22 -0.15 #> 3: 0.28 -0.03 1.00 0.15 -0.35 -0.21 #> 4: 0.46 0.32 0.15 1.00 0.11 -0.09 #> 5: 0.11 0.22 -0.35 0.11 1.00 0.01 #> 6: -0.21 -0.15 -0.21 -0.09 0.01 1.00 #> 7: -0.41 -0.32 -0.36 -0.09 0.10 0.37 #> 8: -0.34 -0.25 -0.33 -0.12 0.06 0.85 #> 9: 0.99 0.90 0.34 0.54 0.10 -0.25 #> interval_coverage_90 interval_coverage_deviation ae_median #> #> 1: -0.41 -0.34 0.99 #> 2: -0.32 -0.25 0.90 #> 3: -0.36 -0.33 0.34 #> 4: -0.09 -0.12 0.54 #> 5: 0.10 0.06 0.10 #> 6: 0.37 0.85 -0.25 #> 7: 1.00 0.64 -0.41 #> 8: 0.64 1.00 -0.38 #> 9: -0.41 -0.38 1.00 #> metric #> #> 1: wis #> 2: overprediction #> 3: underprediction #> 4: dispersion #> 5: bias #> 6: interval_coverage_50 #> 7: interval_coverage_90 #> 8: interval_coverage_deviation #> 9: ae_median"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"validated forecast object quantile-based format (see as_forecast() information), function computes: interval coverage central prediction intervals quantile coverage predictive quantiles deviation desired actual coverage (interval quantile coverage) Coverage values computed specific level grouping, specified argument. default, coverage values computed per model. Interval coverage Interval coverage given interval range defined proportion observations fall within corresponding central prediction intervals. Central prediction intervals symmetric around median formed two quantiles denote lower upper bound. example, 50% central prediction interval interval 0.25 0.75 quantiles predictive distribution. Quantile coverage Quantile coverage given quantile level defined proportion observed values smaller corresponding predictive quantile. example, 0.5 quantile coverage proportion observed values smaller 0.5 quantile predictive distribution. Just , single observation quantile single predictive distribution, value either TRUE FALSE. Coverage deviation coverage deviation difference desired coverage (can either interval quantile coverage) actual coverage. example, desired coverage 90% actual coverage 80%, coverage deviation -0.1.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"","code":"get_coverage(forecast, by = \"model\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()) character vector denotes level grouping coverage values computed. default (\"model\"), one coverage value per model returned.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"data.table columns specified additional columns coverage values described data.table columns \"interval_coverage\", \"interval_coverage_deviation\", \"quantile_coverage\", \"quantile_coverage_deviation\" columns specified .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"","code":"library(magrittr) # pipe operator example_quantile %>% as_forecast() %>% get_coverage(by = \"model\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> model quantile_level interval_range interval_coverage #> #> 1: EuroCOVIDhub-baseline 0.500 0 0.000000000 #> 2: EuroCOVIDhub-baseline 0.450 10 0.085937500 #> 3: EuroCOVIDhub-baseline 0.550 10 0.085937500 #> 4: EuroCOVIDhub-baseline 0.400 20 0.191406250 #> 5: EuroCOVIDhub-baseline 0.600 20 0.191406250 #> 6: EuroCOVIDhub-baseline 0.350 30 0.289062500 #> 7: EuroCOVIDhub-baseline 0.650 30 0.289062500 #> 8: EuroCOVIDhub-baseline 0.300 40 0.375000000 #> 9: EuroCOVIDhub-baseline 0.700 40 0.375000000 #> 10: EuroCOVIDhub-baseline 0.250 50 0.496093750 #> 11: EuroCOVIDhub-baseline 0.750 50 0.496093750 #> 12: EuroCOVIDhub-baseline 0.200 60 0.628906250 #> 13: EuroCOVIDhub-baseline 0.800 60 0.628906250 #> 14: EuroCOVIDhub-baseline 0.150 70 0.773437500 #> 15: EuroCOVIDhub-baseline 0.850 70 0.773437500 #> 16: EuroCOVIDhub-baseline 0.100 80 0.843750000 #> 17: EuroCOVIDhub-baseline 0.900 80 0.843750000 #> 18: EuroCOVIDhub-baseline 0.050 90 0.910156250 #> 19: EuroCOVIDhub-baseline 0.950 90 0.910156250 #> 20: EuroCOVIDhub-baseline 0.025 95 0.925781250 #> 21: EuroCOVIDhub-baseline 0.975 95 0.925781250 #> 22: EuroCOVIDhub-baseline 0.010 98 0.933593750 #> 23: EuroCOVIDhub-baseline 0.990 98 0.933593750 #> 24: EuroCOVIDhub-ensemble 0.500 0 0.003906250 #> 25: EuroCOVIDhub-ensemble 0.450 10 0.148437500 #> 26: EuroCOVIDhub-ensemble 0.550 10 0.148437500 #> 27: EuroCOVIDhub-ensemble 0.400 20 0.250000000 #> 28: EuroCOVIDhub-ensemble 0.600 20 0.250000000 #> 29: EuroCOVIDhub-ensemble 0.350 30 0.386718750 #> 30: EuroCOVIDhub-ensemble 0.650 30 0.386718750 #> 31: EuroCOVIDhub-ensemble 0.300 40 0.519531250 #> 32: EuroCOVIDhub-ensemble 0.700 40 0.519531250 #> 33: EuroCOVIDhub-ensemble 0.250 50 0.632812500 #> 34: EuroCOVIDhub-ensemble 0.750 50 0.632812500 #> 35: EuroCOVIDhub-ensemble 0.200 60 0.667968750 #> 36: EuroCOVIDhub-ensemble 0.800 60 0.667968750 #> 37: EuroCOVIDhub-ensemble 0.150 70 0.753906250 #> 38: EuroCOVIDhub-ensemble 0.850 70 0.753906250 #> 39: EuroCOVIDhub-ensemble 0.100 80 0.816406250 #> 40: EuroCOVIDhub-ensemble 0.900 80 0.816406250 #> 41: EuroCOVIDhub-ensemble 0.050 90 0.902343750 #> 42: EuroCOVIDhub-ensemble 0.950 90 0.902343750 #> 43: EuroCOVIDhub-ensemble 0.025 95 0.941406250 #> 44: EuroCOVIDhub-ensemble 0.975 95 0.941406250 #> 45: EuroCOVIDhub-ensemble 0.010 98 0.968750000 #> 46: EuroCOVIDhub-ensemble 0.990 98 0.968750000 #> 47: epiforecasts-EpiNow2 0.500 0 0.004048583 #> 48: epiforecasts-EpiNow2 0.450 10 0.093117409 #> 49: epiforecasts-EpiNow2 0.550 10 0.093117409 #> 50: epiforecasts-EpiNow2 0.400 20 0.165991903 #> 51: epiforecasts-EpiNow2 0.600 20 0.165991903 #> 52: epiforecasts-EpiNow2 0.350 30 0.230769231 #> 53: epiforecasts-EpiNow2 0.650 30 0.230769231 #> 54: epiforecasts-EpiNow2 0.300 40 0.319838057 #> 55: epiforecasts-EpiNow2 0.700 40 0.319838057 #> 56: epiforecasts-EpiNow2 0.250 50 0.445344130 #> 57: epiforecasts-EpiNow2 0.750 50 0.445344130 #> 58: epiforecasts-EpiNow2 0.200 60 0.538461538 #> 59: epiforecasts-EpiNow2 0.800 60 0.538461538 #> 60: epiforecasts-EpiNow2 0.150 70 0.635627530 #> 61: epiforecasts-EpiNow2 0.850 70 0.635627530 #> 62: epiforecasts-EpiNow2 0.100 80 0.732793522 #> 63: epiforecasts-EpiNow2 0.900 80 0.732793522 #> 64: epiforecasts-EpiNow2 0.050 90 0.846153846 #> 65: epiforecasts-EpiNow2 0.950 90 0.846153846 #> 66: epiforecasts-EpiNow2 0.025 95 0.874493927 #> 67: epiforecasts-EpiNow2 0.975 95 0.874493927 #> 68: epiforecasts-EpiNow2 0.010 98 0.910931174 #> 69: epiforecasts-EpiNow2 0.990 98 0.910931174 #> 70: UMass-MechBayes 0.500 0 0.015625000 #> 71: UMass-MechBayes 0.450 10 0.101562500 #> 72: UMass-MechBayes 0.550 10 0.101562500 #> 73: UMass-MechBayes 0.400 20 0.195312500 #> 74: UMass-MechBayes 0.600 20 0.195312500 #> 75: UMass-MechBayes 0.350 30 0.281250000 #> 76: UMass-MechBayes 0.650 30 0.281250000 #> 77: UMass-MechBayes 0.300 40 0.382812500 #> 78: UMass-MechBayes 0.700 40 0.382812500 #> 79: UMass-MechBayes 0.250 50 0.460937500 #> 80: UMass-MechBayes 0.750 50 0.460937500 #> 81: UMass-MechBayes 0.200 60 0.539062500 #> 82: UMass-MechBayes 0.800 60 0.539062500 #> 83: UMass-MechBayes 0.150 70 0.617187500 #> 84: UMass-MechBayes 0.850 70 0.617187500 #> 85: UMass-MechBayes 0.100 80 0.765625000 #> 86: UMass-MechBayes 0.900 80 0.765625000 #> 87: UMass-MechBayes 0.050 90 0.875000000 #> 88: UMass-MechBayes 0.950 90 0.875000000 #> 89: UMass-MechBayes 0.025 95 0.953125000 #> 90: UMass-MechBayes 0.975 95 0.953125000 #> 91: UMass-MechBayes 0.010 98 0.984375000 #> 92: UMass-MechBayes 0.990 98 0.984375000 #> model quantile_level interval_range interval_coverage #> interval_coverage_deviation quantile_coverage quantile_coverage_deviation #> #> 1: 0.000000000 0.69921875 0.199218750 #> 2: -0.014062500 0.65625000 0.206250000 #> 3: -0.014062500 0.74218750 0.192187500 #> 4: -0.008593750 0.58593750 0.185937500 #> 5: -0.008593750 0.77343750 0.173437500 #> 6: -0.010937500 0.52343750 0.173437500 #> 7: -0.010937500 0.80859375 0.158593750 #> 8: -0.025000000 0.46875000 0.168750000 #> 9: -0.025000000 0.84375000 0.143750000 #> 10: -0.003906250 0.36718750 0.117187500 #> 11: -0.003906250 0.86328125 0.113281250 #> 12: 0.028906250 0.25000000 0.050000000 #> 13: 0.028906250 0.87500000 0.075000000 #> 14: 0.073437500 0.13281250 -0.017187500 #> 15: 0.073437500 0.90625000 0.056250000 #> 16: 0.043750000 0.08203125 -0.017968750 #> 17: 0.043750000 0.92578125 0.025781250 #> 18: 0.010156250 0.04296875 -0.007031250 #> 19: 0.010156250 0.95312500 0.003125000 #> 20: -0.024218750 0.03125000 0.006250000 #> 21: -0.024218750 0.95703125 -0.017968750 #> 22: -0.046406250 0.03125000 0.021250000 #> 23: -0.046406250 0.96484375 -0.025156250 #> 24: 0.003906250 0.53125000 0.031250000 #> 25: 0.048437500 0.46484375 0.014843750 #> 26: 0.048437500 0.60156250 0.051562500 #> 27: 0.050000000 0.40625000 0.006250000 #> 28: 0.050000000 0.65625000 0.056250000 #> 29: 0.086718750 0.33984375 -0.010156250 #> 30: 0.086718750 0.72656250 0.076562500 #> 31: 0.119531250 0.26562500 -0.034375000 #> 32: 0.119531250 0.76562500 0.065625000 #> 33: 0.132812500 0.16406250 -0.085937500 #> 34: 0.132812500 0.79687500 0.046875000 #> 35: 0.067968750 0.14062500 -0.059375000 #> 36: 0.067968750 0.80468750 0.004687500 #> 37: 0.053906250 0.10156250 -0.048437500 #> 38: 0.053906250 0.85546875 0.005468750 #> 39: 0.016406250 0.07812500 -0.021875000 #> 40: 0.016406250 0.89453125 -0.005468750 #> 41: 0.002343750 0.04296875 -0.007031250 #> 42: 0.002343750 0.94531250 -0.004687500 #> 43: -0.008593750 0.03125000 0.006250000 #> 44: -0.008593750 0.97265625 -0.002343750 #> 45: -0.011250000 0.01562500 0.005625000 #> 46: -0.011250000 0.98437500 -0.005625000 #> 47: 0.004048583 0.49392713 -0.006072874 #> 48: -0.006882591 0.43724696 -0.012753036 #> 49: -0.006882591 0.53036437 -0.019635628 #> 50: -0.034008097 0.39676113 -0.003238866 #> 51: -0.034008097 0.55870445 -0.041295547 #> 52: -0.069230769 0.36437247 0.014372470 #> 53: -0.069230769 0.59514170 -0.054858300 #> 54: -0.080161943 0.31983806 0.019838057 #> 55: -0.080161943 0.63967611 -0.060323887 #> 56: -0.054655870 0.26315789 0.013157895 #> 57: -0.054655870 0.70445344 -0.045546559 #> 58: -0.061538462 0.20647773 0.006477733 #> 59: -0.061538462 0.74493927 -0.055060729 #> 60: -0.064372470 0.14574899 -0.004251012 #> 61: -0.064372470 0.78137652 -0.068623482 #> 62: -0.067206478 0.10121457 0.001214575 #> 63: -0.067206478 0.83400810 -0.065991903 #> 64: -0.053846154 0.06072874 0.010728745 #> 65: -0.053846154 0.90688259 -0.043117409 #> 66: -0.075506073 0.04858300 0.023582996 #> 67: -0.075506073 0.92307692 -0.051923077 #> 68: -0.069068826 0.02429150 0.014291498 #> 69: -0.069068826 0.93522267 -0.054777328 #> 70: 0.015625000 0.50000000 0.000000000 #> 71: 0.001562500 0.42187500 -0.028125000 #> 72: 0.001562500 0.52343750 -0.026562500 #> 73: -0.004687500 0.37500000 -0.025000000 #> 74: -0.004687500 0.57031250 -0.029687500 #> 75: -0.018750000 0.35156250 0.001562500 #> 76: -0.018750000 0.61718750 -0.032812500 #> 77: -0.017187500 0.28906250 -0.010937500 #> 78: -0.017187500 0.66406250 -0.035937500 #> 79: -0.039062500 0.27343750 0.023437500 #> 80: -0.039062500 0.71875000 -0.031250000 #> 81: -0.060937500 0.24218750 0.042187500 #> 82: -0.060937500 0.78125000 -0.018750000 #> 83: -0.082812500 0.20312500 0.053125000 #> 84: -0.082812500 0.82031250 -0.029687500 #> 85: -0.034375000 0.12500000 0.025000000 #> 86: -0.034375000 0.87500000 -0.025000000 #> 87: -0.025000000 0.06250000 0.012500000 #> 88: -0.025000000 0.93750000 -0.012500000 #> 89: 0.003125000 0.01562500 -0.009375000 #> 90: 0.003125000 0.96875000 -0.006250000 #> 91: 0.004375000 0.00781250 -0.002187500 #> 92: 0.004375000 0.99218750 0.002187500 #> interval_coverage_deviation quantile_coverage quantile_coverage_deviation"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":null,"dir":"Reference","previous_headings":"","what":"Find duplicate forecasts — get_duplicate_forecasts","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"Helper function identify duplicate forecasts, .e. instances one forecast prediction target.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"","code":"get_duplicate_forecasts(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"data data.frame used score()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"data.frame rows duplicate forecast found","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"","code":"example <- rbind(example_quantile, example_quantile[1000:1010]) get_duplicate_forecasts(example) #> location target_end_date target_type observed location_name forecast_date #> #> 1: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 2: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 3: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 4: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 5: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 6: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 7: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 8: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 9: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 10: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 11: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 12: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 13: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 14: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 15: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 16: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 17: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 18: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 19: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 20: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 21: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 22: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> location target_end_date target_type observed location_name forecast_date #> quantile_level predicted model horizon #> #> 1: 0.950 1464 epiforecasts-EpiNow2 1 #> 2: 0.975 1642 epiforecasts-EpiNow2 1 #> 3: 0.990 1951 epiforecasts-EpiNow2 1 #> 4: 0.010 28999 EuroCOVIDhub-ensemble 3 #> 5: 0.025 32612 EuroCOVIDhub-ensemble 3 #> 6: 0.050 36068 EuroCOVIDhub-ensemble 3 #> 7: 0.100 41484 EuroCOVIDhub-ensemble 3 #> 8: 0.150 47110 EuroCOVIDhub-ensemble 3 #> 9: 0.200 50929 EuroCOVIDhub-ensemble 3 #> 10: 0.250 54561 EuroCOVIDhub-ensemble 3 #> 11: 0.300 57739 EuroCOVIDhub-ensemble 3 #> 12: 0.950 1464 epiforecasts-EpiNow2 1 #> 13: 0.975 1642 epiforecasts-EpiNow2 1 #> 14: 0.990 1951 epiforecasts-EpiNow2 1 #> 15: 0.010 28999 EuroCOVIDhub-ensemble 3 #> 16: 0.025 32612 EuroCOVIDhub-ensemble 3 #> 17: 0.050 36068 EuroCOVIDhub-ensemble 3 #> 18: 0.100 41484 EuroCOVIDhub-ensemble 3 #> 19: 0.150 47110 EuroCOVIDhub-ensemble 3 #> 20: 0.200 50929 EuroCOVIDhub-ensemble 3 #> 21: 0.250 54561 EuroCOVIDhub-ensemble 3 #> 22: 0.300 57739 EuroCOVIDhub-ensemble 3 #> quantile_level predicted model horizon"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":null,"dir":"Reference","previous_headings":"","what":"Count number of available forecasts — get_forecast_counts","title":"Count number of available forecasts — get_forecast_counts","text":"Given data set forecasts, function counts number available forecasts. level grouping can specified using argument (e.g. count number forecasts per model, number forecasts per model location). useful determine whether missing forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Count number of available forecasts — get_forecast_counts","text":"","code":"get_forecast_counts( forecast, by = get_forecast_unit(forecast), collapse = c(\"quantile_level\", \"sample_id\") )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Count number of available forecasts — get_forecast_counts","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()) character vector NULL (default) denotes categories number forecasts counted. default unit single forecast (.e. available columns (apart \"protected\" columns 'predicted' 'observed') plus \"quantile_level\" \"sample_id\" present). collapse character vector (default: c(\"quantile_level\", \"sample_id\") names categories number rows collapsed one counting. example, single forecast usually represented set several quantiles samples collapsing one makes sure single forecast gets counted . Setting collapse = c() mean quantiles / samples counted individual forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Count number of available forecasts — get_forecast_counts","text":"data.table columns specified additional column \"count\" number forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Count number of available forecasts — get_forecast_counts","text":"","code":"# \\dontshow{ data.table::setDTthreads(2) # restricts number of cores used on CRAN # } get_forecast_counts( as_forecast(example_quantile), by = c(\"model\", \"target_type\") ) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Key: #> model target_type count #> #> 1: EuroCOVIDhub-baseline Cases 128 #> 2: EuroCOVIDhub-baseline Deaths 128 #> 3: EuroCOVIDhub-ensemble Cases 128 #> 4: EuroCOVIDhub-ensemble Deaths 128 #> 5: UMass-MechBayes Cases 0 #> 6: UMass-MechBayes Deaths 128 #> 7: epiforecasts-EpiNow2 Cases 128 #> 8: epiforecasts-EpiNow2 Deaths 119"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_type.html","id":null,"dir":"Reference","previous_headings":"","what":"Infer forecast type from data — get_forecast_type","title":"Infer forecast type from data — get_forecast_type","text":"Helper function infer forecast type based data.frame similar forecasts observed values. See details section information different forecast types.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_type.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Infer forecast type from data — get_forecast_type","text":"","code":"get_forecast_type(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_type.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Infer forecast type from data — get_forecast_type","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_type.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Infer forecast type from data — get_forecast_type","text":"Character vector length one either \"binary\", \"quantile\", \"sample\" \"point\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_type.html","id":"forecast-types-and-input-formats","dir":"Reference","previous_headings":"","what":"Forecast types and input formats","title":"Infer forecast type from data — get_forecast_type","text":"Various different forecast types / forecast formats supported. moment, : point forecasts binary forecasts (\"soft binary classification\") Probabilistic forecasts quantile-based format (forecast represented set predictive quantiles) Probabilistic forecasts sample-based format (forecast represented set predictive samples) Forecast types determined based columns present input data. overview required format forecast type: forecast types require data.frame similar columns observed predicted, model. Point forecasts require column observed type numeric column predicted type numeric. Binary forecasts require column observed type factor exactly two levels column predicted type numeric probabilities, corresponding probability observed equal second factor level. See details information. Quantile-based forecasts require column observed type numeric, column predicted type numeric, column quantile_level type numeric quantile-levels (0 1). Sample-based forecasts require column observed type numeric, column predicted type numeric, column sample_id type numeric sample indices. information see vignettes example data (example_quantile, example_sample_continuous, example_sample_discrete, example_point(), example_binary).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":null,"dir":"Reference","previous_headings":"","what":"Get unit of a single forecast — get_forecast_unit","title":"Get unit of a single forecast — get_forecast_unit","text":"Helper function get unit single forecast, .e. column names define single forecast made . just takes columns available data subtracts columns protected, .e. returned get_protected_columns() well names metrics specified scoring, .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get unit of a single forecast — get_forecast_unit","text":"","code":"get_forecast_unit(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get unit of a single forecast — get_forecast_unit","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get unit of a single forecast — get_forecast_unit","text":"character vector column names define unit single forecast","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":"forecast-unit","dir":"Reference","previous_headings":"","what":"Forecast unit","title":"Get unit of a single forecast — get_forecast_unit","text":"order score forecasts, scoringutils needs know rows data belong together jointly form single forecasts. easy e.g. point forecast, one row per forecast. quantile sample-based forecasts, however, multiple rows belong single forecast. forecast unit unit single forecast described combination columns uniquely identify single forecast. example, forecasts made different models various locations different time points, several weeks future. forecast unit described forecast_unit = c(\"model\", \"location\", \"forecast_date\", \"forecast_horizon\"). scoringutils automatically tries determine unit single forecast. uses existing columns , means columns must present unrelated forecast unit. simplistic example, additional row, \"even\", one row number even zero otherwise, mess scoring scoringutils thinks column relevant defining forecast unit. order avoid issues, recommend setting forecast unit explicitly, usually forecast_unit argument as_forecast(). drop unneeded columns, making sure necessary, 'protected columns' like \"predicted\" \"observed\" retained.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Get names of the metrics that were used for scoring — get_metrics","title":"Get names of the metrics that were used for scoring — get_metrics","text":"applying scoring rule via score(), names scoring rules become column names resulting data.table. addition, attribute metrics added output, holding names scores vector. done functions like get_forecast_unit() summarise_scores() can still identify columns part forecast unit hold score. get_metrics() accesses returns metrics attribute. attribute, function return NULL (, error = TRUE produce error instead). addition, checks column names input consistency data stored metrics attribute. Handling missing inconsistent metrics attribute: metrics attribute missing consistent column names data.table, can either run score() , specifying names scoring rules manually, add/update attribute manually using attr(scores, \"metrics\") <- c(\"names\", \"\", \"\", \"scores\") (order matter).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get names of the metrics that were used for scoring — get_metrics","text":"","code":"get_metrics(scores, error = FALSE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get names of the metrics that were used for scoring — get_metrics","text":"scores data.table attribute metrics. error Throw error attribute called metrics? Default FALSE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get names of the metrics that were used for scoring — get_metrics","text":"Character vector names scoring rules used scoring NULL scores computed previously.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":null,"dir":"Reference","previous_headings":"","what":"Obtain pairwise comparisons between models — get_pairwise_comparisons","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"Compare scores obtained different models pairwise tournament. combinations two models compared based overlapping set available forecasts common models. input scores object produced score(). Note adding additional unrelated columns can unpredictably change results, present columns taken account determining set overlapping forecasts two models. output pairwise comparisons set mean score ratios, relative skill scores p-values. following illustrates pairwise comparison process: Mean score ratios every pair two models, mean score ratio computed. simply mean score first model divided mean score second. Mean score ratios computed based set overlapping forecasts two models. means scores targets taken account models submitted forecast. (Scaled) Relative skill scores relative score model geometric mean mean score ratios involve model. baseline provided, scaled relative skill scores calculated well. Scaled relative skill scores simply relative skill score model divided relative skill score baseline model. p-values addition, function computes p-values comparison two models (based set overlapping forecasts). P-values can computed two ways: based nonparametric Wilcoxon signed-rank test (internally using wilcox.test() paired = TRUE) based permutation test. permutation test based difference mean scores two models. default null hypothesis mean score difference zero (see permutation_test()). Adjusted p-values computed calling p.adjust() raw p-values. code pairwise comparisons inspired implementation Johannes Bracher. implementation permutation test follows function permutationTest surveillance package Michael Höhle, Andrea Riebler Michaela Paul.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"","code":"get_pairwise_comparisons( scores, by = \"model\", metric = intersect(c(\"wis\", \"crps\", \"brier_score\"), names(scores)), baseline = NULL, ... )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"scores object class scores (data.table scores additional attribute metrics produced score()). Character vector column names define grouping level pairwise comparisons. default (model), one relative skill score per model. , example, = c(\"model\", \"location\"). get separate relative skill score every model every location. Internally, data.table scores split according (removing \"model\" splitting) pairwise comparisons computed separately split data.tables. metric string name metric relative skill shall computed. default either \"crps\", \"wis\" \"brier_score\" available. baseline string name model. baseline given, scaled relative skill respect baseline returned. default (NULL), relative skill scaled respect baseline model. ... Additional arguments comparison two models. See compare_two_models() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"data.table results pairwise comparisons containing mean score ratios (mean_scores_ratio), unadjusted (pval) adjusted (adj_pval) p-values, relative skill values model (..._relative_skill). baseline model given scaled relative skill reported well (..._scaled_relative_skill).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"Nikos Bosse nikosbosse@gmail.com Johannes Bracher, johannes.bracher@kit.edu","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"","code":"# \\dontshow{ data.table::setDTthreads(2) # restricts number of cores used on CRAN # } scores <- score(as_forecast(example_quantile)) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. pairwise <- get_pairwise_comparisons(scores, by = \"target_type\") pairwise2 <- get_pairwise_comparisons( scores, by = \"target_type\", baseline = \"EuroCOVIDhub-baseline\" ) library(ggplot2) plot_pairwise_comparisons(pairwise, type = \"mean_scores_ratio\") + facet_wrap(~target_type)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit.html","id":null,"dir":"Reference","previous_headings":"","what":"Probability integral transformation (data.frame version) — get_pit","title":"Probability integral transformation (data.frame version) — get_pit","text":"Compute Probability Integral Transformation (PIT) validated forecast objects.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Probability integral transformation (data.frame version) — get_pit","text":"","code":"get_pit(forecast, by, n_replicates = 100)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Probability integral transformation (data.frame version) — get_pit","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()) Character vector columns according PIT values shall grouped. e.g. columns 'model' 'location' input data want PIT histogram every model location, specify = c(\"model\", \"location\"). n_replicates number draws randomised PIT discrete predictions. ignored forecasts continuous.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Probability integral transformation (data.frame version) — get_pit","text":"data.table PIT values according grouping specified .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Probability integral transformation (data.frame version) — get_pit","text":"Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind M. Eggo, W. John Edmunds (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15, doi:10.1371/journal.pcbi.1006785","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Probability integral transformation (data.frame version) — get_pit","text":"","code":"result <- get_pit(as_forecast(example_sample_continuous), by = \"model\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_pit(result) # example with quantile data result <- get_pit(as_forecast(example_quantile), by = \"model\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_pit(result)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_protected_columns.html","id":null,"dir":"Reference","previous_headings":"","what":"Get protected columns from data — get_protected_columns","title":"Get protected columns from data — get_protected_columns","text":"Helper function get names columns data frame protected columns.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_protected_columns.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get protected columns from data — get_protected_columns","text":"","code":"get_protected_columns(data = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_protected_columns.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get protected columns from data — get_protected_columns","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_protected_columns.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get protected columns from data — get_protected_columns","text":"character vector names protected columns data. data NULL (default) returns list columns protected scoringutils.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_range_from_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Get interval range belonging to a quantile — get_range_from_quantile","title":"Get interval range belonging to a quantile — get_range_from_quantile","text":"Every quantile can thought either lower upper bound symmetric central prediction interval. helper function returns range central prediction interval quantile belongs. Due numeric instability sometimes occurred past, ranges rounded 10 decimal places. problem vast majority use cases, something aware .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_range_from_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get interval range belonging to a quantile — get_range_from_quantile","text":"","code":"get_range_from_quantile(quantile_level)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_range_from_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get interval range belonging to a quantile — get_range_from_quantile","text":"quantile_level numeric vector quantile levels size N.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_range_from_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get interval range belonging to a quantile — get_range_from_quantile","text":"numeric vector interval ranges size N","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_type.html","id":null,"dir":"Reference","previous_headings":"","what":"Get type of a vector or matrix of observed values or predictions — get_type","title":"Get type of a vector or matrix of observed values or predictions — get_type","text":"Internal helper function get type vector (usually observed predicted values). function checks whether input factor, else whether integer (can coerced integer) whether continuous.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_type.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get type of a vector or matrix of observed values or predictions — get_type","text":"","code":"get_type(x)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_type.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get type of a vector or matrix of observed values or predictions — get_type","text":"x Input type determined .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_type.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get type of a vector or matrix of observed values or predictions — get_type","text":"Character vector length one either \"classification\", \"integer\", \"continuous\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interpolate_median.html","id":null,"dir":"Reference","previous_headings":"","what":"Helper function to interpolate the median prediction if it is not available — interpolate_median","title":"Helper function to interpolate the median prediction if it is not available — interpolate_median","text":"Internal function interpolate median prediction available given quantile levels. done using linear interpolation two innermost quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interpolate_median.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Helper function to interpolate the median prediction if it is not available — interpolate_median","text":"","code":"interpolate_median(predicted, quantile_level)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interpolate_median.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Helper function to interpolate the median prediction if it is not available — interpolate_median","text":"predicted Vector length N (corresponding number quantiles) holds predictions. quantile_level Vector size N quantile levels predictions made. Note contain median (0.5) median imputed mean two innermost quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interpolate_median.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Helper function to interpolate the median prediction if it is not available — interpolate_median","text":"scalar imputed median prediction","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Interval coverage (for quantile-based forecasts) — interval_coverage","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"Check whether observed value within given central prediction interval. prediction interval defined lower upper bound formed pair predictive quantiles. example, 50% prediction interval formed 0.25 0.75 quantiles predictive distribution.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"","code":"interval_coverage(observed, predicted, quantile_level, interval_range = 50)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made. interval_range single number range prediction interval percent (e.g. 50 50% prediction interval) want compute interval coverage.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"vector length n elements either TRUE, observed value within corresponding prediction interval, FALSE otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"","code":"observed <- c(1, -15, 22) predicted <- rbind( c(-1, 0, 1, 2, 3), c(-2, 1, 2, 2, 4), c(-2, 0, 3, 3, 4) ) quantile_level <- c(0.1, 0.25, 0.5, 0.75, 0.9) interval_coverage(observed, predicted, quantile_level) #> [1] TRUE FALSE FALSE"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage_deviation.html","id":null,"dir":"Reference","previous_headings":"","what":"Interval coverage deviation (for quantile-based forecasts) — interval_coverage_deviation","title":"Interval coverage deviation (for quantile-based forecasts) — interval_coverage_deviation","text":"Check agreement desired actual interval coverage forecast. function similar interval_coverage(), takes provided prediction intervals account compares nominal interval coverage (.e. desired interval coverage) actual observed interval coverage. central symmetric prediction interval defined lower upper bound formed pair predictive quantiles. example, 50% prediction interval formed 0.25 0.75 quantiles predictive distribution. Ideally, forecaster aim cover 50% observed values 50% prediction intervals, 90% observed values 90% prediction intervals, . every prediction interval, deviation computed difference observed interval coverage nominal interval coverage single observed value single prediction interval, coverage always either 0 1 (FALSE TRUE). case single observed value multiple prediction intervals, still make much sense compare nominal (desired) coverage actual coverage single observation. sense coverage deviation really starts make sense metric averaged across multiple observations). Positive values interval coverage deviation indication underconfidence, .e. forecaster likely issued narrower forecast. Negative values indication overconfidence, .e. forecasts narrow. $$ \\textrm{interval coverage deviation} = \\mathbf{1}(\\textrm{observed value falls within interval}) - \\textrm{nominal interval coverage} $$ interval coverage deviation averaged across prediction intervals. median ignored computing coverage deviation.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage_deviation.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Interval coverage deviation (for quantile-based forecasts) — interval_coverage_deviation","text":"","code":"interval_coverage_deviation(observed, predicted, quantile_level)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage_deviation.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Interval coverage deviation (for quantile-based forecasts) — interval_coverage_deviation","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage_deviation.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Interval coverage deviation (for quantile-based forecasts) — interval_coverage_deviation","text":"numeric vector length n interval coverage deviation forecast (forecast comprising one multiple prediction intervals).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage_deviation.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Interval coverage deviation (for quantile-based forecasts) — interval_coverage_deviation","text":"","code":"observed <- c(1, -15, 22) predicted <- rbind( c(-1, 0, 1, 2, 3), c(-2, 1, 2, 2, 4), c(-2, 0, 3, 3, 4) ) quantile_level <- c(0.1, 0.25, 0.5, 0.75, 0.9) interval_coverage_deviation(observed, predicted, quantile_level) #> [1] 0.35 -0.65 -0.65"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":null,"dir":"Reference","previous_headings":"","what":"Interval score — interval_score","title":"Interval score — interval_score","text":"Proper Scoring Rule score quantile predictions, following Gneiting Raftery (2007). Smaller values better. score computed $$ \\textrm{score} = (\\textrm{upper} - \\textrm{lower}) + \\frac{2}{\\alpha}(\\textrm{lower} - \\textrm{observed}) * \\mathbf{1}(\\textrm{observed} < \\textrm{lower}) + \\frac{2}{\\alpha}(\\textrm{observed} - \\textrm{upper}) * \\mathbf{1}(\\textrm{observed} > \\textrm{upper}) $$ \\(\\mathbf{1}()\\) indicator function indicates much outside prediction interval. \\(\\alpha\\) decimal value indicates much outside prediction interval. improve usability, user asked provide interval range percentage terms, .e. interval_range = 90 (percent) 90 percent prediction interval. Correspondingly, user provide 5% 95% quantiles (corresponding alpha 0.1). specific distribution assumed, interval symmetric around median (.e use 0.1 quantile lower bound 0.7 quantile upper bound). Non-symmetric quantiles can scored using function quantile_score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Interval score — interval_score","text":"","code":"interval_score( observed, lower, upper, interval_range, weigh = TRUE, separate_results = FALSE )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Interval score — interval_score","text":"observed vector observed values size n lower Vector size n prediction lower quantile given interval range. upper Vector size n prediction upper quantile given interval range. interval_range Numeric vector (either single number vector size n) range prediction intervals. example, forecasting 0.05 0.95 quantile, interval range 90. interval range corresponds \\((100-\\alpha)/100\\), \\(\\alpha\\) decimal value indicates much outside prediction interval (see e.g. Gneiting Raftery (2007)). weigh Logical. TRUE (default), weigh score \\(\\alpha / 2\\), can averaged interval score , limit (increasing number equally spaced quantiles/prediction intervals), corresponds CRPS. \\(\\alpha\\) value corresponds (\\(\\alpha/2\\)) (\\(1 - \\alpha/2\\)), .e. decimal value represents much outside central prediction interval (E.g. 90 percent central prediction interval, alpha 0.1). separate_results Logical. TRUE (default FALSE), separate parts interval score (dispersion penalty, penalties - -prediction get returned separate elements list). want data.frame instead, simply call .data.frame() output.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Interval score — interval_score","text":"Vector scoring values, list separate entries separate_results TRUE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Interval score — interval_score","text":"Strictly Proper Scoring Rules, Prediction,Estimation, Tilmann Gneiting Adrian E. Raftery, 2007, Journal American Statistical Association, Volume 102, 2007 - Issue 477 Evaluating epidemic forecasts interval format, Johannes Bracher, Evan L. Ray, Tilmann Gneiting Nicholas G. Reich, https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1008618 # nolint","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Interval score — interval_score","text":"","code":"observed <- rnorm(30, mean = 1:30) interval_range <- rep(90, 30) alpha <- (100 - interval_range) / 100 lower <- qnorm(alpha / 2, rnorm(30, mean = 1:30)) upper <- qnorm((1 - alpha / 2), rnorm(30, mean = 11:40)) scoringutils:::interval_score( observed = observed, lower = lower, upper = upper, interval_range = interval_range ) #> [1] 0.6186832 0.6857411 0.6851103 0.6735228 0.7207120 0.6350107 0.6326772 #> [8] 0.7765693 0.9162766 0.7696477 0.5718187 0.5947846 0.6873026 0.6733580 #> [15] 0.6672086 0.7094583 0.5695223 0.6244414 0.7408444 1.4548649 0.6493377 #> [22] 0.9306896 0.6973638 0.7058552 0.8090509 0.7450232 0.5687430 0.7296127 #> [29] 0.7483414 0.6927440 # gives a warning, as the interval_range should likely be 50 instead of 0.5 scoringutils:::interval_score( observed = 4, upper = 8, lower = 2, interval_range = 0.5 ) #> Warning: ! Found interval ranges between 0 and 1. Are you sure that's right? An interval #> range of 0.5 e.g. implies a (49.75%, 50.25%) prediction interval. #> ℹ If you want to score a (25%, 75%) prediction interval, set `interval_range = #> 50`. #> This warning is displayed once per session. #> [1] 2.985 # example with missing values and separate results scoringutils:::interval_score( observed = c(observed, NA), lower = c(lower, NA), upper = c(NA, upper), separate_results = TRUE, interval_range = 90 ) #> $interval_score #> [1] NA 0.5428048 0.7656798 0.5305954 0.6509690 0.6780044 0.5082637 #> [8] 0.7404846 0.8621966 0.7375557 0.5326411 0.5665304 0.6259204 0.5696571 #> [15] 0.6555712 0.5798147 0.6144623 0.5308622 0.6850722 1.4237942 0.6161064 #> [22] 0.8500130 0.6461361 0.6502278 0.7400150 0.6942545 0.5921649 0.5858396 #> [29] 0.6975714 0.7079685 NA #> #> $dispersion #> [1] NA 0.5428048 0.7656798 0.5305954 0.6509690 0.6780044 0.5082637 #> [8] 0.7404846 0.5823934 0.7375557 0.5326411 0.5665304 0.6259204 0.5696571 #> [15] 0.6555712 0.5798147 0.6144623 0.5308622 0.6850722 0.5303743 0.6161064 #> [22] 0.5536698 0.6461361 0.6502278 0.7400150 0.6942545 0.5921649 0.5858396 #> [29] 0.6975714 0.7079685 NA #> #> $underprediction #> [1] NA 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 #> [26] 0 0 0 0 0 NA #> #> $overprediction #> [1] 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 #> [8] 0.0000000 0.2798032 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 #> [15] 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.8934199 0.0000000 #> [22] 0.2963432 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 #> [29] 0.0000000 0.0000000 NA #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Test whether an object is a forecast object — is_forecast","title":"Test whether an object is a forecast object — is_forecast","text":"Generic function test whether object class forecast_*. can also test specific forecast_* class using appropriate is_forecast.forecast_* method. example, check whether object class forecast_quantile, use scoringutils:::is_forecast.forecast_quantile().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Test whether an object is a forecast object — is_forecast","text":"","code":"is_forecast(x, ...) # Default S3 method is_forecast(x, ...) # S3 method for class 'forecast_sample' is_forecast(x, ...) # S3 method for class 'forecast_binary' is_forecast(x, ...) # S3 method for class 'forecast_point' is_forecast(x, ...) # S3 method for class 'forecast_quantile' is_forecast(x, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Test whether an object is a forecast object — is_forecast","text":"x R object. ... Additional arguments","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Test whether an object is a forecast object — is_forecast","text":"TRUE object class forecast_*, FALSE otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Test whether an object is a forecast object — is_forecast","text":"","code":"forecast_binary <- as_forecast(example_binary) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. is_forecast(forecast_binary) #> [1] TRUE"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":null,"dir":"Reference","previous_headings":"","what":"Log transformation with an additive shift — log_shift","title":"Log transformation with an additive shift — log_shift","text":"Function shifts value offset applies natural logarithm .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Log transformation with an additive shift — log_shift","text":"","code":"log_shift(x, offset = 0, base = exp(1))"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Log transformation with an additive shift — log_shift","text":"x vector input values transformed offset Number add input value taking natural logarithm. base positive number: base respect logarithms computed. Defaults e = exp(1).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Log transformation with an additive shift — log_shift","text":"numeric vector transformed values","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Log transformation with an additive shift — log_shift","text":"output computed log(x + offset)","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Log transformation with an additive shift — log_shift","text":"Transformation forecasts evaluating predictive performance epidemiological context Nikos . Bosse, Sam Abbott, Anne Cori, Edwin van Leeuwen, Johannes Bracher, Sebastian Funk medRxiv 2023.01.23.23284722 doi:10.1101/2023.01.23.23284722 https://www.medrxiv.org/content/10.1101/2023.01.23.23284722v1 # nolint","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Log transformation with an additive shift — log_shift","text":"","code":"log_shift(1:10) #> [1] 0.0000000 0.6931472 1.0986123 1.3862944 1.6094379 1.7917595 1.9459101 #> [8] 2.0794415 2.1972246 2.3025851 log_shift(0:9, offset = 1) #> [1] 0.0000000 0.6931472 1.0986123 1.3862944 1.6094379 1.7917595 1.9459101 #> [8] 2.0794415 2.1972246 2.3025851 transform_forecasts( as_forecast(example_quantile)[observed > 0, ], fun = log_shift, offset = 1 ) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Forecast type: quantile #> Forecast unit: #> location, target_end_date, target_type, location_name, forecast_date, model, #> horizon, and scale #> #> location target_end_date target_type observed location_name #> #> 1: DE 2021-01-02 Cases 1.273000e+05 Germany #> 2: DE 2021-01-02 Deaths 4.534000e+03 Germany #> 3: DE 2021-01-09 Cases 1.549220e+05 Germany #> 4: DE 2021-01-09 Deaths 6.117000e+03 Germany #> 5: DE 2021-01-16 Cases 1.101830e+05 Germany #> --- #> 40672: IT 2021-07-24 Deaths 4.369448e+00 Italy #> 40673: IT 2021-07-24 Deaths 4.369448e+00 Italy #> 40674: IT 2021-07-24 Deaths 4.369448e+00 Italy #> 40675: IT 2021-07-24 Deaths 4.369448e+00 Italy #> 40676: IT 2021-07-24 Deaths 4.369448e+00 Italy #> forecast_date quantile_level predicted model horizon #> #> 1: NA NA NA #> 2: NA NA NA #> 3: NA NA NA #> 4: NA NA NA #> 5: NA NA NA #> --- #> 40672: 2021-07-12 0.850 5.866468 epiforecasts-EpiNow2 2 #> 40673: 2021-07-12 0.900 5.986452 epiforecasts-EpiNow2 2 #> 40674: 2021-07-12 0.950 6.214608 epiforecasts-EpiNow2 2 #> 40675: 2021-07-12 0.975 6.416732 epiforecasts-EpiNow2 2 #> 40676: 2021-07-12 0.990 6.579251 epiforecasts-EpiNow2 2 #> scale #> #> 1: natural #> 2: natural #> 3: natural #> 4: natural #> 5: natural #> --- #> 40672: log #> 40673: log #> 40674: log #> 40675: log #> 40676: log"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Logarithmic score (sample-based version) — logs_sample","title":"Logarithmic score (sample-based version) — logs_sample","text":"function wrapper around logs_sample() function scoringRules package. function used score continuous predictions . Log Score theory also applicable discrete forecasts, problem lies implementation: Log score needs kernel density estimation, well defined integer-valued Monte Carlo Samples. Log score can used specific discrete probability distributions. See scoringRules package details.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Logarithmic score (sample-based version) — logs_sample","text":"","code":"logs_sample(observed, predicted, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Logarithmic score (sample-based version) — logs_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. ... Additional arguments passed logs_sample() scoringRules package.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Logarithmic score (sample-based version) — logs_sample","text":"Vector scores.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Logarithmic score (sample-based version) — logs_sample","text":"Alexander Jordan, Fabian Krüger, Sebastian Lerch, Evaluating Probabilistic Forecasts scoringRules, https://www.jstatsoft.org/article/view/v090i12","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Logarithmic score (sample-based version) — logs_sample","text":"","code":"observed <- rpois(30, lambda = 1:30) predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) logs_sample(observed, predicted) #> [1] 0.9877018 1.3179060 2.9308071 3.0967973 1.7353294 2.1375898 3.0671846 #> [8] 2.0115847 2.3128147 2.2232410 2.3023663 6.6464809 2.2126378 2.6212927 #> [15] 2.3262577 2.5675490 3.3007417 2.2614306 2.4344484 2.4467472 3.5018847 #> [22] 2.7833854 2.9574125 2.7755341 2.7756007 3.1608762 3.1106262 2.8047358 #> [29] 2.6202545 2.6531665"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Determine dispersion of a probabilistic forecast — mad_sample","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"Sharpness ability model generate predictions within narrow range dispersion lack thereof. data-independent measure, purely feature forecasts . Dispersion predictive samples corresponding one single observed value measured normalised median absolute deviation median predictive samples. details, see mad() explanations given Funk et al. (2019)","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"","code":"mad_sample(observed = NULL, predicted, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"observed Place holder, argument ignored exists consistency scoring functions. output depend observed values. predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. ... Additional arguments passed mad().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"Vector dispersion values.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"Funk S, Camacho , Kucharski AJ, Lowe R, Eggo RM, Edmunds WJ (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15. PLoS Comput Biol 15(2): e1006785. doi:10.1371/journal.pcbi.1006785","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"","code":"predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) mad_sample(predicted = predicted) #> [1] 1.4826 1.4826 1.4826 2.9652 2.9652 2.9652 2.9652 2.9652 2.9652 2.9652 #> [11] 2.9652 2.9652 4.4478 4.4478 2.9652 3.7065 4.4478 3.7065 4.4478 4.4478 #> [21] 4.4478 5.9304 4.4478 4.4478 4.4478 5.9304 4.4478 4.4478 5.9304 5.9304"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Default metrics and scoring rules for binary forecasts — metrics_binary","title":"Default metrics and scoring rules for binary forecasts — metrics_binary","text":"Helper function returns named list default scoring rules suitable binary forecasts. default scoring rules : \"brier_score\" = brier_score() \"log_score\" = logs_binary()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Default metrics and scoring rules for binary forecasts — metrics_binary","text":"","code":"metrics_binary(select = NULL, exclude = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Default metrics and scoring rules for binary forecasts — metrics_binary","text":"select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Default metrics and scoring rules for binary forecasts — metrics_binary","text":"list scoring rules.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_binary.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Default metrics and scoring rules for binary forecasts — metrics_binary","text":"","code":"metrics_binary() #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> #> #> #> $log_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> logs <- -log(1 - abs(observed - predicted)) #> return(logs) #> } #> #> #> metrics_binary(select = \"brier_score\") #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> #> #> metrics_binary(exclude = \"log_score\") #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> #> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Default metrics and scoring rules for point forecasts — metrics_point","title":"Default metrics and scoring rules for point forecasts — metrics_point","text":"Helper function returns named list default scoring rules suitable point forecasts. default scoring rules : \"ae_point\" = ae() \"se_point\" = se() \"ape\" = ape()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Default metrics and scoring rules for point forecasts — metrics_point","text":"","code":"metrics_point(select = NULL, exclude = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Default metrics and scoring rules for point forecasts — metrics_point","text":"select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Default metrics and scoring rules for point forecasts — metrics_point","text":"list scoring rules.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_point.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Default metrics and scoring rules for point forecasts — metrics_point","text":"","code":"metrics_point() #> $ae_point #> function (actual, predicted) #> { #> return(abs(actual - predicted)) #> } #> #> #> #> $se_point #> function (actual, predicted) #> { #> return((actual - predicted)^2) #> } #> #> #> #> $ape #> function (actual, predicted) #> { #> return(ae(actual, predicted)/abs(actual)) #> } #> #> #> metrics_point(select = \"ape\") #> $ape #> function (actual, predicted) #> { #> return(ae(actual, predicted)/abs(actual)) #> } #> #> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Default metrics and scoring rules for quantile-based forecasts — metrics_quantile","title":"Default metrics and scoring rules for quantile-based forecasts — metrics_quantile","text":"Helper function returns named list default scoring rules suitable forecasts quantile-based format. default scoring rules : \"wis\" = wis \"overprediction\" = overprediction() \"underprediction\" = underprediction() \"dispersion\" = dispersion() \"bias\" = bias_quantile() \"interval_coverage_50\" = interval_coverage() \"interval_coverage_90\" = customise_metric( interval_coverage, interval_range = 90 ) \"interval_coverage_deviation\" = interval_coverage_deviation(), \"ae_median\" = ae_median_quantile() Note: interval_coverage_90 scoring rule created modifying interval_coverage(), making use function customise_metric(). construct allows function deal arbitrary arguments ..., making sure interval_coverage() can accept get passed . interval_range = 90 set function definition, passing argument interval_range = 90 score() mean also get passed interval_coverage_50.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Default metrics and scoring rules for quantile-based forecasts — metrics_quantile","text":"","code":"metrics_quantile(select = NULL, exclude = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Default metrics and scoring rules for quantile-based forecasts — metrics_quantile","text":"select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Default metrics and scoring rules for quantile-based forecasts — metrics_quantile","text":"list scoring rules.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_quantile.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Default metrics and scoring rules for quantile-based forecasts — metrics_quantile","text":"","code":"metrics_quantile() #> $wis #> function (observed, predicted, quantile_level, separate_results = FALSE, #> weigh = TRUE, count_median_twice = FALSE, na.rm = TRUE) #> { #> assert_input_quantile(observed, predicted, quantile_level) #> reformatted <- quantile_to_interval(observed, predicted, #> quantile_level) #> assert_logical(separate_results, len = 1) #> assert_logical(weigh, len = 1) #> assert_logical(count_median_twice, len = 1) #> assert_logical(na.rm, len = 1) #> if (separate_results) { #> cols <- c(\"wis\", \"dispersion\", \"underprediction\", \"overprediction\") #> } #> else { #> cols <- \"wis\" #> } #> reformatted[, `:=`(eval(cols), do.call(interval_score, list(observed = observed, #> lower = lower, upper = upper, interval_range = interval_range, #> weigh = weigh, separate_results = separate_results)))] #> if (count_median_twice) { #> reformatted[, `:=`(weight, 1)] #> } #> else { #> reformatted[, `:=`(weight, ifelse(interval_range == 0, #> 0.5, 1))] #> } #> reformatted <- reformatted[, lapply(.SD, weighted.mean, na.rm = na.rm, #> w = weight), by = \"forecast_id\", .SDcols = colnames(reformatted) %like% #> paste(cols, collapse = \"|\")] #> if (separate_results) { #> return(list(wis = reformatted$wis, dispersion = reformatted$dispersion, #> underprediction = reformatted$underprediction, overprediction = reformatted$overprediction)) #> } #> else { #> return(reformatted$wis) #> } #> } #> #> #> #> $overprediction #> function (observed, predicted, quantile_level, ...) #> { #> args <- list(...) #> args$separate_results <- TRUE #> assert_input_quantile(observed, predicted, quantile_level) #> out <- do.call(wis, c(list(observed), list(predicted), list(quantile_level), #> args)) #> return(out$overprediction) #> } #> #> #> #> $underprediction #> function (observed, predicted, quantile_level, ...) #> { #> args <- list(...) #> args$separate_results <- TRUE #> assert_input_quantile(observed, predicted, quantile_level) #> out <- do.call(wis, c(list(observed), list(predicted), list(quantile_level), #> args)) #> return(out$underprediction) #> } #> #> #> #> $dispersion #> function (observed, predicted, quantile_level, ...) #> { #> args <- list(...) #> args$separate_results <- TRUE #> assert_input_quantile(observed, predicted, quantile_level) #> out <- do.call(wis, c(list(observed), list(predicted), list(quantile_level), #> args)) #> return(out$dispersion) #> } #> #> #> #> $bias #> function (observed, predicted, quantile_level, na.rm = TRUE) #> { #> assert_input_quantile(observed, predicted, quantile_level) #> n <- length(observed) #> N <- length(quantile_level) #> if (is.null(dim(predicted))) { #> dim(predicted) <- c(n, N) #> } #> if (!(0.5 %in% quantile_level)) { #> cli_inform(c(i = \"Median not available, interpolating median from the two\\n innermost quantiles in order to compute bias.\")) #> } #> bias <- sapply(1:n, function(i) { #> bias_quantile_single_vector(observed[i], predicted[i, #> ], quantile_level, na.rm) #> }) #> return(bias) #> } #> #> #> #> $interval_coverage_50 #> function (observed, predicted, quantile_level, interval_range = 50) #> { #> assert_input_quantile(observed, predicted, quantile_level) #> assert_number(interval_range) #> necessary_quantiles <- c((100 - interval_range)/2, 100 - #> (100 - interval_range)/2)/100 #> if (!all(necessary_quantiles %in% quantile_level)) { #> cli_warn(c(`!` = \"To compute the interval coverage for an interval range of\\n {.val {interval_range}%}, the {.val {necessary_quantiles}} quantiles\\n are required.\", #> i = \"Returning {.val {NA}}.\")) #> return(NA) #> } #> r <- interval_range #> reformatted <- quantile_to_interval(observed, predicted, #> quantile_level) #> reformatted <- reformatted[interval_range %in% r] #> reformatted[, `:=`(interval_coverage, (observed >= lower) & #> (observed <= upper))] #> return(reformatted$interval_coverage) #> } #> #> #> #> $interval_coverage_90 #> function (...) #> { #> do.call(metric, c(list(...), dots)) #> } #> #> #> #> $interval_coverage_deviation #> function (observed, predicted, quantile_level) #> { #> assert_input_quantile(observed, predicted, quantile_level) #> available_ranges <- unique(get_range_from_quantile(quantile_level)) #> necessary_quantiles <- unique(c((100 - available_ranges)/2, #> 100 - (100 - available_ranges)/2)/100) #> if (!all(necessary_quantiles %in% quantile_level)) { #> missing <- necessary_quantiles[!necessary_quantiles %in% #> quantile_level] #> cli_warn(c(x = \"To compute interval coverage deviation, all quantiles must form\\n central symmetric prediction intervals.\", #> i = \"Missing quantiles: {.val {missing}}. Returning {.val {NA}}.\")) #> return(NA) #> } #> reformatted <- quantile_to_interval(observed, predicted, #> quantile_level)[interval_range != 0] #> reformatted[, `:=`(interval_coverage, (observed >= lower) & #> (observed <= upper))] #> reformatted[, `:=`(interval_coverage_deviation, interval_coverage - #> interval_range/100)] #> out <- reformatted[, .(interval_coverage_deviation = mean(interval_coverage_deviation)), #> by = \"forecast_id\"] #> return(out$interval_coverage_deviation) #> } #> #> #> #> $ae_median #> function (observed, predicted, quantile_level) #> { #> assert_input_quantile(observed, predicted, quantile_level) #> if (!any(quantile_level == 0.5)) { #> cli_warn(c(x = \"In order to compute the absolute error of the median,\\n {.val 0.5} must be among the quantiles given.\", #> i = \"Returning {.val NA}.\")) #> return(NA_real_) #> } #> if (is.null(dim(predicted))) { #> predicted <- matrix(predicted, nrow = 1) #> } #> predicted <- predicted[, quantile_level == 0.5] #> abs_error_median <- abs(observed - predicted) #> return(abs_error_median) #> } #> #> #> metrics_quantile(select = \"wis\") #> $wis #> function (observed, predicted, quantile_level, separate_results = FALSE, #> weigh = TRUE, count_median_twice = FALSE, na.rm = TRUE) #> { #> assert_input_quantile(observed, predicted, quantile_level) #> reformatted <- quantile_to_interval(observed, predicted, #> quantile_level) #> assert_logical(separate_results, len = 1) #> assert_logical(weigh, len = 1) #> assert_logical(count_median_twice, len = 1) #> assert_logical(na.rm, len = 1) #> if (separate_results) { #> cols <- c(\"wis\", \"dispersion\", \"underprediction\", \"overprediction\") #> } #> else { #> cols <- \"wis\" #> } #> reformatted[, `:=`(eval(cols), do.call(interval_score, list(observed = observed, #> lower = lower, upper = upper, interval_range = interval_range, #> weigh = weigh, separate_results = separate_results)))] #> if (count_median_twice) { #> reformatted[, `:=`(weight, 1)] #> } #> else { #> reformatted[, `:=`(weight, ifelse(interval_range == 0, #> 0.5, 1))] #> } #> reformatted <- reformatted[, lapply(.SD, weighted.mean, na.rm = na.rm, #> w = weight), by = \"forecast_id\", .SDcols = colnames(reformatted) %like% #> paste(cols, collapse = \"|\")] #> if (separate_results) { #> return(list(wis = reformatted$wis, dispersion = reformatted$dispersion, #> underprediction = reformatted$underprediction, overprediction = reformatted$overprediction)) #> } #> else { #> return(reformatted$wis) #> } #> } #> #> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Default metrics and scoring rules sample-based forecasts — metrics_sample","title":"Default metrics and scoring rules sample-based forecasts — metrics_sample","text":"Helper function returns named list default scoring rules suitable forecasts sample-based format. default scoring rules : \"crps\" = crps_sample() \"log_score\" = logs_sample() \"dss\" = dss_sample() \"mad\" = mad_sample() \"bias\" = bias_sample() \"ae_median\" = ae_median_sample() \"se_mean\" = se_mean_sample()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Default metrics and scoring rules sample-based forecasts — metrics_sample","text":"","code":"metrics_sample(select = NULL, exclude = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Default metrics and scoring rules sample-based forecasts — metrics_sample","text":"select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Default metrics and scoring rules sample-based forecasts — metrics_sample","text":"list scoring rules.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Default metrics and scoring rules sample-based forecasts — metrics_sample","text":"","code":"metrics_sample() #> $bias #> function (observed, predicted) #> { #> assert_input_sample(observed, predicted) #> prediction_type <- get_type(predicted) #> n_pred <- ncol(predicted) #> p_x <- rowSums(predicted <= observed)/n_pred #> if (prediction_type == \"continuous\") { #> res <- 1 - 2 * p_x #> return(res) #> } #> else { #> p_xm1 <- rowSums(predicted <= (observed - 1))/n_pred #> res <- 1 - (p_x + p_xm1) #> return(res) #> } #> } #> #> #> #> $dss #> function (observed, predicted, ...) #> { #> assert_input_sample(observed, predicted) #> scoringRules::dss_sample(y = observed, dat = predicted, ...) #> } #> #> #> #> $crps #> function (observed, predicted, ...) #> { #> assert_input_sample(observed, predicted) #> scoringRules::crps_sample(y = observed, dat = predicted, #> ...) #> } #> #> #> #> $log_score #> function (observed, predicted, ...) #> { #> assert_input_sample(observed, predicted) #> scoringRules::logs_sample(y = observed, dat = predicted, #> ...) #> } #> #> #> #> $mad #> function (observed = NULL, predicted, ...) #> { #> assert_input_sample(rep(NA_real_, nrow(predicted)), predicted) #> sharpness <- apply(predicted, MARGIN = 1, mad, ...) #> return(sharpness) #> } #> #> #> #> $ae_median #> function (observed, predicted) #> { #> assert_input_sample(observed, predicted) #> median_predictions <- apply(as.matrix(predicted), MARGIN = 1, #> FUN = median) #> ae_median <- abs(observed - median_predictions) #> return(ae_median) #> } #> #> #> #> $se_mean #> function (observed, predicted) #> { #> assert_input_sample(observed, predicted) #> mean_predictions <- rowMeans(as.matrix(predicted)) #> se_mean <- (observed - mean_predictions)^2 #> return(se_mean) #> } #> #> #> metrics_sample(select = \"mad\") #> $mad #> function (observed = NULL, predicted, ...) #> { #> assert_input_sample(rep(NA_real_, nrow(predicted)), predicted) #> sharpness <- apply(predicted, MARGIN = 1, mad, ...) #> return(sharpness) #> } #> #> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Class constructor for forecast objects — new_forecast","title":"Class constructor for forecast objects — new_forecast","text":"Construct class based data.frame similar. constructor coerces data data.table makes sure column called model exists creates one assigns class","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Class constructor for forecast objects — new_forecast","text":"","code":"new_forecast(data, classname)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Class constructor for forecast objects — new_forecast","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. classname name class created","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Class constructor for forecast objects — new_forecast","text":"object class indicated classname","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Construct an object of class scores — new_scores","title":"Construct an object of class scores — new_scores","text":"function creates object class scores based data.table similar.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Construct an object of class scores — new_scores","text":"","code":"new_scores(scores, metrics, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Construct an object of class scores — new_scores","text":"scores data.table similar scores produced score(). metrics character vector names scores (.e. names scoring rules used scoring). ... Additional arguments .data.table()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Construct an object of class scores — new_scores","text":"object class scores","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Construct an object of class scores — new_scores","text":"","code":"if (FALSE) { df <- data.frame( model = \"A\", wis = \"0.1\" ) new_scores(df, \"wis\") }"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pairwise_comparison_one_group.html","id":null,"dir":"Reference","previous_headings":"","what":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","title":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","text":"function pairwise comparison one set forecasts, multiple models involved. gets called get_pairwise_comparisons(). get_pairwise_comparisons() splits data arbitrary subgroups specified user (e.g. pairwise comparison done separately different forecast targets) actual pairwise comparison subgroup managed pairwise_comparison_one_group(). order actually comparison two models subset common forecasts calls compare_two_models().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pairwise_comparison_one_group.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","text":"","code":"pairwise_comparison_one_group(scores, metric, baseline, by, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pairwise_comparison_one_group.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","text":"scores object class scores (data.table scores additional attribute metrics produced score()). metric string name metric relative skill shall computed. default either \"crps\", \"wis\" \"brier_score\" available. baseline string name model. baseline given, scaled relative skill respect baseline returned. default (NULL), relative skill scaled respect baseline model. Character vector column names define grouping level pairwise comparisons. default (model), one relative skill score per model. , example, = c(\"model\", \"location\"). get separate relative skill score every model every location. Internally, data.table scores split according (removing \"model\" splitting) pairwise comparisons computed separately split data.tables. ... Additional arguments comparison two models. See compare_two_models() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pairwise_comparison_one_group.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","text":"data.table results pairwise comparisons containing mean score ratios (mean_scores_ratio), unadjusted (pval) adjusted (adj_pval) p-values, relative skill values model (..._relative_skill). baseline model given scaled relative skill reported well (..._scaled_relative_skill).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/permutation_test.html","id":null,"dir":"Reference","previous_headings":"","what":"Simple permutation test — permutation_test","title":"Simple permutation test — permutation_test","text":"implementation permutation test follows function permutationTest surveillance package Michael Höhle, Andrea Riebler Michaela Paul. function compares two vectors scores. computes mean vector independently takes either difference ratio two. observed difference ratio compared test statistic based permutations original data. Used get_pairwise_comparisons().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/permutation_test.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Simple permutation test — permutation_test","text":"","code":"permutation_test( scores1, scores2, n_permutation = 999, one_sided = FALSE, comparison_mode = c(\"difference\", \"ratio\") )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/permutation_test.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Simple permutation test — permutation_test","text":"scores1 Vector scores compare another vector scores. scores2 second vector scores compare first n_permutation number replications use permutation test. replications yield exact results, require computation. one_sided Whether compute one-sided test. Default FALSE. comparison_mode compute test statistic comparison two scores. either \"difference\" \"ratio\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/permutation_test.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Simple permutation test — permutation_test","text":"p-value permutation test","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Probability integral transformation (sample-based version) — pit_sample","title":"Probability integral transformation (sample-based version) — pit_sample","text":"Uses Probability integral transformation (PIT) (randomised PIT integer forecasts) assess calibration predictive Monte Carlo samples.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Probability integral transformation (sample-based version) — pit_sample","text":"","code":"pit_sample(observed, predicted, n_replicates = 100)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Probability integral transformation (sample-based version) — pit_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. n_replicates number draws randomised PIT discrete predictions. ignored forecasts continuous.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Probability integral transformation (sample-based version) — pit_sample","text":"vector PIT-values. continuous forecasts, vector correspond length observed. integer forecasts, randomised PIT returned length length(observed) * n_replicates.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Probability integral transformation (sample-based version) — pit_sample","text":"Calibration reliability forecasts ability model correctly identify uncertainty making predictions. model perfect calibration, observed data time point look came predictive probability distribution time. Equivalently, one can inspect probability integral transform predictive distribution time t, $$ u_t = F_t (x_t) $$ \\(x_t\\) observed data point time \\(t \\textrm{ } t_1, …, t_n\\), n number forecasts, \\(F_t\\) (continuous) predictive cumulative probability distribution time t. true probability distribution outcomes time t \\(G_t\\) forecasts \\(F_t\\) said ideal \\(F_t = G_t\\) times t. case, probabilities \\(u_t\\) distributed uniformly. case discrete outcomes incidence counts, PIT longer uniform even forecasts ideal. case randomised PIT can used instead: $$ u_t = P_t(k_t) + v * (P_t(k_t) - P_t(k_t - 1) ) $$ \\(k_t\\) observed count, \\(P_t(x)\\) predictive cumulative probability observing incidence k time t, \\(P_t (-1) = 0\\) definition v standard uniform independent k. \\(P_t\\) true cumulative probability distribution, \\(u_t\\) standard uniform. function checks whether integer continuous forecasts provided. applies (randomised) probability integral tests values \\(u_t\\) uniformity using Anderson-Darling test. rule thumb, evidence suggest forecasting model miscalibrated p-value found greater threshold p >= 0.1, evidence miscalibrated 0.01 < p < 0.1, good evidence miscalibrated p <= 0.01. However, AD-p-values may overly strict actual usefulness may questionable. context noted, though, uniformity PIT necessary sufficient condition calibration.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Probability integral transformation (sample-based version) — pit_sample","text":"Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind M. Eggo, W. John Edmunds (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15, doi:10.1371/journal.pcbi.1006785","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Probability integral transformation (sample-based version) — pit_sample","text":"","code":"# \\dontshow{ data.table::setDTthreads(2) # restricts number of cores used on CRAN # } ## continuous predictions observed <- rnorm(20, mean = 1:20) predicted <- replicate(100, rnorm(n = 20, mean = 1:20)) pit <- pit_sample(observed, predicted) plot_pit(pit) ## integer predictions observed <- rpois(20, lambda = 1:20) predicted <- replicate(100, rpois(n = 20, lambda = 1:20)) pit <- pit_sample(observed, predicted, n_replicates = 30) plot_pit(pit)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot correlation between metrics — plot_correlations","title":"Plot correlation between metrics — plot_correlations","text":"Plots heatmap correlations different metrics.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot correlation between metrics — plot_correlations","text":"","code":"plot_correlations(correlations)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot correlation between metrics — plot_correlations","text":"correlations data.table correlations scores produced get_correlations().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot correlation between metrics — plot_correlations","text":"ggplot object showing coloured matrix correlations metrics. ggplot object visualisation correlations metrics","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot correlation between metrics — plot_correlations","text":"","code":"scores <- score(as_forecast(example_quantile)) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. correlations <- get_correlations( summarise_scores(scores), digits = 2 ) plot_correlations(correlations)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":null,"dir":"Reference","previous_headings":"","what":"Visualise the number of available forecasts — plot_forecast_counts","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"Visualise Forecasts Available.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"","code":"plot_forecast_counts( forecast_counts, x, y = \"model\", x_as_factor = TRUE, show_counts = TRUE )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"forecast_counts data.table (similar) column count holding forecast counts, produced get_forecast_counts(). x Character vector length one denotes name column appear x-axis plot. y Character vector length one denotes name column appear y-axis plot. Default \"model\". x_as_factor Logical (default TRUE). Whether convert variable x-axis factor. effect e.g. dates shown x-axis. show_counts Logical (default TRUE) indicates whether show actual count numbers plot.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"ggplot object plot forecast counts","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"","code":"library(ggplot2) forecast_counts <- get_forecast_counts( as_forecast(example_quantile), by = c(\"model\", \"target_type\", \"target_end_date\") ) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_forecast_counts( forecast_counts, x = \"target_end_date\", show_counts = FALSE ) + facet_wrap(\"target_type\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a heatmap of a scoring metric — plot_heatmap","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"function can used create heatmap one metric across different groups, e.g. interval score obtained several forecasting models different locations.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"","code":"plot_heatmap(scores, y = \"model\", x, metric)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"scores data.frame scores based quantile forecasts produced score(). y variable scores want show y-Axis. default \"model\" x variable scores want show x-Axis. something like \"horizon\", \"location\" metric String, metric determines value colour shown tiles heatmap.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"ggplot object showing heatmap desired metric","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"","code":"scores <- score(as_forecast(example_quantile)) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. scores <- summarise_scores(scores, by = c(\"model\", \"target_type\")) scores <- summarise_scores( scores, by = c(\"model\", \"target_type\"), fun = signif, digits = 2 ) plot_heatmap(scores, x = \"target_type\", metric = \"bias\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot interval coverage — plot_interval_coverage","title":"Plot interval coverage — plot_interval_coverage","text":"Plot interval coverage values (see get_coverage() information).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot interval coverage — plot_interval_coverage","text":"","code":"plot_interval_coverage(coverage, colour = \"model\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot interval coverage — plot_interval_coverage","text":"coverage data frame coverage values produced get_coverage(). colour According variable shall graphs coloured? Default \"model\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot interval coverage — plot_interval_coverage","text":"ggplot object plot interval coverage","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot interval coverage — plot_interval_coverage","text":"","code":"# \\dontshow{ data.table::setDTthreads(2) # restricts number of cores used on CRAN # } coverage <- get_coverage(as_forecast(example_quantile), by = \"model\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_interval_coverage(coverage)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"Creates heatmap ratios pvalues pairwise comparison models.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"","code":"plot_pairwise_comparisons( comparison_result, type = c(\"mean_scores_ratio\", \"pval\") )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"comparison_result data.frame produced get_pairwise_comparisons(). type Character vector length one either \"mean_scores_ratio\" \"pval\". denotes whether visualise ratio p-value pairwise comparison. Default \"mean_scores_ratio\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"ggplot object heatmap mean score ratios pairwise comparisons.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"","code":"library(ggplot2) scores <- score(as_forecast(example_quantile)) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. pairwise <- get_pairwise_comparisons(scores, by = \"target_type\") plot_pairwise_comparisons(pairwise, type = \"mean_scores_ratio\") + facet_wrap(~target_type)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pit.html","id":null,"dir":"Reference","previous_headings":"","what":"PIT histogram — plot_pit","title":"PIT histogram — plot_pit","text":"Make simple histogram probability integral transformed values visually check whether uniform distribution seems likely.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pit.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"PIT histogram — plot_pit","text":"","code":"plot_pit(pit, num_bins = \"auto\", breaks = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pit.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"PIT histogram — plot_pit","text":"pit Either vector PIT values, data.table produced get_pit(). num_bins number bins PIT histogram, default \"auto\". num_bins == \"auto\", plot_pit() either display 10 bins, display bin available quantile case passed data quantile-based format. can control number bins supplying number. fine sample-based pit histograms, may fail quantile-based formats. case preferred supply explicit breaks points using breaks argument. breaks Numeric vector break points bins PIT histogram. preferred creating PIT histogram based quantile-based data. Default NULL breaks determined num_bins. breaks used, num_bins ignored.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pit.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"PIT histogram — plot_pit","text":"ggplot object histogram PIT values","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pit.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"PIT histogram — plot_pit","text":"","code":"# \\dontshow{ data.table::setDTthreads(2) # restricts number of cores used on CRAN # } # PIT histogram in vector based format observed <- rnorm(30, mean = 1:30) predicted <- replicate(200, rnorm(n = 30, mean = 1:30)) pit <- pit_sample(observed, predicted) plot_pit(pit) # quantile-based pit pit <- get_pit(as_forecast(example_quantile), by = \"model\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_pit(pit, breaks = seq(0.1, 1, 0.1)) # sample-based pit pit <- get_pit(as_forecast(example_sample_discrete), by = \"model\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_pit(pit)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot quantile coverage — plot_quantile_coverage","title":"Plot quantile coverage — plot_quantile_coverage","text":"Plot quantile coverage values (see get_coverage() information).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot quantile coverage — plot_quantile_coverage","text":"","code":"plot_quantile_coverage(coverage, colour = \"model\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot quantile coverage — plot_quantile_coverage","text":"coverage data frame coverage values produced get_coverage(). colour String, according variable shall graphs coloured? Default \"model\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot quantile coverage — plot_quantile_coverage","text":"ggplot object plot interval coverage","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot quantile coverage — plot_quantile_coverage","text":"","code":"coverage <- get_coverage(as_forecast(example_quantile), by = \"model\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_quantile_coverage(coverage)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot contributions to the weighted interval score — plot_wis","title":"Plot contributions to the weighted interval score — plot_wis","text":"Visualise components weighted interval score: penalties -prediction, -prediction high dispersion (lack sharpness).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot contributions to the weighted interval score — plot_wis","text":"","code":"plot_wis(scores, x = \"model\", relative_contributions = FALSE, flip = FALSE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot contributions to the weighted interval score — plot_wis","text":"scores data.table scores based quantile forecasts produced score() summarised using summarise_scores(). x variable scores want show x-Axis. Usually \"model\". relative_contributions Logical. Show relative contributions instead absolute contributions? Default FALSE functionality available yet. flip Boolean (default FALSE), whether flip axes.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot contributions to the weighted interval score — plot_wis","text":"ggplot object showing contributions three components weighted interval score. ggplot object visualisation WIS decomposition","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Plot contributions to the weighted interval score — plot_wis","text":"Bracher J, Ray E, Gneiting T, Reich, N (2020) Evaluating epidemic forecasts interval format. https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1008618","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot contributions to the weighted interval score — plot_wis","text":"","code":"library(ggplot2) scores <- score(as_forecast(example_quantile)) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. scores <- summarise_scores(scores, by = c(\"model\", \"target_type\")) plot_wis(scores, x = \"model\", relative_contributions = TRUE ) + facet_wrap(~target_type) plot_wis(scores, x = \"model\", relative_contributions = FALSE ) + facet_wrap(~target_type, scales = \"free_x\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Print information about a forecast object — print.forecast_binary","title":"Print information about a forecast object — print.forecast_binary","text":"function prints information forecast object, including \"Forecast type\", \"Score columns\", \"Forecast unit\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Print information about a forecast object — print.forecast_binary","text":"","code":"# S3 method for class 'forecast_binary' print(x, ...) # S3 method for class 'forecast_quantile' print(x, ...) # S3 method for class 'forecast_point' print(x, ...) # S3 method for class 'forecast_sample' print(x, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Print information about a forecast object — print.forecast_binary","text":"x forecast object (validated data.table predicted observed values, see as_forecast()). ... Additional arguments print().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast_binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Print information about a forecast object — print.forecast_binary","text":"Returns x invisibly.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast_binary.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Print information about a forecast object — print.forecast_binary","text":"","code":"dat <- as_forecast(example_quantile) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. print(dat) #> Forecast type: quantile #> Forecast unit: #> location, target_end_date, target_type, location_name, forecast_date, model, #> and horizon #> #> Key: #> location target_end_date target_type observed location_name #> #> 1: DE 2021-01-02 Cases 127300 Germany #> 2: DE 2021-01-02 Deaths 4534 Germany #> 3: DE 2021-01-09 Cases 154922 Germany #> 4: DE 2021-01-09 Deaths 6117 Germany #> 5: DE 2021-01-16 Cases 110183 Germany #> --- #> 20541: IT 2021-07-24 Deaths 78 Italy #> 20542: IT 2021-07-24 Deaths 78 Italy #> 20543: IT 2021-07-24 Deaths 78 Italy #> 20544: IT 2021-07-24 Deaths 78 Italy #> 20545: IT 2021-07-24 Deaths 78 Italy #> forecast_date quantile_level predicted model horizon #> #> 1: NA NA NA #> 2: NA NA NA #> 3: NA NA NA #> 4: NA NA NA #> 5: NA NA NA #> --- #> 20541: 2021-07-12 0.850 352 epiforecasts-EpiNow2 2 #> 20542: 2021-07-12 0.900 397 epiforecasts-EpiNow2 2 #> 20543: 2021-07-12 0.950 499 epiforecasts-EpiNow2 2 #> 20544: 2021-07-12 0.975 611 epiforecasts-EpiNow2 2 #> 20545: 2021-07-12 0.990 719 epiforecasts-EpiNow2 2"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":null,"dir":"Reference","previous_headings":"","what":"Quantile score — quantile_score","title":"Quantile score — quantile_score","text":"Proper Scoring Rule score quantile predictions. Smaller values better. quantile score closely related interval score (see wis()) quantile equivalent works single quantiles instead central prediction intervals. quantile score, also called pinball loss, single quantile level \\(\\tau\\) defined $$ \\text{QS}_\\tau(F, y) = 2 \\cdot \\{ \\mathbf{1}(y \\leq q_\\tau) - \\tau\\} \\cdot (q_\\tau − y) = \\begin{cases} 2 \\cdot (1 - \\tau) * q_\\tau - y, & \\text{} y \\leq q_\\tau\\\\ 2 \\cdot \\tau * |q_\\tau - y|, & \\text{} y > q_\\tau, \\end{cases} $$ \\(q_\\tau\\) \\(\\tau\\)-quantile predictive distribution \\(F\\), \\(\\mathbf{1}(\\cdot)\\) indicator function. weighted interval score single prediction interval can obtained average quantile scores lower upper quantile prediction interval: $$ \\text{WIS}_\\alpha(F, y) = \\frac{\\text{QS}_{\\alpha/2}(F, y) + \\text{QS}_{1 - \\alpha/2}(F, y)}{2}. $$ See SI Bracher et al. (2021) details. quantile_score() returns average quantile score across quantile levels provided. set quantile levels form pairwise central prediction intervals, quantile score equivalent interval score.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Quantile score — quantile_score","text":"","code":"quantile_score(observed, predicted, quantile_level, weigh = TRUE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Quantile score — quantile_score","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made. weigh Logical. TRUE (default), weigh score \\(\\alpha / 2\\), can averaged interval score , limit (increasing number equally spaced quantiles/prediction intervals), corresponds CRPS. \\(\\alpha\\) value corresponds (\\(\\alpha/2\\)) (\\(1 - \\alpha/2\\)), .e. decimal value represents much outside central prediction interval (E.g. 90 percent central prediction interval, alpha 0.1).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Quantile score — quantile_score","text":"Numeric vector length n quantile score. scores averaged across quantile levels multiple quantile levels provided (result calling rowMeans() matrix quantile scores computed based observed predicted values).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Quantile score — quantile_score","text":"Strictly Proper Scoring Rules, Prediction,Estimation, Tilmann Gneiting Adrian E. Raftery, 2007, Journal American Statistical Association, Volume 102, 2007 - Issue 477 Evaluating epidemic forecasts interval format, Johannes Bracher, Evan L. Ray, Tilmann Gneiting Nicholas G. Reich, 2021, https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1008618","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Quantile score — quantile_score","text":"","code":"observed <- rnorm(10, mean = 1:10) alpha <- 0.5 lower <- qnorm(alpha / 2, observed) upper <- qnorm((1 - alpha / 2), observed) qs_lower <- quantile_score(observed, predicted = matrix(lower), quantile_level = alpha / 2 ) qs_upper <- quantile_score(observed, predicted = matrix(upper), quantile_level = 1 - alpha / 2 ) interval_score <- (qs_lower + qs_upper) / 2 interval_score2 <- quantile_score( observed, predicted = cbind(lower, upper), quantile_level = c(alpha / 2, 1 - alpha / 2) ) # this is the same as the following wis( observed, predicted = cbind(lower, upper), quantile_level = c(alpha / 2, 1 - alpha / 2) ) #> [1] 0.3372449 0.3372449 0.3372449 0.3372449 0.3372449 0.3372449 0.3372449 #> [8] 0.3372449 0.3372449 0.3372449"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_to_interval.html","id":null,"dir":"Reference","previous_headings":"","what":"Transform from a quantile format to an interval format — quantile_to_interval","title":"Transform from a quantile format to an interval format — quantile_to_interval","text":"Internal helper function transform quantile format interval format (longer supported forecast format, still used internally. function mimics S3 generic, actually S3 generic, want functions internal exported.) Quantile format quantile format, prediction characterised one multiple predicted values corresponding quantile levels. example, prediction quantile format represented 0.05, 0.25, 0.5, 0.75 0.95 quantiles predictive distribution. Interval format interval format, two quantiles assumed form prediction interval. Prediction intervals need symmetric around median characterised lower upper bound. lower bound defined lower quantile upper bound defined upper quantile. 90% prediction interval, example, covers 90% probability mass defined 5% 95% quantiles. forecast therefore characterised one multiple prediction intervals, e.g. lower upper bounds 50% 90% prediction intervals (corresponding 0.25 0.75 well 0.05 0.095 quantiles).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_to_interval.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Transform from a quantile format to an interval format — quantile_to_interval","text":"","code":"quantile_to_interval(...) quantile_to_interval_dataframe( forecast, format = \"long\", keep_quantile_col = FALSE, ... ) quantile_to_interval_numeric(observed, predicted, quantile_level, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_to_interval.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Transform from a quantile format to an interval format — quantile_to_interval","text":"... Arguments forecast data.table forecasts quantile-based format (see as_forecast()). format format output. Either \"long\" \"wide\". \"long\" (default), column boundary (values either \"upper\" \"lower\" column interval_range contains range interval. \"wide\", column interval_range two columns lower upper contain lower upper bounds prediction interval, respectively. keep_quantile_col keep quantile_level column final output transformation (default FALSE). works format = \"long\". format = \"wide\", quantile_level column always dropped. observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_to_interval.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Transform from a quantile format to an interval format — quantile_to_interval","text":"data.table forecasts interval format. quantile_to_interval_dataframe: data.table interval format (either \"long\" \"wide\"), without quantile_level column. Rows reordered. quantile_to_interval.numeric: data.table wide interval format columns forecast_id, observed, lower, upper, interval_range. forecast_id column unique identifier forecast. Rows reordered according forecast_id interval_range.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/run_safely.html","id":null,"dir":"Reference","previous_headings":"","what":"Run a function safely — run_safely","title":"Run a function safely — run_safely","text":"wrapper function designed run function safely completely clear arguments passed function. named arguments ... accepted fun removed. unnamed arguments passed function. case fun errors, error converted warning run_safely returns NULL. run_safely can useful constructing functions used metrics score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/run_safely.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Run a function safely — run_safely","text":"","code":"run_safely(..., fun, metric_name)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/run_safely.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Run a function safely — run_safely","text":"... Arguments pass fun. fun function execute. metric_name character string name metric. Used provide informative warning message case fun errors.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/run_safely.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Run a function safely — run_safely","text":"result fun NULL fun errors","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/run_safely.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Run a function safely — run_safely","text":"","code":"f <- function(x) {x} scoringutils:::run_safely(2, fun = f, metric_name = \"f\") #> [1] 2 scoringutils:::run_safely(2, y = 3, fun = f, metric_name = \"f\") #> [1] 2 scoringutils:::run_safely(fun = f, metric_name = \"f\") #> Warning: ! Computation for `f` failed. Error: argument \"x\" is missing, with no default. #> NULL scoringutils:::run_safely(y = 3, fun = f, metric_name = \"f\") #> Warning: ! Computation for `f` failed. Error: argument \"x\" is missing, with no default. #> NULL"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_interval_long.html","id":null,"dir":"Reference","previous_headings":"","what":"Change data from a sample-based format to a long interval range format — sample_to_interval_long","title":"Change data from a sample-based format to a long interval range format — sample_to_interval_long","text":"Transform data format based predictive samples format based interval ranges.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_interval_long.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Change data from a sample-based format to a long interval range format — sample_to_interval_long","text":"","code":"sample_to_interval_long( data, interval_range = c(0, 50, 90), type = 7, keep_quantile_col = TRUE )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_interval_long.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Change data from a sample-based format to a long interval range format — sample_to_interval_long","text":"type Type argument passed quantile function. information, see quantile(). keep_quantile_col keep quantile_level column, default TRUE","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_interval_long.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Change data from a sample-based format to a long interval range format — sample_to_interval_long","text":"data.table long interval interval range format","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Change data from a sample based format to a quantile format — sample_to_quantile","title":"Change data from a sample based format to a quantile format — sample_to_quantile","text":"Transform data format based predictive samples format based plain quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Change data from a sample based format to a quantile format — sample_to_quantile","text":"","code":"sample_to_quantile( forecast, quantile_level = c(0.05, 0.25, 0.5, 0.75, 0.95), type = 7 )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Change data from a sample based format to a quantile format — sample_to_quantile","text":"forecast forecast object class forecast_sample (validated data.table predicted observed values, see as_forecast()). quantile_level numeric vector quantile levels quantiles computed. type Type argument passed quantile function. information, see quantile().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Change data from a sample based format to a quantile format — sample_to_quantile","text":"data.table long interval range format","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_quantile.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Change data from a sample based format to a quantile format — sample_to_quantile","text":"","code":"sample_to_quantile(as_forecast(example_sample_discrete)) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Forecast type: quantile #> Forecast unit: #> location, location_name, target_end_date, target_type, forecast_date, model, #> and horizon #> #> location location_name target_end_date target_type forecast_date #> #> 1: DE Germany 2021-01-02 Cases #> 2: DE Germany 2021-01-02 Cases #> 3: DE Germany 2021-01-02 Cases #> 4: DE Germany 2021-01-02 Cases #> 5: DE Germany 2021-01-02 Cases #> --- #> 5151: IT Italy 2021-07-24 Deaths 2021-07-12 #> 5152: IT Italy 2021-07-24 Deaths 2021-07-12 #> 5153: IT Italy 2021-07-24 Deaths 2021-07-12 #> 5154: IT Italy 2021-07-24 Deaths 2021-07-12 #> 5155: IT Italy 2021-07-24 Deaths 2021-07-12 #> model horizon observed quantile_level predicted #> #> 1: NA 127300 0.05 NA #> 2: NA 127300 0.25 NA #> 3: NA 127300 0.50 NA #> 4: NA 127300 0.75 NA #> 5: NA 127300 0.95 NA #> --- #> 5151: epiforecasts-EpiNow2 2 78 0.05 44.90 #> 5152: epiforecasts-EpiNow2 2 78 0.25 89.25 #> 5153: epiforecasts-EpiNow2 2 78 0.50 151.50 #> 5154: epiforecasts-EpiNow2 2 78 0.75 208.00 #> 5155: epiforecasts-EpiNow2 2 78 0.95 469.10"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":null,"dir":"Reference","previous_headings":"","what":"Evaluate forecasts — score","title":"Evaluate forecasts — score","text":"score() applies selection scoring metrics forecast object (data.table forecasts observations) produced as_forecast(). score() generic dispatches different methods depending class input data. See details section information forecast types input formats. additional help examples, check Getting Started Vignette well paper Evaluating Forecasts scoringutils R.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Evaluate forecasts — score","text":"","code":"score(forecast, metrics, ...) # S3 method for class 'forecast_binary' score(forecast, metrics = metrics_binary(), ...) # S3 method for class 'forecast_point' score(forecast, metrics = metrics_point(), ...) # S3 method for class 'forecast_sample' score(forecast, metrics = metrics_sample(), ...) # S3 method for class 'forecast_quantile' score(forecast, metrics = metrics_quantile(), ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Evaluate forecasts — score","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()) metrics named list scoring functions. Names used column names output. See metrics_point(), metrics_binary(), metrics_quantile(), metrics_sample() information default metrics used. Note want pass arguments given metric, function customise_metric() pass updated list functions custom metric metrics argument score(). ... Additional arguments. Currently unused allows future extensions. want pass arguments individual metrics, use customise_metric().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Evaluate forecasts — score","text":"object class scores. object data.table unsummarised scores (one score per forecast) additional attribute metrics names metrics used scoring. See summarise_scores()) information summarise scores.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"forecast-types-and-input-formats","dir":"Reference","previous_headings":"","what":"Forecast types and input formats","title":"Evaluate forecasts — score","text":"Various different forecast types / forecast formats supported. moment, : point forecasts binary forecasts (\"soft binary classification\") Probabilistic forecasts quantile-based format (forecast represented set predictive quantiles) Probabilistic forecasts sample-based format (forecast represented set predictive samples) Forecast types determined based columns present input data. overview required format forecast type: forecast types require data.frame similar columns observed predicted, model. Point forecasts require column observed type numeric column predicted type numeric. Binary forecasts require column observed type factor exactly two levels column predicted type numeric probabilities, corresponding probability observed equal second factor level. See details information. Quantile-based forecasts require column observed type numeric, column predicted type numeric, column quantile_level type numeric quantile-levels (0 1). Sample-based forecasts require column observed type numeric, column predicted type numeric, column sample_id type numeric sample indices. information see vignettes example data (example_quantile, example_sample_continuous, example_sample_discrete, example_point(), example_binary).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"forecast-unit","dir":"Reference","previous_headings":"","what":"Forecast unit","title":"Evaluate forecasts — score","text":"order score forecasts, scoringutils needs know rows data belong together jointly form single forecasts. easy e.g. point forecast, one row per forecast. quantile sample-based forecasts, however, multiple rows belong single forecast. forecast unit unit single forecast described combination columns uniquely identify single forecast. example, forecasts made different models various locations different time points, several weeks future. forecast unit described forecast_unit = c(\"model\", \"location\", \"forecast_date\", \"forecast_horizon\"). scoringutils automatically tries determine unit single forecast. uses existing columns , means columns must present unrelated forecast unit. simplistic example, additional row, \"even\", one row number even zero otherwise, mess scoring scoringutils thinks column relevant defining forecast unit. order avoid issues, recommend setting forecast unit explicitly, usually forecast_unit argument as_forecast(). drop unneeded columns, making sure necessary, 'protected columns' like \"predicted\" \"observed\" retained.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Evaluate forecasts — score","text":"Bosse NI, Gruson H, Cori , van Leeuwen E, Funk S, Abbott S (2022) Evaluating Forecasts scoringutils R. doi:10.48550/arXiv.2205.07090","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Evaluate forecasts — score","text":"Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Evaluate forecasts — score","text":"","code":"library(magrittr) # pipe operator # \\dontshow{ data.table::setDTthreads(2) # restricts number of cores used on CRAN # } validated <- as_forecast(example_quantile) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. score(validated) %>% summarise_scores(by = c(\"model\", \"target_type\")) #> model target_type wis overprediction underprediction #> #> 1: EuroCOVIDhub-ensemble Cases 17943.82383 10043.121943 4237.177310 #> 2: EuroCOVIDhub-baseline Cases 28483.57465 14096.100883 10284.972826 #> 3: epiforecasts-EpiNow2 Cases 20831.55662 11906.823030 3260.355639 #> 4: EuroCOVIDhub-ensemble Deaths 41.42249 7.138247 4.103261 #> 5: EuroCOVIDhub-baseline Deaths 159.40387 65.899117 2.098505 #> 6: UMass-MechBayes Deaths 52.65195 8.978601 16.800951 #> 7: epiforecasts-EpiNow2 Deaths 66.64282 18.892583 15.893314 #> dispersion bias interval_coverage_50 interval_coverage_90 #> #> 1: 3663.52458 -0.05640625 0.3906250 0.8046875 #> 2: 4102.50094 0.09796875 0.3281250 0.8203125 #> 3: 5664.37795 -0.07890625 0.4687500 0.7890625 #> 4: 30.18099 0.07265625 0.8750000 1.0000000 #> 5: 91.40625 0.33906250 0.6640625 1.0000000 #> 6: 26.87239 -0.02234375 0.4609375 0.8750000 #> 7: 31.85692 -0.00512605 0.4201681 0.9075630 #> interval_coverage_deviation ae_median #> #> 1: -0.10230114 24101.07031 #> 2: -0.11721591 38473.60156 #> 3: -0.06963068 27923.81250 #> 4: 0.20380682 53.13281 #> 5: 0.12142045 233.25781 #> 6: -0.02488636 78.47656 #> 7: -0.04520244 104.74790 # set forecast unit manually (to avoid issues with scoringutils trying to # determine the forecast unit automatically) example_quantile %>% as_forecast( forecast_unit = c( \"location\", \"target_end_date\", \"target_type\", \"horizon\", \"model\" ) ) %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> location target_end_date target_type horizon model #> #> 1: DE 2021-05-08 Cases 1 EuroCOVIDhub-ensemble #> 2: DE 2021-05-08 Cases 1 EuroCOVIDhub-baseline #> 3: DE 2021-05-08 Cases 1 epiforecasts-EpiNow2 #> 4: DE 2021-05-08 Deaths 1 EuroCOVIDhub-ensemble #> 5: DE 2021-05-08 Deaths 1 EuroCOVIDhub-baseline #> --- #> 883: IT 2021-07-24 Deaths 2 EuroCOVIDhub-baseline #> 884: IT 2021-07-24 Deaths 3 UMass-MechBayes #> 885: IT 2021-07-24 Deaths 2 UMass-MechBayes #> 886: IT 2021-07-24 Deaths 3 epiforecasts-EpiNow2 #> 887: IT 2021-07-24 Deaths 2 epiforecasts-EpiNow2 #> wis overprediction underprediction dispersion bias #> #> 1: 7990.854783 2.549870e+03 0.0000000 5440.985217 0.50 #> 2: 16925.046957 1.527583e+04 0.0000000 1649.220870 0.95 #> 3: 25395.960870 1.722226e+04 0.0000000 8173.700000 0.90 #> 4: 53.880000 0.000000e+00 0.6086957 53.271304 -0.10 #> 5: 46.793043 2.130435e+00 0.0000000 44.662609 0.30 #> --- #> 883: 80.336957 3.608696e+00 0.0000000 76.728261 0.20 #> 884: 4.881739 4.347826e-02 0.0000000 4.838261 0.10 #> 885: 25.581739 1.782609e+01 0.0000000 7.755652 0.80 #> 886: 19.762609 5.478261e+00 0.0000000 14.284348 0.50 #> 887: 66.161739 4.060870e+01 0.0000000 25.553043 0.90 #> interval_coverage_50 interval_coverage_90 interval_coverage_deviation #> #> 1: TRUE TRUE 0.05181818 #> 2: FALSE FALSE -0.40272727 #> 3: FALSE TRUE -0.31181818 #> 4: TRUE TRUE 0.41545455 #> 5: TRUE TRUE 0.23363636 #> --- #> 883: TRUE TRUE 0.32454545 #> 884: TRUE TRUE 0.41545455 #> 885: FALSE TRUE -0.22090909 #> 886: TRUE TRUE 0.05181818 #> 887: FALSE TRUE -0.31181818 #> ae_median #> #> 1: 12271 #> 2: 25620 #> 3: 44192 #> 4: 14 #> 5: 15 #> --- #> 883: 53 #> 884: 1 #> 885: 46 #> 886: 26 #> 887: 108 # forecast formats with different metrics if (FALSE) { score(as_forecast(example_binary)) score(as_forecast(example_quantile)) score(as_forecast(example_point)) score(as_forecast(example_sample_discrete)) score(as_forecast(example_sample_continuous)) }"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Metrics for binary outcomes — scoring-functions-binary","title":"Metrics for binary outcomes — scoring-functions-binary","text":"Brier score Brier Score mean squared error probabilistic prediction observed outcome. Brier score proper scoring rule. Small values better (best 0, worst 1). $$ \\textrm{Brier\\_Score} = (\\textrm{prediction} - \\textrm{outcome})^2, $$ \\(\\textrm{outcome} \\\\{0, 1\\}\\), \\(\\textrm{prediction} \\[0, 1]\\) represents probability outcome equal 1. Log score binary outcomes Log Score negative logarithm probability assigned observed value. proper scoring rule. Small values better (best zero, worst infinity).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Metrics for binary outcomes — scoring-functions-binary","text":"","code":"brier_score(observed, predicted) logs_binary(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Metrics for binary outcomes — scoring-functions-binary","text":"observed factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted numeric vector length n, holding probabilities. Values represent probability corresponding outcome equal highest level factor observed.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Metrics for binary outcomes — scoring-functions-binary","text":"numeric vector size n Brier scores numeric vector size n log scores","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Metrics for binary outcomes — scoring-functions-binary","text":"Input formats functions require users provide observed values factor order distinguish input input format required scoring point forecasts. Internally, however, factors converted numeric values. factor observed = factor(c(0, 1, 1, 0, 1) two levels (0 1) internally coerced numeric vector (case result numeric vector c(1, 2, 2, 1, 1)). subtracting 1, resulting vector (c(0, 1, 1, 0) case) used internal calculations. predictions assumed represent probability outcome equal highest factor level (case outcome equal 1). alternatively also provide vector like observed = factor(c(\"\", \"b\", \"b\", \"\")) (two levels, b), result exactly internal representation. Probabilities represent probability outcome equal \"b\". want predictions probabilities outcome \"\", course make observed factor levels swapped, .e. observed = factor(c(\"\", \"b\", \"b\", \"\"), levels = c(\"b\", \"\"))","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Metrics for binary outcomes — scoring-functions-binary","text":"","code":"observed <- factor(sample(c(0, 1), size = 30, replace = TRUE)) predicted <- runif(n = 30, min = 0, max = 1) brier_score(observed, predicted) #> [1] 3.697221e-01 4.333309e-01 8.019571e-01 1.560465e-01 1.214441e-02 #> [6] 4.780794e-01 2.170087e-02 9.225365e-01 4.260066e-01 1.083496e-05 #> [11] 7.643096e-01 2.958275e-02 9.868210e-01 4.245489e-01 9.405384e-01 #> [16] 2.810347e-01 6.433678e-02 7.993802e-01 9.628835e-02 7.492222e-02 #> [21] 4.786024e-03 1.914958e-01 3.273303e-02 3.325011e-01 6.258271e-02 #> [26] 8.713640e-01 5.608322e-01 1.536012e-01 1.239574e-01 4.130782e-01 logs_binary(observed, predicted) #> [1] 0.93661542 1.07375988 2.25876533 0.50257178 0.11676043 1.17581594 #> [7] 0.15936173 3.23114148 1.05754275 0.00329708 2.07344387 0.18873775 #> [13] 5.01896290 1.05432990 3.50036400 0.75529303 0.29255656 2.24507736 #> [19] 0.37150383 0.31981861 0.07169054 0.57554614 0.19957683 0.85950668 #> [25] 0.28790257 2.71008389 1.38185350 0.49744870 0.43398155 1.02921200"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoringutils-package.html","id":null,"dir":"Reference","previous_headings":"","what":"scoringutils: Utilities for Scoring and Assessing Predictions — scoringutils-package","title":"scoringutils: Utilities for Scoring and Assessing Predictions — scoringutils-package","text":"`scoringutils` facilitates evaluation forecasts convenient framework based `data.table`. allows user check forecasts diagnose issues, visualise forecasts missing data, transform data scoring, handle missing forecasts, aggregate scores, visualise results evaluation. package mostly focuses evaluation probabilistic forecasts allows evaluating several different forecast types input formats. Find information package Vignettes well accompanying paper (doi:10.48550/arXiv.2205.07090 ).","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoringutils-package.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"scoringutils: Utilities for Scoring and Assessing Predictions — scoringutils-package","text":"Maintainer: Nikos Bosse nikosbosse@gmail.com (ORCID) Authors: Sam Abbott contact@samabbott.co.uk (ORCID) Hugo Gruson hugo.gruson+R@normalesup.org (ORCID) Sebastian Funk sebastian.funk@lshtm.ac.uk contributors: Johannes Bracher johannes.bracher@kit.edu (ORCID) [contributor] Toshiaki Asakura toshiaki.asa9ra@gmail.com (ORCID) [contributor] James Mba Azam james.azam@lshtm.ac.uk (ORCID) [contributor]","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/se_mean_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Squared error of the mean (sample-based version) — se_mean_sample","title":"Squared error of the mean (sample-based version) — se_mean_sample","text":"Squared error mean calculated $$ \\textrm{mean}(\\textrm{observed} - \\textrm{mean prediction})^2 $$ mean prediction calculated mean predictive samples.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/se_mean_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Squared error of the mean (sample-based version) — se_mean_sample","text":"","code":"se_mean_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/se_mean_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Squared error of the mean (sample-based version) — se_mean_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/se_mean_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Squared error of the mean (sample-based version) — se_mean_sample","text":"","code":"observed <- rnorm(30, mean = 1:30) predicted_values <- matrix(rnorm(30, mean = 1:30)) se_mean_sample(observed, predicted_values) #> [1] 0.173184776 0.023435451 1.610298449 3.571463532 0.287637723 3.456505359 #> [7] 3.104445200 6.416765323 0.014822594 0.262562121 0.002945155 2.290331532 #> [13] 0.062723285 0.492648216 0.047323098 1.489234187 1.959968436 2.103650766 #> [19] 0.086133601 0.894739782 5.993444771 2.978954352 6.778862480 1.779654975 #> [25] 1.712346282 0.843409936 3.193731460 0.514182655 0.134914452 0.485937390"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/select_metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Select metrics from a list of functions — select_metrics","title":"Select metrics from a list of functions — select_metrics","text":"Helper function return scoring rules selected user list possible functions.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/select_metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Select metrics from a list of functions — select_metrics","text":"","code":"select_metrics(metrics, select = NULL, exclude = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/select_metrics.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Select metrics from a list of functions — select_metrics","text":"metrics list scoring functions. select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/select_metrics.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Select metrics from a list of functions — select_metrics","text":"list scoring rules.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/select_metrics.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Select metrics from a list of functions — select_metrics","text":"","code":"select_metrics( metrics = metrics_binary(), select = \"brier_score\" ) #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> #> #> select_metrics( metrics = metrics_binary(), exclude = \"log_score\" ) #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> #> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/set_forecast_unit.html","id":null,"dir":"Reference","previous_headings":"","what":"Set unit of a single forecast manually — set_forecast_unit","title":"Set unit of a single forecast manually — set_forecast_unit","text":"Helper function set unit single forecast (.e. combination columns uniquely define single forecast) manually. simple function keeps columns specified forecast_unit (plus additional protected columns, e.g. observed values, predictions quantile levels) removes duplicate rows. set_forecast_unit() mainly called as_forecast() forecast_unit argument. done explicitly, scoringutils attempts determine unit single forecast automatically simply assuming column names relevant determine forecast unit. may lead unexpected behaviour, setting forecast unit explicitly can help make code easier debug easier read.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/set_forecast_unit.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Set unit of a single forecast manually — set_forecast_unit","text":"","code":"set_forecast_unit(data, forecast_unit)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/set_forecast_unit.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Set unit of a single forecast manually — set_forecast_unit","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. forecast_unit Character vector names columns uniquely identify single forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/set_forecast_unit.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Set unit of a single forecast manually — set_forecast_unit","text":"data.table columns kept relevant scoring denote unit single forecast specified user.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/set_forecast_unit.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Set unit of a single forecast manually — set_forecast_unit","text":"","code":"set_forecast_unit( example_quantile, c(\"location\", \"target_end_date\", \"target_type\", \"horizon\", \"model\") ) #> Key: #> observed quantile_level predicted location target_end_date target_type #> #> 1: 127300 NA NA DE 2021-01-02 Cases #> 2: 4534 NA NA DE 2021-01-02 Deaths #> 3: 154922 NA NA DE 2021-01-09 Cases #> 4: 6117 NA NA DE 2021-01-09 Deaths #> 5: 110183 NA NA DE 2021-01-16 Cases #> --- #> 20541: 78 0.850 352 IT 2021-07-24 Deaths #> 20542: 78 0.900 397 IT 2021-07-24 Deaths #> 20543: 78 0.950 499 IT 2021-07-24 Deaths #> 20544: 78 0.975 611 IT 2021-07-24 Deaths #> 20545: 78 0.990 719 IT 2021-07-24 Deaths #> horizon model #> #> 1: NA #> 2: NA #> 3: NA #> 4: NA #> 5: NA #> --- #> 20541: 2 epiforecasts-EpiNow2 #> 20542: 2 epiforecasts-EpiNow2 #> 20543: 2 epiforecasts-EpiNow2 #> 20544: 2 epiforecasts-EpiNow2 #> 20545: 2 epiforecasts-EpiNow2"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/summarise_scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Summarise scores as produced by score() — summarise_scores","title":"Summarise scores as produced by score() — summarise_scores","text":"Summarise scores produced score(). summarise_scores relies way identify names scores distinguish columns denote unit single forecast. Internally, done via stored attribute, metrics stores names scores. means, however, need careful renaming scores produced score(). , also manually update attribute calling attr(scores, \"metrics\") <- new_names.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/summarise_scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Summarise scores as produced by score() — summarise_scores","text":"","code":"summarise_scores(scores, by = \"model\", across = NULL, fun = mean, ...) summarize_scores(scores, by = \"model\", across = NULL, fun = mean, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/summarise_scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Summarise scores as produced by score() — summarise_scores","text":"scores object class scores (data.table scores additional attribute metrics produced score()). Character vector column names summarise scores . Default model, meaning one score per model output. across Character vector column names summarise scores across (meaning specified columns dropped). alternative specifying directly. across set, ignored. across NULL (default), used. fun function used summarising scores. Default mean(). ... Additional parameters can passed summary function provided fun. information see documentation respective function.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/summarise_scores.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Summarise scores as produced by score() — summarise_scores","text":"data.table summarised scores. Scores summarised according names columns original data specified across using fun passed summarise_scores().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/summarise_scores.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Summarise scores as produced by score() — summarise_scores","text":"","code":"# \\dontshow{ data.table::setDTthreads(2) # restricts number of cores used on CRAN # } library(magrittr) # pipe operator scores <- score(as_forecast(example_sample_continuous)) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. # get scores by model summarise_scores(scores,by = \"model\") #> model bias dss crps log_score mad #> #> 1: EuroCOVIDhub-ensemble 0.009765625 16.40496 9876.95886 10.747811 8763.6176 #> 2: EuroCOVIDhub-baseline 0.177734375 NaN 15309.68627 Inf 9680.3792 #> 3: epiforecasts-EpiNow2 -0.024898785 26.10137 11901.43354 Inf 12999.5404 #> 4: UMass-MechBayes -0.026953125 10.08582 60.19018 5.941622 123.6211 #> ae_median se_mean #> #> 1: 12406.03100 2.103026e+09 #> 2: 18932.50196 2.885063e+09 #> 3: 14680.12285 3.152268e+09 #> 4: 79.66001 1.371418e+04 # get scores by model and target type summarise_scores(scores, by = c(\"model\", \"target_type\")) #> model target_type bias dss crps log_score #> #> 1: EuroCOVIDhub-ensemble Cases -0.04648437 22.89997 19703.05522 15.633420 #> 2: EuroCOVIDhub-baseline Cases 0.03671875 NaN 30453.58346 Inf #> 3: epiforecasts-EpiNow2 Cases -0.03867188 40.87716 22896.51608 Inf #> 4: EuroCOVIDhub-ensemble Deaths 0.06601562 9.90995 50.86249 5.862203 #> 5: EuroCOVIDhub-baseline Deaths 0.31875000 12.99360 165.78907 6.977391 #> 6: UMass-MechBayes Deaths -0.02695313 10.08582 60.19018 5.941622 #> 7: epiforecasts-EpiNow2 Deaths -0.01008403 10.20807 74.79013 6.024092 #> mad ae_median se_mean #> #> 1: 17385.2629 24749.39707 4.206042e+09 #> 2: 18982.2128 37648.01693 5.769964e+09 #> 3: 24929.3438 28233.04536 6.082863e+09 #> 4: 141.9723 62.66492 1.080233e+04 #> 5: 378.5457 216.98699 1.622417e+05 #> 6: 123.6211 79.66001 1.371418e+04 #> 7: 167.4829 102.18939 3.243111e+04 # Get scores summarised across horizon, forecast date, and target end date summarise_scores( scores, across = c(\"horizon\", \"forecast_date\", \"target_end_date\") ) #> location location_name target_type model bias #> #> 1: DE Germany Cases EuroCOVIDhub-ensemble 0.17812500 #> 2: DE Germany Cases EuroCOVIDhub-baseline 0.51093750 #> 3: DE Germany Cases epiforecasts-EpiNow2 0.15937500 #> 4: DE Germany Deaths EuroCOVIDhub-ensemble -0.04531250 #> 5: DE Germany Deaths EuroCOVIDhub-baseline 0.40312500 #> 6: DE Germany Deaths UMass-MechBayes -0.58125000 #> 7: DE Germany Deaths epiforecasts-EpiNow2 -0.34531250 #> 8: FR France Cases EuroCOVIDhub-ensemble 0.01250000 #> 9: FR France Cases EuroCOVIDhub-baseline -0.02968750 #> 10: FR France Cases epiforecasts-EpiNow2 -0.02656250 #> 11: FR France Deaths EuroCOVIDhub-ensemble 0.04687500 #> 12: FR France Deaths EuroCOVIDhub-baseline 0.34218750 #> 13: FR France Deaths UMass-MechBayes 0.05156250 #> 14: FR France Deaths epiforecasts-EpiNow2 -0.05652174 #> 15: GB United Kingdom Cases EuroCOVIDhub-ensemble -0.47656250 #> 16: GB United Kingdom Cases EuroCOVIDhub-baseline -0.61875000 #> 17: GB United Kingdom Cases epiforecasts-EpiNow2 -0.18750000 #> 18: GB United Kingdom Deaths EuroCOVIDhub-ensemble 0.10937500 #> 19: GB United Kingdom Deaths EuroCOVIDhub-baseline 0.08281250 #> 20: GB United Kingdom Deaths UMass-MechBayes 0.62812500 #> 21: GB United Kingdom Deaths epiforecasts-EpiNow2 0.43125000 #> 22: IT Italy Cases EuroCOVIDhub-ensemble 0.10000000 #> 23: IT Italy Cases EuroCOVIDhub-baseline 0.28437500 #> 24: IT Italy Cases epiforecasts-EpiNow2 -0.10000000 #> 25: IT Italy Deaths EuroCOVIDhub-ensemble 0.15312500 #> 26: IT Italy Deaths EuroCOVIDhub-baseline 0.44687500 #> 27: IT Italy Deaths UMass-MechBayes -0.20625000 #> 28: IT Italy Deaths epiforecasts-EpiNow2 -0.08281250 #> location location_name target_type model bias #> dss crps log_score mad ae_median se_mean #> #> 1: 18.913810 7936.62666 10.368232 11207.25397 11618.84123 4.494508e+08 #> 2: 21.878467 15588.64493 11.938333 13450.95537 20043.14918 9.339773e+08 #> 3: 29.470535 13453.07861 22.686464 14195.91295 18498.57965 1.286406e+09 #> 4: 10.257710 56.54768 6.104505 167.78914 67.75772 1.061735e+04 #> 5: 12.536674 155.04973 6.997312 391.89252 219.90763 1.141223e+05 #> 6: 10.620902 76.81642 6.345002 147.52889 113.14713 1.543732e+04 #> 7: 11.020430 98.55426 6.571691 222.78911 140.73457 3.401571e+04 #> 8: 31.176915 47132.62230 28.508977 20503.83242 54298.94987 1.427192e+10 #> 9: NaN 56093.15772 Inf 25117.85238 65048.92899 1.653529e+10 #> 10: 34.493853 54117.03193 32.355909 33004.99213 63535.47737 1.981982e+10 #> 11: 10.729140 67.93297 6.236167 188.67948 85.67433 1.819764e+04 #> 12: 13.289991 202.77900 7.204643 501.10175 259.38228 2.316930e+05 #> 13: 11.322153 85.42422 6.518248 186.16678 98.33634 2.700122e+04 #> 14: 11.196840 119.95348 6.553617 331.22963 160.27973 8.603595e+04 #> 15: 21.106923 17824.87005 11.748949 30723.25562 24247.60751 1.940949e+09 #> 16: 24.877031 40848.86212 13.554332 22622.32113 51882.49530 5.228815e+09 #> 17: 21.224165 20508.11818 11.483762 47677.95848 25915.83258 3.160764e+09 #> 18: 8.839111 25.48464 5.251438 81.18266 26.29047 1.515431e+03 #> 19: 13.061990 109.66110 6.625700 212.20871 110.49682 1.048123e+05 #> 20: 9.040336 40.55896 5.310473 67.03450 62.02151 8.933439e+03 #> 21: 8.859818 29.31789 5.233896 73.15706 35.78943 4.412517e+03 #> 22: 20.402224 5918.10189 11.907522 7106.70962 8832.18969 1.618505e+08 #> 23: 20.338858 9283.66908 10.903144 14737.72225 13617.49426 3.817765e+08 #> 24: 78.320096 3507.83559 Inf 4838.51176 4982.29184 6.446300e+07 #> 25: 9.813839 53.48466 5.856701 130.23793 70.93715 1.287888e+04 #> 26: 13.085745 195.66648 7.081908 408.97976 278.16121 1.983392e+05 #> 27: 9.359876 37.96112 5.592767 93.75413 45.13505 3.484721e+03 #> 28: 10.033302 64.03710 5.886091 88.80946 88.29173 2.033662e+04 #> dss crps log_score mad ae_median se_mean # get standard deviation summarise_scores(scores, by = \"model\", fun = sd) #> model bias dss crps log_score mad #> #> 1: EuroCOVIDhub-ensemble 0.5468290 14.869520 39368.24836 21.510119 19799.1620 #> 2: EuroCOVIDhub-baseline 0.5457971 NA 45020.82814 NaN 13610.4174 #> 3: epiforecasts-EpiNow2 0.6083410 108.130107 44957.07746 NaN 29616.1714 #> 4: UMass-MechBayes 0.6221914 2.248998 49.62465 1.126019 123.3465 #> ae_median se_mean #> #> 1: 42801.64123 1.564286e+10 #> 2: 49458.36446 1.760651e+10 #> 3: 51129.54601 2.209086e+10 #> 4: 76.09471 2.994664e+04 # round digits summarise_scores(scores,by = \"model\") %>% summarise_scores(fun = signif, digits = 2) #> model bias dss crps log_score mad ae_median se_mean #> #> 1: EuroCOVIDhub-ensemble 0.0098 16 9900 11.0 8800 12000 2.1e+09 #> 2: EuroCOVIDhub-baseline 0.1800 NaN 15000 Inf 9700 19000 2.9e+09 #> 3: epiforecasts-EpiNow2 -0.0250 26 12000 Inf 13000 15000 3.2e+09 #> 4: UMass-MechBayes -0.0270 10 60 5.9 120 80 1.4e+04"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_not_present.html","id":null,"dir":"Reference","previous_headings":"","what":"Test whether column names are NOT present in a data.frame — test_columns_not_present","title":"Test whether column names are NOT present in a data.frame — test_columns_not_present","text":"function checks whether column names present. none columns present, function returns TRUE. one columns present, function returns FALSE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_not_present.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Test whether column names are NOT present in a data.frame — test_columns_not_present","text":"","code":"test_columns_not_present(data, columns)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_not_present.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Test whether column names are NOT present in a data.frame — test_columns_not_present","text":"data data.frame similar checked columns character vector column names check","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_not_present.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Test whether column names are NOT present in a data.frame — test_columns_not_present","text":"Returns TRUE none columns present FALSE otherwise","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_present.html","id":null,"dir":"Reference","previous_headings":"","what":"Test whether all column names are present in a data.frame — test_columns_present","title":"Test whether all column names are present in a data.frame — test_columns_present","text":"function checks whether column names present. one columns missing, function returns FALSE. columns present, function returns TRUE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_present.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Test whether all column names are present in a data.frame — test_columns_present","text":"","code":"test_columns_present(data, columns)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_present.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Test whether all column names are present in a data.frame — test_columns_present","text":"data data.frame similar checked columns character vector column names check","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_present.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Test whether all column names are present in a data.frame — test_columns_present","text":"Returns TRUE columns present FALSE otherwise","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Test whether data could be a binary forecast. — test_forecast_type_is_binary","title":"Test whether data could be a binary forecast. — test_forecast_type_is_binary","text":"Checks type necessary columns.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Test whether data could be a binary forecast. — test_forecast_type_is_binary","text":"","code":"test_forecast_type_is_binary(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Test whether data could be a binary forecast. — test_forecast_type_is_binary","text":"data data.frame similar checked","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Test whether data could be a binary forecast. — test_forecast_type_is_binary","text":"Returns TRUE basic requirements satisfied FALSE otherwise","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Test whether data could be a point forecast. — test_forecast_type_is_point","title":"Test whether data could be a point forecast. — test_forecast_type_is_point","text":"Checks type necessary columns.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Test whether data could be a point forecast. — test_forecast_type_is_point","text":"","code":"test_forecast_type_is_point(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Test whether data could be a point forecast. — test_forecast_type_is_point","text":"data data.frame similar checked","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Test whether data could be a point forecast. — test_forecast_type_is_point","text":"Returns TRUE basic requirements satisfied FALSE otherwise","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Test whether data could be a quantile forecast. — test_forecast_type_is_quantile","title":"Test whether data could be a quantile forecast. — test_forecast_type_is_quantile","text":"Checks type necessary columns.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Test whether data could be a quantile forecast. — test_forecast_type_is_quantile","text":"","code":"test_forecast_type_is_quantile(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Test whether data could be a quantile forecast. — test_forecast_type_is_quantile","text":"data data.frame similar checked","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Test whether data could be a quantile forecast. — test_forecast_type_is_quantile","text":"Returns TRUE basic requirements satisfied FALSE otherwise","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Test whether data could be a sample-based forecast. — test_forecast_type_is_sample","title":"Test whether data could be a sample-based forecast. — test_forecast_type_is_sample","text":"Checks type necessary columns.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Test whether data could be a sample-based forecast. — test_forecast_type_is_sample","text":"","code":"test_forecast_type_is_sample(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Test whether data could be a sample-based forecast. — test_forecast_type_is_sample","text":"data data.frame similar checked","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_forecast_type_is_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Test whether data could be a sample-based forecast. — test_forecast_type_is_sample","text":"Returns TRUE basic requirements satisfied FALSE otherwise","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/theme_scoringutils.html","id":null,"dir":"Reference","previous_headings":"","what":"Scoringutils ggplot2 theme — theme_scoringutils","title":"Scoringutils ggplot2 theme — theme_scoringutils","text":"theme ggplot2 plots used scoringutils.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/theme_scoringutils.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Scoringutils ggplot2 theme — theme_scoringutils","text":"","code":"theme_scoringutils()"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/theme_scoringutils.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Scoringutils ggplot2 theme — theme_scoringutils","text":"ggplot2 theme","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":null,"dir":"Reference","previous_headings":"","what":"Transform forecasts and observed values — transform_forecasts","title":"Transform forecasts and observed values — transform_forecasts","text":"Function transform forecasts observed values scoring.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Transform forecasts and observed values — transform_forecasts","text":"","code":"transform_forecasts( forecast, fun = log_shift, append = TRUE, label = \"log\", ... )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Transform forecasts and observed values — transform_forecasts","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()) fun function used transform observed values predictions. default function log_shift(), custom function essentially log(), additional arguments (offset) allows add offset applying logarithm. often helpful natural log transformation defined zero. common, pragmatic solution, add small offset data applying log transformation. work often used offset 1 precise value depend application. append Logical, defaults TRUE. Whether append transformed version data currently existing data (TRUE). selected, data gets transformed appended existing data, making possible use outcome directly score(). additional column, 'scale', gets created denotes rows untransformed ('scale' value \"natural\") transformed ('scale' value passed argument label). label string newly created 'scale' column denote newly transformed values. relevant append = TRUE. ... Additional parameters pass function supplied. default option log_shift() offset argument.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Transform forecasts and observed values — transform_forecasts","text":"forecast object either transformed version data, one untransformed transformed data. includes original data well transformation original data. one additional column, `scale', present set \"natural\" untransformed forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Transform forecasts and observed values — transform_forecasts","text":"reasons, depending circumstances, might desirable (check linked reference info). epidemiology, example, may useful log-transform incidence counts evaluating forecasts using scores weighted interval score (WIS) continuous ranked probability score (CRPS). Log-transforming forecasts observations changes interpretation score measure absolute distance forecast observation score evaluates forecast exponential growth rate. Another motivation can apply variance-stabilising transformation standardise incidence counts population. Note want apply transformation, important transform forecasts observations apply score. Applying transformation score risks losing propriety proper scoring rule.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Transform forecasts and observed values — transform_forecasts","text":"Transformation forecasts evaluating predictive performance epidemiological context Nikos . Bosse, Sam Abbott, Anne Cori, Edwin van Leeuwen, Johannes Bracher, Sebastian Funk medRxiv 2023.01.23.23284722 doi:10.1101/2023.01.23.23284722 https://www.medrxiv.org/content/10.1101/2023.01.23.23284722v1","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Transform forecasts and observed values — transform_forecasts","text":"Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Transform forecasts and observed values — transform_forecasts","text":"","code":"library(magrittr) # pipe operator # transform forecasts using the natural logarithm # negative values need to be handled (here by replacing them with 0) example_quantile %>% .[, observed := ifelse(observed < 0, 0, observed)] %>% as_forecast() %>% # Here we use the default function log_shift() which is essentially the same # as log(), but has an additional arguments (offset) that allows you add an # offset before applying the logarithm. transform_forecasts(append = FALSE) %>% head() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Warning: ! Detected zeros in input values. #> ℹ Try specifying offset = 1 (or any other offset). #> Warning: ! Detected zeros in input values. #> ℹ Try specifying offset = 1 (or any other offset). #> Warning: ! Error in validating forecast object: Error in #> assert_forecast_generic(forecast, verbose) : ! After removing rows with NA #> values in the data, no forecasts are left. . #> Forecast type: quantile #> Forecast unit: #> location, target_end_date, target_type, location_name, forecast_date, model, #> and horizon #> #> Key: #> location target_end_date target_type observed location_name forecast_date #> #> 1: DE 2021-01-02 Cases 11.754302 Germany #> 2: DE 2021-01-02 Deaths 8.419360 Germany #> 3: DE 2021-01-09 Cases 11.950677 Germany #> 4: DE 2021-01-09 Deaths 8.718827 Germany #> 5: DE 2021-01-16 Cases 11.609898 Germany #> 6: DE 2021-01-16 Deaths 8.677099 Germany #> quantile_level predicted model horizon #> #> 1: NA NA NA #> 2: NA NA NA #> 3: NA NA NA #> 4: NA NA NA #> 5: NA NA NA #> 6: NA NA NA # alternatively, integrating the truncation in the transformation function: example_quantile %>% as_forecast() %>% transform_forecasts( fun = function(x) {log_shift(pmax(0, x))}, append = FALSE ) %>% head() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Warning: ! Detected zeros in input values. #> ℹ Try specifying offset = 1 (or any other offset). #> Warning: ! Detected zeros in input values. #> ℹ Try specifying offset = 1 (or any other offset). #> Warning: ! Error in validating forecast object: Error in #> assert_forecast_generic(forecast, verbose) : ! After removing rows with NA #> values in the data, no forecasts are left. . #> Forecast type: quantile #> Forecast unit: #> location, target_end_date, target_type, location_name, forecast_date, model, #> and horizon #> #> Key: #> location target_end_date target_type observed location_name forecast_date #> #> 1: DE 2021-01-02 Cases 11.754302 Germany #> 2: DE 2021-01-02 Deaths 8.419360 Germany #> 3: DE 2021-01-09 Cases 11.950677 Germany #> 4: DE 2021-01-09 Deaths 8.718827 Germany #> 5: DE 2021-01-16 Cases 11.609898 Germany #> 6: DE 2021-01-16 Deaths 8.677099 Germany #> quantile_level predicted model horizon #> #> 1: NA NA NA #> 2: NA NA NA #> 3: NA NA NA #> 4: NA NA NA #> 5: NA NA NA #> 6: NA NA NA # specifying an offset for the log transformation removes the # warning caused by zeros in the data example_quantile %>% as_forecast() %>% .[, observed := ifelse(observed < 0, 0, observed)] %>% transform_forecasts(offset = 1, append = FALSE) %>% head() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Warning: ! Error in validating forecast object: Error in #> assert_forecast_generic(forecast, verbose) : ! After removing rows with NA #> values in the data, no forecasts are left. . #> Forecast type: quantile #> Forecast unit: #> location, target_end_date, target_type, location_name, forecast_date, model, #> and horizon #> #> Key: #> location target_end_date target_type observed location_name forecast_date #> #> 1: DE 2021-01-02 Cases 11.754310 Germany #> 2: DE 2021-01-02 Deaths 8.419580 Germany #> 3: DE 2021-01-09 Cases 11.950683 Germany #> 4: DE 2021-01-09 Deaths 8.718991 Germany #> 5: DE 2021-01-16 Cases 11.609907 Germany #> 6: DE 2021-01-16 Deaths 8.677269 Germany #> quantile_level predicted model horizon #> #> 1: NA NA NA #> 2: NA NA NA #> 3: NA NA NA #> 4: NA NA NA #> 5: NA NA NA #> 6: NA NA NA # adding square root transformed forecasts to the original ones example_quantile %>% .[, observed := ifelse(observed < 0, 0, observed)] %>% as_forecast() %>% transform_forecasts(fun = sqrt, label = \"sqrt\") %>% score() %>% summarise_scores(by = c(\"model\", \"scale\")) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> model scale wis overprediction underprediction #> #> 1: EuroCOVIDhub-ensemble natural 5796.064569 1828.5715014 2120.6402853 #> 2: EuroCOVIDhub-baseline natural 11124.930667 3884.4414062 5143.5356658 #> 3: epiforecasts-EpiNow2 natural 7514.375476 2866.4071466 1697.2341137 #> 4: UMass-MechBayes natural 52.651946 8.9786005 16.8009511 #> 5: EuroCOVIDhub-ensemble sqrt 14.974344 5.5037665 5.1827454 #> 6: EuroCOVIDhub-baseline sqrt 27.742316 10.4190016 9.5936380 #> 7: epiforecasts-EpiNow2 sqrt 17.704899 6.5700431 5.7235785 #> 8: UMass-MechBayes sqrt 1.328653 0.3273746 0.4019195 #> dispersion bias interval_coverage_50 interval_coverage_90 #> #> 1: 1846.8527819 0.00812500 0.6328125 0.9023438 #> 2: 2096.9535954 0.21816406 0.4960938 0.9101562 #> 3: 2950.7342158 -0.04336032 0.4453441 0.8461538 #> 4: 26.8723947 -0.02234375 0.4609375 0.8750000 #> 5: 4.2878323 0.00812500 0.6328125 0.9023438 #> 6: 7.7296761 0.21816406 0.4960938 0.9101562 #> 7: 5.4112770 -0.04336032 0.4453441 0.8461538 #> 8: 0.5993586 -0.02234375 0.4609375 0.8750000 #> interval_coverage_deviation ae_median #> #> 1: 0.050752841 8880.542969 #> 2: 0.003522727 16156.871094 #> 3: -0.057861612 11208.072874 #> 4: -0.024886364 78.476562 #> 5: 0.050752841 22.458900 #> 6: 0.003522727 39.185406 #> 7: -0.057861612 25.585018 #> 8: -0.024886364 2.069103 # adding multiple transformations example_quantile %>% as_forecast() %>% .[, observed := ifelse(observed < 0, 0, observed)] %>% transform_forecasts(fun = log_shift, offset = 1) %>% transform_forecasts(fun = sqrt, label = \"sqrt\") %>% head() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Warning: ! Error in validating forecast object: Error in #> assert_forecast_generic(forecast, verbose) : ! After removing rows with NA #> values in the data, no forecasts are left. . #> Forecast type: quantile #> Forecast unit: #> location, target_end_date, target_type, location_name, forecast_date, model, #> horizon, and scale #> #> location target_end_date target_type observed location_name forecast_date #> #> 1: DE 2021-01-02 Cases 127300 Germany #> 2: DE 2021-01-02 Deaths 4534 Germany #> 3: DE 2021-01-09 Cases 154922 Germany #> 4: DE 2021-01-09 Deaths 6117 Germany #> 5: DE 2021-01-16 Cases 110183 Germany #> 6: DE 2021-01-16 Deaths 5867 Germany #> quantile_level predicted model horizon scale #> #> 1: NA NA NA natural #> 2: NA NA NA natural #> 3: NA NA NA natural #> 4: NA NA NA natural #> 5: NA NA NA natural #> 6: NA NA NA natural"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Re-validate an existing forecast object — validate_forecast","title":"Re-validate an existing forecast object — validate_forecast","text":"function re-validates existing forecast object. similar assert_forecast(), returns input data instead invisible NULL. See as_forecast() details expected input formats.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Re-validate an existing forecast object — validate_forecast","text":"","code":"validate_forecast(forecast, forecast_type = NULL, verbose = TRUE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Re-validate an existing forecast object — validate_forecast","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()) forecast_type (optional) forecast type expect forecasts . forecast type determined scoringutils based input match , error thrown. NULL (default), forecast type inferred data. verbose Logical. FALSE (default TRUE), messages warnings created.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Re-validate an existing forecast object — validate_forecast","text":"Depending forecast type, object class forecast_binary, forecast_point, forecast_sample forecast_quantile.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_forecast.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Re-validate an existing forecast object — validate_forecast","text":"","code":"forecast <- as_forecast(example_binary) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. assert_forecast(forecast) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected."},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Validate metrics — validate_metrics","title":"Validate metrics — validate_metrics","text":"function validates whether list metrics list valid functions. function used score() make sure metrics valid functions.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Validate metrics — validate_metrics","text":"","code":"validate_metrics(metrics)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_metrics.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Validate metrics — validate_metrics","text":"metrics named list metrics. Every element scoring function applied data.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_metrics.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Validate metrics — validate_metrics","text":"named list metrics, filtered valid functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Validate an object of class scores — validate_scores","title":"Validate an object of class scores — validate_scores","text":"function validates object class scores, checking correct class metrics attribute.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Validate an object of class scores — validate_scores","text":"","code":"validate_scores(scores)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Validate an object of class scores — validate_scores","text":"scores data.table similar scores produced score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_scores.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Validate an object of class scores — validate_scores","text":"Returns NULL invisibly","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":null,"dir":"Reference","previous_headings":"","what":"Weighted interval score (WIS) — wis","title":"Weighted interval score (WIS) — wis","text":"WIS proper scoring rule used evaluate forecasts interval- / quantile-based format. See Bracher et al. (2021). Smaller values better. name suggest score assumes forecast comes form one multiple central prediction intervals. prediction interval characterised lower upper bound formed pair predictive quantiles. example, 50% central prediction interval formed 0.25 0.75 quantiles predictive distribution. Interval score interval score () sum three components: overprediction, underprediction dispersion. single prediction interval one components non-zero. single prediction interval observed value lower bound, interval score equal absolute difference lower bound observed value (\"underprediction\"). \"Overprediction\" defined analogously. observed value falls within bounds prediction interval, interval score equal width prediction interval, .e. difference upper lower bound. single interval, therefore : $$ \\textrm{} = (\\textrm{upper} - \\textrm{lower}) + \\frac{2}{\\alpha}(\\textrm{lower} - \\textrm{observed}) * \\mathbf{1}(\\textrm{observed} < \\textrm{lower}) + \\frac{2}{\\alpha}(\\textrm{observed} - \\textrm{upper}) * \\mathbf{1}(\\textrm{observed} > \\textrm{upper}) $$ \\(\\mathbf{1}()\\) indicator function indicates much outside prediction interval. \\(\\alpha\\) decimal value indicates much outside prediction interval. 90% prediction interval, example, \\(\\alpha\\) equal 0.1. specific distribution assumed, interval formed quantiles symmetric around median (.e use 0.1 quantile lower bound 0.7 quantile upper bound). Non-symmetric quantiles can scored using function quantile_score(). Usually interval score weighted factor makes sure average score across increasing number equally spaced quantiles, converges continuous ranked probability score (CRPS). weighted score called weighted interval score (WIS). weight commonly used \\(\\alpha / 2\\). Quantile score addition interval score, also exists quantile score (QS) (see quantile_score()), equal -called pinball loss. quantile score can computed single quantile (whereas interval score requires two quantiles form interval). However, intuitive decomposition overprediction, underprediction dispersion exist quantile score. Two versions weighted interval score two ways conceptualise weighted interval score across several quantiles / prediction intervals median. one view, treat WIS average quantile scores (median 0.5-quantile) (default wis()). another view, treat WIS average several interval scores + difference observed value median forecast. effect contrast first view, median twice much weight (weighted like prediction interval, rather like single quantile). valid ways conceptualise WIS can control behaviour count_median_twice-argument. WIS components: WIS components can computed individually using functions overprediction, underprediction, dispersion.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Weighted interval score (WIS) — wis","text":"","code":"wis( observed, predicted, quantile_level, separate_results = FALSE, weigh = TRUE, count_median_twice = FALSE, na.rm = TRUE ) dispersion(observed, predicted, quantile_level, ...) overprediction(observed, predicted, quantile_level, ...) underprediction(observed, predicted, quantile_level, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Weighted interval score (WIS) — wis","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made. separate_results Logical. TRUE (default FALSE), separate parts interval score (dispersion penalty, penalties - -prediction get returned separate elements list). want data.frame instead, simply call .data.frame() output. weigh Logical. TRUE (default), weigh score \\(\\alpha / 2\\), can averaged interval score , limit (increasing number equally spaced quantiles/prediction intervals), corresponds CRPS. \\(\\alpha\\) value corresponds (\\(\\alpha/2\\)) (\\(1 - \\alpha/2\\)), .e. decimal value represents much outside central prediction interval (E.g. 90 percent central prediction interval, alpha 0.1). count_median_twice TRUE, count median twice score. na.rm TRUE, ignore NA values computing score. ... Additional arguments passed wis() functions overprediction(), underprediction() dispersion().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Weighted interval score (WIS) — wis","text":"wis(): numeric vector WIS values size n (one per observation), list separate entries separate_results TRUE. dispersion(): numeric vector dispersion values (one per observation). overprediction(): numeric vector overprediction values (one per observation). underprediction(): numeric vector underprediction values (one per observation)","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Weighted interval score (WIS) — wis","text":"","code":"observed <- c(1, -15, 22) predicted <- rbind( c(-1, 0, 1, 2, 3), c(-2, 1, 2, 2, 4), c(-2, 0, 3, 3, 4) ) quantile_level <- c(0.1, 0.25, 0.5, 0.75, 0.9) wis(observed, predicted, quantile_level) #> [1] 0.36 15.34 19.14"},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-1229000","dir":"Changelog","previous_headings":"","what":"scoringutils 1.2.2.9000","title":"scoringutils 1.2.2.9000","text":"update represents major rewrite package introduces breaking changes. want keep using older version, can download using remotes::install_github(\"epiforecasts/scoringutils@v1.2\"). update aims make package modular customisable overall cleaner easier work . particular, aimed make suggested workflows evaluating forecasts explicit easier follow (see visualisation ). , clarified input formats made consistent across functions. refactord many functions S3-methods introduced forecast objects separate classes different types forecasts. new function, as_forecast() introduced validate data convert inputs forecast object. Another major update possibility users pass scoring functions score(). updated improved function documentation added new vignettes guide users package. Internally, refactored code, improved input checks, updated notifications (now use cli package) increased test coverage. comprehensive documentation new package rewrite revised version original scoringutils paper.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"score-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"score()","title":"scoringutils 1.2.2.9000","text":"previous columns “true_value” “prediction” renamed. score() now requires columns called “observed” “predicted” “model”. column quantile renamed quantile_level sample renamed sample_id score() now generic. S3 methods classes forecast_point, forecast_binary, forecast_quantile forecast_sample, correspond different forecast types can scored scoringutils. score() now calls na.omit() data, instead removing rows missing values columns observed predicted. NA values columns can also mess e.g. grouping forecasts according unit single forecast. score() many functions now require validated forecast object. forecast objects can created using function as_forecast() (replaces previous check_forecast()). score() now returns objects class scores stored attribute metrics holds names scoring rules used. Users can call get_metrics() access names scoring rules. score() now returns one score per forecast, instead one score per sample quantile. Users can now also use scoring rules (making use metrics argument, takes named list functions). Default scoring rules can accessed using functions metrics_point(), metrics_sample(), metrics_quantile() metrics_binary(), return named list scoring rules suitable respective forecast type. Column names scores output score() correspond names scoring rules (.e. names functions list metrics). Instead supplying arguments score() manipulate individual scoring rules users now manipulate metric list supplied using customise_metric() select_metric().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"creating-a-forecast-object-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Creating a forecast object","title":"scoringutils 1.2.2.9000","text":"function as_forecast() creates forecast object validates . as_forecast() also allows users rename/specify required columns specify forecast unit single step, taking functionality set_forecast_unit() cases.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"updated-workflows-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Updated workflows","title":"scoringutils 1.2.2.9000","text":"example workflow scoring forecast now looks like : Overall, updated suggested workflows users work package. following gives overview (see updated paper details).","code":"forecast_quantile <- as_forecast( example_quantile, observed = \"observed\", predicted = \"predicted\", model = \"model\", quantile_level = \"quantile_level\", forecast_unit = c(\"model\", \"location\", \"target_end_date\", \"forecast_date\", \"target_type\") ) scores <- score(forecast_quantile)"},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"input-formats-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Input formats","title":"scoringutils 1.2.2.9000","text":"standardised input formats score() well scoring rules exported scoreingutils. following plot gives overview expected input formats different forecast types score(). Support interval format mostly dropped (see PR #525 @nikosbosse reviewed @seabbs). co-existence quantile interval format let confusing user experience many duplicated functions providing functionality. decided simplify interface focusing exclusively quantile format. function bias_range() removed (users now use bias_quantile() instead) function interval_score() made internal function rather exported users. recommend using wis() instead.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"re-validating-forecast-objects-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"(Re-)Validating forecast objects","title":"scoringutils 1.2.2.9000","text":"create validate new forecast object, users can use as_forecast(). revalidate existing forecast object users can call assert_forecast() (validates input returns invisible(NULL). assert_forecast() generic methods different forecast types. Alternatively, validate_forecast() can used (calls assert_forecast()), returns input useful pipe. Lastly, users can simply print object obtain additional information. Users can test whether object class forecast_*() using function is_forecast(). Users can also test specific forecast_* class using appropriate is_forecast.forecast_* method. example, check whether object class forecast_quantile, use use scoringutils:::is_forecast.forecast_quantile().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"pairwise-comparisons-and-relative-skill-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Pairwise comparisons and relative skill","title":"scoringutils 1.2.2.9000","text":"functionality computing pairwise comparisons now split summarise_scores(). Instead pairwise comparisons part summarising scores, new function, add_relative_skill(), introduced takes summarised scores input adds columns relative skill scores scaled relative skill scores. function pairwise_comparison() renamed get_pairwise_comparisons(), line get_-functions. Analogously, plot_pairwise_comparison() renamed plot_pairwise_comparisons(). Output columns pairwise comparisons renamed contain name metric used comparing. Replaced warnings errors get_pairwise_comparison avoid returning NULL","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"computing-coverage-values-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Computing coverage values","title":"scoringutils 1.2.2.9000","text":"add_coverage() replaced new function, get_coverage(). function comes updated workflow coverage values computed directly based original data can visualised using plot_interval_coverage() plot_quantile_coverage(). example workflow example_quantile |> as_forecast() |> get_coverage(= \"model\") |> plot_interval_coverage().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"obtaining-and-plotting-forecast-counts-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Obtaining and plotting forecast counts","title":"scoringutils 1.2.2.9000","text":"clarity, output column get_forecast_counts() renamed “Number forecasts” “count”. get_forecast_counts() now also displays combinations 0 forecasts, instead silently dropping corresponding rows. plot_avail_forecasts() renamed plot_forecast_counts() line change function name. x argument longer default value, value depend data provided user.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"renamed-functions-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Renamed functions","title":"scoringutils 1.2.2.9000","text":"function find_duplicates() renamed get_duplicate_forecasts() Renamed interval_coverage_quantile() interval_coverage_dev_quantile() interval_coverage() interval_coverage_deviation(), respectively. “range” consistently renamed “interval_range” code. “range”-format (mostly used internally) renamed “interval”-format Renamed correlation() get_correlations() plot_correlation() plot_correlations() pit() renamed get_pit().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"deleted-functions-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Deleted functions","title":"scoringutils 1.2.2.9000","text":"Removed abs_error squared_error package favour Metrics::ae Metrics::se. Deleted function plot_ranges(). want continue using functionality, can find function code Deprecated-visualisations Vignette. Removed function plot_predictions(), well helper function make_NA(), favour dedicated Vignette shows different ways visualising predictions. future reference, function code can found (Issue #659) Deprecated-visualisations Vignette. Removed function plot_score_table(). can find code Deprecated-visualisations Vignette. Removed function merge_pred_and_obs() used merge two separate data frames forecasts observations. moved contents new “Deprecated functions”-vignette. Removed interval_coverage_sample() users now expected convert quantile format first scoring.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"function-changes-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Function changes","title":"scoringutils 1.2.2.9000","text":"bias_quantile() changed way handles forecasts median missing: median now imputed linear interpolation innermost quantiles. Previously, imputed median simply taking mean innermost quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"internal-package-updates-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Internal package updates","title":"scoringutils 1.2.2.9000","text":"deprecated ..density.. replaced after_stat(density) ggplot calls. Files ending “.Rda” renamed “.rds” appropriate used together saveRDS() readRDS(). Added subsetting [ operator scores, score name attribute gets preserved subsetting. Switched using cli condition handling signalling, added tests check_*() test_*() functions. See #583 @jamesmbaazam reviewed @nikosbosse @seabbs.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"documentation-and-testing-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Documentation and testing","title":"scoringutils 1.2.2.9000","text":"Updates documentation functions made sure functions documented return statements Documentation pkgdown pages now created stable dev versions. Added unit tests many functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-122","dir":"Changelog","previous_headings":"","what":"scoringutils 1.2.2","title":"scoringutils 1.2.2","text":"CRAN release: 2023-11-29","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-2-2","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.2.2","text":"scoringutils now depends R 3.6. change made since packages testthat lifecycle, used scoringutils now require R 3.6. also updated Github action CI check work R 3.6 now. Added new PR template checklist things included PRs facilitate development review process","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"bug-fixes-1-2-2","dir":"Changelog","previous_headings":"","what":"Bug fixes","title":"scoringutils 1.2.2","text":"Fixes bug set_forecast_unit() function worked data.table, data.frame input. metrics table vignette Details metrics implemented scoringutils duplicated entries. fixed removing duplicated rows.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-2-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.2.1","text":"Gets rid preferably package _pkgdown.yml. theme toggle light dark theme didn’t work properly Updates gh pages deploy action v4 also cleans files triggered Introduces gh action automatically render Readme Readme.Rmd Removes links vignettes renamed","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-120","dir":"Changelog","previous_headings":"","what":"scoringutils 1.2.0","title":"scoringutils 1.2.0","text":"major release contains range new features bug fixes introduced minor releases since 1.1.0. important changes : Documentation updated reflect changes since version 1.1.0, including new transform workflow functions. New set_forecast_unit() function allows manual setting forecast unit. summarise_scores() gains new across argument summarizing across variables. New transform_forecasts() log_shift() functions allow forecast transformations. See documentation transform_forecasts() details example use case. Input checks test coverage improved bias functions. Bug fix get_prediction_type() integer matrix input. Links scoringutils paper citation updates. Warning added interval_score() small interval ranges. Linting updates improvements. Thanks @nikosbosse, @seabbs, @sbfnk code review contributions. Thanks @bisaloo suggestion use linting GitHub Action triggers changes, @adrian-lison suggestion add warning interval_score() interval range 0 1.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-2-0","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.2.0","text":"documentation updated reflect recent changes since scoringutils 1.1.0. particular, usage functions set_forecast_unit(), check_forecasts() transform_forecasts() now documented Vignettes. introduction functions enhances overall workflow help make code readable. functions designed used together pipe operator. example, one can now use something like following: Documentation transform_forecasts() also extended. functions allows user easily add transformations forecasts, suggested paper “Scoring epidemiological forecasts transformed scales”. epidemiological context, example, may make sense apply natural logarithm first scoring forecasts, order obtain scores reflect well models able predict exponential growth rates, rather absolute values. Users can now something like following score transformed version data addition original one: use log_shift() function apply logarithmic transformation forecasts. function introduced scoringutils 1.1.2 helper function acts just like log(), additional argument offset can add number every prediction observed value applying log transformation.","code":"example_quantile |> set_forecast_unit(c(\"model\", \"location\", \"forecast_date\", \"horizon\", \"target_type\")) |> check_forecasts() |> score() data <- example_quantile[true_value > 0, ] data |> transform_forecasts(fun = log_shift, offset = 1) |> score() |> summarise_scores(by = c(\"model\", \"scale\"))"},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-1-2-0","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.2.0","text":"Made check_forecasts() score() pipeable (see issue #290). means users can now directly use output check_forecasts() input score(). score() otherwise runs check_forecasts() internally anyway simply makes step explicit helps writing clearer code.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-117","dir":"Changelog","previous_headings":"","what":"scoringutils 1.1.7","title":"scoringutils 1.1.7","text":"Release @seabbs #305. Reviewed @nikosbosse @sbfnk.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"breaking-changes-1-1-7","dir":"Changelog","previous_headings":"","what":"Breaking changes","title":"scoringutils 1.1.7","text":"prediction_type argument get_forecast_unit() changed dropped. Instead new internal function prediction_is_quantile() used detect quantile variable present. Whilst internal function may impact users accessible via `find_duplicates().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-1-7","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.7","text":"Made imputation median bias_range() bias_quantile() obvious user may cause unexpected behaviour. Simplified bias_range() uses bias_quantile() internally. Added additional input checks bias_range(), bias_quantile(), check_predictions() make sure input valid. Improve coverage unit tests bias_range(), bias_quantile(), bias_sample(). Updated pairwise comparison unit tests use realistic data.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"bug-fixes-1-1-7","dir":"Changelog","previous_headings":"","what":"Bug fixes","title":"scoringutils 1.1.7","text":"Fixed bug get_prediction_type() led unable correctly detect integer (instead categorising continuous) forecasts input matrix. issue impacted bias_sample() also score() used integer forecasts resulting lower bias scores expected.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-1-1-6","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.1.6","text":"Added new argument, across, summarise_scores(). argument allows user summarise scores across different forecast units alternative specifying . See documentation summarise_scores() details example use case.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-1-1-5","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.1.5","text":"Added new function, set_forecast_unit() allows user set forecast unit manually. function removes columns relevant uniquely identifying single forecast. done manually, scoringutils attempts determine unit single automatically simply assuming column names relevant determine forecast unit. can lead unexpected behaviour, setting forecast unit explicitly can help make code easier debug easier read (see issue #268). used part workflow, set_forecast_unit() can directly piped check_forecasts() check everything order.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-1-4","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.4","text":"Added links scoringutils paper Evaluating Forecasts scoringutils R package. Updated citation formatting comply newer standards.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-1-3","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.3","text":"Added warning interval_score() interval range 0 1. Thanks @adrian-lison (see #277) suggestion.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-1-3-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.3","text":"Switched linting GitHub Action triggers changes. Inspired @bisaloo recent contribution epinowcast package. Updated package linters extensive. Inspired @bisaloo recent contribution epinowcast package. Resolved flagged linting issues across package.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-1-1-2","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.1.2","text":"Added new function, transform_forecasts() make easy transform forecasts scoring , suggested Bosse et al. (2023), https://www.medrxiv.org/content/10.1101/2023.01.23.23284722v1. Added function, log_shift() implements default transformation function. function allows add offset applying logarithm.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-111","dir":"Changelog","previous_headings":"","what":"scoringutils 1.1.1","title":"scoringutils 1.1.1","text":"Added small change interval_score() explicitly converts logical vector numeric one. happen implicitly anyway, now done explicitly order avoid issues may come input vector type doesn’t allow implicit conversion.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-110","dir":"Changelog","previous_headings":"","what":"scoringutils 1.1.0","title":"scoringutils 1.1.0","text":"CRAN release: 2023-01-30 minor update package bug fixes minor changes.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-1-0","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.0","text":"Removed attach message warned breaking changes 1.0.0. Renamed metric argument summarise_scores() relative_skill_metric. argument now deprecated removed future version package. Please use new argument instead. Updated documentation score() related functions make soft requirement model column input data explicit. Updated documentation score(), pairwise_comparison() summarise_scores() make clearer unit single forecast required computations Simplified function plot_pairwise_comparison() now supports plotting mean score ratios p-values removed hybrid option print time.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"bug-fixes-1-1-0","dir":"Changelog","previous_headings":"","what":"Bug fixes","title":"scoringutils 1.1.0","text":"Missing baseline forecasts pairwise_comparison() now trigger explicit informative error message. requirements table getting started vignette now correct. Added support optional sample column using quantile forecast format. Previously resulted error.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-100","dir":"Changelog","previous_headings":"","what":"scoringutils 1.0.0","title":"scoringutils 1.0.0","text":"CRAN release: 2022-05-13 Major update package package functions lots breaking changes.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-1-0-0","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.0.0","text":"New updated Readme vignette. proposed scoring workflow reworked. Functions changed can easily piped simplified arguments outputs.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"new-functions-and-function-changes-1-0-0","dir":"Changelog","previous_headings":"Feature updates","what":"New functions and function changes","title":"scoringutils 1.0.0","text":"function eval_forecasts() replaced function score() much reduced set function arguments. Functionality summarise scores add relative skill scores moved function summarise_scores() New function check_forecasts() analyse input data scoring New function correlation() compute correlations different metrics New function add_coverage() add coverage specific central prediction intervals. New function avail_forecasts() allows visualise number available forecasts. New function find_duplicates() find duplicate forecasts cause error. plotting functions renamed begin plot_. Arguments simplified. function pit() now works based data.frames. old pit function renamed pit_sample(). PIT p-values removed entirely. function plot_pit() now works directly input produced pit() Many data-handling functions removed input types score() restricted sample-based, quantile-based binary forecasts. function brier_score() now returns brier scores, rather taking mean returning output. crps(), dss() logs() renamed crps_sample(), dss_sample(), logs_sample()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"bug-fixes-1-0-0","dir":"Changelog","previous_headings":"Feature updates","what":"Bug fixes","title":"scoringutils 1.0.0","text":"Testing expanded Minor bugs fixed, example bug sample_to_quantile() function (https://github.com/epiforecasts/scoringutils/pull/223)","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-data-updated-1-0-0","dir":"Changelog","previous_headings":"Feature updates","what":"Package data updated","title":"scoringutils 1.0.0","text":"Package data now based forecasts submitted European Forecast Hub (https://covid19forecasthub.eu/). example data files renamed begin example_. new data set, summary_metrics included contains summary metrics implemented scoringutils.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"other-breaking-changes-1-0-0","dir":"Changelog","previous_headings":"","what":"Other breaking changes","title":"scoringutils 1.0.0","text":"‘sharpness’ component weighted interval score renamed dispersion. done make clear component represents maintain consistency used places.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-8","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.8","text":"Added function check_forecasts() runs basic checks input data provides feedback.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-0172","dir":"Changelog","previous_headings":"","what":"scoringutils 0.1.7.2","title":"scoringutils 0.1.7.2","text":"CRAN release: 2021-07-21","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-7-2","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.7.2","text":"Minor bug fixes (previously, ‘interval_score’ needed among selected metrics). data.tables now returned table[] rather table, don’t called twice display contents.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-017","dir":"Changelog","previous_headings":"","what":"scoringutils 0.1.7","title":"scoringutils 0.1.7","text":"CRAN release: 2021-07-14","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-7","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.7","text":"Added function, pairwise_comparison() runs pairwise comparisons models output eval_forecasts() Added functionality compute relative skill within eval_forecasts(). Added function visualise pairwise comparisons.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-7","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.7","text":"WIS definition change introduced version 0.1.5 partly corrected difference weighting introduced summarising scores different interval ranges. “sharpness” renamed ‘mad’ output [score()] sample-based forecasts.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.","text":"eval_forecasts() can now handle separate forecast truth data set input. eval_forecasts() now supports scoring point forecasts along side quantiles quantile-based format. Currently metric used absolute error.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.","text":"Many functions, especially eval_forecasts() got major rewrite. functionality unchanged, code now easier maintain data-handling functions got renamed, old names supported well now.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-5","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.5","text":"Changed default definition weighted interval score. Previously, median prediction counted twice, counted . want go back old behaviour, can call interval_score function argument count_median_twice = FALSE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-014","dir":"Changelog","previous_headings":"","what":"scoringutils 0.1.4","title":"scoringutils 0.1.4","text":"CRAN release: 2020-11-17","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-4","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.4","text":"Added basic plotting functionality visualise scores. can now easily obtain diagnostic plots based scores produced score. correlation_plot() shows correlation metrics. plot_ranges() shows contribution different prediction intervals chosen metric. plot_heatmap() visualises scores heatmap. plot_score_table() shows coloured summary table scores.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-4","dir":"Changelog","previous_headings":"","what":"package updates","title":"scoringutils 0.1.4","text":"Renamed “calibration” “coverage”. Renamed “true_values” “true_value” data.frames. Renamed “predictions” “prediction” data.frames. Renamed “is_overprediction” “overprediction”. Renamed “is_underprediction” “underprediction”.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"potentially-breaking-changes-0-1-3","dir":"Changelog","previous_headings":"","what":"(Potentially) Breaking changes","title":"scoringutils 0.1.3","text":"argument score now slightly changed meaning. now denotes lowest possible grouping unit, .e. unit one observation needs specified explicitly. default now NULL. reason change metrics need scoring observation level consistent implementation principle. pit function receives grouping now summarise_by. similar spirit, summarise_by specified explicitly e.g. doesn’t assume anymore want ‘range’ included. interval score, weigh = TRUE now default option. Renamed true_values true_value predictions prediction.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-3","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.3","text":"Updated quantile evaluation metrics score. Bias well calibration now take quantiles account. Included option summarise scores according summarise_by argument score() summary can return mean, standard deviation well arbitrary set quantiles. score() can now return pit histograms. Switched ggplot2 plotting.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"potentially-breaking-changes-0-1-2","dir":"Changelog","previous_headings":"","what":"(Potentially) Breaking changes","title":"scoringutils 0.1.2","text":"scores score consistently renamed lower case. Interval_score now interval_score, CRPS now crps etc.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-2","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.2","text":"Included support grouping scores according vector column names score(). Included support passing arguments lower-level functions score() Included support three new metrics score quantiles score(): bias, sharpness calibration","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-2","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.2","text":"Example data now horizon column illustrate use grouping. Documentation updated explain listed changes.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-1","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.1","text":"Included support long well wide input formats quantile forecasts scored score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.1","text":"Updated documentation score(). Added badges README.","code":""}]