vignettes/scoring-rules.Rmd
scoring-rules.Rmd
QSτ(F,y)=2⋅{𝟏(y≤qτ)−τ}⋅(qτ−y)={2⋅(1−τ)*qτ−y,if y≤qτ2⋅τ*|qτ−y|,if y>qτ,\begin{equation} - \text{QS}_\tau(F, y) = 2 \cdot \{ \mathbf{1}(y \leq q_\tau) - \tau\} \cdot (q_\tau − y) = + \text{QS}_\tau(F, y) = 2 \cdot \{ \mathbf{1}(y \leq q_\tau) - \tau\} \cdot (q_\tau - y) = \begin{cases} 2 \cdot (1 - \tau) * q_\tau - y, & \text{if } y \leq q_\tau\\ 2 \cdot \tau * |q_\tau - y|, & \text{if } y > q_\tau, diff --git a/dev/pkgdown.yml b/dev/pkgdown.yml index 1091cf7d..ea16f7ea 100644 --- a/dev/pkgdown.yml +++ b/dev/pkgdown.yml @@ -5,7 +5,7 @@ articles: Deprecated-functions: Deprecated-functions.html Deprecated-visualisations: Deprecated-visualisations.html scoring-rules: scoring-rules.html -last_built: 2024-07-16T15:19Z +last_built: 2024-07-17T10:49Z urls: reference: https://epiforecasts.io/scoringutils/reference article: https://epiforecasts.io/scoringutils/articles diff --git a/dev/reference/metrics_binary.html b/dev/reference/metrics_binary.html index a6e9f4fa..9b2b203a 100644 --- a/dev/reference/metrics_binary.html +++ b/dev/reference/metrics_binary.html @@ -93,7 +93,7 @@ Examples#> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } -#> <bytecode: 0x55703af1ec30> +#> <bytecode: 0x55f0c14d3240> #> <environment: namespace:scoringutils> #> #> $log_score @@ -104,7 +104,7 @@ Examples#> logs <- -log(1 - abs(observed - predicted)) #> return(logs) #> } -#> <bytecode: 0x55703af2d6e8> +#> <bytecode: 0x55f0c14d2408> #> <environment: namespace:scoringutils> #> metrics_binary(select = "brier_score") @@ -116,7 +116,7 @@ Examples#> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } -#> <bytecode: 0x55703af1ec30> +#> <bytecode: 0x55f0c14d3240> #> <environment: namespace:scoringutils> #> metrics_binary(exclude = "log_score") @@ -128,7 +128,7 @@ Examples#> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } -#> <bytecode: 0x55703af1ec30> +#> <bytecode: 0x55f0c14d3240> #> <environment: namespace:scoringutils> #> diff --git a/dev/reference/metrics_point.html b/dev/reference/metrics_point.html index ffd2124f..dbdc2b85 100644 --- a/dev/reference/metrics_point.html +++ b/dev/reference/metrics_point.html @@ -93,7 +93,7 @@ Examples#> { #> return(abs(actual - predicted)) #> } -#> <bytecode: 0x557036394a18> +#> <bytecode: 0x55f0bc2b90e8> #> <environment: namespace:Metrics> #> #> $se_point @@ -101,7 +101,7 @@ Examples#> { #> return((actual - predicted)^2) #> } -#> <bytecode: 0x557036395380> +#> <bytecode: 0x55f0bc2b9a50> #> <environment: namespace:Metrics> #> #> $ape @@ -109,7 +109,7 @@ Examples#> { #> return(ae(actual, predicted)/abs(actual)) #> } -#> <bytecode: 0x557036391ef0> +#> <bytecode: 0x55f0bc2ba3f0> #> <environment: namespace:Metrics> #> metrics_point(select = "ape") @@ -118,7 +118,7 @@ Examples#> { #> return(ae(actual, predicted)/abs(actual)) #> } -#> <bytecode: 0x557036391ef0> +#> <bytecode: 0x55f0bc2ba3f0> #> <environment: namespace:Metrics> #> diff --git a/dev/reference/metrics_quantile.html b/dev/reference/metrics_quantile.html index 4ee0772c..660af3cb 100644 --- a/dev/reference/metrics_quantile.html +++ b/dev/reference/metrics_quantile.html @@ -171,7 +171,7 @@ Examples#> return(reformatted$wis) #> } #> } -#> <bytecode: 0x557035ce32b0> +#> <bytecode: 0x55f0bbc4b868> #> <environment: namespace:scoringutils> #> #> $overprediction @@ -184,7 +184,7 @@ Examples#> args)) #> return(out$overprediction) #> } -#> <bytecode: 0x557035ce2d98> +#> <bytecode: 0x55f0bbc45ad0> #> <environment: namespace:scoringutils> #> #> $underprediction @@ -197,7 +197,7 @@ Examples#> args)) #> return(out$underprediction) #> } -#> <bytecode: 0x557035cde548> +#> <bytecode: 0x55f0bbc43248> #> <environment: namespace:scoringutils> #> #> $dispersion @@ -210,7 +210,7 @@ Examples#> args)) #> return(out$dispersion) #> } -#> <bytecode: 0x557035cdbcc0> +#> <bytecode: 0x55f0bbc40b10> #> <environment: namespace:scoringutils> #> #> $bias @@ -231,7 +231,7 @@ Examples#> }) #> return(bias) #> } -#> <bytecode: 0x55703a8e7168> +#> <bytecode: 0x55f0c0812540> #> <environment: namespace:scoringutils> #> #> $interval_coverage_50 @@ -254,7 +254,7 @@ Examples#> (observed <= upper))] #> return(reformatted$interval_coverage) #> } -#> <bytecode: 0x557035e2d030> +#> <bytecode: 0x55f0bbd87290> #> <environment: namespace:scoringutils> #> #> $interval_coverage_90 @@ -262,8 +262,8 @@ Examples#> { #> do.call(metric, c(list(...), dots)) #> } -#> <bytecode: 0x5570361abe18> -#> <environment: 0x557037a25de8> +#> <bytecode: 0x55f0bc0f5678> +#> <environment: 0x55f0bd9619e8> #> #> $interval_coverage_deviation #> function (observed, predicted, quantile_level) @@ -289,7 +289,7 @@ Examples#> by = "forecast_id"] #> return(out$interval_coverage_deviation) #> } -#> <bytecode: 0x557035cda820> +#> <bytecode: 0x55f0bbc3f670> #> <environment: namespace:scoringutils> #> #> $ae_median @@ -308,7 +308,7 @@ Examples#> abs_error_median <- abs(observed - predicted) #> return(abs_error_median) #> } -#> <bytecode: 0x55703b258b88> +#> <bytecode: 0x55f0c117f970> #> <environment: namespace:scoringutils> #> metrics_quantile(select = "wis") @@ -350,7 +350,7 @@ Examples#> return(reformatted$wis) #> } #> } -#> <bytecode: 0x557035ce32b0> +#> <bytecode: 0x55f0bbc4b868> #> <environment: namespace:scoringutils> #> diff --git a/dev/reference/metrics_sample.html b/dev/reference/metrics_sample.html index 6deafd5a..b15fa54e 100644 --- a/dev/reference/metrics_sample.html +++ b/dev/reference/metrics_sample.html @@ -117,7 +117,7 @@ Examples#> return(res) #> } #> } -#> <bytecode: 0x5570361ffdb8> +#> <bytecode: 0x55f0bc130d10> #> <environment: namespace:scoringutils> #> #> $dss @@ -126,7 +126,7 @@ Examples#> assert_input_sample(observed, predicted) #> scoringRules::dss_sample(y = observed, dat = predicted, ...) #> } -#> <bytecode: 0x55703969e8b8> +#> <bytecode: 0x55f0bf5c1a08> #> <environment: namespace:scoringutils> #> #> $crps @@ -136,7 +136,7 @@ Examples#> scoringRules::crps_sample(y = observed, dat = predicted, #> ...) #> } -#> <bytecode: 0x557038f09800> +#> <bytecode: 0x55f0bee1cbd0> #> <environment: namespace:scoringutils> #> #> $log_score @@ -146,7 +146,7 @@ Examples#> scoringRules::logs_sample(y = observed, dat = predicted, #> ...) #> } -#> <bytecode: 0x557038ab6190> +#> <bytecode: 0x55f0c394cc00> #> <environment: namespace:scoringutils> #> #> $mad @@ -156,7 +156,7 @@ Examples#> sharpness <- apply(predicted, MARGIN = 1, mad, ...) #> return(sharpness) #> } -#> <bytecode: 0x55703ac8e7a8> +#> <bytecode: 0x55f0c15d1498> #> <environment: namespace:scoringutils> #> #> $ae_median @@ -168,7 +168,7 @@ Examples#> ae_median <- abs(observed - median_predictions) #> return(ae_median) #> } -#> <bytecode: 0x5570396b89d0> +#> <bytecode: 0x55f0bf5e00e0> #> <environment: namespace:scoringutils> #> #> $se_mean @@ -179,7 +179,7 @@ Examples#> se_mean <- (observed - mean_predictions)^2 #> return(se_mean) #> } -#> <bytecode: 0x55703a3bc9e0> +#> <bytecode: 0x55f0c1422850> #> <environment: namespace:scoringutils> #> metrics_sample(select = "mad") @@ -190,7 +190,7 @@ Examples#> sharpness <- apply(predicted, MARGIN = 1, mad, ...) #> return(sharpness) #> } -#> <bytecode: 0x55703ac8e7a8> +#> <bytecode: 0x55f0c15d1498> #> <environment: namespace:scoringutils> #> diff --git a/dev/reference/quantile_score.html b/dev/reference/quantile_score.html index 156878e6..c15b3d1d 100644 --- a/dev/reference/quantile_score.html +++ b/dev/reference/quantile_score.html @@ -6,7 +6,7 @@ The quantile score, also called pinball loss, for a single quantile level \(\tau\) is defined as $$ - \text{QS}_\tau(F, y) = 2 \cdot \{ \mathbf{1}(y \leq q_\tau) - \tau\} \cdot (q_\tau − y) = + \text{QS}_\tau(F, y) = 2 \cdot \{ \mathbf{1}(y \leq q_\tau) - \tau\} \cdot (q_\tau - y) = \begin{cases} 2 \cdot (1 - \tau) * q_\tau - y, & \text{if } y \leq q_\tau\\ 2 \cdot \tau * |q_\tau - y|, & \text{if } y > q_\tau, @@ -31,7 +31,7 @@ The quantile score, also called pinball loss, for a single quantile level \(\tau\) is defined as $$ - \text{QS}_\tau(F, y) = 2 \cdot \{ \mathbf{1}(y \leq q_\tau) - \tau\} \cdot (q_\tau − y) = + \text{QS}_\tau(F, y) = 2 \cdot \{ \mathbf{1}(y \leq q_\tau) - \tau\} \cdot (q_\tau - y) = \begin{cases} 2 \cdot (1 - \tau) * q_\tau - y, & \text{if } y \leq q_\tau\\ 2 \cdot \tau * |q_\tau - y|, & \text{if } y > q_\tau, @@ -97,7 +97,7 @@ Quantile score The quantile score, also called pinball loss, for a single quantile level \(\tau\) is defined as $$ - \text{QS}_\tau(F, y) = 2 \cdot \{ \mathbf{1}(y \leq q_\tau) - \tau\} \cdot (q_\tau − y) = + \text{QS}_\tau(F, y) = 2 \cdot \{ \mathbf{1}(y \leq q_\tau) - \tau\} \cdot (q_\tau - y) = \begin{cases} 2 \cdot (1 - \tau) * q_\tau - y, & \text{if } y \leq q_\tau\\ 2 \cdot \tau * |q_\tau - y|, & \text{if } y > q_\tau, diff --git a/dev/reference/select_metrics.html b/dev/reference/select_metrics.html index 2da9c6cc..18e0c6e6 100644 --- a/dev/reference/select_metrics.html +++ b/dev/reference/select_metrics.html @@ -88,7 +88,7 @@ Examples#> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } -#> <bytecode: 0x55703af1ec30> +#> <bytecode: 0x55f0c14d3240> #> <environment: namespace:scoringutils> #> select_metrics( @@ -103,7 +103,7 @@ Examples#> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } -#> <bytecode: 0x55703af1ec30> +#> <bytecode: 0x55f0c14d3240> #> <environment: namespace:scoringutils> #> diff --git a/dev/search.json b/dev/search.json index 2ae7246b..b82d1f38 100644 --- a/dev/search.json +++ b/dev/search.json @@ -1 +1 @@ -[{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"our-pledge","dir":"","previous_headings":"","what":"Our Pledge","title":"Contributor Covenant Code of Conduct","text":"members, contributors, leaders pledge make participation community harassment-free experience everyone, regardless age, body size, visible invisible disability, ethnicity, sex characteristics, gender identity expression, level experience, education, socio-economic status, nationality, personal appearance, race, religion, sexual identity orientation. pledge act interact ways contribute open, welcoming, diverse, inclusive, healthy community.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"our-standards","dir":"","previous_headings":"","what":"Our Standards","title":"Contributor Covenant Code of Conduct","text":"Examples behavior contributes positive environment community include: Demonstrating empathy kindness toward people respectful differing opinions, viewpoints, experiences Giving gracefully accepting constructive feedback Accepting responsibility apologizing affected mistakes, learning experience Focusing best just us individuals, overall community Examples unacceptable behavior include: use sexualized language imagery, sexual attention advances kind Trolling, insulting derogatory comments, personal political attacks Public private harassment Publishing others’ private information, physical email address, without explicit permission conduct reasonably considered inappropriate professional setting","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"enforcement-responsibilities","dir":"","previous_headings":"","what":"Enforcement Responsibilities","title":"Contributor Covenant Code of Conduct","text":"Community leaders responsible clarifying enforcing standards acceptable behavior take appropriate fair corrective action response behavior deem inappropriate, threatening, offensive, harmful. Community leaders right responsibility remove, edit, reject comments, commits, code, wiki edits, issues, contributions aligned Code Conduct, communicate reasons moderation decisions appropriate.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"scope","dir":"","previous_headings":"","what":"Scope","title":"Contributor Covenant Code of Conduct","text":"Code Conduct applies within community spaces, also applies individual officially representing community public spaces. Examples representing community include using official e-mail address, posting via official social media account, acting appointed representative online offline event.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"enforcement","dir":"","previous_headings":"","what":"Enforcement","title":"Contributor Covenant Code of Conduct","text":"Instances abusive, harassing, otherwise unacceptable behavior may reported community leaders responsible enforcement Sebastian.Funk@lshtm.ac.uk. complaints reviewed investigated promptly fairly. community leaders obligated respect privacy security reporter incident.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"enforcement-guidelines","dir":"","previous_headings":"","what":"Enforcement Guidelines","title":"Contributor Covenant Code of Conduct","text":"Community leaders follow Community Impact Guidelines determining consequences action deem violation Code Conduct:","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"id_1-correction","dir":"","previous_headings":"Enforcement Guidelines","what":"1. Correction","title":"Contributor Covenant Code of Conduct","text":"Community Impact: Use inappropriate language behavior deemed unprofessional unwelcome community. Consequence: private, written warning community leaders, providing clarity around nature violation explanation behavior inappropriate. public apology may requested.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"id_2-warning","dir":"","previous_headings":"Enforcement Guidelines","what":"2. Warning","title":"Contributor Covenant Code of Conduct","text":"Community Impact: violation single incident series actions. Consequence: warning consequences continued behavior. interaction people involved, including unsolicited interaction enforcing Code Conduct, specified period time. includes avoiding interactions community spaces well external channels like social media. Violating terms may lead temporary permanent ban.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"id_3-temporary-ban","dir":"","previous_headings":"Enforcement Guidelines","what":"3. Temporary Ban","title":"Contributor Covenant Code of Conduct","text":"Community Impact: serious violation community standards, including sustained inappropriate behavior. Consequence: temporary ban sort interaction public communication community specified period time. public private interaction people involved, including unsolicited interaction enforcing Code Conduct, allowed period. Violating terms may lead permanent ban.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"id_4-permanent-ban","dir":"","previous_headings":"Enforcement Guidelines","what":"4. Permanent Ban","title":"Contributor Covenant Code of Conduct","text":"Community Impact: Demonstrating pattern violation community standards, including sustained inappropriate behavior, harassment individual, aggression toward disparagement classes individuals. Consequence: permanent ban sort public interaction within community.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"attribution","dir":"","previous_headings":"","what":"Attribution","title":"Contributor Covenant Code of Conduct","text":"Code Conduct adapted Contributor Covenant, version 2.0, available https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. Community Impact Guidelines inspired Mozilla’s code conduct enforcement ladder. answers common questions code conduct, see FAQ https://www.contributor-covenant.org/faq. Translations available https://www.contributor-covenant.org/translations.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":null,"dir":"","previous_headings":"","what":"Contributing to scoringutils","title":"Contributing to scoringutils","text":"outlines propose change scoringutils.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"fixing-typos","dir":"","previous_headings":"","what":"Fixing typos","title":"Contributing to scoringutils","text":"can fix typos, spelling mistakes, grammatical errors documentation directly using GitHub web interface, long changes made source file. generally means ’ll need edit roxygen2 comments .R, .Rd file. can find .R file generates .Rd reading comment first line.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"bigger-changes","dir":"","previous_headings":"","what":"Bigger changes","title":"Contributing to scoringutils","text":"want make bigger change, ’s good idea first file issue make sure someone team agrees ’s needed. ’ve found bug, please file issue illustrates bug minimal reprex (also help write unit test, needed).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"pull-request-process","dir":"","previous_headings":"Bigger changes","what":"Pull request process","title":"Contributing to scoringutils","text":"Fork package clone onto computer. haven’t done , recommend using usethis::create_from_github(\"epiforecasts/scoringutils\", fork = TRUE). Install development dependences devtools::install_dev_deps(), make sure package passes R CMD check running devtools::check(). R CMD check doesn’t pass cleanly, ’s good idea ask help continuing. Create Git branch pull request (PR). recommend using usethis::pr_init(\"brief-description--change\"). Make changes, commit git, create PR running usethis::pr_push(), following prompts browser. title PR briefly describe change. body PR contain Fixes #issue-number. user-facing changes, add bullet top NEWS.md (.e. just first header). Follow style described https://style.tidyverse.org/news.html.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"code-style","dir":"","previous_headings":"Bigger changes","what":"Code style","title":"Contributing to scoringutils","text":"New code follow tidyverse style guide. can use styler package apply styles, please don’t restyle code nothing PR. use roxygen2, Markdown syntax, documentation. use testthat unit tests. Contributions test cases included easier accept.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"code-of-conduct","dir":"","previous_headings":"","what":"Code of Conduct","title":"Contributing to scoringutils","text":"Please note scoringutils project released Contributor Code Conduct. contributing project agree abide terms.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/LICENSE.html","id":null,"dir":"","previous_headings":"","what":"MIT License","title":"MIT License","text":"Copyright (c) 2020 EpiForecasts Permission hereby granted, free charge, person obtaining copy software associated documentation files (“Software”), deal Software without restriction, including without limitation rights use, copy, modify, merge, publish, distribute, sublicense, /sell copies Software, permit persons Software furnished , subject following conditions: copyright notice permission notice shall included copies substantial portions Software. SOFTWARE PROVIDED “”, WITHOUT WARRANTY KIND, EXPRESS IMPLIED, INCLUDING LIMITED WARRANTIES MERCHANTABILITY, FITNESS PARTICULAR PURPOSE NONINFRINGEMENT. EVENT SHALL AUTHORS COPYRIGHT HOLDERS LIABLE CLAIM, DAMAGES LIABILITY, WHETHER ACTION CONTRACT, TORT OTHERWISE, ARISING , CONNECTION SOFTWARE USE DEALINGS SOFTWARE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/PULL_REQUEST_TEMPLATE.html","id":"description","dir":"","previous_headings":"","what":"Description","title":"NA","text":"PR closes #. [Describe changes made pull request.]","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/PULL_REQUEST_TEMPLATE.html","id":"checklist","dir":"","previous_headings":"","what":"Checklist","title":"NA","text":"PR based package issue explicitly linked . included target issue issues PR title follows: issue-number: PR title tested changes locally. added updated unit tests necessary. updated documentation required. built package locally run rebuilt docs using roxygen2. code follows established coding standards run lintr::lint_package() check style issues introduced changes. added news item linked PR. reviewed CI checks PR addressed far able.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/Deprecated-functions.html","id":"merge_pred_and_obs","dir":"Articles","previous_headings":"","what":"merge_pred_and_obs()","title":"Deprecated functions","text":"scoringutils requires forecasts observations provided single data frame. forecasts observations two different data frames, merge_pred_and_obs() may help merge two. function mostly wrapper around merge(), additional work deal duplicated column names.","code":"#' @title Merge forecast data and observations #' #' @description #' #' The function more or less provides a wrapper around `merge` that #' aims to handle the merging well if additional columns are present #' in one or both data sets. If in doubt, you should probably merge the #' data sets manually. #' #' @param forecasts A data.frame with the forecast data (as can be passed to #' [score()]). #' @param observations A data.frame with the observations. #' @param join Character, one of `c(\"left\", \"full\", \"right\")`. Determines the #' type of the join. Usually, a left join is appropriate, but sometimes you #' may want to do a full join to keep dates for which there is a forecast, but #' no ground truth data. #' @param by Character vector that denotes the columns by which to merge. Any #' value that is not a column in observations will be removed. #' @return a data.table with forecasts and observations #' @importFrom checkmate assert_subset #' @importFrom data.table as.data.table #' @keywords data-handling #' @export merge_pred_and_obs <- function(forecasts, observations, join = c(\"left\", \"full\", \"right\"), by = NULL) { forecasts <- as.data.table(forecasts) observations <- as.data.table(observations) join <- match.arg(join) assert_subset(by, intersect(names(forecasts), names(observations))) if (is.null(by)) { protected_columns <- c( \"predicted\", \"observed\", \"sample_id\", \"quantile_level\", \"interval_range\", \"boundary\" ) by <- setdiff(colnames(forecasts), protected_columns) } obs_cols <- colnames(observations) by <- intersect(by, obs_cols) join <- match.arg(join) if (join == \"left\") { # do a left_join, where all data in the observations are kept. combined <- merge(observations, forecasts, by = by, all.x = TRUE) } else if (join == \"full\") { # do a full, where all data is kept. combined <- merge(observations, forecasts, by = by, all = TRUE) } else { combined <- merge(observations, forecasts, by = by, all.y = TRUE) } # get colnames that are the same for x and y colnames <- colnames(combined) colnames_x <- colnames[endsWith(colnames, \".x\")] colnames_y <- colnames[endsWith(colnames, \".y\")] # extract basenames basenames_x <- sub(\".x$\", \"\", colnames_x) basenames_y <- sub(\".y$\", \"\", colnames_y) # see whether the column name as well as the content is the same content_x <- as.list(combined[, ..colnames_x]) content_y <- as.list(combined[, ..colnames_y]) overlapping <- (content_x %in% content_y) & (basenames_x == basenames_y) overlap_names <- colnames_x[overlapping] basenames_overlap <- sub(\".x$\", \"\", overlap_names) # delete overlapping columns if (length(basenames_overlap) > 0) { combined[, paste0(basenames_overlap, \".x\") := NULL] combined[, paste0(basenames_overlap, \".y\") := NULL] } return(combined[]) }"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/Deprecated-visualisations.html","id":"functions-plot_predictions-and-make_na","dir":"Articles","previous_headings":"","what":"Functions plot_predictions() and make_na()","title":"Deprecated Visualisations","text":"previous versions scoringutils, forecasts observed values visualised using function plot_predictions() make_na() helper function. following shows function code first example. plot_predictions() actual work producing plot. argument needed user can facet plot correctly user needs specify columns relevant facetting. make_NA() represents form filtering, instead filtering entire rows, relevant entries columns “predicted” “observed” made NA. allows user filter observations forecasts independently. following examples using two functions create plot using scoringutils example data. Visualising median forecasts example data. truth data restricted period 2021-05-01 2021-07-22. forecast data forecast model “EuroCOVIDhub-ensemble” made “2021-06-07”. data set NA, effectively removing plot. plot, variety prediction intervals shown, instead just median. similar plot, time based continuous forecasts. predictions automatically converted quantile-based forecasts plotting. Displaying two forecasts time additional colours:","code":"#\" @title Plot Predictions vs True Values #\" #\" @description #\" Make a plot of observed and predicted values #\" #\" @param data a data.frame that follows the same specifications outlined in #\" [score()]. To customise your plotting, you can filter your data using the #\" function [make_NA()]. #\" @param by character vector with column names that denote categories by which #\" the plot should be stratified. If for example you want to have a facetted #\" plot, this should be a character vector with the columns used in facetting #\" (note that the facetting still needs to be done outside of the function call) #\" @param x character vector of length one that denotes the name of the variable #\" @param interval_range numeric vector indicating the interval ranges to plot. #\" If 0 is included in `interval_range`, the median prediction will be shown. #\" @return ggplot object with a plot of true vs predicted values #\" @importFrom ggplot2 ggplot scale_colour_manual scale_fill_manual theme_light #\" @importFrom ggplot2 facet_wrap facet_grid aes geom_line .data geom_point #\" @importFrom data.table dcast #\" @importFrom ggdist geom_lineribbon #\" @export #\" @examples #\" library(ggplot2) #\" library(magrittr) #\" #\" example_sample_continuous %>% #\" make_NA ( #\" what = \"truth\", #\" target_end_date >= \"2021-07-22\", #\" target_end_date < \"2021-05-01\" #\" ) %>% #\" make_NA ( #\" what = \"forecast\", #\" model != \"EuroCOVIDhub-ensemble\", #\" forecast_date != \"2021-06-07\" #\" ) %>% #\" plot_predictions ( #\" x = \"target_end_date\", #\" by = c(\"target_type\", \"location\"), #\" interval_range = c(0, 50, 90, 95) #\" ) + #\" facet_wrap(~ location + target_type, scales = \"free_y\") + #\" aes(fill = model, color = model) #\" #\" example_sample_continuous %>% #\" make_NA ( #\" what = \"truth\", #\" target_end_date >= \"2021-07-22\", #\" target_end_date < \"2021-05-01\" #\" ) %>% #\" make_NA ( #\" what = \"forecast\", #\" forecast_date != \"2021-06-07\" #\" ) %>% #\" plot_predictions ( #\" x = \"target_end_date\", #\" by = c(\"target_type\", \"location\"), #\" interval_range = 0 #\" ) + #\" facet_wrap(~ location + target_type, scales = \"free_y\") + #\" aes(fill = model, color = model) plot_predictions <- function(data, by = NULL, x = \"date\", interval_range = c(0, 50, 90)) { # split truth data and forecasts in order to apply different filtering truth_data <- data.table::as.data.table(data)[!is.na(observed)] forecasts <- data.table::as.data.table(data)[!is.na(predicted)] del_cols <- colnames(truth_data)[!(colnames(truth_data) %in% c(by, \"observed\", x))] truth_data <- unique(suppressWarnings(truth_data[, eval(del_cols) := NULL])) # find out what type of predictions we have. convert sample based to # interval range data if (get_forecast_type(data) == \"quantile\") { forecasts <- scoringutils:::quantile_to_interval( forecasts, keep_quantile_col = FALSE ) } else if (get_forecast_type(data) == \"sample\") { # using a scoringutils internal function forecasts <- scoringutils:::sample_to_interval_long( as_forecast(forecasts), interval_range = interval_range, keep_quantile_col = FALSE ) } # select appropriate boundaries and pivot wider select <- forecasts$interval_range %in% setdiff(interval_range, 0) intervals <- forecasts[select, ] # delete quantile column in intervals if present. This is important for # pivoting if (\"quantile_level\" %in% names(intervals)) { intervals[, quantile_level := NULL] } plot <- ggplot(data = data, aes(x = .data[[x]])) + theme_scoringutils() + ylab(\"True and predicted values\") if (nrow(intervals) != 0) { # pivot wider and convert range to a factor intervals <- data.table::dcast(intervals, ... ~ boundary, value.var = \"predicted\") # only plot interval ranges if there are interval ranges to plot plot <- plot + ggdist::geom_lineribbon( data = intervals, aes( ymin = lower, ymax = upper, # We use the fill_ramp aesthetic for this instead of the default fill # because we want to keep fill to be able to use it for other # variables fill_ramp = factor( interval_range, levels = sort(unique(interval_range), decreasing = TRUE) ) ), lwd = 0.4 ) + ggdist::scale_fill_ramp_discrete( name = \"interval_range\", # range argument was added to make sure that the line for the median # and the ribbon don\"t have the same opacity, making the line # invisible range = c(0.15, 0.75) ) } # We could treat this step as part of ggdist::geom_lineribbon() but we treat # it separately here to deal with the case when only the median is provided # (in which case ggdist::geom_lineribbon() will fail) if (0 %in% interval_range) { select_median <- forecasts$interval_range == 0 & forecasts$boundary == \"lower\" median <- forecasts[select_median] if (nrow(median) > 0) { plot <- plot + geom_line( data = median, mapping = aes(y = predicted), lwd = 0.4 ) } } # add observed values if (nrow(truth_data) > 0) { plot <- plot + geom_point( data = truth_data, show.legend = FALSE, inherit.aes = FALSE, aes(x = .data[[x]], y = observed), color = \"black\", size = 0.5 ) + geom_line( data = truth_data, inherit.aes = FALSE, show.legend = FALSE, aes(x = .data[[x]], y = observed), linetype = 1, color = \"grey40\", lwd = 0.2 ) } return(plot) } #\" @title Make Rows NA in Data for Plotting #\" #\" @description #\" Filters the data and turns values into `NA` before the data gets passed to #\" [plot_predictions()]. The reason to do this is to this is that it allows to #\" \"filter\" prediction and truth data separately. Any value that is NA will then #\" be removed in the subsequent call to [plot_predictions()]. #\" #\" @inheritParams score #\" @param what character vector that determines which values should be turned #\" into `NA`. If `what = \"truth\"`, values in the column \"observed\" will be #\" turned into `NA`. If `what = \"forecast\"`, values in the column \"prediction\" #\" will be turned into `NA`. If `what = \"both\"`, values in both column will be #\" turned into `NA`. #\" @param ... logical statements used to filter the data #\" @return A data.table #\" @importFrom rlang enexprs #\" @keywords plotting #\" @export #\" #\" @examples #\" make_NA ( #\" example_sample_continuous, #\" what = \"truth\", #\" target_end_date >= \"2021-07-22\", #\" target_end_date < \"2021-05-01\" #\" ) make_NA <- function(data = NULL, what = c(\"truth\", \"forecast\", \"both\"), ...) { stopifnot(is.data.frame(data)) data <- as.data.table(data) what <- match.arg(what) # turn ... arguments into expressions args <- enexprs(...) vars <- NULL if (what %in% c(\"forecast\", \"both\")) { vars <- c(vars, \"predicted\") } if (what %in% c(\"truth\", \"both\")) { vars <- c(vars, \"observed\") } for (expr in args) { data <- data[eval(expr), eval(vars) := NA_real_] } return(data[]) } median_forecasts <- example_quantile[quantile_level == 0.5] median_forecasts %>% make_NA(what = \"truth\", target_end_date <= \"2021-05-01\", target_end_date > \"2021-07-22\") %>% make_NA(what = \"forecast\", model != \"EuroCOVIDhub-ensemble\", forecast_date != \"2021-06-07\") %>% plot_predictions( by = c(\"location\", \"target_type\"), x = \"target_end_date\" ) + facet_wrap(location ~ target_type, scales = \"free_y\") example_quantile %>% make_NA(what = \"truth\", target_end_date <= \"2021-05-01\", target_end_date > \"2021-07-22\") %>% make_NA(what = \"forecast\", model != \"EuroCOVIDhub-ensemble\", forecast_date != \"2021-06-07\") %>% plot_predictions( by = c(\"location\", \"target_type\"), x = \"target_end_date\", interval_range = c(0, 10, 20, 30, 40, 50, 60) ) + facet_wrap(location ~ target_type, scales = \"free_y\") example_sample_continuous %>% make_NA(what = \"truth\", target_end_date <= \"2021-05-01\", target_end_date > \"2021-07-22\") %>% make_NA(what = \"forecast\", model != \"EuroCOVIDhub-ensemble\", forecast_date != \"2021-06-07\") %>% plot_predictions( by = c(\"location\", \"target_type\"), x = \"target_end_date\", interval_range = c(0, 50, 90, 95) ) + facet_wrap(location ~ target_type, scales = \"free_y\") example_quantile %>% make_NA(what = \"truth\", target_end_date > \"2021-07-15\", target_end_date <= \"2021-05-22\") %>% make_NA(what = \"forecast\", !(model %in% c(\"EuroCOVIDhub-ensemble\", \"EuroCOVIDhub-baseline\")), forecast_date != \"2021-06-28\") %>% plot_predictions(x = \"target_end_date\", by = c(\"target_type\", \"location\")) + aes(colour = model, fill = model) + facet_wrap(target_type ~ location, ncol = 4, scales = \"free_y\") + labs(x = \"Target end date\")"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/Deprecated-visualisations.html","id":"function-plot_interval_ranges-formerly-plot_ranges","dir":"Articles","previous_headings":"","what":"Function plot_interval_ranges() (formerly plot_ranges())","title":"Deprecated Visualisations","text":"functionality currently relies hack. previous versions scoringutils, scores computed per interval range/per quantile. Now, scoringutils returns one score per forecast, per interval range/quantile. therefore need add range column, using internal function get_range_from_quantile(). column interpreted one defines unit single forecast scoringutils. also means get warning different number quantile levels different forecasts (0% prediction interval one median forecast, prediction intervals two (lower upper bound)). Plotting dispersion instead WIS:","code":"#\" @title Plot Metrics by Range of the Prediction Interval #\" #\" @description #\" Visualise the metrics by range, e.g. if you are interested how different #\" interval ranges contribute to the overall interval score, or how #\" sharpness / dispersion changes by range. #\" #\" @param scores A data.frame of scores based on quantile forecasts as #\" produced by [score()] or [summarise_scores()]. Note that \"range\" must be included #\" in the `by` argument when running [summarise_scores()] #\" @param y The variable from the scores you want to show on the y-Axis. #\" This could be something like \"wis\" (the default) or \"dispersion\" #\" @param x The variable from the scores you want to show on the x-Axis. #\" Usually this will be \"model\" #\" @param colour Character vector of length one used to determine a variable #\" for colouring dots. The Default is \"range\". #\" @return A ggplot2 object showing a contributions from the three components of #\" the weighted interval score #\" @importFrom ggplot2 ggplot aes aes geom_point geom_line #\" expand_limits theme theme_light element_text scale_color_continuous labs #\" @export #\" @examples #\" library(ggplot2) #\" ex <- data.table::copy(example_quantile) #\" ex$range <- scoringutils:::get_range_from_quantile(ex$quantile) #\" scores <- suppressWarnings(score(as_forecast(ex), metrics = list(\"wis\" = wis))) #\" summarised <- summarise_scores( #\" scores, #\" by = c(\"model\", \"target_type\", \"range\") #\" ) #\" plot_interval_ranges(summarised, x = \"model\") + #\" facet_wrap(~target_type, scales = \"free\") plot_interval_ranges <- function(scores, y = \"wis\", x = \"model\", colour = \"range\") { plot <- ggplot( scores, aes( x = .data[[x]], y = .data[[y]], colour = .data[[colour]] ) ) + geom_point(size = 2) + geom_line(aes(group = range), colour = \"black\", linewidth = 0.01 ) + scale_color_continuous(low = \"steelblue\", high = \"salmon\") + theme_scoringutils() + expand_limits(y = 0) + theme( legend.position = \"right\", axis.text.x = element_text( angle = 90, vjust = 1, hjust = 1 ) ) return(plot) } range_example <- copy(example_quantile) %>% na.omit() %>% .[, range := scoringutils:::get_range_from_quantile(quantile_level)] sum_scores <- range_example %>% as_forecast() %>% score(metrics = list(wis = wis, dispersion = dispersion)) %>% summarise_scores(by = c(\"model\", \"target_type\", \"range\")) %>% suppressWarnings() plot_interval_ranges(sum_scores, x = \"model\") + facet_wrap(~target_type, scales = \"free\") plot_interval_ranges(sum_scores, y = \"dispersion\", x = \"model\") + facet_wrap(~target_type, scales = \"free_y\")"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/Deprecated-visualisations.html","id":"function-plot_score_table","dir":"Articles","previous_headings":"","what":"Function plot_score_table()","title":"Deprecated Visualisations","text":"function allowed users turn table (summarised) scores coloured table. function hard-coded information colour scale pick metric. scoringutils 2.0.0, allowed users assign names metrics use custom scoring functions. stick default names provided scoringutils, function still work. However, functionality easily generalisable, decided deprecate function. main functionality old function provided, scaling scores order obtain reasonable colour shades. per metric, one also pass additional grouping variables argument. allowed users achieve faceting table (note course scores also needed summarised according grouping). function also allowed users combine different facets one, creating combined y-variable. done passing vector column names y argument.","code":"#' @title Plot Coloured Score Table #' #' @description #' Plots a coloured table of summarised scores obtained using #' [score()]. #' #' @param y the variable to be shown on the y-axis. Instead of a single character string, #' you can also specify a vector with column names, e.g. #' `y = c(\"model\", \"location\")`. These column names will be concatenated #' to create a unique row identifier (e.g. \"model1_location1\"). #' @param by A character vector that determines how the colour shading for the #' plot gets computed. By default (`NULL`), shading will be determined per #' metric, but you can provide additional column names (see examples). #' @param metrics A character vector with the metrics to show. If set to #' `NULL` (default), all metrics present in `scores` will be shown. #' #' @return A ggplot object with a coloured table of summarised scores #' @inheritParams get_pairwise_comparisons #' @importFrom ggplot2 ggplot aes element_blank element_text labs coord_cartesian coord_flip #' @importFrom data.table setDT melt #' @importFrom stats sd #' @export #' #' @examples #' library(ggplot2) #' library(magrittr) # pipe operator #' \\dontshow{ #' data.table::setDTthreads(2) # restricts number of cores used on CRAN #' } #' #' scores <- score(as_forecast(example_quantile)) %>% #' summarise_scores(by = c(\"model\", \"target_type\")) %>% #' summarise_scores(by = c(\"model\", \"target_type\"), fun = signif, digits = 2) #' #' plot_score_table(scores, y = \"model\", by = \"target_type\") + #' facet_wrap(~target_type, ncol = 1) #' #' # can also put target description on the y-axis #' plot_score_table(scores, #' y = c(\"model\", \"target_type\"), #' by = \"target_type\") plot_score_table <- function(scores, y = \"model\", by = NULL, metrics = NULL) { # identify metrics ----------------------------------------------------------- id_vars <- get_forecast_unit(scores) metrics <- get_metrics(scores) cols_to_delete <- names(scores)[!(names(scores) %in% c(metrics, id_vars))] suppressWarnings(scores[, eval(cols_to_delete) := NULL]) # compute scaled values ------------------------------------------------------ # scaling is done in order to colour the different scores # for most metrics larger is worse, but others like bias are better if they # are close to zero and deviations in both directions are bad # define which metrics are scaled using min (larger is worse) and # which not (metrics like bias where deviations in both directions are bad) metrics_zero_good <- c(\"bias\", \"interval_coverage_deviation\") metrics_no_color <- \"coverage\" metrics_min_good <- setdiff(metrics, c( metrics_zero_good, metrics_no_color )) # write scale functions that can be used in data.table scale <- function(x) { scaled <- x / sd(x, na.rm = TRUE) return(scaled) } scale_min_good <- function(x) { scaled <- (x - min(x)) / sd(x, na.rm = TRUE) return(scaled) } # pivot longer and add scaled values df <- data.table::melt(scores, value.vars = metrics, id.vars = id_vars, variable.name = \"metric\" ) df[metric %in% metrics_min_good, value_scaled := scale_min_good(value), by = c(\"metric\", by) ] df[metric %in% metrics_zero_good, value_scaled := scale(value), by = c(\"metric\", by) ] df[metric %in% metrics_no_color, value_scaled := 0, by = c(\"metric\", by) ] # create identifier column for plot ------------------------------------------ # if there is only one column, leave column as is. Reason to do that is that # users can then pass in a factor and keep the ordering of that column intact if (length(y) > 1) { df[, identifCol := do.call(paste, c(.SD, sep = \"_\")), .SDcols = y[y %in% names(df)]] } else { setnames(df, old = eval(y), new = \"identifCol\") } # plot ----------------------------------------------------------------------- # make plot with all metrics that are not NA plot <- ggplot( df[!is.na(value), ], aes(y = identifCol, x = metric) ) + geom_tile(aes(fill = value_scaled), colour = \"white\", show.legend = FALSE) + geom_text(aes(y = identifCol, label = value)) + scale_fill_gradient2(low = \"steelblue\", high = \"salmon\") + theme_scoringutils() + theme( legend.title = element_blank(), legend.position = \"none\", axis.text.x = element_text( angle = 90, vjust = 1, hjust = 1 ) ) + labs(x = \"\", y = \"\") + coord_cartesian(expand = FALSE) return(plot) } scores <- score(as_forecast(example_quantile)) %>% summarise_scores(by = c(\"model\", \"target_type\")) %>% summarise_scores(by = c(\"model\", \"target_type\"), fun = signif, digits = 2) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_score_table(scores, y = \"model\", by = \"target_type\") + facet_wrap(~target_type, ncol = 1) # can also put target description on the y-axis plot_score_table(scores, y = c(\"model\", \"target_type\"), by = \"target_type\")"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"introduction","dir":"Articles","previous_headings":"","what":"Introduction","title":"Scoring rules in `scoringutils`","text":"vignette gives overview default scoring rules made available scoringutils package. can, course, also use scoring rules, provided follow format. want obtain detailed information pacakge works, look revised version scoringutils paper. can distinguish two types forecasts: point forecasts probabilistic forecasts. point forecast single number representing single outcome. probabilistic forecast full predictive probability distribution multiple possible outcomes. contrast point forecasts, probabilistic forecasts incorporate uncertainty different possible outcomes. Scoring rules functions take forecast observation input return single numeric value. point forecasts, take form S(ŷ,y)S(\\hat{y}, y), ŷ\\hat{y} forecast yy observation. probabilistic forecasts, usually take form S(F,y)S(F, y), FF cumulative density function (CDF) predictive distribution yy observation. convention, scoring rules usually negatively oriented, meaning smaller values better (best possible score usually zero). sense, score can understood penalty. Many scoring rules probabilistic forecasts -called (strictly) proper scoring rules. Essentially, means “cheated”: forecaster evaluated strictly proper scoring rule always incentivised report honest best belief future , expectation, improve score reporting something else. formal definition following: Let GG true, unobserved data-generating distribution. scoring rule said proper, GG ideal forecast F=GF = G, forecast F′≠FF' \\neq F expectation receives better score FF. scoring rule considered strictly proper , GG, forecast F′F' expectation receives score better FF.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"metrics-for-point-forecasts","dir":"Articles","previous_headings":"","what":"Metrics for point forecasts","title":"Scoring rules in `scoringutils`","text":"See list default metrics point forecasts calling ?metrics_point(). overview input output formats point forecasts: Input output formats: metrics point.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"a-note-of-caution","dir":"Articles","previous_headings":"Metrics for point forecasts","what":"A note of caution","title":"Scoring rules in `scoringutils`","text":"Scoring point forecasts can tricky business. Depending choice scoring rule, forecaster clearly worse another, might consistently receive better scores (see Gneiting (2011) illustrative example). Every scoring rule point forecast implicitly minimised specific aspect predictive distribution. mean squared error, example, meaningful scoring rule forecaster actually reported mean predictive distribution point forecast. forecaster reported median, mean absolute error appropriate scoring rule. scoring rule predictive task align, misleading results ensue. Consider following example:","code":"set.seed(123) n <- 1000 observed <- rnorm(n, 5, 4)^2 predicted_mu <- mean(observed) predicted_not_mu <- predicted_mu - rnorm(n, 10, 2) mean(Metrics::ae(observed, predicted_mu)) #> [1] 34.45981 mean(Metrics::ae(observed, predicted_not_mu)) #> [1] 32.54821 mean(Metrics::se(observed, predicted_mu)) #> [1] 2171.089 mean(Metrics::se(observed, predicted_not_mu)) #> [1] 2290.155"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"absolute-error","dir":"Articles","previous_headings":"Metrics for point forecasts","what":"Absolute error","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: ŷ\\hat{y}, real number, median forecaster’s predictive distribution. absolute error absolute difference predicted observed values. See ?Metrics::ae. ae=|y−ŷ|\\text{ae} = |y - \\hat{y}| absolute error appropriate rule ŷ\\hat{y} corresponds median forecaster’s predictive distribution. Otherwise, results misleading (see Gneiting (2011)).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"squared-error","dir":"Articles","previous_headings":"Metrics for point forecasts","what":"Squared error","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: ŷ\\hat{y}, real number, mean forecaster’s predictive distribution. squared error squared difference predicted observed values. See ?Metrics::se. se=(y−ŷ)2\\text{se} = (y - \\hat{y})^2 squared error appropriate rule ŷ\\hat{y} corresponds mean forecaster’s predictive distribution. Otherwise, results misleading (see Gneiting (2011)).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"absolute-percentage-error","dir":"Articles","previous_headings":"Metrics for point forecasts","what":"Absolute percentage error","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: ŷ\\hat{y}, real number absolute percentage error absolute percent difference predicted observed values. See ?Metrics::ape. ape=|y−ŷ||y|\\text{ape} = \\frac{|y - \\hat{y}|}{|y|} absolute percentage error appropriate rule ŷ\\hat{y} corresponds β\\beta-median forecaster’s predictive distribution β=−1\\beta = -1. β\\beta-median, med(β)(F)\\text{med}^{(\\beta)}(F), median random variable whose density proportional yβf(y)y^\\beta f(y). specific β\\beta-median corresponds absolute percentage error med(−1)(F)\\text{med}^{(-1)}(F). Otherwise, results misleading (see Gneiting (2011)).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"binary-forecasts","dir":"Articles","previous_headings":"","what":"Binary forecasts","title":"Scoring rules in `scoringutils`","text":"See list default metrics point forecasts calling ?metrics_binary(). overview input output formats point forecasts: Input output formats: metrics binary forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"brier-score","dir":"Articles","previous_headings":"Binary forecasts","what":"Brier score","title":"Scoring rules in `scoringutils`","text":"Observation: yy, either 0 1 Forecast: pp, probability observed outcome 1. Brier score strictly proper scoring rule. computed mean squared error probabilistic prediction observed outcome. BS(p,y)=(p−y)2={p2,y=1(1−p)2,y=0\\begin{equation} \\text{BS}(p, y) = (p - y)^2 = \\begin{cases} p^2, & \\text{} y = 1\\\\ (1 - p)^2, & \\text{} y = 0 \\end{cases} \\end{equation} Brier score logarithmic score (see ) differ penalise - underconfidence (see Machete (2012)). Brier score penalises overconfidence underconfidence probability space . Consider following example: See ?brier_score() information.","code":"n <- 1e6 p_true <- 0.7 observed <- factor(rbinom(n = n, size = 1, prob = p_true), levels = c(0, 1)) p_over <- p_true + 0.15 p_under <- p_true - 0.15 abs(mean(brier_score(observed, p_true)) - mean(brier_score(observed, p_over))) #> [1] 0.0223866 abs(mean(brier_score(observed, p_true)) - mean(brier_score(observed, p_under))) #> [1] 0.0226134"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"logarithmic-score","dir":"Articles","previous_headings":"Binary forecasts","what":"Logarithmic score","title":"Scoring rules in `scoringutils`","text":"Observation: yy, either 0 1 Forecast: pp, probability observed outcome 1. logarithmic score (log score) strictly proper scoring rule. computed negative logarithm probability assigned observed outcome. Log score(p,y)=−log(1−|y−p|)={−log(p),y=1−log(1−p),y=0\\begin{equation} \\text{Log score}(p, y) = - \\log(1 - |y - p|) = \\begin{cases} -\\log (p), & \\text{} y = 1\\\\ -\\log (1 - p), & \\text{} y = 0 \\end{cases} \\end{equation} log score penalises overconfidence strongly underconfidence (probability space). Consider following example: See ?logs_binary() information.","code":"abs(mean(logs_binary(observed, p_true)) - mean(logs_binary(observed, p_over))) #> [1] 0.07169954 abs(mean(logs_binary(observed, p_true)) - mean(logs_binary(observed, p_under))) #> [1] 0.04741833"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"sample-based-forecasts","dir":"Articles","previous_headings":"","what":"Sample-based forecasts","title":"Scoring rules in `scoringutils`","text":"See list default metrics sample-based forecasts calling ?metrics_sample(). overview input output formats quantile forecasts: Input output formats: metrics sample-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"crps","dir":"Articles","previous_headings":"Sample-based forecasts","what":"CRPS","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number (discrete number). Forecast: continuous (FF) discrete (PP) forecast. continuous ranked probability score (CRPS) popular fields meteorology epidemiology. CRPS defined CRPS(F,y)=∫−∞∞(F(x)−1(x≥y))2dx,\\text{CRPS}(F, y) = \\int_{-\\infty}^\\infty \\left( F(x) - 1(x \\geq y) \\right)^2 dx, yy observed value FF CDF predictive distribution. discrete forecasts, example count data, ranked probability score (RPS) can used instead commonly defined : RPS(P,y)=∑x=0∞(P(x)−1(x≥y))2, \\text{RPS}(P, y) = \\sum_{x = 0}^\\infty (P(x) - 1(x \\geq y))^2, PP cumulative probability mass function (PMF) predictive distribution. CRPS can understood generalisation absolute error predictive distributions (Gneiting Raftery 2007). can also understood integral Brier score binary probability forecasts implied CDF possible observed values. CRPS also related Cramér-distance two distributions equals special case one distributions concentrated single point (see e.g. Ziel (2021)). CRPS global scoring rule, meaning entire predictive distribution taken account determining quality forecast. scoringutils re-exports crps_sample() function scoringRules package, assumes forecast represented set samples predictive distribution. See ?crps_sample() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"log-score","dir":"Articles","previous_headings":"Sample-based forecasts","what":"Log score","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number (discrete number). Forecast: continuous (FF) discrete (PP) forecast. logarithmic scoring rule simply negative logarithm density predictive distribution evaluated observed value: log score(F,y)=−logf(y), \\text{log score}(F, y) = -\\log f(y), ff predictive probability density function (PDF) corresponding Forecast FF yy observed value. discrete forecasts, log score can computed log score(F,y)=−logpy, \\text{log score}(F, y) = -\\log p_y, pyp_y probability assigned observed outcome yy forecast FF. logarithmic scoring rule can produce large penalties observed value takes values f(y)f(y) (pyp_y) close zero. therefore considered sensitive outlier forecasts. may desirable applications, also means scores can easily dominated extreme values. logarithmic scoring rule local scoring rule, meaning score depends probability assigned actual outcome. often regarded desirable property example context Bayesian inference . implies example, ranking forecasters invariant monotone transformations predictive distribution target. scoringutils re-exports logs_sample() function scoringRules package, assumes forecast represented set samples predictive distribution. One implications currently advisable use log score discrete forecasts. reason scoringRules::logs_sample() estimates predictive density samples, can problematic discrete forecasts. See ?logs_sample() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"dawid-sebastiani-score","dir":"Articles","previous_headings":"Sample-based forecasts","what":"Dawid-Sebastiani score","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number (discrete number). Forecast: FF. predictive distribution mean μ\\mu standard deviation σ\\sigma. Dawid-Sebastiani score proper scoring rule relies first moments predictive distribution therefore easy compute. given dss(F,y)=(y−μσ)2+2⋅logσ.\\text{dss}(F, y) = \\left( \\frac{y - \\mu}{\\sigma} \\right)^2 + 2 \\cdot \\log \\sigma. scoringutils re-exports implementation DSS scoringRules package. assumes forecast represented set samples drawn predictive distribution. See ?dss_sample() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"dispersion---median-absolute-deviation-mad","dir":"Articles","previous_headings":"Sample-based forecasts","what":"Dispersion - Median Absolute Deviation (MAD)","title":"Scoring rules in `scoringutils`","text":"Observation: required. Forecast: FF, predictive distribution. Dispersion (also called sharpness) ability produce narrow forecasts. feature forecasts depend observations. Dispersion therefore interest conditional calibration: precise forecast useful clearly wrong. One way measure sharpness (suggested Funk et al. (2019)) normalised median absolute deviation median (MADN) ). computed S(F)=10.675⋅median(|F−median(F)|). S(F) = \\frac{1}{0.675} \\cdot \\text{median}(|F - \\text{median(F)}|). forecast FF follows normal distribution, sharpness equal standard deviation FF.details, see ?mad_sample().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"bias","dir":"Articles","previous_headings":"Sample-based forecasts","what":"Bias","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number (discrete number). Forecast: continuous (FF) discrete (PP) forecast. Bias measure tendency forecaster - underpredict. continuous forecasts, scoringutils implementation calculates bias B(F,y)=1−2⋅F(y),B(F, y) = 1 - 2 \\cdot F (y), F(y)F(y) cumulative distribution function forecast. discrete forecasts, calculate bias B(P,y)=1−(P(y)+P(y+1)).B(P, y) = 1 - (P(y) + P(y + 1)). P(y)P(y) cumulative probability assigned outcomes smaller equal yy, .e. cumulative probability mass function. Bias bound -1 1 represents tendency forecasts biased rather absolute amount - underprediction (e.g. case weighted interval score (WIS, see ).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"absolute-error-of-the-median","dir":"Articles","previous_headings":"Sample-based forecasts","what":"Absolute error of the median","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number (discrete number). Forecast: forecast FF aemedian(F,y)=|median(F)−y|.\\text{ae}_{\\text{median}}(F, y) = |\\text{median} (F) - y|. See section note caution Gneiting (2011) discussion correspondence absolute error median.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"squared-error-of-the-mean","dir":"Articles","previous_headings":"Sample-based forecasts","what":"Squared error of the mean","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number (discrete number). Forecast: forecast FF semedn(F,y)=(mean(F)−y)2.\\text{se}_{\\text{medn}}(F, y) = (\\text{mean} (F) - y)^2. See section note caution Gneiting (2011) discussion correspondence squared error mean.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"quantile-based-forecasts","dir":"Articles","previous_headings":"","what":"Quantile-based forecasts","title":"Scoring rules in `scoringutils`","text":"See list default metrics quantile-based forecasts calling ?metrics_quantile(). overview input output formats quantile forecasts: Input output formats: metrics quantile-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"weighted-interval-score-wis","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Weighted interval score (WIS)","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: FF. CDF predictive distribution represented set quantiles. quantiles form lower (ll) upper (uu) bounds central prediction intervals. weighted interval score (WIS) strictly proper scoring rule can understood approximation CRPS forecasts quantile format (turn represents generalisation absolute error). Quantiles assumed lower upper bounds prediction intervals symmetric around median. single interval, interval score ISα(F,y)=(u−l)⏟dispersion+2α⋅(l−y)⋅𝟏(y≤l)⏟overprediction+2α⋅(y−u)⋅𝟏(y≥u)⏟underprediction,IS_\\alpha(F,y) = \\underbrace{(u-l)}_\\text{dispersion} + \\underbrace{\\frac{2}{\\alpha} \\cdot (l-y) \\cdot \\mathbf{1}(y \\leq l)}_{\\text{overprediction}} + \\underbrace{\\frac{2}{\\alpha} \\cdot (y-u) \\cdot \\mathbf{1}(y \\geq u)}_{\\text{underprediction}}, 𝟏()\\mathbf{1}() indicator function, ll uu α2\\frac{\\alpha}{2} 1−α21 - \\frac{\\alpha}{2} quantiles predictive distribution FF. ll uu together form prediction interval. interval score can understood sum three components: dispersion, overprediction underprediction. set KK prediction intervals median mm, score given weighted sum individual interval scores, .e. WIS=1K+0.5⋅(w0⋅|y−m|+∑k=1Kwk⋅ISαk(F,y)),WIS = \\frac{1}{K + 0.5} \\cdot \\left(w_0 \\cdot |y - m| + \\sum_{k = 1}^{K} w_k \\cdot IS_{\\alpha_{k}}(F, y)\\right), mm median forecast wkw_k weight assigned every interval. weights set wk=αk2w_k = \\frac{\\alpha_k}{2} w0=0.5w_0 = 0.5, WIS converges CRPS increasing number equally spaced quantiles. See ?wis() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"overprediction-underprediction-and-dispersion","dir":"Articles","previous_headings":"Quantile-based forecasts > Weighted interval score (WIS)","what":"Overprediction, underprediction and dispersion","title":"Scoring rules in `scoringutils`","text":"individual components WIS. See ?overprediction(), ?underprediction() ?dispersion() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"bias-1","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Bias","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: FF. CDF predictive distribution represented set quantiles, QQ. Bias can measured B(F,y)={(1−2⋅max{α|qα∈Q∧qα≤y}),yq0.5(underprediction)0,y=q0.5,\\begin{equation} \\text{B}(F, y) = \\begin{cases} (1 - 2 \\cdot \\max \\{\\alpha | q_\\alpha \\Q \\land q_\\alpha \\leq y\\}), & \\text{} y < q_{0.5} \\quad \\text{(overprediction)}\\\\ (1 - 2 \\cdot \\min \\{\\alpha | q_\\alpha \\Q_t \\land q_\\alpha \\geq y\\}, & \\text{} y > q_{0.5} \\quad \\text{(underprediction)}\\\\ 0, & \\text{} y = q_{0.5}, \\\\ \\end{cases} \\end{equation} qαq_\\alpha α\\alpha-quantile predictive distribution. consistency, define QQ (set quantiles form predictive distribution FF) always includes element q0=−∞q_0 = -\\infty q1=∞q_1 = \\infty. clearer terms, bias : 1−(2×1 - (2 \\times maximum percentile rank corresponding quantile still observed value), observed value smaller median predictive distribution. 1−(2×1 - (2 \\times minimum percentile rank corresponding quantile still larger observed value) observed value larger median predictive distribution.. 00if observed value exactly median. Bias can assume values -1 (underprediction) 1 (overpredictin) 0 ideally (.e. unbiased). increasing number quantiles, percentile rank equal proportion predictive samples observed value, bias metric coincides one continuous forecasts (see ). See ?bias_quantile() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"interval-coverage","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Interval coverage","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: FF. CDF predictive distribution represented set quantiles. quantiles form central prediction intervals. Interval coverage given interval range defined proportion observations fall within corresponding central prediction intervals. Central prediction intervals symmetric around median formed two quantiles denote lower upper bound. example, 50% central prediction interval interval 0.25 0.75 quantiles predictive distribution.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"interval-coverage-deviation","dir":"Articles","previous_headings":"Quantile-based forecasts > Interval coverage","what":"Interval coverage deviation","title":"Scoring rules in `scoringutils`","text":"interval coverage deviation difference observed interval coverage nominal interval coverage. example, observed interval coverage 50% central prediction interval 0.6, interval coverage deviation 0.6=−0.5=0.1.0.6 = - 0.5 = 0.1. interval coverage deviation=observed interval coverage−nominal interval coverage\\text{interval coverage deviation} = \\text{observed interval coverage} - \\text{nominal interval coverage}","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"absolute-error-of-the-median-1","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Absolute error of the median","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: FF. CDF predictive distribution represented set quantiles. absolute error median absolute difference median predictive distribution observed value. aemedian=|median(F)−y|\\text{ae}_\\text{median} = |\\text{median}(F) - y| See section note caution Gneiting (2011) discussion correspondence absolute error median.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"quantile-score","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Quantile score","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: FF. CDF predictive distribution represented set quantiles. quantile score, also called pinball loss, single quantile level τ\\tau defined QSτ(F,y)=2⋅{𝟏(y≤qτ)−τ}⋅(qτ−y)={2⋅(1−τ)*qτ−y,y≤qτ2⋅τ*|qτ−y|,y>qτ,\\begin{equation} \\text{QS}_\\tau(F, y) = 2 \\cdot \\{ \\mathbf{1}(y \\leq q_\\tau) - \\tau\\} \\cdot (q_\\tau − y) = \\begin{cases} 2 \\cdot (1 - \\tau) * q_\\tau - y, & \\text{} y \\leq q_\\tau\\\\ 2 \\cdot \\tau * |q_\\tau - y|, & \\text{} y > q_\\tau, \\end{cases} \\end{equation} qτq_\\tau τ\\tau-quantile predictive distribution FF, 𝟏(⋅)\\mathbf{1}(\\cdot) indicator function. (unweighted) interval score (see ) 1−α1 - \\alpha prediction interval can computed quantile scores levels α/2\\alpha/2 1−α/21 - \\alpha/2 ISα(F,y)=QSα/2(F,y)+QS1−α/2(F,y)α\\text{}_\\alpha(F, y) = \\frac{\\text{QS}_{\\alpha/2}(F, y) + \\text{QS}_{1 - \\alpha/2}(F, y)}{\\alpha}. weighted interval score can obtained simple average quantile scores: WISα(F,y)=QSα/2(F,y)+QS1−α/2(F,y)2\\text{WIS}_\\alpha(F, y) = \\frac{\\text{QS}_{\\alpha/2}(F, y) + \\text{QS}_{1 - \\alpha/2}(F, y)}{2}. See ?quantile_score Bracher et al. (2021) details.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"quantile-coverage","dir":"Articles","previous_headings":"Additional metrics","what":"Quantile coverage","title":"Scoring rules in `scoringutils`","text":"Quantile coverage given quantile level defined proportion observed values smaller corresponding predictive quantiles. example, 0.5 quantile coverage proportion observed values smaller 0.5-quantiles predictive distribution.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/authors.html","id":null,"dir":"","previous_headings":"","what":"Authors","title":"Authors and Citation","text":"Nikos Bosse. Author, maintainer. Sam Abbott. Author. Hugo Gruson. Author. Johannes Bracher. Contributor. Toshiaki Asakura. Contributor. James Mba Azam. Contributor. Sebastian Funk. Author.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/authors.html","id":"citation","dir":"","previous_headings":"","what":"Citation","title":"Authors and Citation","text":"Nikos . Bosse, Hugo Gruson, Sebastian Funk, Anne Cori, Edwin van Leeuwen, Sam Abbott (2022). Evaluating Forecasts scoringutils R, arXiv. DOI: 10.48550/ARXIV.2205.07090 Alexander Jordan, Fabian Krueger, Sebastian Lerch (2019). Evaluating Probabilistic Forecasts scoringRules. Journal Statistical Software, 90(12), 1-37. DOI 10.18637/jss.v090.i12","code":"@Article{, title = {Evaluating Forecasts with scoringutils in R}, author = {Nikos I. Bosse and Hugo Gruson and Anne Cori and Edwin {van Leeuwen} and Sebastian Funk and Sam Abbott}, journal = {arXiv}, url = {https://arxiv.org/abs/2205.07090}, year = {2022}, doi = {10.48550/ARXIV.2205.07090}, } @Article{, title = {Evaluating Probabilistic Forecasts with {scoringRules}}, author = {Alexander Jordan and Fabian Kr\\\"uger and Sebastian Lerch}, journal = {Journal of Statistical Software}, year = {2019}, volume = {90}, number = {12}, pages = {1--37}, doi = {10.18637/jss.v090.i12}, }"},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"scoringutils-utilities-for-scoring-and-assessing-predictions","dir":"","previous_headings":"","what":"Utilities for Scoring and Assessing Predictions","title":"Utilities for Scoring and Assessing Predictions","text":"Note: documentation refers development version scoringutils. can also view documentation stable version. scoringutils package facilitates process evaluating forecasts R, using convenient flexible data.table-based framework. provides broad functionality check input data diagnose issues, visualise forecasts missing data, transform data scoring, handle missing forecasts, aggregate scores, visualise results evaluation. package easily extendable, meaning users can supply scoring rules extend existing classes handle new types forecasts. package underwent major re-write. comprehensive documentation updated package revised version original scoringutils paper. Another good starting point vignettes Getting started, Details metrics implemented Scoring forecasts directly. details specific issue transforming forecasts scoring see: Nikos . Bosse, Sam Abbott, Anne Cori, Edwin van Leeuwen, Johannes Bracher* Sebastian Funk* (*: equal contribution) (2023). Scoring epidemiological forecasts transformed scales, PLoS Comput Biol 19(8): e1011393 https://doi.org/10.1371/journal.pcbi.1011393","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"installation","dir":"","previous_headings":"","what":"Installation","title":"Utilities for Scoring and Assessing Predictions","text":"Install CRAN version package using Install unstable development version GitHub using","code":"install.packages(\"scoringutils\") remotes::install_github(\"epiforecasts/scoringutils\", dependencies = TRUE)"},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"forecast-types","dir":"","previous_headings":"Quick start","what":"Forecast types","title":"Utilities for Scoring and Assessing Predictions","text":"scoringutils currently supports scoring following forecast types: - binary: probability binary (yes/) outcome variable. - point: forecast continuous discrete outcome variable represented single number. - quantile: probabilistic forecast continuous discrete outcome variable, forecast distribution represented set predictive quantiles. - sample: probabilistic forecast continuous discrete outcome variable, forecast represented finite set samples drawn predictive distribution.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"input-formats-and-input-validation","dir":"","previous_headings":"Quick start","what":"Input formats and input validation","title":"Utilities for Scoring and Assessing Predictions","text":"expected input format generally data.frame (similar) required columns observed, predicted, model holds forecasts observed values. Exact requirements depend forecast type. information, look paper, call ?as_forecast(), look example data provided package (example_binary, example_point, example_quantile, example_sample_continuous, example_sample_discrete). scoring, input data needs validated transformed forecast object using function as_forecast().","code":"forecast_quantile <- example_quantile |> as_forecast( forecast_unit = c( \"location\", \"forecast_date\", \"target_end_date\", \"target_type\", \"model\", \"horizon\" ), forecast_type = \"quantile\" ) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. print(forecast_quantile, 2) #> Forecast type: quantile #> Forecast unit: #> location, forecast_date, target_end_date, target_type, model, and horizon #> #> Key: #> observed quantile_level predicted location forecast_date target_end_date #> #> 1: 127300 NA NA DE 2021-01-02 #> 2: 4534 NA NA DE 2021-01-02 #> --- #> 20544: 78 0.975 611 IT 2021-07-12 2021-07-24 #> 20545: 78 0.990 719 IT 2021-07-12 2021-07-24 #> target_type model horizon #> #> 1: Cases NA #> 2: Deaths NA #> --- #> 20544: Deaths epiforecasts-EpiNow2 2 #> 20545: Deaths epiforecasts-EpiNow2 2"},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"the-forecast-unit","dir":"","previous_headings":"Quick start","what":"The forecast unit","title":"Utilities for Scoring and Assessing Predictions","text":"quantile-based sample-based forecasts, single prediction represented set several quantiles (samples) predictive distribution, .e. several rows input data. scoringutils therefore needs group rows together form single forecast. scoringutils uses existing columns input data achieve - values columns uniquely identify single forecast. Additional columns unrelated forecast unit can mess . forecast_unit argument as_forecast() makes sure columns retained relevant defining unit single forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"scoring-forecasts","dir":"","previous_headings":"Quick start","what":"Scoring forecasts","title":"Utilities for Scoring and Assessing Predictions","text":"Forecasts can scored calling score() validated forecast object. score() takes additional argument, metrics, list scoring rules. Every forecast type default list metrics. can easily add scoring functions, long conform format forecast type. See paper information. can summarise scores using function summarise_scores(). argument used specify desired level summary. fun let’s specify summary function, although recommended stick mean primary summary function, functions can lead improper scores.","code":"scores <- forecast_quantile |> score() scores |> summarise_scores(by = c(\"model\", \"target_type\")) |> summarise_scores(by = c(\"model\", \"target_type\"), fun = signif, digits = 3) #> model target_type wis overprediction underprediction #> #> 1: EuroCOVIDhub-ensemble Cases 17900.0 10000.00 4240.0 #> 2: EuroCOVIDhub-baseline Cases 28500.0 14100.00 10300.0 #> 3: epiforecasts-EpiNow2 Cases 20800.0 11900.00 3260.0 #> 4: EuroCOVIDhub-ensemble Deaths 41.4 7.14 4.1 #> 5: EuroCOVIDhub-baseline Deaths 159.0 65.90 2.1 #> 6: UMass-MechBayes Deaths 52.7 8.98 16.8 #> 7: epiforecasts-EpiNow2 Deaths 66.6 18.90 15.9 #> dispersion bias interval_coverage_50 interval_coverage_90 #> #> 1: 3660.0 -0.05640 0.391 0.805 #> 2: 4100.0 0.09800 0.328 0.820 #> 3: 5660.0 -0.07890 0.469 0.789 #> 4: 30.2 0.07270 0.875 1.000 #> 5: 91.4 0.33900 0.664 1.000 #> 6: 26.9 -0.02230 0.461 0.875 #> 7: 31.9 -0.00513 0.420 0.908 #> interval_coverage_deviation ae_median #> #> 1: -0.1020 24100.0 #> 2: -0.1170 38500.0 #> 3: -0.0696 27900.0 #> 4: 0.2040 53.1 #> 5: 0.1210 233.0 #> 6: -0.0249 78.5 #> 7: -0.0452 105.0"},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"package-workflow","dir":"","previous_headings":"","what":"Package workflow","title":"Utilities for Scoring and Assessing Predictions","text":"following depicts suggested workflow evaluating forecasts scoringutils (sections refer paper). Please find information paper, function documentation vignettes.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"citation","dir":"","previous_headings":"","what":"Citation","title":"Utilities for Scoring and Assessing Predictions","text":"using scoringutils work please consider citing using output citation(\"scoringutils\") (print(citation(\"scoringutils\"), bibtex = TRUE)):","code":"#> To cite scoringutils in publications use the following. If you use the #> CRPS, DSS, or Log Score, please also cite scoringRules. #> #> Nikos I. Bosse, Hugo Gruson, Sebastian Funk, Anne Cori, Edwin van #> Leeuwen, and Sam Abbott (2022). Evaluating Forecasts with #> scoringutils in R, arXiv. DOI: 10.48550/ARXIV.2205.07090 #> #> To cite scoringRules in publications use: #> #> Alexander Jordan, Fabian Krueger, Sebastian Lerch (2019). Evaluating #> Probabilistic Forecasts with scoringRules. Journal of Statistical #> Software, 90(12), 1-37. DOI 10.18637/jss.v090.i12 #> #> To see these entries in BibTeX format, use 'print(, #> bibtex=TRUE)', 'toBibtex(.)', or set #> 'options(citation.bibtex.max=999)'."},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"how-to-make-a-bug-report-or-feature-request","dir":"","previous_headings":"","what":"How to make a bug report or feature request","title":"Utilities for Scoring and Assessing Predictions","text":"Please briefly describe problem output expect issue. question, please don’t open issue. Instead, ask Q page.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"contributing","dir":"","previous_headings":"","what":"Contributing","title":"Utilities for Scoring and Assessing Predictions","text":"welcome contributions new contributors! particularly appreciate help priority problems issues. Please check add issues, /add pull request.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"code-of-conduct","dir":"","previous_headings":"","what":"Code of Conduct","title":"Utilities for Scoring and Assessing Predictions","text":"Please note scoringutils project released Contributor Code Conduct. contributing project, agree abide terms.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/add_relative_skill.html","id":null,"dir":"Reference","previous_headings":"","what":"Add relative skill scores based on pairwise comparisons — add_relative_skill","title":"Add relative skill scores based on pairwise comparisons — add_relative_skill","text":"Adds columns relative skills computed running pairwise comparisons scores. information computation relative skill, see get_pairwise_comparisons(). Relative skill calculated aggregation level specified .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/add_relative_skill.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Add relative skill scores based on pairwise comparisons — add_relative_skill","text":"","code":"add_relative_skill( scores, by = \"model\", metric = intersect(c(\"wis\", \"crps\", \"brier_score\"), names(scores)), baseline = NULL )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/add_relative_skill.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Add relative skill scores based on pairwise comparisons — add_relative_skill","text":"scores object class scores (data.table scores additional attribute metrics produced score()). Character vector column names define grouping level pairwise comparisons. default (model), one relative skill score per model. , example, = c(\"model\", \"location\"). get separate relative skill score every model every location. Internally, data.table scores split according (removing \"model\" splitting) pairwise comparisons computed separately split data.tables. metric string name metric relative skill shall computed. default either \"crps\", \"wis\" \"brier_score\" available. baseline string name model. baseline given, scaled relative skill respect baseline returned. default (NULL), relative skill scaled respect baseline model.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Absolute error of the median (quantile-based version) — ae_median_quantile","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"Compute absolute error median calculated $$ \\textrm{abs}(\\textrm{observed} - \\textrm{median prediction}) $$ median prediction predicted value quantile_level == 0.5, function therefore requires 0.5 among quantile levels quantile_level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"","code":"ae_median_quantile(observed, predicted, quantile_level)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"Numeric vector length N absolute error median.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"","code":"observed <- rnorm(30, mean = 1:30) predicted_values <- replicate(3, rnorm(30, mean = 1:30)) ae_median_quantile( observed, predicted_values, quantile_level = c(0.2, 0.5, 0.8) ) #> [1] 2.47438940 0.92040530 3.55121603 0.24032512 1.79911603 2.12426222 #> [7] 2.88687498 0.37899594 0.73282842 1.41674512 0.91703692 0.34483170 #> [13] 0.72770448 1.86768569 0.80586643 2.38692128 1.12876056 0.05733376 #> [19] 0.37081463 0.82374754 1.45618892 0.93544150 2.05333481 0.18155199 #> [25] 2.43676219 1.20798000 1.67648698 0.13974346 1.26067874 1.13044854"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Absolute error of the median (sample-based version) — ae_median_sample","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"Absolute error median calculated $$% \\textrm{abs}(\\textrm{observevd} - \\textrm{median\\_prediction}) $$","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"","code":"ae_median_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"vector scoring values","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"","code":"observed <- rnorm(30, mean = 1:30) predicted_values <- matrix(rnorm(30, mean = 1:30)) ae_median_sample(observed, predicted_values) #> [1] 1.61022189 0.32735036 2.52982645 0.98458168 0.94495454 0.65538891 #> [7] 0.56511146 0.09373061 1.31110818 0.61226219 0.75386115 0.08959962 #> [13] 0.39077113 1.56818369 0.84567980 1.24260044 0.27781917 0.65054779 #> [19] 1.18084954 0.45036469 0.05976767 0.14675942 0.60583332 0.19442459 #> [25] 0.21123533 0.28585022 0.64582375 1.78993469 1.20347916 0.67902801"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/apply_metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Apply a list of functions to a data table of forecasts — apply_metrics","title":"Apply a list of functions to a data table of forecasts — apply_metrics","text":"helper function applies scoring rules (stored list functions) data table forecasts. apply_metrics used within score() apply scoring rules data. Scoring rules wrapped run_safely() catch errors make sure arguments passed scoring rule actually accepted .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/apply_metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Apply a list of functions to a data table of forecasts — apply_metrics","text":"","code":"apply_metrics(forecast, metrics, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/apply_metrics.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Apply a list of functions to a data table of forecasts — apply_metrics","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()) metrics named list scoring functions. Names used column names output. See metrics_point(), metrics_binary(), metrics_quantile(), metrics_sample() information default metrics used. Note want pass arguments given metric, function customise_metric() pass updated list functions custom metric metrics argument score(). ... Additional arguments passed scoring rules. Note currently used, calls apply_scores currently avoid passing arguments via ... instead expect metrics directly modified using customise_metric().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/apply_metrics.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Apply a list of functions to a data table of forecasts — apply_metrics","text":"data table forecasts calculated metrics.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a forecast object — as_forecast","title":"Create a forecast object — as_forecast","text":"Process validate data.frame (similar) similar forecasts observations. input passes input checks, converted forecast object. forecast object data.table class forecast additional class depends forecast type. See details section information expected input formats. as_forecast() gives users control data parsed. Using arguments observed, predicted, model, users can rename existing columns input data match required columns forecast object. Using argument forecast_unit, users can specify columns uniquely identify single forecast (remove others, see set_forecast_unit() details).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a forecast object — as_forecast","text":"","code":"as_forecast(data, ...) # Default S3 method as_forecast( data, forecast_unit = NULL, forecast_type = NULL, observed = NULL, predicted = NULL, model = NULL, quantile_level = NULL, sample_id = NULL, ... )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a forecast object — as_forecast","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. ... Additional arguments forecast_unit (optional) Name columns data (renaming columns done as_forecast()) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. forecast_type (optional) forecast type expect forecasts . forecast type determined scoringutils based input match , error thrown. NULL (default), forecast type inferred data. observed (optional) Name column data contains observed values. column renamed \"observed\". predicted (optional) Name column data contains predicted values. column renamed \"predicted\". model (optional) Name column data contains names models/forecasters generated predicted values. column renamed \"model\". quantile_level (optional) Name column data contains quantile level predicted values. column renamed \"quantile_level\". applicable quantile-based forecasts. sample_id (optional) Name column data contains sample id. column renamed \"sample_id\". applicable sample-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Create a forecast object — as_forecast","text":"Depending forecast type, object following class returned: forecast_binary binary forecasts forecast_point point forecasts forecast_sample sample-based forecasts forecast_quantile quantile-based forecasts","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"forecast-types-and-input-formats","dir":"Reference","previous_headings":"","what":"Forecast types and input formats","title":"Create a forecast object — as_forecast","text":"Various different forecast types / forecast formats supported. moment, : point forecasts binary forecasts (\"soft binary classification\") Probabilistic forecasts quantile-based format (forecast represented set predictive quantiles) Probabilistic forecasts sample-based format (forecast represented set predictive samples) Forecast types determined based columns present input data. overview required format forecast type: forecast types require data.frame similar columns observed predicted, model. Point forecasts require column observed type numeric column predicted type numeric. Binary forecasts require column observed type factor exactly two levels column predicted type numeric probabilities, corresponding probability observed equal second factor level. See details information. Quantile-based forecasts require column observed type numeric, column predicted type numeric, column quantile_level type numeric quantile-levels (0 1). Sample-based forecasts require column observed type numeric, column predicted type numeric, column sample_id type numeric sample indices. information see vignettes example data (example_quantile, example_sample_continuous, example_sample_discrete, example_point(), example_binary).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"forecast-unit","dir":"Reference","previous_headings":"","what":"Forecast unit","title":"Create a forecast object — as_forecast","text":"order score forecasts, scoringutils needs know rows data belong together jointly form single forecasts. easy e.g. point forecast, one row per forecast. quantile sample-based forecasts, however, multiple rows belong single forecast. forecast unit unit single forecast described combination columns uniquely identify single forecast. example, forecasts made different models various locations different time points, several weeks future. forecast unit described forecast_unit = c(\"model\", \"location\", \"forecast_date\", \"forecast_horizon\"). scoringutils automatically tries determine unit single forecast. uses existing columns , means columns must present unrelated forecast unit. simplistic example, additional row, \"even\", one row number even zero otherwise, mess scoring scoringutils thinks column relevant defining forecast unit. order avoid issues, recommend setting forecast unit explicitly, usually forecast_unit argument as_forecast(). drop unneeded columns, making sure necessary, 'protected columns' like \"predicted\" \"observed\" retained.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Create a forecast object — as_forecast","text":"","code":"as_forecast(example_binary) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Forecast type: binary #> Forecast unit: #> location, location_name, target_end_date, target_type, forecast_date, model, #> and horizon #> #> location location_name target_end_date target_type forecast_date #> #> 1: DE Germany 2021-01-02 Cases #> 2: DE Germany 2021-01-02 Deaths #> 3: DE Germany 2021-01-09 Cases #> 4: DE Germany 2021-01-09 Deaths #> 5: DE Germany 2021-01-16 Cases #> --- #> 1027: IT Italy 2021-07-24 Deaths 2021-07-12 #> 1028: IT Italy 2021-07-24 Deaths 2021-07-05 #> 1029: IT Italy 2021-07-24 Deaths 2021-07-12 #> 1030: IT Italy 2021-07-24 Deaths 2021-07-05 #> 1031: IT Italy 2021-07-24 Deaths 2021-07-12 #> model horizon predicted observed #> #> 1: NA NA #> 2: NA NA #> 3: NA NA #> 4: NA NA #> 5: NA NA #> --- #> 1027: EuroCOVIDhub-baseline 2 0.250 0 #> 1028: UMass-MechBayes 3 0.475 0 #> 1029: UMass-MechBayes 2 0.450 0 #> 1030: epiforecasts-EpiNow2 3 0.375 0 #> 1031: epiforecasts-EpiNow2 2 0.300 0 as_forecast( example_quantile, forecast_unit = c(\"model\", \"target_type\", \"target_end_date\", \"horizon\", \"location\") ) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Forecast type: quantile #> Forecast unit: #> model, target_type, target_end_date, horizon, and location #> #> Key: #> observed quantile_level predicted model target_type #> #> 1: 127300 NA NA Cases #> 2: 4534 NA NA Deaths #> 3: 154922 NA NA Cases #> 4: 6117 NA NA Deaths #> 5: 110183 NA NA Cases #> --- #> 20541: 78 0.850 352 epiforecasts-EpiNow2 Deaths #> 20542: 78 0.900 397 epiforecasts-EpiNow2 Deaths #> 20543: 78 0.950 499 epiforecasts-EpiNow2 Deaths #> 20544: 78 0.975 611 epiforecasts-EpiNow2 Deaths #> 20545: 78 0.990 719 epiforecasts-EpiNow2 Deaths #> target_end_date horizon location #> #> 1: 2021-01-02 NA DE #> 2: 2021-01-02 NA DE #> 3: 2021-01-09 NA DE #> 4: 2021-01-09 NA DE #> 5: 2021-01-16 NA DE #> --- #> 20541: 2021-07-24 2 IT #> 20542: 2021-07-24 2 IT #> 20543: 2021-07-24 2 IT #> 20544: 2021-07-24 2 IT #> 20545: 2021-07-24 2 IT"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Create an object of class scores from data — as_scores","title":"Create an object of class scores from data — as_scores","text":"convenience function wraps new_scores() validates scores object.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create an object of class scores from data — as_scores","text":"","code":"as_scores(scores, metrics)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create an object of class scores from data — as_scores","text":"scores data.table similar scores produced score(). metrics character vector names scores (.e. names scoring rules used scoring).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_scores.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Create an object of class scores from data — as_scores","text":"object class scores","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_dims_ok_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","title":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","text":"Function assesses whether input dimensions match. following, n number observations / forecasts. Scalar values may repeated match length input. Allowed options therefore: observed vector length 1 length n predicted : vector length 1 length n matrix n rows 1 column","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_dims_ok_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","text":"","code":"assert_dims_ok_point(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_dims_ok_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","text":"observed Input checked. factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_dims_ok_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that input is a forecast object and passes validations — assert_forecast","title":"Assert that input is a forecast object and passes validations — assert_forecast","text":"Assert object forecast object (.e. data.table class forecast additional class forecast_* corresponding forecast type).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that input is a forecast object and passes validations — assert_forecast","text":"","code":"assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) # Default S3 method assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) # S3 method for class 'forecast_binary' assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) # S3 method for class 'forecast_point' assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) # S3 method for class 'forecast_quantile' assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) # S3 method for class 'forecast_sample' assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that input is a forecast object and passes validations — assert_forecast","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()) forecast_type (optional) forecast type expect forecasts . forecast type determined scoringutils based input match , error thrown. NULL (default), forecast type inferred data. verbose Logical. FALSE (default TRUE), messages warnings created. ... Additional arguments","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that input is a forecast object and passes validations — assert_forecast","text":"Returns NULL invisibly.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"forecast-types-and-input-formats","dir":"Reference","previous_headings":"","what":"Forecast types and input formats","title":"Assert that input is a forecast object and passes validations — assert_forecast","text":"Various different forecast types / forecast formats supported. moment, : point forecasts binary forecasts (\"soft binary classification\") Probabilistic forecasts quantile-based format (forecast represented set predictive quantiles) Probabilistic forecasts sample-based format (forecast represented set predictive samples) Forecast types determined based columns present input data. overview required format forecast type: forecast types require data.frame similar columns observed predicted, model. Point forecasts require column observed type numeric column predicted type numeric. Binary forecasts require column observed type factor exactly two levels column predicted type numeric probabilities, corresponding probability observed equal second factor level. See details information. Quantile-based forecasts require column observed type numeric, column predicted type numeric, column quantile_level type numeric quantile-levels (0 1). Sample-based forecasts require column observed type numeric, column predicted type numeric, column sample_id type numeric sample indices. information see vignettes example data (example_quantile, example_sample_continuous, example_sample_discrete, example_point(), example_binary).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Assert that input is a forecast object and passes validations — assert_forecast","text":"","code":"forecast <- as_forecast(example_binary) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. assert_forecast(forecast) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected."},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_generic.html","id":null,"dir":"Reference","previous_headings":"","what":"Validation common to all forecast types — assert_forecast_generic","title":"Validation common to all forecast types — assert_forecast_generic","text":"function runs input checks apply input data, regardless forecast type. function asserts forecast data.table columns observed predicted, well column called model. checks forecast type forecast unit checks duplicate forecasts appropriate, checks number samples / quantiles forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_generic.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Validation common to all forecast types — assert_forecast_generic","text":"","code":"assert_forecast_generic(data, verbose = TRUE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_generic.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Validation common to all forecast types — assert_forecast_generic","text":"data data.table forecasts observed values validated. verbose Logical. FALSE (default TRUE), messages warnings created.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_generic.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Validation common to all forecast types — assert_forecast_generic","text":"returns input","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_type.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that forecast type is as expected — assert_forecast_type","title":"Assert that forecast type is as expected — assert_forecast_type","text":"Assert forecast type expected","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_type.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that forecast type is as expected — assert_forecast_type","text":"","code":"assert_forecast_type(data, actual = get_forecast_type(data), desired = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_type.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that forecast type is as expected — assert_forecast_type","text":"data forecast object produced as_forecast(). actual actual forecast type data desired desired forecast type data","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_type.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that forecast type is as expected — assert_forecast_type","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for binary forecast — assert_input_binary","title":"Assert that inputs are correct for binary forecast — assert_input_binary","text":"Function assesses whether inputs correspond requirements scoring binary forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for binary forecast — assert_input_binary","text":"","code":"assert_input_binary(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for binary forecast — assert_input_binary","text":"observed Input checked. factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for binary forecast — assert_input_binary","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_interval.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for interval-based forecast — assert_input_interval","title":"Assert that inputs are correct for interval-based forecast — assert_input_interval","text":"Function assesses whether inputs correspond requirements scoring interval-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_interval.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for interval-based forecast — assert_input_interval","text":"","code":"assert_input_interval(observed, lower, upper, interval_range)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_interval.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for interval-based forecast — assert_input_interval","text":"observed Input checked. numeric vector observed values size n. lower Input checked. numeric vector size n holds predicted value lower bounds prediction intervals. upper Input checked. numeric vector size n holds predicted value upper bounds prediction intervals. interval_range Input checked. vector size n denotes interval range percent. E.g. value 50 denotes (25%, 75%) prediction interval.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_interval.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for interval-based forecast — assert_input_interval","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for point forecast — assert_input_point","title":"Assert that inputs are correct for point forecast — assert_input_point","text":"Function assesses whether inputs correspond requirements scoring point forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for point forecast — assert_input_point","text":"","code":"assert_input_point(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for point forecast — assert_input_point","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. numeric vector predicted values size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for point forecast — assert_input_point","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","title":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","text":"Function assesses whether inputs correspond requirements scoring quantile-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","text":"","code":"assert_input_quantile( observed, predicted, quantile_level, unique_quantile_levels = TRUE )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. nxN matrix predictive quantiles, n (number rows) number data points N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Input checked. vector size N denotes quantile levels corresponding columns prediction matrix. unique_quantile_levels Whether quantile levels required unique (TRUE, default) (FALSE).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for sample-based forecast — assert_input_sample","title":"Assert that inputs are correct for sample-based forecast — assert_input_sample","text":"Function assesses whether inputs correspond requirements scoring sample-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for sample-based forecast — assert_input_sample","text":"","code":"assert_input_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for sample-based forecast — assert_input_sample","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. numeric nxN matrix predictive samples, n (number rows) number data points N (number columns) number samples per forecast. observed just single number, predicted values can just vector size N.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for sample-based forecast — assert_input_sample","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Determines bias of quantile forecasts — bias_quantile","title":"Determines bias of quantile forecasts — bias_quantile","text":"Determines bias quantile forecasts. increasing number quantiles measure converges sample based bias version integer continuous forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Determines bias of quantile forecasts — bias_quantile","text":"","code":"bias_quantile(observed, predicted, quantile_level, na.rm = TRUE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Determines bias of quantile forecasts — bias_quantile","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made. Note contain median (0.5) median imputed mean two innermost quantiles. na.rm Logical. missing values removed?","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Determines bias of quantile forecasts — bias_quantile","text":"scalar quantile bias single quantile prediction","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Determines bias of quantile forecasts — bias_quantile","text":"quantile forecasts, bias measured $$ B_t = (1 - 2 \\cdot \\max \\{| q_{t,} \\Q_t \\land q_{t,} \\leq x_t\\}) \\mathbf{1}( x_t \\leq q_{t, 0.5}) \\\\ + (1 - 2 \\cdot \\min \\{| q_{t,} \\Q_t \\land q_{t,} \\geq x_t\\}) 1( x_t \\geq q_{t, 0.5}),$$ \\(Q_t\\) set quantiles form predictive distribution time \\(t\\) \\(x_t\\) observed value. consistency, define \\(Q_t\\) always includes element \\(q_{t, 0} = - \\infty\\) \\(q_{t,1} = \\infty\\). \\(1()\\) indicator function \\(1\\) condition satisfied \\(0\\) otherwise. clearer terms, bias \\(B_t\\) : \\(1 - 2 \\cdot\\) maximum percentile rank corresponding quantile still smaller equal observed value, observed value smaller median predictive distribution. \\(1 - 2 \\cdot\\) minimum percentile rank corresponding quantile still larger equal observed value observed value larger median predictive distribution.. \\(0\\) observed value exactly median (terms cancel ) Bias can assume values -1 1 0 ideally (.e. unbiased). Note given quantiles contain median, median imputed mean two innermost quantiles. large enough number quantiles, percentile rank equal proportion predictive samples observed value, bias metric coincides one continuous forecasts (see bias_sample()).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Determines bias of quantile forecasts — bias_quantile","text":"","code":"predicted <- matrix(c(1.5:23.5, 3.3:25.3), nrow = 2, byrow = TRUE) quantile_level <- c(0.01, 0.025, seq(0.05, 0.95, 0.05), 0.975, 0.99) observed <- c(15, 12.4) bias_quantile(observed, predicted, quantile_level) #> [1] -0.3 0.2"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile_single_vector.html","id":null,"dir":"Reference","previous_headings":"","what":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","title":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","text":"Internal function compute bias single observed value, vector predicted values vector quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile_single_vector.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","text":"","code":"bias_quantile_single_vector(observed, predicted, quantile_level, na.rm)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile_single_vector.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","text":"observed Scalar observed value. predicted Vector length N (corresponding number quantiles) holds predictions. quantile_level Vector size N quantile levels predictions made. Note contain median (0.5) median imputed mean two innermost quantiles. na.rm Logical. missing values removed?","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile_single_vector.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","text":"scalar quantile bias single quantile prediction","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Determine bias of forecasts — bias_sample","title":"Determine bias of forecasts — bias_sample","text":"Determines bias predictive Monte-Carlo samples. function automatically recognises, whether forecasts continuous integer valued adapts Bias function accordingly.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Determine bias of forecasts — bias_sample","text":"","code":"bias_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Determine bias of forecasts — bias_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Determine bias of forecasts — bias_sample","text":"Numeric vector length n biases predictive samples respect observed values.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Determine bias of forecasts — bias_sample","text":"continuous forecasts, Bias measured $$ B_t (P_t, x_t) = 1 - 2 * (P_t (x_t)) $$ \\(P_t\\) empirical cumulative distribution function prediction observed value \\(x_t\\). Computationally, \\(P_t (x_t)\\) just calculated fraction predictive samples \\(x_t\\) smaller \\(x_t\\). integer valued forecasts, Bias measured $$ B_t (P_t, x_t) = 1 - (P_t (x_t) + P_t (x_t + 1)) $$ adjust integer nature forecasts. cases, Bias can assume values -1 1 0 ideally.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Determine bias of forecasts — bias_sample","text":"integer valued Bias function discussed Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15 Funk S, Camacho , Kucharski AJ, Lowe R, Eggo RM, et al. (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15. PLOS Computational Biology 15(2): e1006785. doi:10.1371/journal.pcbi.1006785","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Determine bias of forecasts — bias_sample","text":"","code":"## integer valued forecasts observed <- rpois(30, lambda = 1:30) predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) bias_sample(observed, predicted) #> [1] 0.660 -0.025 0.710 -0.930 0.870 0.435 -0.885 -0.940 -0.515 -0.790 #> [11] 0.975 -0.975 -0.620 -0.740 -0.640 0.395 0.695 -0.765 -0.935 -0.680 #> [21] -0.725 -0.320 0.355 0.730 0.250 0.995 -0.650 0.235 0.250 0.850 ## continuous forecasts observed <- rnorm(30, mean = 1:30) predicted <- replicate(200, rnorm(30, mean = 1:30)) bias_sample(observed, predicted) #> [1] -0.46 0.02 0.02 0.12 -0.18 -0.07 0.96 -0.60 0.16 -0.31 0.79 0.49 #> [13] -0.74 -0.48 0.26 -0.56 0.82 0.89 0.41 -0.31 0.19 0.47 -0.85 0.32 #> [25] 0.15 -0.16 0.34 -0.30 0.80 0.27"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_columns_present.html","id":null,"dir":"Reference","previous_headings":"","what":"Check column names are present in a data.frame — check_columns_present","title":"Check column names are present in a data.frame — check_columns_present","text":"functions loops column names checks whether present. issue encountered, function immediately stops returns message first issue encountered.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_columns_present.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check column names are present in a data.frame — check_columns_present","text":"","code":"check_columns_present(data, columns)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_columns_present.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check column names are present in a data.frame — check_columns_present","text":"data data.frame similar checked columns character vector column names check","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_columns_present.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check column names are present in a data.frame — check_columns_present","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_dims_ok_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Check Inputs Have Matching Dimensions — check_dims_ok_point","title":"Check Inputs Have Matching Dimensions — check_dims_ok_point","text":"Function assesses whether input dimensions match. following, n number observations / forecasts. Scalar values may repeated match length input. Allowed options therefore: observed vector length 1 length n predicted : vector length 1 length n matrix n rows 1 column","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_dims_ok_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check Inputs Have Matching Dimensions — check_dims_ok_point","text":"","code":"check_dims_ok_point(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_dims_ok_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check Inputs Have Matching Dimensions — check_dims_ok_point","text":"observed Input checked. factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_dims_ok_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check Inputs Have Matching Dimensions — check_dims_ok_point","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_duplicates.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that there are no duplicate forecasts — check_duplicates","title":"Check that there are no duplicate forecasts — check_duplicates","text":"Runs get_duplicate_forecasts() returns message issue encountered","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_duplicates.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that there are no duplicate forecasts — check_duplicates","text":"","code":"check_duplicates(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_duplicates.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that there are no duplicate forecasts — check_duplicates","text":"data data.frame used score()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_duplicates.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that there are no duplicate forecasts — check_duplicates","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for binary forecast — check_input_binary","title":"Check that inputs are correct for binary forecast — check_input_binary","text":"Function assesses whether inputs correspond requirements scoring binary forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for binary forecast — check_input_binary","text":"","code":"check_input_binary(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for binary forecast — check_input_binary","text":"observed Input checked. factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for binary forecast — check_input_binary","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_interval.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for interval-based forecast — check_input_interval","title":"Check that inputs are correct for interval-based forecast — check_input_interval","text":"Function assesses whether inputs correspond requirements scoring interval-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_interval.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for interval-based forecast — check_input_interval","text":"","code":"check_input_interval(observed, lower, upper, interval_range)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_interval.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for interval-based forecast — check_input_interval","text":"observed Input checked. numeric vector observed values size n. lower Input checked. numeric vector size n holds predicted value lower bounds prediction intervals. upper Input checked. numeric vector size n holds predicted value upper bounds prediction intervals. interval_range Input checked. vector size n denotes interval range percent. E.g. value 50 denotes (25%, 75%) prediction interval.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_interval.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for interval-based forecast — check_input_interval","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for point forecast — check_input_point","title":"Check that inputs are correct for point forecast — check_input_point","text":"Function assesses whether inputs correspond requirements scoring point forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for point forecast — check_input_point","text":"","code":"check_input_point(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for point forecast — check_input_point","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. numeric vector predicted values size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for point forecast — check_input_point","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for quantile-based forecast — check_input_quantile","title":"Check that inputs are correct for quantile-based forecast — check_input_quantile","text":"Function assesses whether inputs correspond requirements scoring quantile-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for quantile-based forecast — check_input_quantile","text":"","code":"check_input_quantile(observed, predicted, quantile_level)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for quantile-based forecast — check_input_quantile","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. nxN matrix predictive quantiles, n (number rows) number data points N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Input checked. vector size N denotes quantile levels corresponding columns prediction matrix.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for quantile-based forecast — check_input_quantile","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for sample-based forecast — check_input_sample","title":"Check that inputs are correct for sample-based forecast — check_input_sample","text":"Function assesses whether inputs correspond requirements scoring sample-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for sample-based forecast — check_input_sample","text":"","code":"check_input_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for sample-based forecast — check_input_sample","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. numeric nxN matrix predictive samples, n (number rows) number data points N (number columns) number samples per forecast. observed just single number, predicted values can just vector size N.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for sample-based forecast — check_input_sample","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_number_per_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that all forecasts have the same number of quantiles or samples — check_number_per_forecast","title":"Check that all forecasts have the same number of quantiles or samples — check_number_per_forecast","text":"Function checks number quantiles samples per forecast. number quantiles samples forecasts, returns TRUE string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_number_per_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that all forecasts have the same number of quantiles or samples — check_number_per_forecast","text":"","code":"check_number_per_forecast(data, forecast_unit)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_number_per_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that all forecasts have the same number of quantiles or samples — check_number_per_forecast","text":"data data.frame similar checked forecast_unit Character vector denoting unit single forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_number_per_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that all forecasts have the same number of quantiles or samples — check_number_per_forecast","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_numeric_vector.html","id":null,"dir":"Reference","previous_headings":"","what":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","title":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","text":"Helper function check whether input numeric vector.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_numeric_vector.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","text":"","code":"check_numeric_vector(x, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_numeric_vector.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","text":"x input check ... Arguments passed checkmate::check_numeric lower [numeric(1)] Lower value elements x must greater equal . upper [numeric(1)] Upper value elements x must lower equal . finite [logical(1)] Check finite values? Default FALSE. .missing [logical(1)] vectors missing values allowed? Default TRUE. .missing [logical(1)] vectors non-missing values allowed? Default TRUE. Note empty vectors non-missing values. len [integer(1)] Exact expected length x. min.len [integer(1)] Minimal length x. max.len [integer(1)] Maximal length x. unique [logical(1)] Must values unique? Default FALSE. sorted [logical(1)] Elements must sorted ascending order. Missing values ignored. names [character(1)] Check names. See checkNamed possible values. Default “” performs check . Note can use checkSubset check specific set names. typed.missing [logical(1)] set FALSE (default), types missing values (NA, NA_integer_, NA_real_, NA_character_ NA_character_) well empty vectors allowed type-checking atomic input. Set TRUE enable strict type checking. null.ok [logical(1)] set TRUE, x may also NULL. case type check x performed, additional checks disabled.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_numeric_vector.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_try.html","id":null,"dir":"Reference","previous_headings":"","what":"Helper function to convert assert statements into checks — check_try","title":"Helper function to convert assert statements into checks — check_try","text":"Tries execute expression. Internally, used see whether assertions fail checking inputs (.e. convert assert_*() statement check). expression fails, error message returned. expression succeeds, TRUE returned.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_try.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Helper function to convert assert statements into checks — check_try","text":"","code":"check_try(expr)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_try.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Helper function to convert assert statements into checks — check_try","text":"expr expression evaluated","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_try.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Helper function to convert assert statements into checks — check_try","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/clean_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Clean forecast object — clean_forecast","title":"Clean forecast object — clean_forecast","text":"function makes possible silently validate object. addition, can return copy data remove rows missing values.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/clean_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Clean forecast object — clean_forecast","text":"","code":"clean_forecast(forecast, copy = FALSE, na.omit = FALSE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/clean_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Clean forecast object — clean_forecast","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()) copy Logical, default FALSE. TRUE, copy input data created. na.omit Logical, default FALSE. TRUE, rows missing values removed.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_two_models.html","id":null,"dir":"Reference","previous_headings":"","what":"Compare two models based on subset of common forecasts — compare_two_models","title":"Compare two models based on subset of common forecasts — compare_two_models","text":"function compares two models based subset forecasts models made prediction. gets called pairwise_comparison_one_group(), handles comparison multiple models single set forecasts (subsets forecasts distinguished). pairwise_comparison_one_group() turn gets called get_pairwise_comparisons() can handle pairwise comparisons set forecasts multiple subsets, e.g. pairwise comparisons one set forecasts, done separately two different forecast targets.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_two_models.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Compare two models based on subset of common forecasts — compare_two_models","text":"","code":"compare_two_models( scores, name_model1, name_model2, metric, one_sided = FALSE, test_type = c(\"non_parametric\", \"permutation\"), n_permutations = 999 )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_two_models.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Compare two models based on subset of common forecasts — compare_two_models","text":"scores object class scores (data.table scores additional attribute metrics produced score()). name_model1 Character, name first model name_model2 Character, name model compare metric string name metric relative skill shall computed. default either \"crps\", \"wis\" \"brier_score\" available. one_sided Boolean, default FALSE, whether two conduct one-sided instead two-sided test determine significance pairwise comparison. test_type Character, either \"non_parametric\" (default) \"permutation\". determines kind test shall conducted determine p-values. n_permutations Numeric, number permutations permutation test. Default 999.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_two_models.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Compare two models based on subset of common forecasts — compare_two_models","text":"list mean score ratios p-values comparison two models","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_two_models.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Compare two models based on subset of common forecasts — compare_two_models","text":"Johannes Bracher, johannes.bracher@kit.edu Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"(Continuous) ranked probability score — crps_sample","title":"(Continuous) ranked probability score — crps_sample","text":"Wrapper around crps_sample() function scoringRules package. Can used continuous well integer valued forecasts","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"(Continuous) ranked probability score — crps_sample","text":"","code":"crps_sample(observed, predicted, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"(Continuous) ranked probability score — crps_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. ... Additional arguments passed crps_sample() scoringRules package.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"(Continuous) ranked probability score — crps_sample","text":"Vector scores.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"(Continuous) ranked probability score — crps_sample","text":"Alexander Jordan, Fabian Krüger, Sebastian Lerch, Evaluating Probabilistic Forecasts scoringRules, https://www.jstatsoft.org/article/view/v090i12","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"(Continuous) ranked probability score — crps_sample","text":"","code":"observed <- rpois(30, lambda = 1:30) predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) crps_sample(observed, predicted) #> [1] 0.231225 0.329850 1.060100 0.412625 0.991550 0.793850 2.847025 1.126200 #> [9] 0.690675 0.992875 1.118300 4.911975 4.135800 1.411050 0.967825 3.425125 #> [17] 0.839600 2.404700 1.376075 1.665525 2.771375 1.168550 1.244700 2.377250 #> [25] 2.847875 2.019175 1.827975 2.074850 1.868775 3.171950"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/customise_metric.html","id":null,"dir":"Reference","previous_headings":"","what":"Customises a metric function with additional arguments. — customise_metric","title":"Customises a metric function with additional arguments. — customise_metric","text":"function takes metric function additional arguments, returns new function includes additional arguments calling original metric function. expected way pass additional arguments metric evaluating forecast using score(): evaluate forecast using metric additional argument, need create custom version scoring function argument included. need create updated version list scoring functions includes customised metric pass score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/customise_metric.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Customises a metric function with additional arguments. — customise_metric","text":"","code":"customise_metric(metric, ...) customize_metric(metric, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/customise_metric.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Customises a metric function with additional arguments. — customise_metric","text":"metric metric function customised. ... Additional arguments included calling metric function.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/customise_metric.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Customises a metric function with additional arguments. — customise_metric","text":"customised metric function.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/customise_metric.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Customises a metric function with additional arguments. — customise_metric","text":"","code":"# Create a customised metric function custom_metric <- customise_metric(mean, na.rm = TRUE) # Use the customised metric function values <- c(1, 2, NA, 4, 5) custom_metric(values) #> [1] 3 # Updating metrics list to calculate 70% coverage in `score()` interval_coverage_70 <- customise_metric( interval_coverage, interval_range = 70 ) updated_metrics <- c( metrics_quantile(), \"interval_coverage_70\" = interval_coverage_70 ) score( as_forecast(example_quantile), metrics = updated_metrics ) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> location target_end_date target_type location_name forecast_date #> #> 1: DE 2021-05-08 Cases Germany 2021-05-03 #> 2: DE 2021-05-08 Cases Germany 2021-05-03 #> 3: DE 2021-05-08 Cases Germany 2021-05-03 #> 4: DE 2021-05-08 Deaths Germany 2021-05-03 #> 5: DE 2021-05-08 Deaths Germany 2021-05-03 #> --- #> 883: IT 2021-07-24 Deaths Italy 2021-07-12 #> 884: IT 2021-07-24 Deaths Italy 2021-07-05 #> 885: IT 2021-07-24 Deaths Italy 2021-07-12 #> 886: IT 2021-07-24 Deaths Italy 2021-07-05 #> 887: IT 2021-07-24 Deaths Italy 2021-07-12 #> model horizon wis overprediction underprediction #> #> 1: EuroCOVIDhub-ensemble 1 7990.854783 2.549870e+03 0.0000000 #> 2: EuroCOVIDhub-baseline 1 16925.046957 1.527583e+04 0.0000000 #> 3: epiforecasts-EpiNow2 1 25395.960870 1.722226e+04 0.0000000 #> 4: EuroCOVIDhub-ensemble 1 53.880000 0.000000e+00 0.6086957 #> 5: EuroCOVIDhub-baseline 1 46.793043 2.130435e+00 0.0000000 #> --- #> 883: EuroCOVIDhub-baseline 2 80.336957 3.608696e+00 0.0000000 #> 884: UMass-MechBayes 3 4.881739 4.347826e-02 0.0000000 #> 885: UMass-MechBayes 2 25.581739 1.782609e+01 0.0000000 #> 886: epiforecasts-EpiNow2 3 19.762609 5.478261e+00 0.0000000 #> 887: epiforecasts-EpiNow2 2 66.161739 4.060870e+01 0.0000000 #> dispersion bias interval_coverage_50 interval_coverage_90 #> #> 1: 5440.985217 0.50 TRUE TRUE #> 2: 1649.220870 0.95 FALSE FALSE #> 3: 8173.700000 0.90 FALSE TRUE #> 4: 53.271304 -0.10 TRUE TRUE #> 5: 44.662609 0.30 TRUE TRUE #> --- #> 883: 76.728261 0.20 TRUE TRUE #> 884: 4.838261 0.10 TRUE TRUE #> 885: 7.755652 0.80 FALSE TRUE #> 886: 14.284348 0.50 TRUE TRUE #> 887: 25.553043 0.90 FALSE TRUE #> interval_coverage_deviation ae_median interval_coverage_70 #> #> 1: 0.05181818 12271 TRUE #> 2: -0.40272727 25620 FALSE #> 3: -0.31181818 44192 FALSE #> 4: 0.41545455 14 TRUE #> 5: 0.23363636 15 TRUE #> --- #> 883: 0.32454545 53 TRUE #> 884: 0.41545455 1 TRUE #> 885: -0.22090909 46 FALSE #> 886: 0.05181818 26 TRUE #> 887: -0.31181818 108 FALSE"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_assert_functions.html","id":null,"dir":"Reference","previous_headings":"","what":"Documentation template for assert functions — document_assert_functions","title":"Documentation template for assert functions — document_assert_functions","text":"Documentation template assert functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_assert_functions.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Documentation template for assert functions — document_assert_functions","text":"observed Input checked. numeric vector observed values size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_assert_functions.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Documentation template for assert functions — document_assert_functions","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_check_functions.html","id":null,"dir":"Reference","previous_headings":"","what":"Documentation template for check functions — document_check_functions","title":"Documentation template for check functions — document_check_functions","text":"Documentation template check functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_check_functions.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Documentation template for check functions — document_check_functions","text":"data data.frame similar checked observed Input checked. numeric vector observed values size n. columns character vector column names check","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_check_functions.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Documentation template for check functions — document_check_functions","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_test_functions.html","id":null,"dir":"Reference","previous_headings":"","what":"Documentation template for test functions — document_test_functions","title":"Documentation template for test functions — document_test_functions","text":"Documentation template test functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_test_functions.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Documentation template for test functions — document_test_functions","text":"Returns TRUE check successful FALSE otherwise","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Dawid-Sebastiani score — dss_sample","title":"Dawid-Sebastiani score — dss_sample","text":"Wrapper around dss_sample() function scoringRules package.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Dawid-Sebastiani score — dss_sample","text":"","code":"dss_sample(observed, predicted, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Dawid-Sebastiani score — dss_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. ... Additional arguments passed dss_sample() scoringRules package.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Dawid-Sebastiani score — dss_sample","text":"Vector scores.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Dawid-Sebastiani score — dss_sample","text":"Alexander Jordan, Fabian Krüger, Sebastian Lerch, Evaluating Probabilistic Forecasts scoringRules, https://www.jstatsoft.org/article/view/v090i12","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Dawid-Sebastiani score — dss_sample","text":"","code":"observed <- rpois(30, lambda = 1:30) predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) dss_sample(observed, predicted) #> [1] -0.06179111 2.81530809 2.46742176 1.79360855 1.61740613 2.39081466 #> [7] 1.92865409 15.58374079 3.88246189 4.97070116 2.24551342 2.71528477 #> [13] 2.79485162 2.62249405 2.80770087 4.30015607 3.23096351 3.16329747 #> [19] 3.23001192 7.67628290 2.92052357 5.37555852 4.78857945 3.12865561 #> [25] 3.61999703 4.32028466 3.38593521 6.87116360 4.71995903 4.99107110"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_data.table.html","id":null,"dir":"Reference","previous_headings":"","what":"Ensure that an object is a data.table — ensure_data.table","title":"Ensure that an object is a data.table — ensure_data.table","text":"function ensures object data table. object data table, converted one. object data table, copy object returned.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_data.table.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Ensure that an object is a data.table — ensure_data.table","text":"","code":"ensure_data.table(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_data.table.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Ensure that an object is a data.table — ensure_data.table","text":"data object ensure data table.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_data.table.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Ensure that an object is a data.table — ensure_data.table","text":"data.table/copy existing data.table.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_model_column.html","id":null,"dir":"Reference","previous_headings":"","what":"Assure that data has a model column — ensure_model_column","title":"Assure that data has a model column — ensure_model_column","text":"Check whether data.table column called model. , column called model added value Unspecified model.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_model_column.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assure that data has a model column — ensure_model_column","text":"","code":"ensure_model_column(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_model_column.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assure that data has a model column — ensure_model_column","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_model_column.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assure that data has a model column — ensure_model_column","text":"data.table column called model","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Binary forecast example data — example_binary","title":"Binary forecast example data — example_binary","text":"data set binary predictions COVID-19 cases deaths constructed data submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Binary forecast example data — example_binary","text":"","code":"example_binary"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Binary forecast example data — example_binary","text":"data frame following columns: location country prediction made location_name name country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed factor observed values forecast_date date prediction made model name model generated forecasts horizon forecast horizon weeks predicted predicted value","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Binary forecast example data — example_binary","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Binary forecast example data — example_binary","text":"Predictions data set constructed based continuous example data looking number samples mean prediction. outcome constructed whether actually observed value mean prediction. understood sound statistical practice, rather practical way create example data set. data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Point forecast example data — example_point","title":"Point forecast example data — example_point","text":"data set predictions COVID-19 cases deaths submitted European Forecast Hub. data set like quantile example data, median replaced point forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Point forecast example data — example_point","text":"","code":"example_point"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Point forecast example data — example_point","text":"data frame following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed observed values location_name name country prediction made forecast_date date prediction made predicted predicted value model name model generated forecasts horizon forecast horizon weeks","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Point forecast example data — example_point","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Point forecast example data — example_point","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Quantile example data — example_quantile","title":"Quantile example data — example_quantile","text":"data set predictions COVID-19 cases deaths submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Quantile example data — example_quantile","text":"","code":"example_quantile"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Quantile example data — example_quantile","text":"data frame following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed Numeric: observed values location_name name country prediction made forecast_date date prediction made quantile_level quantile level corresponding prediction predicted predicted value model name model generated forecasts horizon forecast horizon weeks","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Quantile example data — example_quantile","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Quantile example data — example_quantile","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":null,"dir":"Reference","previous_headings":"","what":"Continuous forecast example data — example_sample_continuous","title":"Continuous forecast example data — example_sample_continuous","text":"data set continuous predictions COVID-19 cases deaths constructed data submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Continuous forecast example data — example_sample_continuous","text":"","code":"example_sample_continuous"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Continuous forecast example data — example_sample_continuous","text":"data frame following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed observed values location_name name country prediction made forecast_date date prediction made model name model generated forecasts horizon forecast horizon weeks predicted predicted value sample_id id corresponding sample","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Continuous forecast example data — example_sample_continuous","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Continuous forecast example data — example_sample_continuous","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":null,"dir":"Reference","previous_headings":"","what":"Discrete forecast example data — example_sample_discrete","title":"Discrete forecast example data — example_sample_discrete","text":"data set integer predictions COVID-19 cases deaths constructed data submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Discrete forecast example data — example_sample_discrete","text":"","code":"example_sample_discrete"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Discrete forecast example data — example_sample_discrete","text":"data frame following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed observed values location_name name country prediction made forecast_date date prediction made model name model generated forecasts horizon forecast horizon weeks predicted predicted value sample_id id corresponding sample","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Discrete forecast example data — example_sample_discrete","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Discrete forecast example data — example_sample_discrete","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/forecast_types.html","id":null,"dir":"Reference","previous_headings":"","what":"Documentation template for forecast types — forecast_types","title":"Documentation template for forecast types — forecast_types","text":"Documentation template forecast types","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/forecast_types.html","id":"forecast-types-and-input-formats","dir":"Reference","previous_headings":"","what":"Forecast types and input formats","title":"Documentation template for forecast types — forecast_types","text":"Various different forecast types / forecast formats supported. moment, : point forecasts binary forecasts (\"soft binary classification\") Probabilistic forecasts quantile-based format (forecast represented set predictive quantiles) Probabilistic forecasts sample-based format (forecast represented set predictive samples) Forecast types determined based columns present input data. overview required format forecast type: forecast types require data.frame similar columns observed predicted, model. Point forecasts require column observed type numeric column predicted type numeric. Binary forecasts require column observed type factor exactly two levels column predicted type numeric probabilities, corresponding probability observed equal second factor level. See details information. Quantile-based forecasts require column observed type numeric, column predicted type numeric, column quantile_level type numeric quantile-levels (0 1). Sample-based forecasts require column observed type numeric, column predicted type numeric, column sample_id type numeric sample indices. information see vignettes example data (example_quantile, example_sample_continuous, example_sample_discrete, example_point(), example_binary).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/forecast_types.html","id":"forecast-unit","dir":"Reference","previous_headings":"","what":"Forecast unit","title":"Documentation template for forecast types — forecast_types","text":"order score forecasts, scoringutils needs know rows data belong together jointly form single forecasts. easy e.g. point forecast, one row per forecast. quantile sample-based forecasts, however, multiple rows belong single forecast. forecast unit unit single forecast described combination columns uniquely identify single forecast. example, forecasts made different models various locations different time points, several weeks future. forecast unit described forecast_unit = c(\"model\", \"location\", \"forecast_date\", \"forecast_horizon\"). scoringutils automatically tries determine unit single forecast. uses existing columns , means columns must present unrelated forecast unit. simplistic example, additional row, \"even\", one row number even zero otherwise, mess scoring scoringutils thinks column relevant defining forecast unit. order avoid issues, recommend setting forecast unit explicitly, usually forecast_unit argument as_forecast(). drop unneeded columns, making sure necessary, 'protected columns' like \"predicted\" \"observed\" retained.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":null,"dir":"Reference","previous_headings":"","what":"Calculate geometric mean — geometric_mean","title":"Calculate geometric mean — geometric_mean","text":"Calculate geometric mean","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Calculate geometric mean — geometric_mean","text":"","code":"geometric_mean(x)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Calculate geometric mean — geometric_mean","text":"x Numeric vector values calculate geometric mean.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Calculate geometric mean — geometric_mean","text":"geometric mean values x. NA values ignored.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Calculate geometric mean — geometric_mean","text":"Used get_pairwise_comparisons().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":null,"dir":"Reference","previous_headings":"","what":"Calculate correlation between metrics — get_correlations","title":"Calculate correlation between metrics — get_correlations","text":"Calculate correlation different metrics data.frame scores produced score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Calculate correlation between metrics — get_correlations","text":"","code":"get_correlations(scores, metrics = get_metrics(scores), ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Calculate correlation between metrics — get_correlations","text":"scores object class scores (data.table scores additional attribute metrics produced score()). metrics character vector metrics show. set NULL (default), metrics present scores shown. ... Additional arguments pass cor().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Calculate correlation between metrics — get_correlations","text":"object class scores (data.table additional attribute metrics holding names scores) correlations different metrics","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Calculate correlation between metrics — get_correlations","text":"","code":"scores <- score(as_forecast(example_quantile)) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. get_correlations(scores) #> wis overprediction underprediction dispersion bias #> #> 1: 1.0000000 0.94297565 0.28377361 0.45566303 0.10545891 #> 2: 0.9429757 1.00000000 -0.03310356 0.32493799 0.21532161 #> 3: 0.2837736 -0.03310356 1.00000000 0.14580143 -0.35123801 #> 4: 0.4556630 0.32493799 0.14580143 1.00000000 0.11118365 #> 5: 0.1054589 0.21532161 -0.35123801 0.11118365 1.00000000 #> 6: -0.2076649 -0.14556039 -0.21392764 -0.09400664 0.01338140 #> 7: -0.4075613 -0.31824017 -0.35756699 -0.08614678 0.09802725 #> 8: -0.3425972 -0.25332123 -0.32605710 -0.11957701 0.06001718 #> 9: 0.9886108 0.90326672 0.33589892 0.53809741 0.09578751 #> interval_coverage_50 interval_coverage_90 interval_coverage_deviation #> #> 1: -0.20766492 -0.40756133 -0.34259724 #> 2: -0.14556039 -0.31824017 -0.25332123 #> 3: -0.21392764 -0.35756699 -0.32605710 #> 4: -0.09400664 -0.08614678 -0.11957701 #> 5: 0.01338140 0.09802725 0.06001718 #> 6: 1.00000000 0.37245118 0.84683642 #> 7: 0.37245118 1.00000000 0.63647831 #> 8: 0.84683642 0.63647831 1.00000000 #> 9: -0.24559356 -0.41079097 -0.37689441 #> ae_median metric #> #> 1: 0.98861080 wis #> 2: 0.90326672 overprediction #> 3: 0.33589892 underprediction #> 4: 0.53809741 dispersion #> 5: 0.09578751 bias #> 6: -0.24559356 interval_coverage_50 #> 7: -0.41079097 interval_coverage_90 #> 8: -0.37689441 interval_coverage_deviation #> 9: 1.00000000 ae_median"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"validated forecast object quantile-based format (see as_forecast() information), function computes: interval coverage central prediction intervals quantile coverage predictive quantiles deviation desired actual coverage (interval quantile coverage) Coverage values computed specific level grouping, specified argument. default, coverage values computed per model. Interval coverage Interval coverage given interval range defined proportion observations fall within corresponding central prediction intervals. Central prediction intervals symmetric around median formed two quantiles denote lower upper bound. example, 50% central prediction interval interval 0.25 0.75 quantiles predictive distribution. Quantile coverage Quantile coverage given quantile level defined proportion observed values smaller corresponding predictive quantile. example, 0.5 quantile coverage proportion observed values smaller 0.5 quantile predictive distribution. Just , single observation quantile single predictive distribution, value either TRUE FALSE. Coverage deviation coverage deviation difference desired coverage (can either interval quantile coverage) actual coverage. example, desired coverage 90% actual coverage 80%, coverage deviation -0.1.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"","code":"get_coverage(forecast, by = \"model\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()) character vector denotes level grouping coverage values computed. default (\"model\"), one coverage value per model returned.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"data.table columns specified additional columns coverage values described data.table columns \"interval_coverage\", \"interval_coverage_deviation\", \"quantile_coverage\", \"quantile_coverage_deviation\" columns specified .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"","code":"library(magrittr) # pipe operator example_quantile %>% as_forecast() %>% get_coverage(by = \"model\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> model quantile_level interval_range interval_coverage #> #> 1: EuroCOVIDhub-baseline 0.500 0 0.000000000 #> 2: EuroCOVIDhub-baseline 0.450 10 0.085937500 #> 3: EuroCOVIDhub-baseline 0.550 10 0.085937500 #> 4: EuroCOVIDhub-baseline 0.400 20 0.191406250 #> 5: EuroCOVIDhub-baseline 0.600 20 0.191406250 #> 6: EuroCOVIDhub-baseline 0.350 30 0.289062500 #> 7: EuroCOVIDhub-baseline 0.650 30 0.289062500 #> 8: EuroCOVIDhub-baseline 0.300 40 0.375000000 #> 9: EuroCOVIDhub-baseline 0.700 40 0.375000000 #> 10: EuroCOVIDhub-baseline 0.250 50 0.496093750 #> 11: EuroCOVIDhub-baseline 0.750 50 0.496093750 #> 12: EuroCOVIDhub-baseline 0.200 60 0.628906250 #> 13: EuroCOVIDhub-baseline 0.800 60 0.628906250 #> 14: EuroCOVIDhub-baseline 0.150 70 0.773437500 #> 15: EuroCOVIDhub-baseline 0.850 70 0.773437500 #> 16: EuroCOVIDhub-baseline 0.100 80 0.843750000 #> 17: EuroCOVIDhub-baseline 0.900 80 0.843750000 #> 18: EuroCOVIDhub-baseline 0.050 90 0.910156250 #> 19: EuroCOVIDhub-baseline 0.950 90 0.910156250 #> 20: EuroCOVIDhub-baseline 0.025 95 0.925781250 #> 21: EuroCOVIDhub-baseline 0.975 95 0.925781250 #> 22: EuroCOVIDhub-baseline 0.010 98 0.933593750 #> 23: EuroCOVIDhub-baseline 0.990 98 0.933593750 #> 24: EuroCOVIDhub-ensemble 0.500 0 0.003906250 #> 25: EuroCOVIDhub-ensemble 0.450 10 0.148437500 #> 26: EuroCOVIDhub-ensemble 0.550 10 0.148437500 #> 27: EuroCOVIDhub-ensemble 0.400 20 0.250000000 #> 28: EuroCOVIDhub-ensemble 0.600 20 0.250000000 #> 29: EuroCOVIDhub-ensemble 0.350 30 0.386718750 #> 30: EuroCOVIDhub-ensemble 0.650 30 0.386718750 #> 31: EuroCOVIDhub-ensemble 0.300 40 0.519531250 #> 32: EuroCOVIDhub-ensemble 0.700 40 0.519531250 #> 33: EuroCOVIDhub-ensemble 0.250 50 0.632812500 #> 34: EuroCOVIDhub-ensemble 0.750 50 0.632812500 #> 35: EuroCOVIDhub-ensemble 0.200 60 0.667968750 #> 36: EuroCOVIDhub-ensemble 0.800 60 0.667968750 #> 37: EuroCOVIDhub-ensemble 0.150 70 0.753906250 #> 38: EuroCOVIDhub-ensemble 0.850 70 0.753906250 #> 39: EuroCOVIDhub-ensemble 0.100 80 0.816406250 #> 40: EuroCOVIDhub-ensemble 0.900 80 0.816406250 #> 41: EuroCOVIDhub-ensemble 0.050 90 0.902343750 #> 42: EuroCOVIDhub-ensemble 0.950 90 0.902343750 #> 43: EuroCOVIDhub-ensemble 0.025 95 0.941406250 #> 44: EuroCOVIDhub-ensemble 0.975 95 0.941406250 #> 45: EuroCOVIDhub-ensemble 0.010 98 0.968750000 #> 46: EuroCOVIDhub-ensemble 0.990 98 0.968750000 #> 47: epiforecasts-EpiNow2 0.500 0 0.004048583 #> 48: epiforecasts-EpiNow2 0.450 10 0.093117409 #> 49: epiforecasts-EpiNow2 0.550 10 0.093117409 #> 50: epiforecasts-EpiNow2 0.400 20 0.165991903 #> 51: epiforecasts-EpiNow2 0.600 20 0.165991903 #> 52: epiforecasts-EpiNow2 0.350 30 0.230769231 #> 53: epiforecasts-EpiNow2 0.650 30 0.230769231 #> 54: epiforecasts-EpiNow2 0.300 40 0.319838057 #> 55: epiforecasts-EpiNow2 0.700 40 0.319838057 #> 56: epiforecasts-EpiNow2 0.250 50 0.445344130 #> 57: epiforecasts-EpiNow2 0.750 50 0.445344130 #> 58: epiforecasts-EpiNow2 0.200 60 0.538461538 #> 59: epiforecasts-EpiNow2 0.800 60 0.538461538 #> 60: epiforecasts-EpiNow2 0.150 70 0.635627530 #> 61: epiforecasts-EpiNow2 0.850 70 0.635627530 #> 62: epiforecasts-EpiNow2 0.100 80 0.732793522 #> 63: epiforecasts-EpiNow2 0.900 80 0.732793522 #> 64: epiforecasts-EpiNow2 0.050 90 0.846153846 #> 65: epiforecasts-EpiNow2 0.950 90 0.846153846 #> 66: epiforecasts-EpiNow2 0.025 95 0.874493927 #> 67: epiforecasts-EpiNow2 0.975 95 0.874493927 #> 68: epiforecasts-EpiNow2 0.010 98 0.910931174 #> 69: epiforecasts-EpiNow2 0.990 98 0.910931174 #> 70: UMass-MechBayes 0.500 0 0.015625000 #> 71: UMass-MechBayes 0.450 10 0.101562500 #> 72: UMass-MechBayes 0.550 10 0.101562500 #> 73: UMass-MechBayes 0.400 20 0.195312500 #> 74: UMass-MechBayes 0.600 20 0.195312500 #> 75: UMass-MechBayes 0.350 30 0.281250000 #> 76: UMass-MechBayes 0.650 30 0.281250000 #> 77: UMass-MechBayes 0.300 40 0.382812500 #> 78: UMass-MechBayes 0.700 40 0.382812500 #> 79: UMass-MechBayes 0.250 50 0.460937500 #> 80: UMass-MechBayes 0.750 50 0.460937500 #> 81: UMass-MechBayes 0.200 60 0.539062500 #> 82: UMass-MechBayes 0.800 60 0.539062500 #> 83: UMass-MechBayes 0.150 70 0.617187500 #> 84: UMass-MechBayes 0.850 70 0.617187500 #> 85: UMass-MechBayes 0.100 80 0.765625000 #> 86: UMass-MechBayes 0.900 80 0.765625000 #> 87: UMass-MechBayes 0.050 90 0.875000000 #> 88: UMass-MechBayes 0.950 90 0.875000000 #> 89: UMass-MechBayes 0.025 95 0.953125000 #> 90: UMass-MechBayes 0.975 95 0.953125000 #> 91: UMass-MechBayes 0.010 98 0.984375000 #> 92: UMass-MechBayes 0.990 98 0.984375000 #> model quantile_level interval_range interval_coverage #> interval_coverage_deviation quantile_coverage quantile_coverage_deviation #> #> 1: 0.000000000 0.69921875 0.199218750 #> 2: -0.014062500 0.65625000 0.206250000 #> 3: -0.014062500 0.74218750 0.192187500 #> 4: -0.008593750 0.58593750 0.185937500 #> 5: -0.008593750 0.77343750 0.173437500 #> 6: -0.010937500 0.52343750 0.173437500 #> 7: -0.010937500 0.80859375 0.158593750 #> 8: -0.025000000 0.46875000 0.168750000 #> 9: -0.025000000 0.84375000 0.143750000 #> 10: -0.003906250 0.36718750 0.117187500 #> 11: -0.003906250 0.86328125 0.113281250 #> 12: 0.028906250 0.25000000 0.050000000 #> 13: 0.028906250 0.87500000 0.075000000 #> 14: 0.073437500 0.13281250 -0.017187500 #> 15: 0.073437500 0.90625000 0.056250000 #> 16: 0.043750000 0.08203125 -0.017968750 #> 17: 0.043750000 0.92578125 0.025781250 #> 18: 0.010156250 0.04296875 -0.007031250 #> 19: 0.010156250 0.95312500 0.003125000 #> 20: -0.024218750 0.03125000 0.006250000 #> 21: -0.024218750 0.95703125 -0.017968750 #> 22: -0.046406250 0.03125000 0.021250000 #> 23: -0.046406250 0.96484375 -0.025156250 #> 24: 0.003906250 0.53125000 0.031250000 #> 25: 0.048437500 0.46484375 0.014843750 #> 26: 0.048437500 0.60156250 0.051562500 #> 27: 0.050000000 0.40625000 0.006250000 #> 28: 0.050000000 0.65625000 0.056250000 #> 29: 0.086718750 0.33984375 -0.010156250 #> 30: 0.086718750 0.72656250 0.076562500 #> 31: 0.119531250 0.26562500 -0.034375000 #> 32: 0.119531250 0.76562500 0.065625000 #> 33: 0.132812500 0.16406250 -0.085937500 #> 34: 0.132812500 0.79687500 0.046875000 #> 35: 0.067968750 0.14062500 -0.059375000 #> 36: 0.067968750 0.80468750 0.004687500 #> 37: 0.053906250 0.10156250 -0.048437500 #> 38: 0.053906250 0.85546875 0.005468750 #> 39: 0.016406250 0.07812500 -0.021875000 #> 40: 0.016406250 0.89453125 -0.005468750 #> 41: 0.002343750 0.04296875 -0.007031250 #> 42: 0.002343750 0.94531250 -0.004687500 #> 43: -0.008593750 0.03125000 0.006250000 #> 44: -0.008593750 0.97265625 -0.002343750 #> 45: -0.011250000 0.01562500 0.005625000 #> 46: -0.011250000 0.98437500 -0.005625000 #> 47: 0.004048583 0.49392713 -0.006072874 #> 48: -0.006882591 0.43724696 -0.012753036 #> 49: -0.006882591 0.53036437 -0.019635628 #> 50: -0.034008097 0.39676113 -0.003238866 #> 51: -0.034008097 0.55870445 -0.041295547 #> 52: -0.069230769 0.36437247 0.014372470 #> 53: -0.069230769 0.59514170 -0.054858300 #> 54: -0.080161943 0.31983806 0.019838057 #> 55: -0.080161943 0.63967611 -0.060323887 #> 56: -0.054655870 0.26315789 0.013157895 #> 57: -0.054655870 0.70445344 -0.045546559 #> 58: -0.061538462 0.20647773 0.006477733 #> 59: -0.061538462 0.74493927 -0.055060729 #> 60: -0.064372470 0.14574899 -0.004251012 #> 61: -0.064372470 0.78137652 -0.068623482 #> 62: -0.067206478 0.10121457 0.001214575 #> 63: -0.067206478 0.83400810 -0.065991903 #> 64: -0.053846154 0.06072874 0.010728745 #> 65: -0.053846154 0.90688259 -0.043117409 #> 66: -0.075506073 0.04858300 0.023582996 #> 67: -0.075506073 0.92307692 -0.051923077 #> 68: -0.069068826 0.02429150 0.014291498 #> 69: -0.069068826 0.93522267 -0.054777328 #> 70: 0.015625000 0.50000000 0.000000000 #> 71: 0.001562500 0.42187500 -0.028125000 #> 72: 0.001562500 0.52343750 -0.026562500 #> 73: -0.004687500 0.37500000 -0.025000000 #> 74: -0.004687500 0.57031250 -0.029687500 #> 75: -0.018750000 0.35156250 0.001562500 #> 76: -0.018750000 0.61718750 -0.032812500 #> 77: -0.017187500 0.28906250 -0.010937500 #> 78: -0.017187500 0.66406250 -0.035937500 #> 79: -0.039062500 0.27343750 0.023437500 #> 80: -0.039062500 0.71875000 -0.031250000 #> 81: -0.060937500 0.24218750 0.042187500 #> 82: -0.060937500 0.78125000 -0.018750000 #> 83: -0.082812500 0.20312500 0.053125000 #> 84: -0.082812500 0.82031250 -0.029687500 #> 85: -0.034375000 0.12500000 0.025000000 #> 86: -0.034375000 0.87500000 -0.025000000 #> 87: -0.025000000 0.06250000 0.012500000 #> 88: -0.025000000 0.93750000 -0.012500000 #> 89: 0.003125000 0.01562500 -0.009375000 #> 90: 0.003125000 0.96875000 -0.006250000 #> 91: 0.004375000 0.00781250 -0.002187500 #> 92: 0.004375000 0.99218750 0.002187500 #> interval_coverage_deviation quantile_coverage quantile_coverage_deviation"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":null,"dir":"Reference","previous_headings":"","what":"Find duplicate forecasts — get_duplicate_forecasts","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"Helper function identify duplicate forecasts, .e. instances one forecast prediction target.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"","code":"get_duplicate_forecasts(data, counts = FALSE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"data data.frame used score() counts output show number duplicates per forecast unit instead individual duplicated rows? Default FALSE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"data.frame rows duplicate forecast found","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"","code":"example <- rbind(example_quantile, example_quantile[1000:1010]) get_duplicate_forecasts(example) #> location target_end_date target_type observed location_name forecast_date #> #> 1: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 2: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 3: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 4: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 5: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 6: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 7: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 8: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 9: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 10: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 11: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 12: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 13: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 14: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 15: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 16: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 17: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 18: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 19: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 20: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 21: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 22: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> location target_end_date target_type observed location_name forecast_date #> quantile_level predicted model horizon #> #> 1: 0.950 1464 epiforecasts-EpiNow2 1 #> 2: 0.950 1464 epiforecasts-EpiNow2 1 #> 3: 0.975 1642 epiforecasts-EpiNow2 1 #> 4: 0.975 1642 epiforecasts-EpiNow2 1 #> 5: 0.990 1951 epiforecasts-EpiNow2 1 #> 6: 0.990 1951 epiforecasts-EpiNow2 1 #> 7: 0.010 28999 EuroCOVIDhub-ensemble 3 #> 8: 0.010 28999 EuroCOVIDhub-ensemble 3 #> 9: 0.025 32612 EuroCOVIDhub-ensemble 3 #> 10: 0.025 32612 EuroCOVIDhub-ensemble 3 #> 11: 0.050 36068 EuroCOVIDhub-ensemble 3 #> 12: 0.050 36068 EuroCOVIDhub-ensemble 3 #> 13: 0.100 41484 EuroCOVIDhub-ensemble 3 #> 14: 0.100 41484 EuroCOVIDhub-ensemble 3 #> 15: 0.150 47110 EuroCOVIDhub-ensemble 3 #> 16: 0.150 47110 EuroCOVIDhub-ensemble 3 #> 17: 0.200 50929 EuroCOVIDhub-ensemble 3 #> 18: 0.200 50929 EuroCOVIDhub-ensemble 3 #> 19: 0.250 54561 EuroCOVIDhub-ensemble 3 #> 20: 0.250 54561 EuroCOVIDhub-ensemble 3 #> 21: 0.300 57739 EuroCOVIDhub-ensemble 3 #> 22: 0.300 57739 EuroCOVIDhub-ensemble 3 #> quantile_level predicted model horizon"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":null,"dir":"Reference","previous_headings":"","what":"Count number of available forecasts — get_forecast_counts","title":"Count number of available forecasts — get_forecast_counts","text":"Given data set forecasts, function counts number available forecasts. level grouping can specified using argument (e.g. count number forecasts per model, number forecasts per model location). useful determine whether missing forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Count number of available forecasts — get_forecast_counts","text":"","code":"get_forecast_counts( forecast, by = get_forecast_unit(forecast), collapse = c(\"quantile_level\", \"sample_id\") )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Count number of available forecasts — get_forecast_counts","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()) character vector NULL (default) denotes categories number forecasts counted. default unit single forecast (.e. available columns (apart \"protected\" columns 'predicted' 'observed') plus \"quantile_level\" \"sample_id\" present). collapse character vector (default: c(\"quantile_level\", \"sample_id\") names categories number rows collapsed one counting. example, single forecast usually represented set several quantiles samples collapsing one makes sure single forecast gets counted . Setting collapse = c() mean quantiles / samples counted individual forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Count number of available forecasts — get_forecast_counts","text":"data.table columns specified additional column \"count\" number forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Count number of available forecasts — get_forecast_counts","text":"","code":"get_forecast_counts( as_forecast(example_quantile), by = c(\"model\", \"target_type\") ) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Key: #> model target_type count #> #> 1: EuroCOVIDhub-baseline Cases 128 #> 2: EuroCOVIDhub-baseline Deaths 128 #> 3: EuroCOVIDhub-ensemble Cases 128 #> 4: EuroCOVIDhub-ensemble Deaths 128 #> 5: UMass-MechBayes Cases 0 #> 6: UMass-MechBayes Deaths 128 #> 7: epiforecasts-EpiNow2 Cases 128 #> 8: epiforecasts-EpiNow2 Deaths 119"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_type.html","id":null,"dir":"Reference","previous_headings":"","what":"Infer forecast type from data — get_forecast_type","title":"Infer forecast type from data — get_forecast_type","text":"Helper function infer forecast type based data.frame similar forecasts observed values. See details section information different forecast types.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_type.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Infer forecast type from data — get_forecast_type","text":"","code":"get_forecast_type(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_type.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Infer forecast type from data — get_forecast_type","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_type.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Infer forecast type from data — get_forecast_type","text":"Character vector length one either \"binary\", \"quantile\", \"sample\" \"point\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_type.html","id":"forecast-types-and-input-formats","dir":"Reference","previous_headings":"","what":"Forecast types and input formats","title":"Infer forecast type from data — get_forecast_type","text":"Various different forecast types / forecast formats supported. moment, : point forecasts binary forecasts (\"soft binary classification\") Probabilistic forecasts quantile-based format (forecast represented set predictive quantiles) Probabilistic forecasts sample-based format (forecast represented set predictive samples) Forecast types determined based columns present input data. overview required format forecast type: forecast types require data.frame similar columns observed predicted, model. Point forecasts require column observed type numeric column predicted type numeric. Binary forecasts require column observed type factor exactly two levels column predicted type numeric probabilities, corresponding probability observed equal second factor level. See details information. Quantile-based forecasts require column observed type numeric, column predicted type numeric, column quantile_level type numeric quantile-levels (0 1). Sample-based forecasts require column observed type numeric, column predicted type numeric, column sample_id type numeric sample indices. information see vignettes example data (example_quantile, example_sample_continuous, example_sample_discrete, example_point(), example_binary).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":null,"dir":"Reference","previous_headings":"","what":"Get unit of a single forecast — get_forecast_unit","title":"Get unit of a single forecast — get_forecast_unit","text":"Helper function get unit single forecast, .e. column names define single forecast made . just takes columns available data subtracts columns protected, .e. returned get_protected_columns() well names metrics specified scoring, .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get unit of a single forecast — get_forecast_unit","text":"","code":"get_forecast_unit(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get unit of a single forecast — get_forecast_unit","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get unit of a single forecast — get_forecast_unit","text":"character vector column names define unit single forecast","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":"forecast-unit","dir":"Reference","previous_headings":"","what":"Forecast unit","title":"Get unit of a single forecast — get_forecast_unit","text":"order score forecasts, scoringutils needs know rows data belong together jointly form single forecasts. easy e.g. point forecast, one row per forecast. quantile sample-based forecasts, however, multiple rows belong single forecast. forecast unit unit single forecast described combination columns uniquely identify single forecast. example, forecasts made different models various locations different time points, several weeks future. forecast unit described forecast_unit = c(\"model\", \"location\", \"forecast_date\", \"forecast_horizon\"). scoringutils automatically tries determine unit single forecast. uses existing columns , means columns must present unrelated forecast unit. simplistic example, additional row, \"even\", one row number even zero otherwise, mess scoring scoringutils thinks column relevant defining forecast unit. order avoid issues, recommend setting forecast unit explicitly, usually forecast_unit argument as_forecast(). drop unneeded columns, making sure necessary, 'protected columns' like \"predicted\" \"observed\" retained.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Get names of the metrics that were used for scoring — get_metrics","title":"Get names of the metrics that were used for scoring — get_metrics","text":"applying scoring rule via score(), names scoring rules become column names resulting data.table. addition, attribute metrics added output, holding names scores vector. done functions like get_forecast_unit() summarise_scores() can still identify columns part forecast unit hold score. get_metrics() accesses returns metrics attribute. attribute, function return NULL (, error = TRUE produce error instead). addition, checks column names input consistency data stored metrics attribute. Handling missing inconsistent metrics attribute: metrics attribute missing consistent column names data.table, can either run score() , specifying names scoring rules manually, add/update attribute manually using attr(scores, \"metrics\") <- c(\"names\", \"\", \"\", \"scores\") (order matter).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get names of the metrics that were used for scoring — get_metrics","text":"","code":"get_metrics(scores, error = FALSE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get names of the metrics that were used for scoring — get_metrics","text":"scores data.table attribute metrics. error Throw error attribute called metrics? Default FALSE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get names of the metrics that were used for scoring — get_metrics","text":"Character vector names scoring rules used scoring NULL scores computed previously.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":null,"dir":"Reference","previous_headings":"","what":"Obtain pairwise comparisons between models — get_pairwise_comparisons","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"Compare scores obtained different models pairwise tournament. combinations two models compared based overlapping set available forecasts common models. input scores object produced score(). Note adding additional unrelated columns can unpredictably change results, present columns taken account determining set overlapping forecasts two models. output pairwise comparisons set mean score ratios, relative skill scores p-values. following illustrates pairwise comparison process: Mean score ratios every pair two models, mean score ratio computed. simply mean score first model divided mean score second. Mean score ratios computed based set overlapping forecasts two models. means scores targets taken account models submitted forecast. (Scaled) Relative skill scores relative score model geometric mean mean score ratios involve model. baseline provided, scaled relative skill scores calculated well. Scaled relative skill scores simply relative skill score model divided relative skill score baseline model. p-values addition, function computes p-values comparison two models (based set overlapping forecasts). P-values can computed two ways: based nonparametric Wilcoxon signed-rank test (internally using wilcox.test() paired = TRUE) based permutation test. permutation test based difference mean scores two models. default null hypothesis mean score difference zero (see permutation_test()). Adjusted p-values computed calling p.adjust() raw p-values. code pairwise comparisons inspired implementation Johannes Bracher. implementation permutation test follows function permutationTest surveillance package Michael Höhle, Andrea Riebler Michaela Paul.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"","code":"get_pairwise_comparisons( scores, by = \"model\", metric = intersect(c(\"wis\", \"crps\", \"brier_score\"), names(scores)), baseline = NULL, ... )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"scores object class scores (data.table scores additional attribute metrics produced score()). Character vector column names define grouping level pairwise comparisons. default (model), one relative skill score per model. , example, = c(\"model\", \"location\"). get separate relative skill score every model every location. Internally, data.table scores split according (removing \"model\" splitting) pairwise comparisons computed separately split data.tables. metric string name metric relative skill shall computed. default either \"crps\", \"wis\" \"brier_score\" available. baseline string name model. baseline given, scaled relative skill respect baseline returned. default (NULL), relative skill scaled respect baseline model. ... Additional arguments comparison two models. See compare_two_models() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"data.table results pairwise comparisons containing mean score ratios (mean_scores_ratio), unadjusted (pval) adjusted (adj_pval) p-values, relative skill values model (..._relative_skill). baseline model given scaled relative skill reported well (..._scaled_relative_skill).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"Nikos Bosse nikosbosse@gmail.com Johannes Bracher, johannes.bracher@kit.edu","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"","code":"scores <- score(as_forecast(example_quantile)) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. pairwise <- get_pairwise_comparisons(scores, by = \"target_type\") pairwise2 <- get_pairwise_comparisons( scores, by = \"target_type\", baseline = \"EuroCOVIDhub-baseline\" ) library(ggplot2) plot_pairwise_comparisons(pairwise, type = \"mean_scores_ratio\") + facet_wrap(~target_type)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit.html","id":null,"dir":"Reference","previous_headings":"","what":"Probability integral transformation (data.frame version) — get_pit","title":"Probability integral transformation (data.frame version) — get_pit","text":"Compute Probability Integral Transformation (PIT) validated forecast objects.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Probability integral transformation (data.frame version) — get_pit","text":"","code":"get_pit(forecast, by, n_replicates = 100)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Probability integral transformation (data.frame version) — get_pit","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()) Character vector columns according PIT values shall grouped. e.g. columns 'model' 'location' input data want PIT histogram every model location, specify = c(\"model\", \"location\"). n_replicates number draws randomised PIT discrete predictions. ignored forecasts continuous.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Probability integral transformation (data.frame version) — get_pit","text":"data.table PIT values according grouping specified .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Probability integral transformation (data.frame version) — get_pit","text":"Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind M. Eggo, W. John Edmunds (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15, doi:10.1371/journal.pcbi.1006785","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Probability integral transformation (data.frame version) — get_pit","text":"","code":"result <- get_pit(as_forecast(example_sample_continuous), by = \"model\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_pit(result) # example with quantile data result <- get_pit(as_forecast(example_quantile), by = \"model\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_pit(result)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_protected_columns.html","id":null,"dir":"Reference","previous_headings":"","what":"Get protected columns from data — get_protected_columns","title":"Get protected columns from data — get_protected_columns","text":"Helper function get names columns data frame protected columns.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_protected_columns.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get protected columns from data — get_protected_columns","text":"","code":"get_protected_columns(data = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_protected_columns.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get protected columns from data — get_protected_columns","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_protected_columns.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get protected columns from data — get_protected_columns","text":"character vector names protected columns data. data NULL (default) returns list columns protected scoringutils.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_range_from_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Get interval range belonging to a quantile — get_range_from_quantile","title":"Get interval range belonging to a quantile — get_range_from_quantile","text":"Every quantile can thought either lower upper bound symmetric central prediction interval. helper function returns range central prediction interval quantile belongs. Due numeric instability sometimes occurred past, ranges rounded 10 decimal places. problem vast majority use cases, something aware .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_range_from_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get interval range belonging to a quantile — get_range_from_quantile","text":"","code":"get_range_from_quantile(quantile_level)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_range_from_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get interval range belonging to a quantile — get_range_from_quantile","text":"quantile_level numeric vector quantile levels size N.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_range_from_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get interval range belonging to a quantile — get_range_from_quantile","text":"numeric vector interval ranges size N","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_type.html","id":null,"dir":"Reference","previous_headings":"","what":"Get type of a vector or matrix of observed values or predictions — get_type","title":"Get type of a vector or matrix of observed values or predictions — get_type","text":"Internal helper function get type vector (usually observed predicted values). function checks whether input factor, else whether integer (can coerced integer) whether continuous.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_type.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get type of a vector or matrix of observed values or predictions — get_type","text":"","code":"get_type(x)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_type.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get type of a vector or matrix of observed values or predictions — get_type","text":"x Input type determined .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_type.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get type of a vector or matrix of observed values or predictions — get_type","text":"Character vector length one either \"classification\", \"integer\", \"continuous\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interpolate_median.html","id":null,"dir":"Reference","previous_headings":"","what":"Helper function to interpolate the median prediction if it is not available — interpolate_median","title":"Helper function to interpolate the median prediction if it is not available — interpolate_median","text":"Internal function interpolate median prediction available given quantile levels. done using linear interpolation two innermost quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interpolate_median.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Helper function to interpolate the median prediction if it is not available — interpolate_median","text":"","code":"interpolate_median(predicted, quantile_level)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interpolate_median.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Helper function to interpolate the median prediction if it is not available — interpolate_median","text":"predicted Vector length N (corresponding number quantiles) holds predictions. quantile_level Vector size N quantile levels predictions made. Note contain median (0.5) median imputed mean two innermost quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interpolate_median.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Helper function to interpolate the median prediction if it is not available — interpolate_median","text":"scalar imputed median prediction","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Interval coverage (for quantile-based forecasts) — interval_coverage","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"Check whether observed value within given central prediction interval. prediction interval defined lower upper bound formed pair predictive quantiles. example, 50% prediction interval formed 0.25 0.75 quantiles predictive distribution.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"","code":"interval_coverage(observed, predicted, quantile_level, interval_range = 50)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made. interval_range single number range prediction interval percent (e.g. 50 50% prediction interval) want compute interval coverage.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"vector length n elements either TRUE, observed value within corresponding prediction interval, FALSE otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"","code":"observed <- c(1, -15, 22) predicted <- rbind( c(-1, 0, 1, 2, 3), c(-2, 1, 2, 2, 4), c(-2, 0, 3, 3, 4) ) quantile_level <- c(0.1, 0.25, 0.5, 0.75, 0.9) interval_coverage(observed, predicted, quantile_level) #> [1] TRUE FALSE FALSE"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage_deviation.html","id":null,"dir":"Reference","previous_headings":"","what":"Interval coverage deviation (for quantile-based forecasts) — interval_coverage_deviation","title":"Interval coverage deviation (for quantile-based forecasts) — interval_coverage_deviation","text":"Check agreement desired actual interval coverage forecast. function similar interval_coverage(), takes provided prediction intervals account compares nominal interval coverage (.e. desired interval coverage) actual observed interval coverage. central symmetric prediction interval defined lower upper bound formed pair predictive quantiles. example, 50% prediction interval formed 0.25 0.75 quantiles predictive distribution. Ideally, forecaster aim cover 50% observed values 50% prediction intervals, 90% observed values 90% prediction intervals, . every prediction interval, deviation computed difference observed interval coverage nominal interval coverage single observed value single prediction interval, coverage always either 0 1 (FALSE TRUE). case single observed value multiple prediction intervals, still make much sense compare nominal (desired) coverage actual coverage single observation. sense coverage deviation really starts make sense metric averaged across multiple observations). Positive values interval coverage deviation indication underconfidence, .e. forecaster likely issued narrower forecast. Negative values indication overconfidence, .e. forecasts narrow. $$ \\textrm{interval coverage deviation} = \\mathbf{1}(\\textrm{observed value falls within interval}) - \\textrm{nominal interval coverage} $$ interval coverage deviation averaged across prediction intervals. median ignored computing coverage deviation.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage_deviation.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Interval coverage deviation (for quantile-based forecasts) — interval_coverage_deviation","text":"","code":"interval_coverage_deviation(observed, predicted, quantile_level)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage_deviation.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Interval coverage deviation (for quantile-based forecasts) — interval_coverage_deviation","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage_deviation.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Interval coverage deviation (for quantile-based forecasts) — interval_coverage_deviation","text":"numeric vector length n interval coverage deviation forecast (forecast comprising one multiple prediction intervals).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage_deviation.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Interval coverage deviation (for quantile-based forecasts) — interval_coverage_deviation","text":"","code":"observed <- c(1, -15, 22) predicted <- rbind( c(-1, 0, 1, 2, 3), c(-2, 1, 2, 2, 4), c(-2, 0, 3, 3, 4) ) quantile_level <- c(0.1, 0.25, 0.5, 0.75, 0.9) interval_coverage_deviation(observed, predicted, quantile_level) #> [1] 0.35 -0.65 -0.65"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":null,"dir":"Reference","previous_headings":"","what":"Interval score — interval_score","title":"Interval score — interval_score","text":"Proper Scoring Rule score quantile predictions, following Gneiting Raftery (2007). Smaller values better. score computed $$ \\textrm{score} = (\\textrm{upper} - \\textrm{lower}) + \\frac{2}{\\alpha}(\\textrm{lower} - \\textrm{observed}) * \\mathbf{1}(\\textrm{observed} < \\textrm{lower}) + \\frac{2}{\\alpha}(\\textrm{observed} - \\textrm{upper}) * \\mathbf{1}(\\textrm{observed} > \\textrm{upper}) $$ \\(\\mathbf{1}()\\) indicator function indicates much outside prediction interval. \\(\\alpha\\) decimal value indicates much outside prediction interval. improve usability, user asked provide interval range percentage terms, .e. interval_range = 90 (percent) 90 percent prediction interval. Correspondingly, user provide 5% 95% quantiles (corresponding alpha 0.1). specific distribution assumed, interval symmetric around median (.e use 0.1 quantile lower bound 0.7 quantile upper bound). Non-symmetric quantiles can scored using function quantile_score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Interval score — interval_score","text":"","code":"interval_score( observed, lower, upper, interval_range, weigh = TRUE, separate_results = FALSE )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Interval score — interval_score","text":"observed vector observed values size n lower Vector size n prediction lower quantile given interval range. upper Vector size n prediction upper quantile given interval range. interval_range Numeric vector (either single number vector size n) range prediction intervals. example, forecasting 0.05 0.95 quantile, interval range 90. interval range corresponds \\((100-\\alpha)/100\\), \\(\\alpha\\) decimal value indicates much outside prediction interval (see e.g. Gneiting Raftery (2007)). weigh Logical. TRUE (default), weigh score \\(\\alpha / 2\\), can averaged interval score , limit (increasing number equally spaced quantiles/prediction intervals), corresponds CRPS. \\(\\alpha\\) value corresponds (\\(\\alpha/2\\)) (\\(1 - \\alpha/2\\)), .e. decimal value represents much outside central prediction interval (E.g. 90 percent central prediction interval, alpha 0.1). separate_results Logical. TRUE (default FALSE), separate parts interval score (dispersion penalty, penalties - -prediction get returned separate elements list). want data.frame instead, simply call .data.frame() output.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Interval score — interval_score","text":"Vector scoring values, list separate entries separate_results TRUE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Interval score — interval_score","text":"Strictly Proper Scoring Rules, Prediction,Estimation, Tilmann Gneiting Adrian E. Raftery, 2007, Journal American Statistical Association, Volume 102, 2007 - Issue 477 Evaluating epidemic forecasts interval format, Johannes Bracher, Evan L. Ray, Tilmann Gneiting Nicholas G. Reich, https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1008618 # nolint","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Interval score — interval_score","text":"","code":"observed <- rnorm(30, mean = 1:30) interval_range <- rep(90, 30) alpha <- (100 - interval_range) / 100 lower <- qnorm(alpha / 2, rnorm(30, mean = 1:30)) upper <- qnorm((1 - alpha / 2), rnorm(30, mean = 11:40)) scoringutils:::interval_score( observed = observed, lower = lower, upper = upper, interval_range = interval_range ) #> [1] 0.6186832 0.6857411 0.6851103 0.6735228 0.7207120 0.6350107 0.6326772 #> [8] 0.7765693 0.9162766 0.7696477 0.5718187 0.5947846 0.6873026 0.6733580 #> [15] 0.6672086 0.7094583 0.5695223 0.6244414 0.7408444 1.4548649 0.6493377 #> [22] 0.9306896 0.6973638 0.7058552 0.8090509 0.7450232 0.5687430 0.7296127 #> [29] 0.7483414 0.6927440 # gives a warning, as the interval_range should likely be 50 instead of 0.5 scoringutils:::interval_score( observed = 4, upper = 8, lower = 2, interval_range = 0.5 ) #> Warning: ! Found interval ranges between 0 and 1. Are you sure that's right? An interval #> range of 0.5 e.g. implies a (49.75%, 50.25%) prediction interval. #> ℹ If you want to score a (25%, 75%) prediction interval, set `interval_range = #> 50`. #> This warning is displayed once per session. #> [1] 2.985 # example with missing values and separate results scoringutils:::interval_score( observed = c(observed, NA), lower = c(lower, NA), upper = c(NA, upper), separate_results = TRUE, interval_range = 90 ) #> $interval_score #> [1] NA 0.5428048 0.7656798 0.5305954 0.6509690 0.6780044 0.5082637 #> [8] 0.7404846 0.8621966 0.7375557 0.5326411 0.5665304 0.6259204 0.5696571 #> [15] 0.6555712 0.5798147 0.6144623 0.5308622 0.6850722 1.4237942 0.6161064 #> [22] 0.8500130 0.6461361 0.6502278 0.7400150 0.6942545 0.5921649 0.5858396 #> [29] 0.6975714 0.7079685 NA #> #> $dispersion #> [1] NA 0.5428048 0.7656798 0.5305954 0.6509690 0.6780044 0.5082637 #> [8] 0.7404846 0.5823934 0.7375557 0.5326411 0.5665304 0.6259204 0.5696571 #> [15] 0.6555712 0.5798147 0.6144623 0.5308622 0.6850722 0.5303743 0.6161064 #> [22] 0.5536698 0.6461361 0.6502278 0.7400150 0.6942545 0.5921649 0.5858396 #> [29] 0.6975714 0.7079685 NA #> #> $underprediction #> [1] NA 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 #> [26] 0 0 0 0 0 NA #> #> $overprediction #> [1] 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 #> [8] 0.0000000 0.2798032 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 #> [15] 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.8934199 0.0000000 #> [22] 0.2963432 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 #> [29] 0.0000000 0.0000000 NA #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Test whether an object is a forecast object — is_forecast","title":"Test whether an object is a forecast object — is_forecast","text":"Test whether object forecast object (see as_forecast() information). can test specific forecast_* class using appropriate is_forecast_* function.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Test whether an object is a forecast object — is_forecast","text":"","code":"is_forecast(x) is_forecast_sample(x) is_forecast_binary(x) is_forecast_point(x) is_forecast_quantile(x)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Test whether an object is a forecast object — is_forecast","text":"x R object.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Test whether an object is a forecast object — is_forecast","text":"is_forecast: TRUE object class forecast, FALSE otherwise. is_forecast_*: TRUE object class forecast_* addition class forecast, FALSE otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Test whether an object is a forecast object — is_forecast","text":"","code":"forecast_binary <- as_forecast(example_binary) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. is_forecast(forecast_binary) #> [1] TRUE"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":null,"dir":"Reference","previous_headings":"","what":"Log transformation with an additive shift — log_shift","title":"Log transformation with an additive shift — log_shift","text":"Function shifts value offset applies natural logarithm .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Log transformation with an additive shift — log_shift","text":"","code":"log_shift(x, offset = 0, base = exp(1))"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Log transformation with an additive shift — log_shift","text":"x vector input values transformed offset Number add input value taking natural logarithm. base positive number: base respect logarithms computed. Defaults e = exp(1).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Log transformation with an additive shift — log_shift","text":"numeric vector transformed values","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Log transformation with an additive shift — log_shift","text":"output computed log(x + offset)","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Log transformation with an additive shift — log_shift","text":"Transformation forecasts evaluating predictive performance epidemiological context Nikos . Bosse, Sam Abbott, Anne Cori, Edwin van Leeuwen, Johannes Bracher, Sebastian Funk medRxiv 2023.01.23.23284722 doi:10.1101/2023.01.23.23284722 https://www.medrxiv.org/content/10.1101/2023.01.23.23284722v1 # nolint","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Log transformation with an additive shift — log_shift","text":"","code":"log_shift(1:10) #> [1] 0.0000000 0.6931472 1.0986123 1.3862944 1.6094379 1.7917595 1.9459101 #> [8] 2.0794415 2.1972246 2.3025851 log_shift(0:9, offset = 1) #> [1] 0.0000000 0.6931472 1.0986123 1.3862944 1.6094379 1.7917595 1.9459101 #> [8] 2.0794415 2.1972246 2.3025851 transform_forecasts( as_forecast(example_quantile)[observed > 0, ], fun = log_shift, offset = 1 ) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Forecast type: quantile #> Forecast unit: #> location, target_end_date, target_type, location_name, forecast_date, model, #> horizon, and scale #> #> location target_end_date target_type observed location_name #> #> 1: DE 2021-01-02 Cases 1.273000e+05 Germany #> 2: DE 2021-01-02 Deaths 4.534000e+03 Germany #> 3: DE 2021-01-09 Cases 1.549220e+05 Germany #> 4: DE 2021-01-09 Deaths 6.117000e+03 Germany #> 5: DE 2021-01-16 Cases 1.101830e+05 Germany #> --- #> 40672: IT 2021-07-24 Deaths 4.369448e+00 Italy #> 40673: IT 2021-07-24 Deaths 4.369448e+00 Italy #> 40674: IT 2021-07-24 Deaths 4.369448e+00 Italy #> 40675: IT 2021-07-24 Deaths 4.369448e+00 Italy #> 40676: IT 2021-07-24 Deaths 4.369448e+00 Italy #> forecast_date quantile_level predicted model horizon #> #> 1: NA NA NA #> 2: NA NA NA #> 3: NA NA NA #> 4: NA NA NA #> 5: NA NA NA #> --- #> 40672: 2021-07-12 0.850 5.866468 epiforecasts-EpiNow2 2 #> 40673: 2021-07-12 0.900 5.986452 epiforecasts-EpiNow2 2 #> 40674: 2021-07-12 0.950 6.214608 epiforecasts-EpiNow2 2 #> 40675: 2021-07-12 0.975 6.416732 epiforecasts-EpiNow2 2 #> 40676: 2021-07-12 0.990 6.579251 epiforecasts-EpiNow2 2 #> scale #> #> 1: natural #> 2: natural #> 3: natural #> 4: natural #> 5: natural #> --- #> 40672: log #> 40673: log #> 40674: log #> 40675: log #> 40676: log"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Logarithmic score (sample-based version) — logs_sample","title":"Logarithmic score (sample-based version) — logs_sample","text":"function wrapper around logs_sample() function scoringRules package. function used score continuous predictions . Log Score theory also applicable discrete forecasts, problem lies implementation: Log score needs kernel density estimation, well defined integer-valued Monte Carlo Samples. Log score can used specific discrete probability distributions. See scoringRules package details.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Logarithmic score (sample-based version) — logs_sample","text":"","code":"logs_sample(observed, predicted, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Logarithmic score (sample-based version) — logs_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. ... Additional arguments passed logs_sample() scoringRules package.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Logarithmic score (sample-based version) — logs_sample","text":"Vector scores.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Logarithmic score (sample-based version) — logs_sample","text":"Alexander Jordan, Fabian Krüger, Sebastian Lerch, Evaluating Probabilistic Forecasts scoringRules, https://www.jstatsoft.org/article/view/v090i12","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Logarithmic score (sample-based version) — logs_sample","text":"","code":"observed <- rpois(30, lambda = 1:30) predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) logs_sample(observed, predicted) #> [1] 0.9877018 1.3179060 2.9308071 3.0967973 1.7353294 2.1375898 3.0671846 #> [8] 2.0115847 2.3128147 2.2232410 2.3023663 6.6464809 2.2126378 2.6212927 #> [15] 2.3262577 2.5675490 3.3007417 2.2614306 2.4344484 2.4467472 3.5018847 #> [22] 2.7833854 2.9574125 2.7755341 2.7756007 3.1608762 3.1106262 2.8047358 #> [29] 2.6202545 2.6531665"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Determine dispersion of a probabilistic forecast — mad_sample","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"Sharpness ability model generate predictions within narrow range dispersion lack thereof. data-independent measure, purely feature forecasts . Dispersion predictive samples corresponding one single observed value measured normalised median absolute deviation median predictive samples. details, see mad() explanations given Funk et al. (2019)","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"","code":"mad_sample(observed = NULL, predicted, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"observed Place holder, argument ignored exists consistency scoring functions. output depend observed values. predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. ... Additional arguments passed mad().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"Vector dispersion values.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"Funk S, Camacho , Kucharski AJ, Lowe R, Eggo RM, Edmunds WJ (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15. PLoS Comput Biol 15(2): e1006785. doi:10.1371/journal.pcbi.1006785","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"","code":"predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) mad_sample(predicted = predicted) #> [1] 1.4826 1.4826 1.4826 2.9652 2.9652 2.9652 2.9652 2.9652 2.9652 2.9652 #> [11] 2.9652 2.9652 4.4478 4.4478 2.9652 3.7065 4.4478 3.7065 4.4478 4.4478 #> [21] 4.4478 5.9304 4.4478 4.4478 4.4478 5.9304 4.4478 4.4478 5.9304 5.9304"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Default metrics and scoring rules for binary forecasts — metrics_binary","title":"Default metrics and scoring rules for binary forecasts — metrics_binary","text":"Helper function returns named list default scoring rules suitable binary forecasts. default scoring rules : \"brier_score\" = brier_score() \"log_score\" = logs_binary()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Default metrics and scoring rules for binary forecasts — metrics_binary","text":"","code":"metrics_binary(select = NULL, exclude = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Default metrics and scoring rules for binary forecasts — metrics_binary","text":"select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Default metrics and scoring rules for binary forecasts — metrics_binary","text":"list scoring rules.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_binary.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Default metrics and scoring rules for binary forecasts — metrics_binary","text":"","code":"metrics_binary() #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> #> #> #> $log_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> logs <- -log(1 - abs(observed - predicted)) #> return(logs) #> } #> #> #> metrics_binary(select = \"brier_score\") #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> #> #> metrics_binary(exclude = \"log_score\") #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> #> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Default metrics and scoring rules for point forecasts — metrics_point","title":"Default metrics and scoring rules for point forecasts — metrics_point","text":"Helper function returns named list default scoring rules suitable point forecasts. default scoring rules : \"ae_point\" = ae() \"se_point\" = se() \"ape\" = ape()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Default metrics and scoring rules for point forecasts — metrics_point","text":"","code":"metrics_point(select = NULL, exclude = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Default metrics and scoring rules for point forecasts — metrics_point","text":"select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Default metrics and scoring rules for point forecasts — metrics_point","text":"list scoring rules.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_point.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Default metrics and scoring rules for point forecasts — metrics_point","text":"","code":"metrics_point() #> $ae_point #> function (actual, predicted) #> { #> return(abs(actual - predicted)) #> } #> #> #> #> $se_point #> function (actual, predicted) #> { #> return((actual - predicted)^2) #> } #> #> #> #> $ape #> function (actual, predicted) #> { #> return(ae(actual, predicted)/abs(actual)) #> } #> #> #> metrics_point(select = \"ape\") #> $ape #> function (actual, predicted) #> { #> return(ae(actual, predicted)/abs(actual)) #> } #> #> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Default metrics and scoring rules for quantile-based forecasts — metrics_quantile","title":"Default metrics and scoring rules for quantile-based forecasts — metrics_quantile","text":"Helper function returns named list default scoring rules suitable forecasts quantile-based format. default scoring rules : \"wis\" = wis \"overprediction\" = overprediction() \"underprediction\" = underprediction() \"dispersion\" = dispersion() \"bias\" = bias_quantile() \"interval_coverage_50\" = interval_coverage() \"interval_coverage_90\" = customise_metric( interval_coverage, interval_range = 90 ) \"interval_coverage_deviation\" = interval_coverage_deviation(), \"ae_median\" = ae_median_quantile() Note: interval_coverage_90 scoring rule created modifying interval_coverage(), making use function customise_metric(). construct allows function deal arbitrary arguments ..., making sure interval_coverage() can accept get passed . interval_range = 90 set function definition, passing argument interval_range = 90 score() mean also get passed interval_coverage_50.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Default metrics and scoring rules for quantile-based forecasts — metrics_quantile","text":"","code":"metrics_quantile(select = NULL, exclude = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Default metrics and scoring rules for quantile-based forecasts — metrics_quantile","text":"select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Default metrics and scoring rules for quantile-based forecasts — metrics_quantile","text":"list scoring rules.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_quantile.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Default metrics and scoring rules for quantile-based forecasts — metrics_quantile","text":"","code":"metrics_quantile() #> $wis #> function (observed, predicted, quantile_level, separate_results = FALSE, #> weigh = TRUE, count_median_twice = FALSE, na.rm = TRUE) #> { #> assert_input_quantile(observed, predicted, quantile_level) #> reformatted <- quantile_to_interval(observed, predicted, #> quantile_level) #> assert_logical(separate_results, len = 1) #> assert_logical(weigh, len = 1) #> assert_logical(count_median_twice, len = 1) #> assert_logical(na.rm, len = 1) #> if (separate_results) { #> cols <- c(\"wis\", \"dispersion\", \"underprediction\", \"overprediction\") #> } #> else { #> cols <- \"wis\" #> } #> reformatted[, `:=`(eval(cols), do.call(interval_score, list(observed = observed, #> lower = lower, upper = upper, interval_range = interval_range, #> weigh = weigh, separate_results = separate_results)))] #> if (count_median_twice) { #> reformatted[, `:=`(weight, 1)] #> } #> else { #> reformatted[, `:=`(weight, ifelse(interval_range == 0, #> 0.5, 1))] #> } #> reformatted <- reformatted[, lapply(.SD, weighted.mean, na.rm = na.rm, #> w = weight), by = \"forecast_id\", .SDcols = colnames(reformatted) %like% #> paste(cols, collapse = \"|\")] #> if (separate_results) { #> return(list(wis = reformatted$wis, dispersion = reformatted$dispersion, #> underprediction = reformatted$underprediction, overprediction = reformatted$overprediction)) #> } #> else { #> return(reformatted$wis) #> } #> } #> #> #> #> $overprediction #> function (observed, predicted, quantile_level, ...) #> { #> args <- list(...) #> args$separate_results <- TRUE #> assert_input_quantile(observed, predicted, quantile_level) #> out <- do.call(wis, c(list(observed), list(predicted), list(quantile_level), #> args)) #> return(out$overprediction) #> } #> #> #> #> $underprediction #> function (observed, predicted, quantile_level, ...) #> { #> args <- list(...) #> args$separate_results <- TRUE #> assert_input_quantile(observed, predicted, quantile_level) #> out <- do.call(wis, c(list(observed), list(predicted), list(quantile_level), #> args)) #> return(out$underprediction) #> } #> #> #> #> $dispersion #> function (observed, predicted, quantile_level, ...) #> { #> args <- list(...) #> args$separate_results <- TRUE #> assert_input_quantile(observed, predicted, quantile_level) #> out <- do.call(wis, c(list(observed), list(predicted), list(quantile_level), #> args)) #> return(out$dispersion) #> } #> #> #> #> $bias #> function (observed, predicted, quantile_level, na.rm = TRUE) #> { #> assert_input_quantile(observed, predicted, quantile_level) #> n <- length(observed) #> N <- length(quantile_level) #> if (is.null(dim(predicted))) { #> dim(predicted) <- c(n, N) #> } #> if (!(0.5 %in% quantile_level)) { #> cli_inform(c(i = \"Median not available, interpolating median from the two\\n innermost quantiles in order to compute bias.\")) #> } #> bias <- sapply(1:n, function(i) { #> bias_quantile_single_vector(observed[i], predicted[i, #> ], quantile_level, na.rm) #> }) #> return(bias) #> } #> #> #> #> $interval_coverage_50 #> function (observed, predicted, quantile_level, interval_range = 50) #> { #> assert_input_quantile(observed, predicted, quantile_level) #> assert_number(interval_range) #> necessary_quantiles <- c((100 - interval_range)/2, 100 - #> (100 - interval_range)/2)/100 #> if (!all(necessary_quantiles %in% quantile_level)) { #> cli_warn(c(`!` = \"To compute the interval coverage for an interval range of\\n {.val {interval_range}%}, the {.val {necessary_quantiles}} quantiles\\n are required.\", #> i = \"Returning {.val {NA}}.\")) #> return(NA) #> } #> r <- interval_range #> reformatted <- quantile_to_interval(observed, predicted, #> quantile_level) #> reformatted <- reformatted[interval_range %in% r] #> reformatted[, `:=`(interval_coverage, (observed >= lower) & #> (observed <= upper))] #> return(reformatted$interval_coverage) #> } #> #> #> #> $interval_coverage_90 #> function (...) #> { #> do.call(metric, c(list(...), dots)) #> } #> #> #> #> $interval_coverage_deviation #> function (observed, predicted, quantile_level) #> { #> assert_input_quantile(observed, predicted, quantile_level) #> available_ranges <- unique(get_range_from_quantile(quantile_level)) #> necessary_quantiles <- unique(c((100 - available_ranges)/2, #> 100 - (100 - available_ranges)/2)/100) #> if (!all(necessary_quantiles %in% quantile_level)) { #> missing <- necessary_quantiles[!necessary_quantiles %in% #> quantile_level] #> cli_warn(c(x = \"To compute interval coverage deviation, all quantiles must form\\n central symmetric prediction intervals.\", #> i = \"Missing quantiles: {.val {missing}}. Returning {.val {NA}}.\")) #> return(NA) #> } #> reformatted <- quantile_to_interval(observed, predicted, #> quantile_level)[interval_range != 0] #> reformatted[, `:=`(interval_coverage, (observed >= lower) & #> (observed <= upper))] #> reformatted[, `:=`(interval_coverage_deviation, interval_coverage - #> interval_range/100)] #> out <- reformatted[, .(interval_coverage_deviation = mean(interval_coverage_deviation)), #> by = \"forecast_id\"] #> return(out$interval_coverage_deviation) #> } #> #> #> #> $ae_median #> function (observed, predicted, quantile_level) #> { #> assert_input_quantile(observed, predicted, quantile_level) #> if (!any(quantile_level == 0.5)) { #> cli_warn(c(x = \"In order to compute the absolute error of the median,\\n {.val 0.5} must be among the quantiles given.\", #> i = \"Returning {.val NA}.\")) #> return(NA_real_) #> } #> if (is.null(dim(predicted))) { #> predicted <- matrix(predicted, nrow = 1) #> } #> predicted <- predicted[, quantile_level == 0.5] #> abs_error_median <- abs(observed - predicted) #> return(abs_error_median) #> } #> #> #> metrics_quantile(select = \"wis\") #> $wis #> function (observed, predicted, quantile_level, separate_results = FALSE, #> weigh = TRUE, count_median_twice = FALSE, na.rm = TRUE) #> { #> assert_input_quantile(observed, predicted, quantile_level) #> reformatted <- quantile_to_interval(observed, predicted, #> quantile_level) #> assert_logical(separate_results, len = 1) #> assert_logical(weigh, len = 1) #> assert_logical(count_median_twice, len = 1) #> assert_logical(na.rm, len = 1) #> if (separate_results) { #> cols <- c(\"wis\", \"dispersion\", \"underprediction\", \"overprediction\") #> } #> else { #> cols <- \"wis\" #> } #> reformatted[, `:=`(eval(cols), do.call(interval_score, list(observed = observed, #> lower = lower, upper = upper, interval_range = interval_range, #> weigh = weigh, separate_results = separate_results)))] #> if (count_median_twice) { #> reformatted[, `:=`(weight, 1)] #> } #> else { #> reformatted[, `:=`(weight, ifelse(interval_range == 0, #> 0.5, 1))] #> } #> reformatted <- reformatted[, lapply(.SD, weighted.mean, na.rm = na.rm, #> w = weight), by = \"forecast_id\", .SDcols = colnames(reformatted) %like% #> paste(cols, collapse = \"|\")] #> if (separate_results) { #> return(list(wis = reformatted$wis, dispersion = reformatted$dispersion, #> underprediction = reformatted$underprediction, overprediction = reformatted$overprediction)) #> } #> else { #> return(reformatted$wis) #> } #> } #> #> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Default metrics and scoring rules sample-based forecasts — metrics_sample","title":"Default metrics and scoring rules sample-based forecasts — metrics_sample","text":"Helper function returns named list default scoring rules suitable forecasts sample-based format. default scoring rules : \"crps\" = crps_sample() \"log_score\" = logs_sample() \"dss\" = dss_sample() \"mad\" = mad_sample() \"bias\" = bias_sample() \"ae_median\" = ae_median_sample() \"se_mean\" = se_mean_sample()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Default metrics and scoring rules sample-based forecasts — metrics_sample","text":"","code":"metrics_sample(select = NULL, exclude = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Default metrics and scoring rules sample-based forecasts — metrics_sample","text":"select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Default metrics and scoring rules sample-based forecasts — metrics_sample","text":"list scoring rules.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/metrics_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Default metrics and scoring rules sample-based forecasts — metrics_sample","text":"","code":"metrics_sample() #> $bias #> function (observed, predicted) #> { #> assert_input_sample(observed, predicted) #> prediction_type <- get_type(predicted) #> n_pred <- ncol(predicted) #> p_x <- rowSums(predicted <= observed)/n_pred #> if (prediction_type == \"continuous\") { #> res <- 1 - 2 * p_x #> return(res) #> } #> else { #> p_xm1 <- rowSums(predicted <= (observed - 1))/n_pred #> res <- 1 - (p_x + p_xm1) #> return(res) #> } #> } #> #> #> #> $dss #> function (observed, predicted, ...) #> { #> assert_input_sample(observed, predicted) #> scoringRules::dss_sample(y = observed, dat = predicted, ...) #> } #> #> #> #> $crps #> function (observed, predicted, ...) #> { #> assert_input_sample(observed, predicted) #> scoringRules::crps_sample(y = observed, dat = predicted, #> ...) #> } #> #> #> #> $log_score #> function (observed, predicted, ...) #> { #> assert_input_sample(observed, predicted) #> scoringRules::logs_sample(y = observed, dat = predicted, #> ...) #> } #> #> #> #> $mad #> function (observed = NULL, predicted, ...) #> { #> assert_input_sample(rep(NA_real_, nrow(predicted)), predicted) #> sharpness <- apply(predicted, MARGIN = 1, mad, ...) #> return(sharpness) #> } #> #> #> #> $ae_median #> function (observed, predicted) #> { #> assert_input_sample(observed, predicted) #> median_predictions <- apply(as.matrix(predicted), MARGIN = 1, #> FUN = median) #> ae_median <- abs(observed - median_predictions) #> return(ae_median) #> } #> #> #> #> $se_mean #> function (observed, predicted) #> { #> assert_input_sample(observed, predicted) #> mean_predictions <- rowMeans(as.matrix(predicted)) #> se_mean <- (observed - mean_predictions)^2 #> return(se_mean) #> } #> #> #> metrics_sample(select = \"mad\") #> $mad #> function (observed = NULL, predicted, ...) #> { #> assert_input_sample(rep(NA_real_, nrow(predicted)), predicted) #> sharpness <- apply(predicted, MARGIN = 1, mad, ...) #> return(sharpness) #> } #> #> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Class constructor for forecast objects — new_forecast","title":"Class constructor for forecast objects — new_forecast","text":"Construct class based data.frame similar. constructor coerces data data.table makes sure column called model exists creates one assigns class","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Class constructor for forecast objects — new_forecast","text":"","code":"new_forecast(data, classname)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Class constructor for forecast objects — new_forecast","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. classname name class created","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Class constructor for forecast objects — new_forecast","text":"object class indicated classname","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Construct an object of class scores — new_scores","title":"Construct an object of class scores — new_scores","text":"function creates object class scores based data.table similar.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Construct an object of class scores — new_scores","text":"","code":"new_scores(scores, metrics, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Construct an object of class scores — new_scores","text":"scores data.table similar scores produced score(). metrics character vector names scores (.e. names scoring rules used scoring). ... Additional arguments .data.table()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Construct an object of class scores — new_scores","text":"object class scores","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Construct an object of class scores — new_scores","text":"","code":"if (FALSE) { # \\dontrun{ df <- data.frame( model = \"A\", wis = \"0.1\" ) new_scores(df, \"wis\") } # }"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pairwise_comparison_one_group.html","id":null,"dir":"Reference","previous_headings":"","what":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","title":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","text":"function pairwise comparison one set forecasts, multiple models involved. gets called get_pairwise_comparisons(). get_pairwise_comparisons() splits data arbitrary subgroups specified user (e.g. pairwise comparison done separately different forecast targets) actual pairwise comparison subgroup managed pairwise_comparison_one_group(). order actually comparison two models subset common forecasts calls compare_two_models().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pairwise_comparison_one_group.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","text":"","code":"pairwise_comparison_one_group(scores, metric, baseline, by, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pairwise_comparison_one_group.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","text":"scores object class scores (data.table scores additional attribute metrics produced score()). metric string name metric relative skill shall computed. default either \"crps\", \"wis\" \"brier_score\" available. baseline string name model. baseline given, scaled relative skill respect baseline returned. default (NULL), relative skill scaled respect baseline model. Character vector column names define grouping level pairwise comparisons. default (model), one relative skill score per model. , example, = c(\"model\", \"location\"). get separate relative skill score every model every location. Internally, data.table scores split according (removing \"model\" splitting) pairwise comparisons computed separately split data.tables. ... Additional arguments comparison two models. See compare_two_models() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pairwise_comparison_one_group.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","text":"data.table results pairwise comparisons containing mean score ratios (mean_scores_ratio), unadjusted (pval) adjusted (adj_pval) p-values, relative skill values model (..._relative_skill). baseline model given scaled relative skill reported well (..._scaled_relative_skill).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/permutation_test.html","id":null,"dir":"Reference","previous_headings":"","what":"Simple permutation test — permutation_test","title":"Simple permutation test — permutation_test","text":"implementation permutation test follows function permutationTest surveillance package Michael Höhle, Andrea Riebler Michaela Paul. function compares two vectors scores. computes mean vector independently takes either difference ratio two. observed difference ratio compared test statistic based permutations original data. Used get_pairwise_comparisons().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/permutation_test.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Simple permutation test — permutation_test","text":"","code":"permutation_test( scores1, scores2, n_permutation = 999, one_sided = FALSE, comparison_mode = c(\"difference\", \"ratio\") )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/permutation_test.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Simple permutation test — permutation_test","text":"scores1 Vector scores compare another vector scores. scores2 second vector scores compare first n_permutation number replications use permutation test. replications yield exact results, require computation. one_sided Whether compute one-sided test. Default FALSE. comparison_mode compute test statistic comparison two scores. either \"difference\" \"ratio\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/permutation_test.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Simple permutation test — permutation_test","text":"p-value permutation test","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Probability integral transformation (sample-based version) — pit_sample","title":"Probability integral transformation (sample-based version) — pit_sample","text":"Uses Probability integral transformation (PIT) (randomised PIT integer forecasts) assess calibration predictive Monte Carlo samples.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Probability integral transformation (sample-based version) — pit_sample","text":"","code":"pit_sample(observed, predicted, n_replicates = 100)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Probability integral transformation (sample-based version) — pit_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. n_replicates number draws randomised PIT discrete predictions. ignored forecasts continuous.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Probability integral transformation (sample-based version) — pit_sample","text":"vector PIT-values. continuous forecasts, vector correspond length observed. integer forecasts, randomised PIT returned length length(observed) * n_replicates.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Probability integral transformation (sample-based version) — pit_sample","text":"Calibration reliability forecasts ability model correctly identify uncertainty making predictions. model perfect calibration, observed data time point look came predictive probability distribution time. Equivalently, one can inspect probability integral transform predictive distribution time t, $$ u_t = F_t (x_t) $$ \\(x_t\\) observed data point time \\(t \\textrm{ } t_1, …, t_n\\), n number forecasts, \\(F_t\\) (continuous) predictive cumulative probability distribution time t. true probability distribution outcomes time t \\(G_t\\) forecasts \\(F_t\\) said ideal \\(F_t = G_t\\) times t. case, probabilities \\(u_t\\) distributed uniformly. case discrete outcomes incidence counts, PIT longer uniform even forecasts ideal. case randomised PIT can used instead: $$ u_t = P_t(k_t) + v * (P_t(k_t) - P_t(k_t - 1) ) $$ \\(k_t\\) observed count, \\(P_t(x)\\) predictive cumulative probability observing incidence k time t, \\(P_t (-1) = 0\\) definition v standard uniform independent k. \\(P_t\\) true cumulative probability distribution, \\(u_t\\) standard uniform. function checks whether integer continuous forecasts provided. applies (randomised) probability integral tests values \\(u_t\\) uniformity using Anderson-Darling test. rule thumb, evidence suggest forecasting model miscalibrated p-value found greater threshold p >= 0.1, evidence miscalibrated 0.01 < p < 0.1, good evidence miscalibrated p <= 0.01. However, AD-p-values may overly strict actual usefulness may questionable. context noted, though, uniformity PIT necessary sufficient condition calibration.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Probability integral transformation (sample-based version) — pit_sample","text":"Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind M. Eggo, W. John Edmunds (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15, doi:10.1371/journal.pcbi.1006785","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Probability integral transformation (sample-based version) — pit_sample","text":"","code":"## continuous predictions observed <- rnorm(20, mean = 1:20) predicted <- replicate(100, rnorm(n = 20, mean = 1:20)) pit <- pit_sample(observed, predicted) plot_pit(pit) ## integer predictions observed <- rpois(20, lambda = 1:20) predicted <- replicate(100, rpois(n = 20, lambda = 1:20)) pit <- pit_sample(observed, predicted, n_replicates = 30) plot_pit(pit)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot correlation between metrics — plot_correlations","title":"Plot correlation between metrics — plot_correlations","text":"Plots heatmap correlations different metrics.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot correlation between metrics — plot_correlations","text":"","code":"plot_correlations(correlations, digits = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot correlation between metrics — plot_correlations","text":"correlations data.table correlations scores produced get_correlations(). digits number indicating many decimal places correlations rounded . default (digits = NULL) rounding takes place.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot correlation between metrics — plot_correlations","text":"ggplot object showing coloured matrix correlations metrics. ggplot object visualisation correlations metrics","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot correlation between metrics — plot_correlations","text":"","code":"scores <- score(as_forecast(example_quantile)) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. correlations <- get_correlations( summarise_scores(scores) ) plot_correlations(correlations, digits = 2)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":null,"dir":"Reference","previous_headings":"","what":"Visualise the number of available forecasts — plot_forecast_counts","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"Visualise Forecasts Available.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"","code":"plot_forecast_counts( forecast_counts, x, y = \"model\", x_as_factor = TRUE, show_counts = TRUE )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"forecast_counts data.table (similar) column count holding forecast counts, produced get_forecast_counts(). x Character vector length one denotes name column appear x-axis plot. y Character vector length one denotes name column appear y-axis plot. Default \"model\". x_as_factor Logical (default TRUE). Whether convert variable x-axis factor. effect e.g. dates shown x-axis. show_counts Logical (default TRUE) indicates whether show actual count numbers plot.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"ggplot object plot forecast counts","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"","code":"library(ggplot2) forecast_counts <- get_forecast_counts( as_forecast(example_quantile), by = c(\"model\", \"target_type\", \"target_end_date\") ) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_forecast_counts( forecast_counts, x = \"target_end_date\", show_counts = FALSE ) + facet_wrap(\"target_type\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a heatmap of a scoring metric — plot_heatmap","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"function can used create heatmap one metric across different groups, e.g. interval score obtained several forecasting models different locations.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"","code":"plot_heatmap(scores, y = \"model\", x, metric)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"scores data.frame scores based quantile forecasts produced score(). y variable scores want show y-Axis. default \"model\" x variable scores want show x-Axis. something like \"horizon\", \"location\" metric String, metric determines value colour shown tiles heatmap.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"ggplot object showing heatmap desired metric","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"","code":"scores <- score(as_forecast(example_quantile)) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. scores <- summarise_scores(scores, by = c(\"model\", \"target_type\")) scores <- summarise_scores( scores, by = c(\"model\", \"target_type\"), fun = signif, digits = 2 ) plot_heatmap(scores, x = \"target_type\", metric = \"bias\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot interval coverage — plot_interval_coverage","title":"Plot interval coverage — plot_interval_coverage","text":"Plot interval coverage values (see get_coverage() information).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot interval coverage — plot_interval_coverage","text":"","code":"plot_interval_coverage(coverage, colour = \"model\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot interval coverage — plot_interval_coverage","text":"coverage data frame coverage values produced get_coverage(). colour According variable shall graphs coloured? Default \"model\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot interval coverage — plot_interval_coverage","text":"ggplot object plot interval coverage","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot interval coverage — plot_interval_coverage","text":"","code":"coverage <- get_coverage(as_forecast(example_quantile), by = \"model\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_interval_coverage(coverage)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"Creates heatmap ratios pvalues pairwise comparison models.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"","code":"plot_pairwise_comparisons( comparison_result, type = c(\"mean_scores_ratio\", \"pval\") )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"comparison_result data.frame produced get_pairwise_comparisons(). type Character vector length one either \"mean_scores_ratio\" \"pval\". denotes whether visualise ratio p-value pairwise comparison. Default \"mean_scores_ratio\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"ggplot object heatmap mean score ratios pairwise comparisons.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"","code":"library(ggplot2) scores <- score(as_forecast(example_quantile)) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. pairwise <- get_pairwise_comparisons(scores, by = \"target_type\") plot_pairwise_comparisons(pairwise, type = \"mean_scores_ratio\") + facet_wrap(~target_type)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pit.html","id":null,"dir":"Reference","previous_headings":"","what":"PIT histogram — plot_pit","title":"PIT histogram — plot_pit","text":"Make simple histogram probability integral transformed values visually check whether uniform distribution seems likely.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pit.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"PIT histogram — plot_pit","text":"","code":"plot_pit(pit, num_bins = \"auto\", breaks = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pit.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"PIT histogram — plot_pit","text":"pit Either vector PIT values, data.table produced get_pit(). num_bins number bins PIT histogram, default \"auto\". num_bins == \"auto\", plot_pit() either display 10 bins, display bin available quantile case passed data quantile-based format. can control number bins supplying number. fine sample-based pit histograms, may fail quantile-based formats. case preferred supply explicit breaks points using breaks argument. breaks Numeric vector break points bins PIT histogram. preferred creating PIT histogram based quantile-based data. Default NULL breaks determined num_bins. breaks used, num_bins ignored.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pit.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"PIT histogram — plot_pit","text":"ggplot object histogram PIT values","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pit.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"PIT histogram — plot_pit","text":"","code":"# PIT histogram in vector based format observed <- rnorm(30, mean = 1:30) predicted <- replicate(200, rnorm(n = 30, mean = 1:30)) pit <- pit_sample(observed, predicted) plot_pit(pit) # quantile-based pit pit <- get_pit(as_forecast(example_quantile), by = \"model\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_pit(pit, breaks = seq(0.1, 1, 0.1)) # sample-based pit pit <- get_pit(as_forecast(example_sample_discrete), by = \"model\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_pit(pit)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot quantile coverage — plot_quantile_coverage","title":"Plot quantile coverage — plot_quantile_coverage","text":"Plot quantile coverage values (see get_coverage() information).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot quantile coverage — plot_quantile_coverage","text":"","code":"plot_quantile_coverage(coverage, colour = \"model\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot quantile coverage — plot_quantile_coverage","text":"coverage data frame coverage values produced get_coverage(). colour String, according variable shall graphs coloured? Default \"model\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot quantile coverage — plot_quantile_coverage","text":"ggplot object plot interval coverage","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot quantile coverage — plot_quantile_coverage","text":"","code":"coverage <- get_coverage(as_forecast(example_quantile), by = \"model\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_quantile_coverage(coverage)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot contributions to the weighted interval score — plot_wis","title":"Plot contributions to the weighted interval score — plot_wis","text":"Visualise components weighted interval score: penalties -prediction, -prediction high dispersion (lack sharpness).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot contributions to the weighted interval score — plot_wis","text":"","code":"plot_wis(scores, x = \"model\", relative_contributions = FALSE, flip = FALSE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot contributions to the weighted interval score — plot_wis","text":"scores data.table scores based quantile forecasts produced score() summarised using summarise_scores(). x variable scores want show x-Axis. Usually \"model\". relative_contributions Logical. Show relative contributions instead absolute contributions? Default FALSE functionality available yet. flip Boolean (default FALSE), whether flip axes.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot contributions to the weighted interval score — plot_wis","text":"ggplot object showing contributions three components weighted interval score. ggplot object visualisation WIS decomposition","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Plot contributions to the weighted interval score — plot_wis","text":"Bracher J, Ray E, Gneiting T, Reich, N (2020) Evaluating epidemic forecasts interval format. https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1008618","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot contributions to the weighted interval score — plot_wis","text":"","code":"library(ggplot2) scores <- score(as_forecast(example_quantile)) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. scores <- summarise_scores(scores, by = c(\"model\", \"target_type\")) plot_wis(scores, x = \"model\", relative_contributions = TRUE ) + facet_wrap(~target_type) plot_wis(scores, x = \"model\", relative_contributions = FALSE ) + facet_wrap(~target_type, scales = \"free_x\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Print information about a forecast object — print.forecast","title":"Print information about a forecast object — print.forecast","text":"function prints information forecast object, including \"Forecast type\", \"Score columns\", \"Forecast unit\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Print information about a forecast object — print.forecast","text":"","code":"# S3 method for class 'forecast' print(x, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Print information about a forecast object — print.forecast","text":"x forecast object (validated data.table predicted observed values, see as_forecast()). ... Additional arguments print().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Print information about a forecast object — print.forecast","text":"Returns x invisibly.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Print information about a forecast object — print.forecast","text":"","code":"dat <- as_forecast(example_quantile) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. print(dat) #> Forecast type: quantile #> Forecast unit: #> location, target_end_date, target_type, location_name, forecast_date, model, #> and horizon #> #> Key: #> location target_end_date target_type observed location_name #> #> 1: DE 2021-01-02 Cases 127300 Germany #> 2: DE 2021-01-02 Deaths 4534 Germany #> 3: DE 2021-01-09 Cases 154922 Germany #> 4: DE 2021-01-09 Deaths 6117 Germany #> 5: DE 2021-01-16 Cases 110183 Germany #> --- #> 20541: IT 2021-07-24 Deaths 78 Italy #> 20542: IT 2021-07-24 Deaths 78 Italy #> 20543: IT 2021-07-24 Deaths 78 Italy #> 20544: IT 2021-07-24 Deaths 78 Italy #> 20545: IT 2021-07-24 Deaths 78 Italy #> forecast_date quantile_level predicted model horizon #>
The quantile score, also called pinball loss, for a single quantile level \(\tau\) is defined as $$ - \text{QS}_\tau(F, y) = 2 \cdot \{ \mathbf{1}(y \leq q_\tau) - \tau\} \cdot (q_\tau − y) = + \text{QS}_\tau(F, y) = 2 \cdot \{ \mathbf{1}(y \leq q_\tau) - \tau\} \cdot (q_\tau - y) = \begin{cases} 2 \cdot (1 - \tau) * q_\tau - y, & \text{if } y \leq q_\tau\\ 2 \cdot \tau * |q_\tau - y|, & \text{if } y > q_\tau, diff --git a/dev/reference/select_metrics.html b/dev/reference/select_metrics.html index 2da9c6cc..18e0c6e6 100644 --- a/dev/reference/select_metrics.html +++ b/dev/reference/select_metrics.html @@ -88,7 +88,7 @@