diff --git a/dev/articles/scoring-rules.html b/dev/articles/scoring-rules.html index e4b7c7d2e..ad9fca7f2 100644 --- a/dev/articles/scoring-rules.html +++ b/dev/articles/scoring-rules.html @@ -63,7 +63,7 @@

Scoring rules in `scoringutils`

Nikos Bosse

-

2024-10-09

+

2024-10-14

Source: vignettes/scoring-rules.Rmd
scoring-rules.Rmd
diff --git a/dev/pkgdown.yml b/dev/pkgdown.yml index 8b2834d71..09d830846 100644 --- a/dev/pkgdown.yml +++ b/dev/pkgdown.yml @@ -5,7 +5,7 @@ articles: Deprecated-functions: Deprecated-functions.html Deprecated-visualisations: Deprecated-visualisations.html scoring-rules: scoring-rules.html -last_built: 2024-10-09T15:33Z +last_built: 2024-10-14T11:46Z urls: reference: https://epiforecasts.io/scoringutils/reference article: https://epiforecasts.io/scoringutils/articles diff --git a/dev/reference/figures/input-formats-scoring-rules.png b/dev/reference/figures/input-formats-scoring-rules.png new file mode 100644 index 000000000..478abb191 Binary files /dev/null and b/dev/reference/figures/input-formats-scoring-rules.png differ diff --git a/dev/reference/figures/pairwise-illustration.png b/dev/reference/figures/pairwise-illustration.png index f469de0ea..95b6f5723 100644 Binary files a/dev/reference/figures/pairwise-illustration.png and b/dev/reference/figures/pairwise-illustration.png differ diff --git a/dev/reference/figures/workflow.png b/dev/reference/figures/workflow.png index 1fa067b37..34cf59a3c 100644 Binary files a/dev/reference/figures/workflow.png and b/dev/reference/figures/workflow.png differ diff --git a/dev/reference/get_metrics.forecast_binary.html b/dev/reference/get_metrics.forecast_binary.html index f0b37477b..782267f51 100644 --- a/dev/reference/get_metrics.forecast_binary.html +++ b/dev/reference/get_metrics.forecast_binary.html @@ -112,7 +112,7 @@

Examples#> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } -#> <bytecode: 0x55dfc21ce738> +#> <bytecode: 0x55bae1b9b798> #> <environment: namespace:scoringutils> #> #> $log_score @@ -123,7 +123,7 @@

Examples#> logs <- -log(1 - abs(observed - predicted)) #> return(logs) #> } -#> <bytecode: 0x55dfc21cd8c8> +#> <bytecode: 0x55bae1b9e790> #> <environment: namespace:scoringutils> #> get_metrics(example_binary, select = "brier_score") @@ -135,7 +135,7 @@

Examples#> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } -#> <bytecode: 0x55dfc21ce738> +#> <bytecode: 0x55bae1b9b798> #> <environment: namespace:scoringutils> #> get_metrics(example_binary, exclude = "log_score") @@ -147,7 +147,7 @@

Examples#> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } -#> <bytecode: 0x55dfc21ce738> +#> <bytecode: 0x55bae1b9b798> #> <environment: namespace:scoringutils> #> diff --git a/dev/reference/get_metrics.forecast_nominal.html b/dev/reference/get_metrics.forecast_nominal.html index 2d094661a..dbeee1fe3 100644 --- a/dev/reference/get_metrics.forecast_nominal.html +++ b/dev/reference/get_metrics.forecast_nominal.html @@ -105,7 +105,7 @@

Examples#> logs <- -log(pred_for_observed) #> return(logs) #> } -#> <bytecode: 0x55dfbcdde508> +#> <bytecode: 0x55bade54e758> #> <environment: namespace:scoringutils> #> diff --git a/dev/reference/get_metrics.forecast_point.html b/dev/reference/get_metrics.forecast_point.html index 3116b420a..592be9dd3 100644 --- a/dev/reference/get_metrics.forecast_point.html +++ b/dev/reference/get_metrics.forecast_point.html @@ -143,7 +143,7 @@

Examples#> { #> return(ae(actual, predicted)/abs(actual)) #> } -#> <bytecode: 0x55dfc1530420> +#> <bytecode: 0x55bae09b8d68> #> <environment: namespace:Metrics> #> diff --git a/dev/reference/get_metrics.forecast_quantile.html b/dev/reference/get_metrics.forecast_quantile.html index d5ac2b86f..2c6ce73f5 100644 --- a/dev/reference/get_metrics.forecast_quantile.html +++ b/dev/reference/get_metrics.forecast_quantile.html @@ -191,7 +191,7 @@

Examples#> return(reformatted$wis) #> } #> } -#> <bytecode: 0x55dfc11a1180> +#> <bytecode: 0x55bae1608340> #> <environment: namespace:scoringutils> #> diff --git a/dev/reference/get_metrics.forecast_sample.html b/dev/reference/get_metrics.forecast_sample.html index d48fe1e3a..795756ebc 100644 --- a/dev/reference/get_metrics.forecast_sample.html +++ b/dev/reference/get_metrics.forecast_sample.html @@ -141,7 +141,7 @@

Examples#> return(res) #> } #> } -#> <bytecode: 0x55dfc138f110> +#> <bytecode: 0x55bae2e13a78> #> <environment: namespace:scoringutils> #> #> $dss @@ -150,7 +150,7 @@

Examples#> assert_input_sample(observed, predicted) #> scoringRules::dss_sample(y = observed, dat = predicted, ...) #> } -#> <bytecode: 0x55dfbfde9d80> +#> <bytecode: 0x55bae19cafc8> #> <environment: namespace:scoringutils> #> #> $crps @@ -182,7 +182,7 @@

Examples#> return(crps) #> } #> } -#> <bytecode: 0x55dfbe87a408> +#> <bytecode: 0x55bae04149b8> #> <environment: namespace:scoringutils> #> #> $overprediction @@ -192,7 +192,7 @@

Examples#> ...) #> return(crps$overprediction) #> } -#> <bytecode: 0x55dfc00d6940> +#> <bytecode: 0x55bae1a63fe0> #> <environment: namespace:scoringutils> #> #> $underprediction @@ -202,7 +202,7 @@

Examples#> ...) #> return(crps$underprediction) #> } -#> <bytecode: 0x55dfc00d9c10> +#> <bytecode: 0x55bae1a634b8> #> <environment: namespace:scoringutils> #> #> $dispersion @@ -212,7 +212,7 @@

Examples#> ...) #> return(crps$dispersion) #> } -#> <bytecode: 0x55dfc00d8ef0> +#> <bytecode: 0x55bae1a62990> #> <environment: namespace:scoringutils> #> #> $log_score @@ -222,7 +222,7 @@

Examples#> scoringRules::logs_sample(y = observed, dat = predicted, #> ...) #> } -#> <bytecode: 0x55dfc00dc1f8> +#> <bytecode: 0x55bae1a65c98> #> <environment: namespace:scoringutils> #> #> $ae_median @@ -234,7 +234,7 @@

Examples#> ae_median <- abs(observed - median_predictions) #> return(ae_median) #> } -#> <bytecode: 0x55dfbfeed730> +#> <bytecode: 0x55bae18f0080> #> <environment: namespace:scoringutils> #> #> $se_mean @@ -245,7 +245,7 @@

Examples#> se_mean <- (observed - mean_predictions)^2 #> return(se_mean) #> } -#> <bytecode: 0x55dfc00dd510> +#> <bytecode: 0x55bae1a64370> #> <environment: namespace:scoringutils> #> diff --git a/dev/reference/select_metrics.html b/dev/reference/select_metrics.html index b3c4a4dba..051851e7f 100644 --- a/dev/reference/select_metrics.html +++ b/dev/reference/select_metrics.html @@ -88,7 +88,7 @@

Examples#> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } -#> <bytecode: 0x55dfc21ce738> +#> <bytecode: 0x55bae1b9b798> #> <environment: namespace:scoringutils> #> select_metrics( @@ -103,7 +103,7 @@

Examples#> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } -#> <bytecode: 0x55dfc21ce738> +#> <bytecode: 0x55bae1b9b798> #> <environment: namespace:scoringutils> #> diff --git a/dev/search.json b/dev/search.json index 62313eca9..07409621d 100644 --- a/dev/search.json +++ b/dev/search.json @@ -1 +1 @@ -[{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"our-pledge","dir":"","previous_headings":"","what":"Our Pledge","title":"Contributor Covenant Code of Conduct","text":"members, contributors, leaders pledge make participation community harassment-free experience everyone, regardless age, body size, visible invisible disability, ethnicity, sex characteristics, gender identity expression, level experience, education, socio-economic status, nationality, personal appearance, race, religion, sexual identity orientation. pledge act interact ways contribute open, welcoming, diverse, inclusive, healthy community.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"our-standards","dir":"","previous_headings":"","what":"Our Standards","title":"Contributor Covenant Code of Conduct","text":"Examples behavior contributes positive environment community include: Demonstrating empathy kindness toward people respectful differing opinions, viewpoints, experiences Giving gracefully accepting constructive feedback Accepting responsibility apologizing affected mistakes, learning experience Focusing best just us individuals, overall community Examples unacceptable behavior include: use sexualized language imagery, sexual attention advances kind Trolling, insulting derogatory comments, personal political attacks Public private harassment Publishing others’ private information, physical email address, without explicit permission conduct reasonably considered inappropriate professional setting","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"enforcement-responsibilities","dir":"","previous_headings":"","what":"Enforcement Responsibilities","title":"Contributor Covenant Code of Conduct","text":"Community leaders responsible clarifying enforcing standards acceptable behavior take appropriate fair corrective action response behavior deem inappropriate, threatening, offensive, harmful. Community leaders right responsibility remove, edit, reject comments, commits, code, wiki edits, issues, contributions aligned Code Conduct, communicate reasons moderation decisions appropriate.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"scope","dir":"","previous_headings":"","what":"Scope","title":"Contributor Covenant Code of Conduct","text":"Code Conduct applies within community spaces, also applies individual officially representing community public spaces. Examples representing community include using official e-mail address, posting via official social media account, acting appointed representative online offline event.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"enforcement","dir":"","previous_headings":"","what":"Enforcement","title":"Contributor Covenant Code of Conduct","text":"Instances abusive, harassing, otherwise unacceptable behavior may reported community leaders responsible enforcement Sebastian.Funk@lshtm.ac.uk. complaints reviewed investigated promptly fairly. community leaders obligated respect privacy security reporter incident.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"enforcement-guidelines","dir":"","previous_headings":"","what":"Enforcement Guidelines","title":"Contributor Covenant Code of Conduct","text":"Community leaders follow Community Impact Guidelines determining consequences action deem violation Code Conduct:","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"id_1-correction","dir":"","previous_headings":"Enforcement Guidelines","what":"1. Correction","title":"Contributor Covenant Code of Conduct","text":"Community Impact: Use inappropriate language behavior deemed unprofessional unwelcome community. Consequence: private, written warning community leaders, providing clarity around nature violation explanation behavior inappropriate. public apology may requested.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"id_2-warning","dir":"","previous_headings":"Enforcement Guidelines","what":"2. Warning","title":"Contributor Covenant Code of Conduct","text":"Community Impact: violation single incident series actions. Consequence: warning consequences continued behavior. interaction people involved, including unsolicited interaction enforcing Code Conduct, specified period time. includes avoiding interactions community spaces well external channels like social media. Violating terms may lead temporary permanent ban.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"id_3-temporary-ban","dir":"","previous_headings":"Enforcement Guidelines","what":"3. Temporary Ban","title":"Contributor Covenant Code of Conduct","text":"Community Impact: serious violation community standards, including sustained inappropriate behavior. Consequence: temporary ban sort interaction public communication community specified period time. public private interaction people involved, including unsolicited interaction enforcing Code Conduct, allowed period. Violating terms may lead permanent ban.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"id_4-permanent-ban","dir":"","previous_headings":"Enforcement Guidelines","what":"4. Permanent Ban","title":"Contributor Covenant Code of Conduct","text":"Community Impact: Demonstrating pattern violation community standards, including sustained inappropriate behavior, harassment individual, aggression toward disparagement classes individuals. Consequence: permanent ban sort public interaction within community.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"attribution","dir":"","previous_headings":"","what":"Attribution","title":"Contributor Covenant Code of Conduct","text":"Code Conduct adapted Contributor Covenant, version 2.0, available https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. Community Impact Guidelines inspired Mozilla’s code conduct enforcement ladder. answers common questions code conduct, see FAQ https://www.contributor-covenant.org/faq. Translations available https://www.contributor-covenant.org/translations.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":null,"dir":"","previous_headings":"","what":"Contributing to scoringutils","title":"Contributing to scoringutils","text":"outlines propose change scoringutils.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"fixing-typos","dir":"","previous_headings":"","what":"Fixing typos","title":"Contributing to scoringutils","text":"can fix typos, spelling mistakes, grammatical errors documentation directly using GitHub web interface, long changes made source file. generally means ’ll need edit roxygen2 comments .R, .Rd file. can find .R file generates .Rd reading comment first line.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"bigger-changes","dir":"","previous_headings":"","what":"Bigger changes","title":"Contributing to scoringutils","text":"want make bigger change, ’s good idea first file issue make sure someone team agrees ’s needed. ’ve found bug, please file issue illustrates bug minimal reprex (also help write unit test, needed).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"pull-request-process","dir":"","previous_headings":"Bigger changes","what":"Pull request process","title":"Contributing to scoringutils","text":"Fork package clone onto computer. haven’t done , recommend using usethis::create_from_github(\"epiforecasts/scoringutils\", fork = TRUE). Install development dependences devtools::install_dev_deps(), make sure package passes R CMD check running devtools::check(). R CMD check doesn’t pass cleanly, ’s good idea ask help continuing. Create Git branch pull request (PR). recommend using usethis::pr_init(\"brief-description--change\"). Make changes, commit git, create PR running usethis::pr_push(), following prompts browser. title PR briefly describe change. body PR contain Fixes #issue-number. user-facing changes, add bullet top NEWS.md (.e. just first header). Follow style described https://style.tidyverse.org/news.html.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"code-style","dir":"","previous_headings":"Bigger changes","what":"Code style","title":"Contributing to scoringutils","text":"New code follow tidyverse style guide. can use styler package apply styles, please don’t restyle code nothing PR. use roxygen2, Markdown syntax, documentation. use testthat unit tests. Contributions test cases included easier accept.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"code-of-conduct","dir":"","previous_headings":"","what":"Code of Conduct","title":"Contributing to scoringutils","text":"Please note scoringutils project released Contributor Code Conduct. contributing project agree abide terms.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/LICENSE.html","id":null,"dir":"","previous_headings":"","what":"MIT License","title":"MIT License","text":"Copyright (c) 2020 EpiForecasts Permission hereby granted, free charge, person obtaining copy software associated documentation files (“Software”), deal Software without restriction, including without limitation rights use, copy, modify, merge, publish, distribute, sublicense, /sell copies Software, permit persons Software furnished , subject following conditions: copyright notice permission notice shall included copies substantial portions Software. SOFTWARE PROVIDED “”, WITHOUT WARRANTY KIND, EXPRESS IMPLIED, INCLUDING LIMITED WARRANTIES MERCHANTABILITY, FITNESS PARTICULAR PURPOSE NONINFRINGEMENT. EVENT SHALL AUTHORS COPYRIGHT HOLDERS LIABLE CLAIM, DAMAGES LIABILITY, WHETHER ACTION CONTRACT, TORT OTHERWISE, ARISING , CONNECTION SOFTWARE USE DEALINGS SOFTWARE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/PULL_REQUEST_TEMPLATE.html","id":"description","dir":"","previous_headings":"","what":"Description","title":"NA","text":"PR closes #. [Describe changes made pull request.]","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/PULL_REQUEST_TEMPLATE.html","id":"checklist","dir":"","previous_headings":"","what":"Checklist","title":"NA","text":"PR based package issue explicitly linked . included target issue issues PR title follows: issue-number: PR title tested changes locally. added updated unit tests necessary. updated documentation required. built package locally run rebuilt docs using roxygen2. code follows established coding standards run lintr::lint_package() check style issues introduced changes. added news item linked PR. reviewed CI checks PR addressed far able.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/Deprecated-functions.html","id":"merge_pred_and_obs","dir":"Articles","previous_headings":"","what":"merge_pred_and_obs()","title":"Deprecated functions","text":"scoringutils requires forecasts observations provided single data frame. forecasts observations two different data frames, merge_pred_and_obs() may help merge two. function mostly wrapper around merge(), additional work deal duplicated column names.","code":"#' @title Merge forecast data and observations #' #' @description #' #' The function more or less provides a wrapper around `merge` that #' aims to handle the merging well if additional columns are present #' in one or both data sets. If in doubt, you should probably merge the #' data sets manually. #' #' @param forecasts A data.frame with the forecast data (as can be passed to #' [score()]). #' @param observations A data.frame with the observations. #' @param join Character, one of `c(\"left\", \"full\", \"right\")`. Determines the #' type of the join. Usually, a left join is appropriate, but sometimes you #' may want to do a full join to keep dates for which there is a forecast, but #' no ground truth data. #' @param by Character vector that denotes the columns by which to merge. Any #' value that is not a column in observations will be removed. #' @return a data.table with forecasts and observations #' @importFrom checkmate assert_subset #' @importFrom data.table as.data.table #' @keywords data-handling #' @export merge_pred_and_obs <- function(forecasts, observations, join = c(\"left\", \"full\", \"right\"), by = NULL) { forecasts <- as.data.table(forecasts) observations <- as.data.table(observations) join <- match.arg(join) assert_subset(by, intersect(names(forecasts), names(observations))) if (is.null(by)) { protected_columns <- c( \"predicted\", \"observed\", \"sample_id\", \"quantile_level\", \"interval_range\", \"boundary\" ) by <- setdiff(colnames(forecasts), protected_columns) } obs_cols <- colnames(observations) by <- intersect(by, obs_cols) join <- match.arg(join) if (join == \"left\") { # do a left_join, where all data in the observations are kept. combined <- merge(observations, forecasts, by = by, all.x = TRUE) } else if (join == \"full\") { # do a full, where all data is kept. combined <- merge(observations, forecasts, by = by, all = TRUE) } else { combined <- merge(observations, forecasts, by = by, all.y = TRUE) } # get colnames that are the same for x and y colnames <- colnames(combined) colnames_x <- colnames[endsWith(colnames, \".x\")] colnames_y <- colnames[endsWith(colnames, \".y\")] # extract basenames basenames_x <- sub(\".x$\", \"\", colnames_x) basenames_y <- sub(\".y$\", \"\", colnames_y) # see whether the column name as well as the content is the same content_x <- as.list(combined[, ..colnames_x]) content_y <- as.list(combined[, ..colnames_y]) overlapping <- (content_x %in% content_y) & (basenames_x == basenames_y) overlap_names <- colnames_x[overlapping] basenames_overlap <- sub(\".x$\", \"\", overlap_names) # delete overlapping columns if (length(basenames_overlap) > 0) { combined[, paste0(basenames_overlap, \".x\") := NULL] combined[, paste0(basenames_overlap, \".y\") := NULL] } return(combined[]) }"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/Deprecated-visualisations.html","id":"functions-plot_predictions-and-make_na","dir":"Articles","previous_headings":"","what":"Functions plot_predictions() and make_na()","title":"Deprecated Visualisations","text":"previous versions scoringutils, forecasts observed values visualised using function plot_predictions() make_na() helper function. following shows function code first example. plot_predictions() actual work producing plot. argument needed user can facet plot correctly user needs specify columns relevant facetting. make_NA() represents form filtering, instead filtering entire rows, relevant entries columns “predicted” “observed” made NA. allows user filter observations forecasts independently. following examples using two functions create plot using scoringutils example data. Visualising median forecasts example data. truth data restricted period 2021-05-01 2021-07-22. forecast data forecast model “EuroCOVIDhub-ensemble” made “2021-06-07”. data set NA, effectively removing plot. plot, variety prediction intervals shown, instead just median. similar plot, time based continuous forecasts. predictions automatically converted quantile-based forecasts plotting. Displaying two forecasts time additional colours:","code":"#\" @title Plot Predictions vs True Values #\" #\" @description #\" Make a plot of observed and predicted values #\" #\" @param data a data.frame that follows the same specifications outlined in #\" [score()]. To customise your plotting, you can filter your data using the #\" function [make_NA()]. #\" @param by character vector with column names that denote categories by which #\" the plot should be stratified. If for example you want to have a facetted #\" plot, this should be a character vector with the columns used in facetting #\" (note that the facetting still needs to be done outside of the function call) #\" @param x character vector of length one that denotes the name of the variable #\" @param interval_range numeric vector indicating the interval ranges to plot. #\" If 0 is included in `interval_range`, the median prediction will be shown. #\" @return ggplot object with a plot of true vs predicted values #\" @importFrom ggplot2 ggplot scale_colour_manual scale_fill_manual theme_light #\" @importFrom ggplot2 facet_wrap facet_grid aes geom_line .data geom_point #\" @importFrom data.table dcast #\" @importFrom ggdist geom_lineribbon #\" @export #\" @examples #\" library(ggplot2) #\" library(magrittr) #\" #\" example_sample_continuous %>% #\" make_NA ( #\" what = \"truth\", #\" target_end_date >= \"2021-07-22\", #\" target_end_date < \"2021-05-01\" #\" ) %>% #\" make_NA ( #\" what = \"forecast\", #\" model != \"EuroCOVIDhub-ensemble\", #\" forecast_date != \"2021-06-07\" #\" ) %>% #\" plot_predictions ( #\" x = \"target_end_date\", #\" by = c(\"target_type\", \"location\"), #\" interval_range = c(0, 50, 90, 95) #\" ) + #\" facet_wrap(~ location + target_type, scales = \"free_y\") + #\" aes(fill = model, color = model) #\" #\" example_sample_continuous %>% #\" make_NA ( #\" what = \"truth\", #\" target_end_date >= \"2021-07-22\", #\" target_end_date < \"2021-05-01\" #\" ) %>% #\" make_NA ( #\" what = \"forecast\", #\" forecast_date != \"2021-06-07\" #\" ) %>% #\" plot_predictions ( #\" x = \"target_end_date\", #\" by = c(\"target_type\", \"location\"), #\" interval_range = 0 #\" ) + #\" facet_wrap(~ location + target_type, scales = \"free_y\") + #\" aes(fill = model, color = model) library(ggdist) plot_predictions <- function(data, by = NULL, x = \"date\", interval_range = c(0, 50, 90)) { # split truth data and forecasts in order to apply different filtering truth_data <- data.table::as.data.table(data)[!is.na(observed)] forecasts <- data.table::as.data.table(data)[!is.na(predicted)] del_cols <- colnames(truth_data)[!(colnames(truth_data) %in% c(by, \"observed\", x))] truth_data <- unique(suppressWarnings(truth_data[, eval(del_cols) := NULL])) # find out what type of predictions we have. convert sample based to # interval range data if (\"quantile_level\" %in% colnames(data)) { forecasts <- scoringutils:::quantile_to_interval( forecasts, keep_quantile_col = FALSE ) } else if (\"sample_id\" %in% colnames(data)) { # using a scoringutils internal function forecasts <- scoringutils:::sample_to_interval_long( as_forecast_sample(forecasts), interval_range = interval_range, keep_quantile_col = FALSE ) } # select appropriate boundaries and pivot wider select <- forecasts$interval_range %in% setdiff(interval_range, 0) intervals <- forecasts[select, ] # delete quantile column in intervals if present. This is important for # pivoting if (\"quantile_level\" %in% names(intervals)) { intervals[, quantile_level := NULL] } plot <- ggplot(data = data, aes(x = .data[[x]])) + theme_scoringutils() + ylab(\"True and predicted values\") if (nrow(intervals) != 0) { # pivot wider and convert range to a factor intervals <- data.table::dcast(intervals, ... ~ boundary, value.var = \"predicted\") # only plot interval ranges if there are interval ranges to plot plot <- plot + ggdist::geom_lineribbon( data = intervals, aes( ymin = lower, ymax = upper, # We use the fill_ramp aesthetic for this instead of the default fill # because we want to keep fill to be able to use it for other # variables fill_ramp = factor( interval_range, levels = sort(unique(interval_range), decreasing = TRUE) ) ), lwd = 0.4 ) + ggdist::scale_fill_ramp_discrete( name = \"interval_range\", # range argument was added to make sure that the line for the median # and the ribbon don\"t have the same opacity, making the line # invisible range = c(0.15, 0.75) ) } # We could treat this step as part of ggdist::geom_lineribbon() but we treat # it separately here to deal with the case when only the median is provided # (in which case ggdist::geom_lineribbon() will fail) if (0 %in% interval_range) { select_median <- forecasts$interval_range == 0 & forecasts$boundary == \"lower\" median <- forecasts[select_median] if (nrow(median) > 0) { plot <- plot + geom_line( data = median, mapping = aes(y = predicted), lwd = 0.4 ) } } # add observed values if (nrow(truth_data) > 0) { plot <- plot + geom_point( data = truth_data, show.legend = FALSE, inherit.aes = FALSE, aes(x = .data[[x]], y = observed), color = \"black\", size = 0.5 ) + geom_line( data = truth_data, inherit.aes = FALSE, show.legend = FALSE, aes(x = .data[[x]], y = observed), linetype = 1, color = \"grey40\", lwd = 0.2 ) } return(plot) } #\" @title Make Rows NA in Data for Plotting #\" #\" @description #\" Filters the data and turns values into `NA` before the data gets passed to #\" [plot_predictions()]. The reason to do this is to this is that it allows to #\" \"filter\" prediction and truth data separately. Any value that is NA will then #\" be removed in the subsequent call to [plot_predictions()]. #\" #\" @inheritParams score #\" @param what character vector that determines which values should be turned #\" into `NA`. If `what = \"truth\"`, values in the column \"observed\" will be #\" turned into `NA`. If `what = \"forecast\"`, values in the column \"prediction\" #\" will be turned into `NA`. If `what = \"both\"`, values in both column will be #\" turned into `NA`. #\" @param ... logical statements used to filter the data #\" @return A data.table #\" @importFrom rlang enexprs #\" @keywords plotting #\" @export #\" #\" @examples #\" make_NA ( #\" example_sample_continuous, #\" what = \"truth\", #\" target_end_date >= \"2021-07-22\", #\" target_end_date < \"2021-05-01\" #\" ) make_NA <- function(data = NULL, what = c(\"truth\", \"forecast\", \"both\"), ...) { stopifnot(is.data.frame(data)) data <- as.data.table(data) what <- match.arg(what) # turn ... arguments into expressions args <- enexprs(...) vars <- NULL if (what %in% c(\"forecast\", \"both\")) { vars <- c(vars, \"predicted\") } if (what %in% c(\"truth\", \"both\")) { vars <- c(vars, \"observed\") } for (expr in args) { data <- data[eval(expr), eval(vars) := NA_real_] } return(data[]) } median_forecasts <- example_quantile[quantile_level == 0.5] median_forecasts %>% make_NA(what = \"truth\", target_end_date <= \"2021-05-01\", target_end_date > \"2021-07-22\") %>% make_NA(what = \"forecast\", model != \"EuroCOVIDhub-ensemble\", forecast_date != \"2021-06-07\") %>% plot_predictions( by = c(\"location\", \"target_type\"), x = \"target_end_date\" ) + facet_wrap(location ~ target_type, scales = \"free_y\") example_quantile %>% make_NA(what = \"truth\", target_end_date <= \"2021-05-01\", target_end_date > \"2021-07-22\") %>% make_NA(what = \"forecast\", model != \"EuroCOVIDhub-ensemble\", forecast_date != \"2021-06-07\") %>% plot_predictions( by = c(\"location\", \"target_type\"), x = \"target_end_date\", interval_range = c(0, 10, 20, 30, 40, 50, 60) ) + facet_wrap(location ~ target_type, scales = \"free_y\") example_sample_continuous %>% make_NA(what = \"truth\", target_end_date <= \"2021-05-01\", target_end_date > \"2021-07-22\") %>% make_NA(what = \"forecast\", model != \"EuroCOVIDhub-ensemble\", forecast_date != \"2021-06-07\") %>% plot_predictions( by = c(\"location\", \"target_type\"), x = \"target_end_date\", interval_range = c(0, 50, 90, 95) ) + facet_wrap(location ~ target_type, scales = \"free_y\") example_quantile %>% make_NA(what = \"truth\", target_end_date > \"2021-07-15\", target_end_date <= \"2021-05-22\") %>% make_NA(what = \"forecast\", !(model %in% c(\"EuroCOVIDhub-ensemble\", \"EuroCOVIDhub-baseline\")), forecast_date != \"2021-06-28\") %>% plot_predictions(x = \"target_end_date\", by = c(\"target_type\", \"location\")) + aes(colour = model, fill = model) + facet_wrap(target_type ~ location, ncol = 4, scales = \"free_y\") + labs(x = \"Target end date\")"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/Deprecated-visualisations.html","id":"function-plot_interval_ranges-formerly-plot_ranges","dir":"Articles","previous_headings":"","what":"Function plot_interval_ranges() (formerly plot_ranges())","title":"Deprecated Visualisations","text":"functionality currently relies hack. previous versions scoringutils, scores computed per interval range/per quantile. Now, scoringutils returns one score per forecast, per interval range/quantile. therefore need add range column, using internal function get_range_from_quantile(). column interpreted one defines unit single forecast scoringutils. also means get warning different number quantile levels different forecasts (0% prediction interval one median forecast, prediction intervals two (lower upper bound)). Plotting dispersion instead WIS:","code":"#\" @title Plot Metrics by Range of the Prediction Interval #\" #\" @description #\" Visualise the metrics by range, e.g. if you are interested how different #\" interval ranges contribute to the overall interval score, or how #\" sharpness / dispersion changes by range. #\" #\" @param scores A data.frame of scores based on quantile forecasts as #\" produced by [score()] or [summarise_scores()]. Note that \"range\" must be included #\" in the `by` argument when running [summarise_scores()] #\" @param y The variable from the scores you want to show on the y-Axis. #\" This could be something like \"wis\" (the default) or \"dispersion\" #\" @param x The variable from the scores you want to show on the x-Axis. #\" Usually this will be \"model\" #\" @param colour Character vector of length one used to determine a variable #\" for colouring dots. The Default is \"range\". #\" @return A ggplot2 object showing a contributions from the three components of #\" the weighted interval score #\" @importFrom ggplot2 ggplot aes aes geom_point geom_line #\" expand_limits theme theme_light element_text scale_color_continuous labs #\" @export #\" @examples #\" library(ggplot2) #\" ex <- data.table::copy(example_quantile) #\" ex$range <- scoringutils:::get_range_from_quantile(ex$quantile) #\" scores <- suppressWarnings(score(as_forecast_quantile(ex), metrics = list(\"wis\" = wis))) #\" summarised <- summarise_scores( #\" scores, #\" by = c(\"model\", \"target_type\", \"range\") #\" ) #\" plot_interval_ranges(summarised, x = \"model\") + #\" facet_wrap(~target_type, scales = \"free\") plot_interval_ranges <- function(scores, y = \"wis\", x = \"model\", colour = \"range\") { plot <- ggplot( scores, aes( x = .data[[x]], y = .data[[y]], colour = .data[[colour]] ) ) + geom_point(size = 2) + geom_line(aes(group = range), colour = \"black\", linewidth = 0.01 ) + scale_color_continuous(low = \"steelblue\", high = \"salmon\") + theme_scoringutils() + expand_limits(y = 0) + theme( legend.position = \"right\", axis.text.x = element_text( angle = 90, vjust = 1, hjust = 1 ) ) return(plot) } range_example <- copy(example_quantile) %>% na.omit() %>% .[, range := scoringutils:::get_range_from_quantile(quantile_level)] sum_scores <- range_example %>% as_forecast_quantile() %>% score(metrics = list(wis = wis, dispersion = dispersion_quantile)) %>% summarise_scores(by = c(\"model\", \"target_type\", \"range\")) %>% suppressWarnings() plot_interval_ranges(sum_scores, x = \"model\") + facet_wrap(~target_type, scales = \"free\") plot_interval_ranges(sum_scores, y = \"dispersion\", x = \"model\") + facet_wrap(~target_type, scales = \"free_y\")"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/Deprecated-visualisations.html","id":"function-plot_score_table","dir":"Articles","previous_headings":"","what":"Function plot_score_table()","title":"Deprecated Visualisations","text":"function allowed users turn table (summarised) scores coloured table. function hard-coded information colour scale pick metric. scoringutils 2.0.0, allowed users assign names metrics use custom scoring functions. stick default names provided scoringutils, function still work. However, functionality easily generalisable, decided deprecate function. main functionality old function provided, scaling scores order obtain reasonable colour shades. per metric, one also pass additional grouping variables argument. allowed users achieve faceting table (note course scores also needed summarised according grouping). function also allowed users combine different facets one, creating combined y-variable. done passing vector column names y argument.","code":"#' @title Plot Coloured Score Table #' #' @description #' Plots a coloured table of summarised scores obtained using #' [score()]. #' #' @param y the variable to be shown on the y-axis. Instead of a single character string, #' you can also specify a vector with column names, e.g. #' `y = c(\"model\", \"location\")`. These column names will be concatenated #' to create a unique row identifier (e.g. \"model1_location1\"). #' @param by A character vector that determines how the colour shading for the #' plot gets computed. By default (`NULL`), shading will be determined per #' metric, but you can provide additional column names (see examples). #' @param metrics A character vector with the metrics to show. If set to #' `NULL` (default), all metrics present in `scores` will be shown. #' #' @return A ggplot object with a coloured table of summarised scores #' @inheritParams get_pairwise_comparisons #' @importFrom ggplot2 ggplot aes element_blank element_text labs coord_cartesian coord_flip #' @importFrom data.table setDT melt #' @importFrom stats sd #' @export #' #' @examples #' library(ggplot2) #' library(magrittr) # pipe operator #' \\dontshow{ #' data.table::setDTthreads(2) # restricts number of cores used on CRAN #' } #' #' scores <- score(as_forecast_quantile(example_quantile)) %>% #' summarise_scores(by = c(\"model\", \"target_type\")) %>% #' summarise_scores(by = c(\"model\", \"target_type\"), fun = signif, digits = 2) #' #' plot_score_table(scores, y = \"model\", by = \"target_type\") + #' facet_wrap(~target_type, ncol = 1) #' #' # can also put target description on the y-axis #' plot_score_table(scores, #' y = c(\"model\", \"target_type\"), #' by = \"target_type\") plot_score_table <- function(scores, y = \"model\", by = NULL, metrics = NULL) { # identify metrics ----------------------------------------------------------- id_vars <- get_forecast_unit(scores) metrics <- get_metrics(scores) cols_to_delete <- names(scores)[!(names(scores) %in% c(metrics, id_vars))] suppressWarnings(scores[, eval(cols_to_delete) := NULL]) # compute scaled values ------------------------------------------------------ # scaling is done in order to colour the different scores # for most metrics larger is worse, but others like bias are better if they # are close to zero and deviations in both directions are bad # define which metrics are scaled using min (larger is worse) and # which not (metrics like bias where deviations in both directions are bad) metrics_zero_good <- c(\"bias\", \"interval_coverage_deviation\") metrics_no_color <- \"coverage\" metrics_min_good <- setdiff(metrics, c( metrics_zero_good, metrics_no_color )) # write scale functions that can be used in data.table scale <- function(x) { scaled <- x / sd(x, na.rm = TRUE) return(scaled) } scale_min_good <- function(x) { scaled <- (x - min(x)) / sd(x, na.rm = TRUE) return(scaled) } # pivot longer and add scaled values df <- data.table::melt(scores, value.vars = metrics, id.vars = id_vars, variable.name = \"metric\" ) df[metric %in% metrics_min_good, value_scaled := scale_min_good(value), by = c(\"metric\", by) ] df[metric %in% metrics_zero_good, value_scaled := scale(value), by = c(\"metric\", by) ] df[metric %in% metrics_no_color, value_scaled := 0, by = c(\"metric\", by) ] # create identifier column for plot ------------------------------------------ # if there is only one column, leave column as is. Reason to do that is that # users can then pass in a factor and keep the ordering of that column intact if (length(y) > 1) { df[, identifCol := do.call(paste, c(.SD, sep = \"_\")), .SDcols = y[y %in% names(df)]] } else { setnames(df, old = eval(y), new = \"identifCol\") } # plot ----------------------------------------------------------------------- # make plot with all metrics that are not NA plot <- ggplot( df[!is.na(value), ], aes(y = identifCol, x = metric) ) + geom_tile(aes(fill = value_scaled), colour = \"white\", show.legend = FALSE) + geom_text(aes(y = identifCol, label = value)) + scale_fill_gradient2(low = \"steelblue\", high = \"salmon\") + theme_scoringutils() + theme( legend.title = element_blank(), legend.position = \"none\", axis.text.x = element_text( angle = 90, vjust = 1, hjust = 1 ) ) + labs(x = \"\", y = \"\") + coord_cartesian(expand = FALSE) return(plot) } scores <- score(as_forecast_quantile(example_quantile)) %>% summarise_scores(by = c(\"model\", \"target_type\")) %>% summarise_scores(by = c(\"model\", \"target_type\"), fun = signif, digits = 2) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_score_table(scores, y = \"model\", by = \"target_type\") + facet_wrap(~target_type, ncol = 1) # can also put target description on the y-axis plot_score_table(scores, y = c(\"model\", \"target_type\"), by = \"target_type\")"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"introduction","dir":"Articles","previous_headings":"","what":"Introduction","title":"Scoring rules in `scoringutils`","text":"vignette gives overview default scoring rules made available scoringutils package. can, course, also use scoring rules, provided follow format. want obtain detailed information pacakge works, look revised version scoringutils paper. can distinguish two types forecasts: point forecasts probabilistic forecasts. point forecast single number representing single outcome. probabilistic forecast full predictive probability distribution multiple possible outcomes. contrast point forecasts, probabilistic forecasts incorporate uncertainty different possible outcomes. Scoring rules functions take forecast observation input return single numeric value. point forecasts, take form S(ŷ,y)S(\\hat{y}, y), ŷ\\hat{y} forecast yy observation. probabilistic forecasts, usually take form S(F,y)S(F, y), FF cumulative density function (CDF) predictive distribution yy observation. convention, scoring rules usually negatively oriented, meaning smaller values better (best possible score usually zero). sense, score can understood penalty. Many scoring rules probabilistic forecasts -called (strictly) proper scoring rules. Essentially, means “cheated”: forecaster evaluated strictly proper scoring rule always incentivised report honest best belief future , expectation, improve score reporting something else. formal definition following: Let GG true, unobserved data-generating distribution. scoring rule said proper, GG ideal forecast F=GF = G, forecast F′≠FF' \\neq F expectation receives better score FF. scoring rule considered strictly proper , GG, forecast F′F' expectation receives score better FF.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"metrics-for-point-forecasts","dir":"Articles","previous_headings":"","what":"Metrics for point forecasts","title":"Scoring rules in `scoringutils`","text":"See list default metrics point forecasts calling get_metrics(example_point). overview input output formats point forecasts: Input output formats: metrics point.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"a-note-of-caution","dir":"Articles","previous_headings":"Metrics for point forecasts","what":"A note of caution","title":"Scoring rules in `scoringutils`","text":"Scoring point forecasts can tricky business. Depending choice scoring rule, forecaster clearly worse another, might consistently receive better scores (see Gneiting (2011) illustrative example). Every scoring rule point forecast implicitly minimised specific aspect predictive distribution. mean squared error, example, meaningful scoring rule forecaster actually reported mean predictive distribution point forecast. forecaster reported median, mean absolute error appropriate scoring rule. scoring rule predictive task align, misleading results ensue. Consider following example:","code":"set.seed(123) n <- 1000 observed <- rnorm(n, 5, 4)^2 predicted_mu <- mean(observed) predicted_not_mu <- predicted_mu - rnorm(n, 10, 2) mean(Metrics::ae(observed, predicted_mu)) #> [1] 34.45981 mean(Metrics::ae(observed, predicted_not_mu)) #> [1] 32.54821 mean(Metrics::se(observed, predicted_mu)) #> [1] 2171.089 mean(Metrics::se(observed, predicted_not_mu)) #> [1] 2290.155"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"absolute-error","dir":"Articles","previous_headings":"Metrics for point forecasts","what":"Absolute error","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: ŷ\\hat{y}, real number, median forecaster’s predictive distribution. absolute error absolute difference predicted observed values. See ?Metrics::ae. ae=|y−ŷ|\\text{ae} = |y - \\hat{y}| absolute error appropriate rule ŷ\\hat{y} corresponds median forecaster’s predictive distribution. Otherwise, results misleading (see Gneiting (2011)).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"squared-error","dir":"Articles","previous_headings":"Metrics for point forecasts","what":"Squared error","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: ŷ\\hat{y}, real number, mean forecaster’s predictive distribution. squared error squared difference predicted observed values. See ?Metrics::se. se=(y−ŷ)2\\text{se} = (y - \\hat{y})^2 squared error appropriate rule ŷ\\hat{y} corresponds mean forecaster’s predictive distribution. Otherwise, results misleading (see Gneiting (2011)).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"absolute-percentage-error","dir":"Articles","previous_headings":"Metrics for point forecasts","what":"Absolute percentage error","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: ŷ\\hat{y}, real number absolute percentage error absolute percent difference predicted observed values. See ?Metrics::ape. ape=|y−ŷ||y|\\text{ape} = \\frac{|y - \\hat{y}|}{|y|} absolute percentage error appropriate rule ŷ\\hat{y} corresponds β\\beta-median forecaster’s predictive distribution β=−1\\beta = -1. β\\beta-median, med(β)(F)\\text{med}^{(\\beta)}(F), median random variable whose density proportional yβf(y)y^\\beta f(y). specific β\\beta-median corresponds absolute percentage error med(−1)(F)\\text{med}^{(-1)}(F). Otherwise, results misleading (see Gneiting (2011)).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"binary-forecasts","dir":"Articles","previous_headings":"","what":"Binary forecasts","title":"Scoring rules in `scoringutils`","text":"See list default metrics point forecasts calling ?get_metrics(example_binary). overview input output formats point forecasts: Input output formats: metrics binary forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"brier-score","dir":"Articles","previous_headings":"Binary forecasts","what":"Brier score","title":"Scoring rules in `scoringutils`","text":"Observation: yy, either 0 1 Forecast: pp, probability observed outcome 1. Brier score strictly proper scoring rule. computed mean squared error probabilistic prediction observed outcome. BS(p,y)=(p−y)2={p2,y=1(1−p)2,y=0\\begin{equation} \\text{BS}(p, y) = (p - y)^2 = \\begin{cases} p^2, & \\text{} y = 1\\\\ (1 - p)^2, & \\text{} y = 0 \\end{cases} \\end{equation} Brier score logarithmic score (see ) differ penalise - underconfidence (see Machete (2012)). Brier score penalises overconfidence underconfidence probability space . Consider following example: See ?brier_score() information.","code":"n <- 1e6 p_true <- 0.7 observed <- factor(rbinom(n = n, size = 1, prob = p_true), levels = c(0, 1)) p_over <- p_true + 0.15 p_under <- p_true - 0.15 abs(mean(brier_score(observed, p_true)) - mean(brier_score(observed, p_over))) #> [1] 0.0223866 abs(mean(brier_score(observed, p_true)) - mean(brier_score(observed, p_under))) #> [1] 0.0226134"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"logarithmic-score","dir":"Articles","previous_headings":"Binary forecasts","what":"Logarithmic score","title":"Scoring rules in `scoringutils`","text":"Observation: yy, either 0 1 Forecast: pp, probability observed outcome 1. logarithmic score (log score) strictly proper scoring rule. computed negative logarithm probability assigned observed outcome. Log score(p,y)=−log(1−|y−p|)={−log(p),y=1−log(1−p),y=0\\begin{equation} \\text{Log score}(p, y) = - \\log(1 - |y - p|) = \\begin{cases} -\\log (p), & \\text{} y = 1\\\\ -\\log (1 - p), & \\text{} y = 0 \\end{cases} \\end{equation} log score penalises overconfidence strongly underconfidence (probability space). Consider following example: See ?logs_binary() information.","code":"abs(mean(logs_binary(observed, p_true)) - mean(logs_binary(observed, p_over))) #> [1] 0.07169954 abs(mean(logs_binary(observed, p_true)) - mean(logs_binary(observed, p_under))) #> [1] 0.04741833"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"sample-based-forecasts","dir":"Articles","previous_headings":"","what":"Sample-based forecasts","title":"Scoring rules in `scoringutils`","text":"See list default metrics sample-based forecasts calling get_metrics(example_sample_continuous). overview input output formats quantile forecasts: Input output formats: metrics sample-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"crps","dir":"Articles","previous_headings":"Sample-based forecasts","what":"CRPS","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number (discrete number). Forecast: continuous (FF) discrete (PP) forecast. continuous ranked probability score (CRPS) popular fields meteorology epidemiology. CRPS defined CRPS(F,y)=∫−∞∞(F(x)−1(x≥y))2dx,\\text{CRPS}(F, y) = \\int_{-\\infty}^\\infty \\left( F(x) - 1(x \\geq y) \\right)^2 dx, yy observed value FF CDF predictive distribution. discrete forecasts, example count data, ranked probability score (RPS) can used instead commonly defined : RPS(P,y)=∑x=0∞(P(x)−1(x≥y))2, \\text{RPS}(P, y) = \\sum_{x = 0}^\\infty (P(x) - 1(x \\geq y))^2, PP cumulative probability mass function (PMF) predictive distribution. CRPS can understood generalisation absolute error predictive distributions (Gneiting Raftery 2007). can also understood integral Brier score binary probability forecasts implied CDF possible observed values. CRPS also related Cramér-distance two distributions equals special case one distributions concentrated single point (see e.g. Ziel (2021)). CRPS global scoring rule, meaning entire predictive distribution taken account determining quality forecast. scoringutils re-exports crps_sample() function scoringRules package, assumes forecast represented set samples predictive distribution. See ?crps_sample() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"overprediction-underprediction-and-dispersion","dir":"Articles","previous_headings":"Sample-based forecasts > CRPS","what":"Overprediction, underprediction and dispersion","title":"Scoring rules in `scoringutils`","text":"CRPS can interpreted sum dispersion, overprediction underprediction component. mm median forecast dispersion component CRPS(F,m),\\text{CRPS}(F, m), overprediction component {m>yCRPS(F,y)−CRPS(F,m)m≤y0 \\begin{cases} m > y & CRPS(F, y) - CRPS(F, m)\\\\ m \\leq y & 0\\\\ \\end{cases} underprediction component {m Weighted interval score (WIS)","what":"Overprediction, underprediction and dispersion","title":"Scoring rules in `scoringutils`","text":"individual components WIS. See ?overprediction_quantile(), ?underprediction_quantile() ?dispersion_quantile() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"bias-1","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Bias","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: FF. CDF predictive distribution represented set quantiles, QQ. Bias can measured B(F,y)={(1−2⋅max{α|qα∈Q∧qα≤y}),yq0.5(underprediction)0,y=q0.5,\\begin{equation} \\text{B}(F, y) = \\begin{cases} (1 - 2 \\cdot \\max \\{\\alpha | q_\\alpha \\Q \\land q_\\alpha \\leq y\\}), & \\text{} y < q_{0.5} \\quad \\text{(overprediction)}\\\\ (1 - 2 \\cdot \\min \\{\\alpha | q_\\alpha \\Q_t \\land q_\\alpha \\geq y\\}, & \\text{} y > q_{0.5} \\quad \\text{(underprediction)}\\\\ 0, & \\text{} y = q_{0.5}, \\\\ \\end{cases} \\end{equation} qαq_\\alpha α\\alpha-quantile predictive distribution. consistency, define QQ (set quantiles form predictive distribution FF) always includes element q0=−∞q_0 = -\\infty q1=∞q_1 = \\infty. clearer terms, bias : 1−(2×1 - (2 \\times maximum percentile rank corresponding quantile still observed value), observed value smaller median predictive distribution. 1−(2×1 - (2 \\times minimum percentile rank corresponding quantile still larger observed value) observed value larger median predictive distribution.. 00if observed value exactly median. Bias can assume values -1 (underprediction) 1 (overpredictin) 0 ideally (.e. unbiased). increasing number quantiles, percentile rank equal proportion predictive samples observed value, bias metric coincides one continuous forecasts (see ). See ?bias_quantile() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"interval-coverage","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Interval coverage","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: FF. CDF predictive distribution represented set quantiles. quantiles form central prediction intervals. Interval coverage given interval range defined proportion observations fall within corresponding central prediction intervals. Central prediction intervals symmetric around median formed two quantiles denote lower upper bound. example, 50% central prediction interval interval 0.25 0.75 quantiles predictive distribution.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"interval-coverage-deviation","dir":"Articles","previous_headings":"Quantile-based forecasts > Interval coverage","what":"Interval coverage deviation","title":"Scoring rules in `scoringutils`","text":"interval coverage deviation difference observed interval coverage nominal interval coverage. example, observed interval coverage 50% central prediction interval 0.6, interval coverage deviation 0.6=−0.5=0.1.0.6 = - 0.5 = 0.1. interval coverage deviation=observed interval coverage−nominal interval coverage\\text{interval coverage deviation} = \\text{observed interval coverage} - \\text{nominal interval coverage}","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"absolute-error-of-the-median-1","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Absolute error of the median","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: FF. CDF predictive distribution represented set quantiles. absolute error median absolute difference median predictive distribution observed value. aemedian=|median(F)−y|\\text{ae}_\\text{median} = |\\text{median}(F) - y| See section note caution Gneiting (2011) discussion correspondence absolute error median.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"quantile-score","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Quantile score","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: FF. CDF predictive distribution represented set quantiles. quantile score, also called pinball loss, single quantile level τ\\tau defined QSτ(F,y)=2⋅{𝟏(y≤qτ)−τ}⋅(qτ−y)={2⋅(1−τ)*qτ−y,y≤qτ2⋅τ*|qτ−y|,y>qτ,\\begin{equation} \\text{QS}_\\tau(F, y) = 2 \\cdot \\{ \\mathbf{1}(y \\leq q_\\tau) - \\tau\\} \\cdot (q_\\tau - y) = \\begin{cases} 2 \\cdot (1 - \\tau) * q_\\tau - y, & \\text{} y \\leq q_\\tau\\\\ 2 \\cdot \\tau * |q_\\tau - y|, & \\text{} y > q_\\tau, \\end{cases} \\end{equation} qτq_\\tau τ\\tau-quantile predictive distribution FF, 𝟏(⋅)\\mathbf{1}(\\cdot) indicator function. (unweighted) interval score (see ) 1−α1 - \\alpha prediction interval can computed quantile scores levels α/2\\alpha/2 1−α/21 - \\alpha/2 ISα(F,y)=QSα/2(F,y)+QS1−α/2(F,y)α\\text{}_\\alpha(F, y) = \\frac{\\text{QS}_{\\alpha/2}(F, y) + \\text{QS}_{1 - \\alpha/2}(F, y)}{\\alpha}. weighted interval score can obtained simple average quantile scores: WISα(F,y)=QSα/2(F,y)+QS1−α/2(F,y)2\\text{WIS}_\\alpha(F, y) = \\frac{\\text{QS}_{\\alpha/2}(F, y) + \\text{QS}_{1 - \\alpha/2}(F, y)}{2}. See ?quantile_score Bracher et al. (2021) details.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"quantile-coverage","dir":"Articles","previous_headings":"Additional metrics","what":"Quantile coverage","title":"Scoring rules in `scoringutils`","text":"Quantile coverage given quantile level defined proportion observed values smaller corresponding predictive quantiles. example, 0.5 quantile coverage proportion observed values smaller 0.5-quantiles predictive distribution.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/authors.html","id":null,"dir":"","previous_headings":"","what":"Authors","title":"Authors and Citation","text":"Nikos Bosse. Author, maintainer. Sam Abbott. Author. Hugo Gruson. Author. Johannes Bracher. Contributor. Toshiaki Asakura. Contributor. James Mba Azam. Contributor. Sebastian Funk. Author. Michael Chirico. Contributor.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/authors.html","id":"citation","dir":"","previous_headings":"","what":"Citation","title":"Authors and Citation","text":"Nikos . Bosse, Hugo Gruson, Sebastian Funk, Anne Cori, Edwin van Leeuwen, Sam Abbott (2022). Evaluating Forecasts scoringutils R, arXiv. DOI: 10.48550/ARXIV.2205.07090 Alexander Jordan, Fabian Krueger, Sebastian Lerch (2019). Evaluating Probabilistic Forecasts scoringRules. Journal Statistical Software, 90(12), 1-37. DOI 10.18637/jss.v090.i12","code":"@Article{, title = {Evaluating Forecasts with scoringutils in R}, author = {Nikos I. Bosse and Hugo Gruson and Anne Cori and Edwin {van Leeuwen} and Sebastian Funk and Sam Abbott}, journal = {arXiv}, url = {https://arxiv.org/abs/2205.07090}, year = {2022}, doi = {10.48550/ARXIV.2205.07090}, } @Article{, title = {Evaluating Probabilistic Forecasts with {scoringRules}}, author = {Alexander Jordan and Fabian Kr\\\"uger and Sebastian Lerch}, journal = {Journal of Statistical Software}, year = {2019}, volume = {90}, number = {12}, pages = {1--37}, doi = {10.18637/jss.v090.i12}, }"},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"scoringutils-utilities-for-scoring-and-assessing-predictions","dir":"","previous_headings":"","what":"Utilities for Scoring and Assessing Predictions","title":"Utilities for Scoring and Assessing Predictions","text":"Note: documentation refers development version scoringutils. can also view documentation stable version. scoringutils package facilitates process evaluating forecasts R, using convenient flexible data.table-based framework. provides broad functionality check input data diagnose issues, visualise forecasts missing data, transform data scoring, handle missing forecasts, aggregate scores, visualise results evaluation. package easily extendable, meaning users can supply scoring rules extend existing classes handle new types forecasts. package underwent major re-write. comprehensive documentation updated package revised version original scoringutils paper. Another good starting point vignettes Getting started, Details metrics implemented Scoring forecasts directly. details specific issue transforming forecasts scoring see: Nikos . Bosse, Sam Abbott, Anne Cori, Edwin van Leeuwen, Johannes Bracher* Sebastian Funk* (*: equal contribution) (2023). Scoring epidemiological forecasts transformed scales, PLoS Comput Biol 19(8): e1011393 https://doi.org/10.1371/journal.pcbi.1011393","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"installation","dir":"","previous_headings":"","what":"Installation","title":"Utilities for Scoring and Assessing Predictions","text":"Install CRAN version package using Install unstable development version GitHub using","code":"install.packages(\"scoringutils\") remotes::install_github(\"epiforecasts/scoringutils\", dependencies = TRUE)"},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"forecast-types","dir":"","previous_headings":"Quick start","what":"Forecast types","title":"Utilities for Scoring and Assessing Predictions","text":"scoringutils currently supports scoring following forecast types: - binary: probability binary (yes/) outcome variable. - point: forecast continuous discrete outcome variable represented single number. - quantile: probabilistic forecast continuous discrete outcome variable, forecast distribution represented set predictive quantiles. - sample: probabilistic forecast continuous discrete outcome variable, forecast represented finite set samples drawn predictive distribution.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"input-formats-and-input-validation","dir":"","previous_headings":"Quick start","what":"Input formats and input validation","title":"Utilities for Scoring and Assessing Predictions","text":"expected input format generally data.frame (similar) required columns observed, predicted, model holds forecasts observed values. Exact requirements depend forecast type. information, look paper, call ?as_forecast(), look example data provided package (example_binary, example_point, example_quantile, example_sample_continuous, example_sample_discrete). scoring, input data needs validated transformed forecast object using function as_forecast().","code":"forecast_quantile <- example_quantile |> as_forecast_quantile( forecast_unit = c( \"location\", \"forecast_date\", \"target_end_date\", \"target_type\", \"model\", \"horizon\" ) ) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. print(forecast_quantile, 2) #> Forecast type: quantile #> Forecast unit: #> location, forecast_date, target_end_date, target_type, model, and horizon #> #> Key: #> observed quantile_level predicted location forecast_date target_end_date #> #> 1: 127300 NA NA DE 2021-01-02 #> 2: 4534 NA NA DE 2021-01-02 #> --- #> 20544: 78 0.975 611 IT 2021-07-12 2021-07-24 #> 20545: 78 0.990 719 IT 2021-07-12 2021-07-24 #> target_type model horizon #> #> 1: Cases NA #> 2: Deaths NA #> --- #> 20544: Deaths epiforecasts-EpiNow2 2 #> 20545: Deaths epiforecasts-EpiNow2 2"},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"the-forecast-unit","dir":"","previous_headings":"Quick start","what":"The forecast unit","title":"Utilities for Scoring and Assessing Predictions","text":"quantile-based sample-based forecasts, single prediction represented set several quantiles (samples) predictive distribution, .e. several rows input data. scoringutils therefore needs group rows together form single forecast. scoringutils uses existing columns input data achieve - values columns uniquely identify single forecast. Additional columns unrelated forecast unit can mess . forecast_unit argument as_forecast_() makes sure columns retained relevant defining unit single forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"scoring-forecasts","dir":"","previous_headings":"Quick start","what":"Scoring forecasts","title":"Utilities for Scoring and Assessing Predictions","text":"Forecasts can scored calling score() validated forecast object. score() takes additional argument, metrics, list scoring rules. Every forecast type default list metrics. can easily add scoring functions, long conform format forecast type. See paper information. can summarise scores using function summarise_scores(). argument used specify desired level summary. fun let’s specify summary function, although recommended stick mean primary summary function, functions can lead improper scores.","code":"scores <- forecast_quantile |> score() scores |> summarise_scores(by = c(\"model\", \"target_type\")) |> summarise_scores(by = c(\"model\", \"target_type\"), fun = signif, digits = 3) #> model target_type wis overprediction underprediction #> #> 1: EuroCOVIDhub-ensemble Cases 17900.0 10000.00 4240.0 #> 2: EuroCOVIDhub-baseline Cases 28500.0 14100.00 10300.0 #> 3: epiforecasts-EpiNow2 Cases 20800.0 11900.00 3260.0 #> 4: EuroCOVIDhub-ensemble Deaths 41.4 7.14 4.1 #> 5: EuroCOVIDhub-baseline Deaths 159.0 65.90 2.1 #> 6: UMass-MechBayes Deaths 52.7 8.98 16.8 #> 7: epiforecasts-EpiNow2 Deaths 66.6 18.90 15.9 #> dispersion bias interval_coverage_50 interval_coverage_90 ae_median #> #> 1: 3660.0 -0.05640 0.391 0.805 24100.0 #> 2: 4100.0 0.09800 0.328 0.820 38500.0 #> 3: 5660.0 -0.07890 0.469 0.789 27900.0 #> 4: 30.2 0.07270 0.875 1.000 53.1 #> 5: 91.4 0.33900 0.664 1.000 233.0 #> 6: 26.9 -0.02230 0.461 0.875 78.5 #> 7: 31.9 -0.00513 0.420 0.908 105.0"},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"package-workflow","dir":"","previous_headings":"","what":"Package workflow","title":"Utilities for Scoring and Assessing Predictions","text":"following depicts suggested workflow evaluating forecasts scoringutils (sections refer paper). Please find information paper, function documentation vignettes.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"citation","dir":"","previous_headings":"","what":"Citation","title":"Utilities for Scoring and Assessing Predictions","text":"using scoringutils work please consider citing using output citation(\"scoringutils\") (print(citation(\"scoringutils\"), bibtex = TRUE)):","code":"#> To cite scoringutils in publications use the following. If you use the #> CRPS, DSS, or Log Score, please also cite scoringRules. #> #> Nikos I. Bosse, Hugo Gruson, Sebastian Funk, Anne Cori, Edwin van #> Leeuwen, and Sam Abbott (2022). Evaluating Forecasts with #> scoringutils in R, arXiv. DOI: 10.48550/ARXIV.2205.07090 #> #> To cite scoringRules in publications use: #> #> Alexander Jordan, Fabian Krueger, Sebastian Lerch (2019). Evaluating #> Probabilistic Forecasts with scoringRules. Journal of Statistical #> Software, 90(12), 1-37. DOI 10.18637/jss.v090.i12 #> #> To see these entries in BibTeX format, use 'print(, #> bibtex=TRUE)', 'toBibtex(.)', or set #> 'options(citation.bibtex.max=999)'."},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"how-to-make-a-bug-report-or-feature-request","dir":"","previous_headings":"","what":"How to make a bug report or feature request","title":"Utilities for Scoring and Assessing Predictions","text":"Please briefly describe problem output expect issue. question, please don’t open issue. Instead, ask Q page.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"contributing","dir":"","previous_headings":"","what":"Contributing","title":"Utilities for Scoring and Assessing Predictions","text":"welcome contributions new contributors! particularly appreciate help priority problems issues. Please check add issues, /add pull request.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"code-of-conduct","dir":"","previous_headings":"","what":"Code of Conduct","title":"Utilities for Scoring and Assessing Predictions","text":"Please note scoringutils project released Contributor Code Conduct. contributing project, agree abide terms.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"funding","dir":"","previous_headings":"","what":"Funding","title":"Utilities for Scoring and Assessing Predictions","text":"development scoringutils funded via Health Protection Research Unit (grant code NIHR200908) Wellcome Trust (grant: 210758/Z/18/Z). work also supported US National Institutes General Medical Sciences (R35GM119582). content solely responsibility authors necessarily represent official views NIGMS, National Institutes Health.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"contributors","dir":"","previous_headings":"","what":"Contributors","title":"Utilities for Scoring and Assessing Predictions","text":"contributions project gratefully acknowledged using allcontributors package following -contributors specification. Contributions kind welcome!","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"code","dir":"","previous_headings":"Contributors","what":"Code","title":"Utilities for Scoring and Assessing Predictions","text":"nikosbosse, seabbs, jamesmbaazam, Bisaloo, sbfnk, actions-user, toshiakiasakura, MichaelChirico, nickreich, jhellewell14, damonbayer","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"issue-authors","dir":"","previous_headings":"Contributors","what":"Issue Authors","title":"Utilities for Scoring and Assessing Predictions","text":"DavideMagno, mbojan, dshemetov, elray1","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"issue-contributors","dir":"","previous_headings":"Contributors","what":"Issue Contributors","title":"Utilities for Scoring and Assessing Predictions","text":"jbracher, dylanhmorris, jonathonmellor, kathsherratt","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/add_relative_skill.html","id":null,"dir":"Reference","previous_headings":"","what":"Add relative skill scores based on pairwise comparisons — add_relative_skill","title":"Add relative skill scores based on pairwise comparisons — add_relative_skill","text":"Adds columns relative skills computed running pairwise comparisons scores. information computation relative skill, see get_pairwise_comparisons(). Relative skill calculated aggregation level specified .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/add_relative_skill.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Add relative skill scores based on pairwise comparisons — add_relative_skill","text":"","code":"add_relative_skill( scores, compare = \"model\", by = NULL, metric = intersect(c(\"wis\", \"crps\", \"brier_score\"), names(scores)), baseline = NULL )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/add_relative_skill.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Add relative skill scores based on pairwise comparisons — add_relative_skill","text":"scores object class scores (data.table scores additional attribute metrics produced score()). compare Character vector single colum name defines elements pairwise comparison. example, set \"model\" (default), elements \"model\" column compared. Character vector column names define grouping levels pairwise comparisons. default NULL one relative skill score per distinct entry column selected compare. columns given , example, = \"location\" compare = \"model\", one separate relative skill score calculated every model every location. metric string name metric relative skill shall computed. default either \"crps\", \"wis\" \"brier_score\" available. baseline string name model. baseline given, scaled relative skill respect baseline returned. default (NULL), relative skill scaled respect baseline model.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Absolute error of the median (quantile-based version) — ae_median_quantile","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"Compute absolute error median calculated $$ \\textrm{abs}(\\textrm{observed} - \\textrm{median prediction}) $$ median prediction predicted value quantile_level == 0.5, function therefore requires 0.5 among quantile levels quantile_level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"","code":"ae_median_quantile(observed, predicted, quantile_level)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"Numeric vector length N absolute error median.","code":""},{"path":[]},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"","code":"observed <- rnorm(30, mean = 1:30) predicted_values <- replicate(3, rnorm(30, mean = 1:30)) ae_median_quantile( observed, predicted_values, quantile_level = c(0.2, 0.5, 0.8) ) #> [1] 2.47438940 0.92040530 3.55121603 0.24032512 1.79911603 2.12426222 #> [7] 2.88687498 0.37899594 0.73282842 1.41674512 0.91703692 0.34483170 #> [13] 0.72770448 1.86768569 0.80586643 2.38692128 1.12876056 0.05733376 #> [19] 0.37081463 0.82374754 1.45618892 0.93544150 2.05333481 0.18155199 #> [25] 2.43676219 1.20798000 1.67648698 0.13974346 1.26067874 1.13044854"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Absolute error of the median (sample-based version) — ae_median_sample","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"Absolute error median calculated $$% \\textrm{abs}(\\textrm{observevd} - \\textrm{median\\_prediction}) $$","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"","code":"ae_median_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"vector scoring values","code":""},{"path":[]},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"","code":"observed <- rnorm(30, mean = 1:30) predicted_values <- matrix(rnorm(30, mean = 1:30)) ae_median_sample(observed, predicted_values) #> [1] 1.61022189 0.32735036 2.52982645 0.98458168 0.94495454 0.65538891 #> [7] 0.56511146 0.09373061 1.31110818 0.61226219 0.75386115 0.08959962 #> [13] 0.39077113 1.56818369 0.84567980 1.24260044 0.27781917 0.65054779 #> [19] 1.18084954 0.45036469 0.05976767 0.14675942 0.60583332 0.19442459 #> [25] 0.21123533 0.28585022 0.64582375 1.78993469 1.20347916 0.67902801"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/apply_metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Apply a list of functions to a data table of forecasts — apply_metrics","title":"Apply a list of functions to a data table of forecasts — apply_metrics","text":"helper function applies scoring rules (stored list functions) data table forecasts. apply_metrics used within score() apply scoring rules data. Scoring rules wrapped run_safely() catch errors make sure arguments passed scoring rule actually accepted .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/apply_metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Apply a list of functions to a data table of forecasts — apply_metrics","text":"","code":"apply_metrics(forecast, metrics, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/apply_metrics.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Apply a list of functions to a data table of forecasts — apply_metrics","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()). metrics named list scoring functions. Names used column names output. See get_metrics() information default metrics used. See Customising metrics section information pass custom arguments scoring functions. ... Additional arguments passed scoring rules. Note currently used, calls apply_scores currently avoid passing arguments via ... instead expect metrics directly modified using purrr::partial().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/apply_metrics.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Apply a list of functions to a data table of forecasts — apply_metrics","text":"data table forecasts calculated metrics.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"General information on creating a forecast object — as_forecast","title":"General information on creating a forecast object — as_forecast","text":"several as_forecast_() functions process validate data.frame (similar) similar forecasts observations. input passes input checks, functions converted forecast object. forecast object data.table class forecast additional class depends forecast type. Every forecast type as_forecast_() function. See details section information expected input formats. as_forecast_() functions give users control data parsed. Using arguments observed, predicted, etc. users can rename existing columns input data match required columns forecast object. Using argument forecast_unit, users can specify columns uniquely identify single forecast (remove others, see docs internal set_forecast_unit() details). following functions available: as_forecast_point() as_forecast_binary() as_forecast_sample() as_forecast_quantile()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"General information on creating a forecast object — as_forecast","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. observed (optional) Name column data contains observed values. column renamed \"observed\". predicted (optional) Name column data contains predicted values. column renamed \"predicted\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"General information on creating a forecast object — as_forecast","text":"Depending forecast type, object following class returned: forecast_binary binary forecasts forecast_point point forecasts forecast_sample sample-based forecasts forecast_quantile quantile-based forecasts","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"forecast-types-and-input-formats","dir":"Reference","previous_headings":"","what":"Forecast types and input formats","title":"General information on creating a forecast object — as_forecast","text":"Various different forecast types / forecast formats supported. moment, : point forecasts binary forecasts (\"soft binary classification\") nominal forecasts (\"soft classification multiple unordered classes\") Probabilistic forecasts quantile-based format (forecast represented set predictive quantiles) Probabilistic forecasts sample-based format (forecast represented set predictive samples) Forecast types determined based columns present input data. overview required format forecast type: forecast types require data.frame similar columns observed predicted, model. Point forecasts require column observed type numeric column predicted type numeric. Binary forecasts require column observed type factor exactly two levels column predicted type numeric probabilities, corresponding probability observed equal second factor level. See details information. Nominal forecasts require column observed type factor N levels, (N number possible outcomes), column predicted type numeric probabilities (sum one across possible outcomes), column predicted_label type factor N levels, denoting outcome probability given. Forecasts must complete, .e. must probability assigned every possible outcome. Quantile-based forecasts require column observed type numeric, column predicted type numeric, column quantile_level type numeric quantile-levels (0 1). Sample-based forecasts require column observed type numeric, column predicted type numeric, column sample_id type numeric sample indices. information see vignettes example data (example_quantile, example_sample_continuous, example_sample_discrete, example_point(), example_binary, example_nominal).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"forecast-unit","dir":"Reference","previous_headings":"","what":"Forecast unit","title":"General information on creating a forecast object — as_forecast","text":"order score forecasts, scoringutils needs know rows data belong together jointly form single forecasts. easy e.g. point forecast, one row per forecast. quantile sample-based forecasts, however, multiple rows belong single forecast. forecast unit unit single forecast described combination columns uniquely identify single forecast. example, forecasts made different models various locations different time points, several weeks future. forecast unit described forecast_unit = c(\"model\", \"location\", \"forecast_date\", \"forecast_horizon\"). scoringutils automatically tries determine unit single forecast. uses existing columns , means columns must present unrelated forecast unit. simplistic example, additional row, \"even\", one row number even zero otherwise, mess scoring scoringutils thinks column relevant defining forecast unit. order avoid issues, recommend setting forecast unit explicitly, usually forecast_unit argument as_forecast() functions. drop unneeded columns, making sure necessary, 'protected columns' like \"predicted\" \"observed\" retained.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"General information on creating a forecast object — as_forecast","text":"","code":"as_forecast_binary(example_binary) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Forecast type: binary #> Forecast unit: #> location, location_name, target_end_date, target_type, forecast_date, model, #> and horizon #> #> location location_name target_end_date target_type forecast_date #> #> 1: DE Germany 2021-01-02 Cases #> 2: DE Germany 2021-01-02 Deaths #> 3: DE Germany 2021-01-09 Cases #> 4: DE Germany 2021-01-09 Deaths #> 5: DE Germany 2021-01-16 Cases #> --- #> 1027: IT Italy 2021-07-24 Deaths 2021-07-12 #> 1028: IT Italy 2021-07-24 Deaths 2021-07-05 #> 1029: IT Italy 2021-07-24 Deaths 2021-07-12 #> 1030: IT Italy 2021-07-24 Deaths 2021-07-05 #> 1031: IT Italy 2021-07-24 Deaths 2021-07-12 #> model horizon predicted observed #> #> 1: NA NA #> 2: NA NA #> 3: NA NA #> 4: NA NA #> 5: NA NA #> --- #> 1027: EuroCOVIDhub-baseline 2 0.250 0 #> 1028: UMass-MechBayes 3 0.475 0 #> 1029: UMass-MechBayes 2 0.450 0 #> 1030: epiforecasts-EpiNow2 3 0.375 0 #> 1031: epiforecasts-EpiNow2 2 0.300 0 as_forecast_quantile( example_quantile, forecast_unit = c(\"model\", \"target_type\", \"target_end_date\", \"horizon\", \"location\") ) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Forecast type: quantile #> Forecast unit: #> model, target_type, target_end_date, horizon, and location #> #> Key: #> observed quantile_level predicted model target_type #> #> 1: 127300 NA NA Cases #> 2: 4534 NA NA Deaths #> 3: 154922 NA NA Cases #> 4: 6117 NA NA Deaths #> 5: 110183 NA NA Cases #> --- #> 20541: 78 0.850 352 epiforecasts-EpiNow2 Deaths #> 20542: 78 0.900 397 epiforecasts-EpiNow2 Deaths #> 20543: 78 0.950 499 epiforecasts-EpiNow2 Deaths #> 20544: 78 0.975 611 epiforecasts-EpiNow2 Deaths #> 20545: 78 0.990 719 epiforecasts-EpiNow2 Deaths #> target_end_date horizon location #> #> 1: 2021-01-02 NA DE #> 2: 2021-01-02 NA DE #> 3: 2021-01-09 NA DE #> 4: 2021-01-09 NA DE #> 5: 2021-01-16 NA DE #> --- #> 20541: 2021-07-24 2 IT #> 20542: 2021-07-24 2 IT #> 20543: 2021-07-24 2 IT #> 20544: 2021-07-24 2 IT #> 20545: 2021-07-24 2 IT"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a forecast object for binary forecasts — as_forecast_binary","title":"Create a forecast object for binary forecasts — as_forecast_binary","text":"Create forecast object binary forecasts. See information forecast types expected input formats calling ?as_forecast().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a forecast object for binary forecasts — as_forecast_binary","text":"","code":"as_forecast_binary( data, forecast_unit = NULL, observed = NULL, predicted = NULL )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a forecast object for binary forecasts — as_forecast_binary","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. observed (optional) Name column data contains observed values. column renamed \"observed\". predicted (optional) Name column data contains predicted values. column renamed \"predicted\".","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_generic.html","id":null,"dir":"Reference","previous_headings":"","what":"Common functionality for as_forecast_ functions — as_forecast_generic","title":"Common functionality for as_forecast_ functions — as_forecast_generic","text":"Common functionality as_forecast_ functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_generic.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Common functionality for as_forecast_ functions — as_forecast_generic","text":"","code":"as_forecast_generic( data, forecast_unit = NULL, observed = NULL, predicted = NULL )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_generic.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Common functionality for as_forecast_ functions — as_forecast_generic","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. observed (optional) Name column data contains observed values. column renamed \"observed\". predicted (optional) Name column data contains predicted values. column renamed \"predicted\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_generic.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Common functionality for as_forecast_ functions — as_forecast_generic","text":"function splits part functionality as_forecast_ as_forecast_ functions. renames required columns, appropriate, sets forecast unit.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_nominal.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a forecast object for nominal forecasts — as_forecast_nominal","title":"Create a forecast object for nominal forecasts — as_forecast_nominal","text":"Nominal forecasts form categorical forecasts possible outcomes observed values can assume ordered. sense, Nominal forecasts represent generalisation binary forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_nominal.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a forecast object for nominal forecasts — as_forecast_nominal","text":"","code":"as_forecast_nominal( data, forecast_unit = NULL, observed = NULL, predicted = NULL, predicted_label = NULL )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_nominal.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a forecast object for nominal forecasts — as_forecast_nominal","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. observed (optional) Name column data contains observed values. column renamed \"observed\". predicted (optional) Name column data contains predicted values. column renamed \"predicted\". predicted_label (optional) Name column data denotes outcome predicted probability corresponds . column renamed \"predicted_label\". applicable nominal forecasts.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a forecast object for point forecasts — as_forecast_point","title":"Create a forecast object for point forecasts — as_forecast_point","text":"Create forecast object point forecasts. See information forecast types expected input formats calling ?as_forecast(). converting forecast_quantile object forecast_point object, 0.5 quantile extracted returned point forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a forecast object for point forecasts — as_forecast_point","text":"","code":"as_forecast_point(data, ...) # Default S3 method as_forecast_point( data, forecast_unit = NULL, observed = NULL, predicted = NULL, ... ) # S3 method for class 'forecast_quantile' as_forecast_point(data, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a forecast object for point forecasts — as_forecast_point","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. ... Unused forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. observed (optional) Name column data contains observed values. column renamed \"observed\". predicted (optional) Name column data contains predicted values. column renamed \"predicted\".","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a forecast object for quantile-based forecasts — as_forecast_quantile","title":"Create a forecast object for quantile-based forecasts — as_forecast_quantile","text":"Create forecast object quantile-based forecasts. See information forecast types expected input formats calling ?as_forecast(). creating forecast_quantile object forecast_sample object, quantiles estimated computing empircal quantiles samples via quantile(). Note empirical quantiles biased estimator true quantiles particular tails distribution number available samples low.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a forecast object for quantile-based forecasts — as_forecast_quantile","text":"","code":"as_forecast_quantile(data, ...) # Default S3 method as_forecast_quantile( data, forecast_unit = NULL, observed = NULL, predicted = NULL, quantile_level = NULL, ... ) # S3 method for class 'forecast_sample' as_forecast_quantile( data, probs = c(0.05, 0.25, 0.5, 0.75, 0.95), type = 7, ... )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a forecast object for quantile-based forecasts — as_forecast_quantile","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. ... Unused forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. observed (optional) Name column data contains observed values. column renamed \"observed\". predicted (optional) Name column data contains predicted values. column renamed \"predicted\". quantile_level (optional) Name column data contains quantile level predicted values. column renamed \"quantile_level\". applicable quantile-based forecasts. probs numeric vector quantile levels quantiles computed. Corresponds probs argument quantile(). type Type argument passed quantile function. information, see quantile().","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a forecast object for sample-based forecasts — as_forecast_sample","title":"Create a forecast object for sample-based forecasts — as_forecast_sample","text":"Create forecast object sample-based forecasts","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a forecast object for sample-based forecasts — as_forecast_sample","text":"","code":"as_forecast_sample( data, forecast_unit = NULL, observed = NULL, predicted = NULL, sample_id = NULL )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a forecast object for sample-based forecasts — as_forecast_sample","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. observed (optional) Name column data contains observed values. column renamed \"observed\". predicted (optional) Name column data contains predicted values. column renamed \"predicted\". sample_id (optional) Name column data contains sample id. column renamed \"sample_id\". applicable sample-based forecasts.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Create an object of class scores from data — as_scores","title":"Create an object of class scores from data — as_scores","text":"convenience function wraps new_scores() validates scores object.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create an object of class scores from data — as_scores","text":"","code":"as_scores(scores, metrics)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create an object of class scores from data — as_scores","text":"scores data.table similar scores produced score(). metrics character vector names scores (.e. names scoring rules used scoring).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_scores.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Create an object of class scores from data — as_scores","text":"object class scores","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_dims_ok_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","title":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","text":"Function assesses whether input dimensions match. following, n number observations / forecasts. Scalar values may repeated match length input. Allowed options therefore: observed vector length 1 length n predicted : vector length 1 length n matrix n rows 1 column","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_dims_ok_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","text":"","code":"assert_dims_ok_point(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_dims_ok_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","text":"observed Input checked. factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_dims_ok_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that input is a forecast object and passes validations — assert_forecast.forecast_binary","title":"Assert that input is a forecast object and passes validations — assert_forecast.forecast_binary","text":"Assert object forecast object (.e. data.table class forecast additional class forecast_* corresponding forecast type).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that input is a forecast object and passes validations — assert_forecast.forecast_binary","text":"","code":"# S3 method for class 'forecast_binary' assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) # S3 method for class 'forecast_point' assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) # S3 method for class 'forecast_quantile' assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) # S3 method for class 'forecast_sample' assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) # Default S3 method assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that input is a forecast object and passes validations — assert_forecast.forecast_binary","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()). forecast_type (optional) forecast type expect forecasts . forecast type determined scoringutils based input match , error thrown. NULL (default), forecast type inferred data. verbose Logical. FALSE (default TRUE), messages warnings created. ... Currently unused. pass additional arguments scoring functions via .... See Customising metrics section details use purrr::partial() pass arguments individual metrics.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that input is a forecast object and passes validations — assert_forecast.forecast_binary","text":"Returns NULL invisibly.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"forecast-types-and-input-formats","dir":"Reference","previous_headings":"","what":"Forecast types and input formats","title":"Assert that input is a forecast object and passes validations — assert_forecast.forecast_binary","text":"Various different forecast types / forecast formats supported. moment, : point forecasts binary forecasts (\"soft binary classification\") nominal forecasts (\"soft classification multiple unordered classes\") Probabilistic forecasts quantile-based format (forecast represented set predictive quantiles) Probabilistic forecasts sample-based format (forecast represented set predictive samples) Forecast types determined based columns present input data. overview required format forecast type: forecast types require data.frame similar columns observed predicted, model. Point forecasts require column observed type numeric column predicted type numeric. Binary forecasts require column observed type factor exactly two levels column predicted type numeric probabilities, corresponding probability observed equal second factor level. See details information. Nominal forecasts require column observed type factor N levels, (N number possible outcomes), column predicted type numeric probabilities (sum one across possible outcomes), column predicted_label type factor N levels, denoting outcome probability given. Forecasts must complete, .e. must probability assigned every possible outcome. Quantile-based forecasts require column observed type numeric, column predicted type numeric, column quantile_level type numeric quantile-levels (0 1). Sample-based forecasts require column observed type numeric, column predicted type numeric, column sample_id type numeric sample indices. information see vignettes example data (example_quantile, example_sample_continuous, example_sample_discrete, example_point(), example_binary, example_nominal).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Assert that input is a forecast object and passes validations — assert_forecast.forecast_binary","text":"","code":"forecast <- as_forecast_binary(example_binary) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. assert_forecast(forecast) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected."},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_generic.html","id":null,"dir":"Reference","previous_headings":"","what":"Validation common to all forecast types — assert_forecast_generic","title":"Validation common to all forecast types — assert_forecast_generic","text":"function runs input checks apply input data, regardless forecast type. function asserts forecast data.table columns observed predicted checks forecast type forecast unit checks duplicate forecasts appropriate, checks number samples / quantiles forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_generic.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Validation common to all forecast types — assert_forecast_generic","text":"","code":"assert_forecast_generic(data, verbose = TRUE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_generic.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Validation common to all forecast types — assert_forecast_generic","text":"data data.table forecasts observed values validated. verbose Logical. FALSE (default TRUE), messages warnings created.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_generic.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Validation common to all forecast types — assert_forecast_generic","text":"returns input","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_type.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that forecast type is as expected — assert_forecast_type","title":"Assert that forecast type is as expected — assert_forecast_type","text":"Assert forecast type expected","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_type.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that forecast type is as expected — assert_forecast_type","text":"","code":"assert_forecast_type(data, actual = get_forecast_type(data), desired = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_type.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that forecast type is as expected — assert_forecast_type","text":"data forecast object (see as_forecast()). actual actual forecast type data desired desired forecast type data","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_type.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that forecast type is as expected — assert_forecast_type","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for binary forecast — assert_input_binary","title":"Assert that inputs are correct for binary forecast — assert_input_binary","text":"Function assesses whether inputs correspond requirements scoring binary forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for binary forecast — assert_input_binary","text":"","code":"assert_input_binary(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for binary forecast — assert_input_binary","text":"observed Input checked. factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for binary forecast — assert_input_binary","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_interval.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for interval-based forecast — assert_input_interval","title":"Assert that inputs are correct for interval-based forecast — assert_input_interval","text":"Function assesses whether inputs correspond requirements scoring interval-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_interval.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for interval-based forecast — assert_input_interval","text":"","code":"assert_input_interval(observed, lower, upper, interval_range)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_interval.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for interval-based forecast — assert_input_interval","text":"observed Input checked. numeric vector observed values size n. lower Input checked. numeric vector size n holds predicted value lower bounds prediction intervals. upper Input checked. numeric vector size n holds predicted value upper bounds prediction intervals. interval_range Input checked. vector size n denotes interval range percent. E.g. value 50 denotes (25%, 75%) prediction interval.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_interval.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for interval-based forecast — assert_input_interval","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_nominal.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for nominal forecasts — assert_input_nominal","title":"Assert that inputs are correct for nominal forecasts — assert_input_nominal","text":"Function assesses whether inputs correspond requirements scoring nominal forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_nominal.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for nominal forecasts — assert_input_nominal","text":"","code":"assert_input_nominal(observed, predicted, predicted_label)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_nominal.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for nominal forecasts — assert_input_nominal","text":"observed Input checked. factor length n N levels holding observed values. n number observations N number possible outcomes observed values can assume. output) predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level. predicted_label Factor length N N levels, N number possible outcomes observed values can assume.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_nominal.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for nominal forecasts — assert_input_nominal","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for point forecast — assert_input_point","title":"Assert that inputs are correct for point forecast — assert_input_point","text":"Function assesses whether inputs correspond requirements scoring point forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for point forecast — assert_input_point","text":"","code":"assert_input_point(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for point forecast — assert_input_point","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. numeric vector predicted values size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for point forecast — assert_input_point","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","title":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","text":"Function assesses whether inputs correspond requirements scoring quantile-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","text":"","code":"assert_input_quantile( observed, predicted, quantile_level, unique_quantile_levels = TRUE )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. nxN matrix predictive quantiles, n (number rows) number data points N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Input checked. vector size N denotes quantile levels corresponding columns prediction matrix. unique_quantile_levels Whether quantile levels required unique (TRUE, default) (FALSE).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for sample-based forecast — assert_input_sample","title":"Assert that inputs are correct for sample-based forecast — assert_input_sample","text":"Function assesses whether inputs correspond requirements scoring sample-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for sample-based forecast — assert_input_sample","text":"","code":"assert_input_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for sample-based forecast — assert_input_sample","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. numeric nxN matrix predictive samples, n (number rows) number data points N (number columns) number samples per forecast. observed just single number, predicted values can just vector size N.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for sample-based forecast — assert_input_sample","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Validate an object of class scores — assert_scores","title":"Validate an object of class scores — assert_scores","text":"function validates object class scores, checking correct class metrics attribute.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Validate an object of class scores — assert_scores","text":"","code":"assert_scores(scores)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Validate an object of class scores — assert_scores","text":"scores data.table similar scores produced score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_scores.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Validate an object of class scores — assert_scores","text":"Returns NULL invisibly","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Determines bias of quantile forecasts — bias_quantile","title":"Determines bias of quantile forecasts — bias_quantile","text":"Determines bias quantile forecasts. increasing number quantiles measure converges sample based bias version integer continuous forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Determines bias of quantile forecasts — bias_quantile","text":"","code":"bias_quantile(observed, predicted, quantile_level, na.rm = TRUE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Determines bias of quantile forecasts — bias_quantile","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made. Note contain median (0.5) median imputed mean two innermost quantiles. na.rm Logical. missing values removed?","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Determines bias of quantile forecasts — bias_quantile","text":"scalar quantile bias single quantile prediction","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Determines bias of quantile forecasts — bias_quantile","text":"quantile forecasts, bias measured $$ B_t = (1 - 2 \\cdot \\max \\{| q_{t,} \\Q_t \\land q_{t,} \\leq x_t\\}) \\mathbf{1}( x_t \\leq q_{t, 0.5}) \\\\ + (1 - 2 \\cdot \\min \\{| q_{t,} \\Q_t \\land q_{t,} \\geq x_t\\}) 1( x_t \\geq q_{t, 0.5}),$$ \\(Q_t\\) set quantiles form predictive distribution time \\(t\\) \\(x_t\\) observed value. consistency, define \\(Q_t\\) always includes element \\(q_{t, 0} = - \\infty\\) \\(q_{t,1} = \\infty\\). \\(1()\\) indicator function \\(1\\) condition satisfied \\(0\\) otherwise. clearer terms, bias \\(B_t\\) : \\(1 - 2 \\cdot\\) maximum percentile rank corresponding quantile still smaller equal observed value, observed value smaller median predictive distribution. \\(1 - 2 \\cdot\\) minimum percentile rank corresponding quantile still larger equal observed value observed value larger median predictive distribution.. \\(0\\) observed value exactly median (terms cancel ) Bias can assume values -1 1 0 ideally (.e. unbiased). Note given quantiles contain median, median imputed linear interpolation two innermost quantiles. median available imputed, error thrown. Note order compute bias, quantiles must non-decreasing increasing quantile levels. large enough number quantiles, percentile rank equal proportion predictive samples observed value, bias metric coincides one continuous forecasts (see bias_sample()).","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Determines bias of quantile forecasts — bias_quantile","text":"","code":"predicted <- matrix(c(1.5:23.5, 3.3:25.3), nrow = 2, byrow = TRUE) quantile_level <- c(0.01, 0.025, seq(0.05, 0.95, 0.05), 0.975, 0.99) observed <- c(15, 12.4) bias_quantile(observed, predicted, quantile_level) #> [1] -0.3 0.2"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile_single_vector.html","id":null,"dir":"Reference","previous_headings":"","what":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","title":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","text":"Internal function compute bias single observed value, vector predicted values vector quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile_single_vector.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","text":"","code":"bias_quantile_single_vector(observed, predicted, quantile_level, na.rm)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile_single_vector.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","text":"observed Scalar observed value. predicted Vector length N (corresponding number quantiles) holds predictions. quantile_level Vector size N quantile levels predictions made. Note contain median (0.5) median imputed mean two innermost quantiles. na.rm Logical. missing values removed?","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile_single_vector.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","text":"scalar quantile bias single quantile prediction","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Determine bias of forecasts — bias_sample","title":"Determine bias of forecasts — bias_sample","text":"Determines bias predictive Monte-Carlo samples. function automatically recognises whether forecasts continuous integer valued adapts Bias function accordingly.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Determine bias of forecasts — bias_sample","text":"","code":"bias_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Determine bias of forecasts — bias_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Determine bias of forecasts — bias_sample","text":"Numeric vector length n biases predictive samples respect observed values.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Determine bias of forecasts — bias_sample","text":"continuous forecasts, Bias measured $$ B_t (P_t, x_t) = 1 - 2 * (P_t (x_t)) $$ \\(P_t\\) empirical cumulative distribution function prediction observed value \\(x_t\\). Computationally, \\(P_t (x_t)\\) just calculated fraction predictive samples \\(x_t\\) smaller \\(x_t\\). integer valued forecasts, Bias measured $$ B_t (P_t, x_t) = 1 - (P_t (x_t) + P_t (x_t + 1)) $$ adjust integer nature forecasts. cases, Bias can assume values -1 1 0 ideally.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Determine bias of forecasts — bias_sample","text":"integer valued Bias function discussed Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15 Funk S, Camacho , Kucharski AJ, Lowe R, Eggo RM, et al. (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15. PLOS Computational Biology 15(2): e1006785. doi:10.1371/journal.pcbi.1006785","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Determine bias of forecasts — bias_sample","text":"","code":"## integer valued forecasts observed <- rpois(30, lambda = 1:30) predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) bias_sample(observed, predicted) #> [1] 0.660 -0.025 0.710 -0.930 0.870 0.435 -0.885 -0.940 -0.515 -0.790 #> [11] 0.975 -0.975 -0.620 -0.740 -0.640 0.395 0.695 -0.765 -0.935 -0.680 #> [21] -0.725 -0.320 0.355 0.730 0.250 0.995 -0.650 0.235 0.250 0.850 ## continuous forecasts observed <- rnorm(30, mean = 1:30) predicted <- replicate(200, rnorm(30, mean = 1:30)) bias_sample(observed, predicted) #> [1] -0.46 0.02 0.02 0.12 -0.18 -0.07 0.96 -0.60 0.16 -0.31 0.79 0.49 #> [13] -0.74 -0.48 0.26 -0.56 0.82 0.89 0.41 -0.31 0.19 0.47 -0.85 0.32 #> [25] 0.15 -0.16 0.34 -0.30 0.80 0.27"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_columns_present.html","id":null,"dir":"Reference","previous_headings":"","what":"Check column names are present in a data.frame — check_columns_present","title":"Check column names are present in a data.frame — check_columns_present","text":"functions loops column names checks whether present. issue encountered, function immediately stops returns message first issue encountered.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_columns_present.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check column names are present in a data.frame — check_columns_present","text":"","code":"check_columns_present(data, columns)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_columns_present.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check column names are present in a data.frame — check_columns_present","text":"data data.frame similar checked columns character vector column names check","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_columns_present.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check column names are present in a data.frame — check_columns_present","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_dims_ok_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Check Inputs Have Matching Dimensions — check_dims_ok_point","title":"Check Inputs Have Matching Dimensions — check_dims_ok_point","text":"Function assesses whether input dimensions match. following, n number observations / forecasts. Scalar values may repeated match length input. Allowed options therefore: observed vector length 1 length n predicted : vector length 1 length n matrix n rows 1 column","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_dims_ok_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check Inputs Have Matching Dimensions — check_dims_ok_point","text":"","code":"check_dims_ok_point(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_dims_ok_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check Inputs Have Matching Dimensions — check_dims_ok_point","text":"observed Input checked. factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_dims_ok_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check Inputs Have Matching Dimensions — check_dims_ok_point","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_duplicates.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that there are no duplicate forecasts — check_duplicates","title":"Check that there are no duplicate forecasts — check_duplicates","text":"Runs get_duplicate_forecasts() returns message issue encountered","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_duplicates.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that there are no duplicate forecasts — check_duplicates","text":"","code":"check_duplicates(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_duplicates.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that there are no duplicate forecasts — check_duplicates","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_duplicates.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that there are no duplicate forecasts — check_duplicates","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for binary forecast — check_input_binary","title":"Check that inputs are correct for binary forecast — check_input_binary","text":"Function assesses whether inputs correspond requirements scoring binary forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for binary forecast — check_input_binary","text":"","code":"check_input_binary(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for binary forecast — check_input_binary","text":"observed Input checked. factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for binary forecast — check_input_binary","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_interval.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for interval-based forecast — check_input_interval","title":"Check that inputs are correct for interval-based forecast — check_input_interval","text":"Function assesses whether inputs correspond requirements scoring interval-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_interval.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for interval-based forecast — check_input_interval","text":"","code":"check_input_interval(observed, lower, upper, interval_range)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_interval.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for interval-based forecast — check_input_interval","text":"observed Input checked. numeric vector observed values size n. lower Input checked. numeric vector size n holds predicted value lower bounds prediction intervals. upper Input checked. numeric vector size n holds predicted value upper bounds prediction intervals. interval_range Input checked. vector size n denotes interval range percent. E.g. value 50 denotes (25%, 75%) prediction interval.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_interval.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for interval-based forecast — check_input_interval","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for point forecast — check_input_point","title":"Check that inputs are correct for point forecast — check_input_point","text":"Function assesses whether inputs correspond requirements scoring point forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for point forecast — check_input_point","text":"","code":"check_input_point(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for point forecast — check_input_point","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. numeric vector predicted values size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for point forecast — check_input_point","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for quantile-based forecast — check_input_quantile","title":"Check that inputs are correct for quantile-based forecast — check_input_quantile","text":"Function assesses whether inputs correspond requirements scoring quantile-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for quantile-based forecast — check_input_quantile","text":"","code":"check_input_quantile(observed, predicted, quantile_level)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for quantile-based forecast — check_input_quantile","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. nxN matrix predictive quantiles, n (number rows) number data points N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Input checked. vector size N denotes quantile levels corresponding columns prediction matrix.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for quantile-based forecast — check_input_quantile","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for sample-based forecast — check_input_sample","title":"Check that inputs are correct for sample-based forecast — check_input_sample","text":"Function assesses whether inputs correspond requirements scoring sample-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for sample-based forecast — check_input_sample","text":"","code":"check_input_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for sample-based forecast — check_input_sample","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. numeric nxN matrix predictive samples, n (number rows) number data points N (number columns) number samples per forecast. observed just single number, predicted values can just vector size N.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for sample-based forecast — check_input_sample","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_number_per_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that all forecasts have the same number of rows — check_number_per_forecast","title":"Check that all forecasts have the same number of rows — check_number_per_forecast","text":"Helper function checks number rows (corresponding e.g quantiles samples) per forecast. number quantiles samples forecasts, returns TRUE string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_number_per_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that all forecasts have the same number of rows — check_number_per_forecast","text":"","code":"check_number_per_forecast(data, forecast_unit)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_number_per_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that all forecasts have the same number of rows — check_number_per_forecast","text":"data data.frame similar checked forecast_unit Character vector denoting unit single forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_number_per_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that all forecasts have the same number of rows — check_number_per_forecast","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_numeric_vector.html","id":null,"dir":"Reference","previous_headings":"","what":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","title":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","text":"Helper function check whether input numeric vector.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_numeric_vector.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","text":"","code":"check_numeric_vector(x, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_numeric_vector.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","text":"x input check ... Arguments passed checkmate::check_numeric lower [numeric(1)] Lower value elements x must greater equal . upper [numeric(1)] Upper value elements x must lower equal . finite [logical(1)] Check finite values? Default FALSE. .missing [logical(1)] vectors missing values allowed? Default TRUE. .missing [logical(1)] vectors non-missing values allowed? Default TRUE. Note empty vectors non-missing values. len [integer(1)] Exact expected length x. min.len [integer(1)] Minimal length x. max.len [integer(1)] Maximal length x. unique [logical(1)] Must values unique? Default FALSE. sorted [logical(1)] Elements must sorted ascending order. Missing values ignored. names [character(1)] Check names. See checkNamed possible values. Default “” performs check . Note can use checkSubset check specific set names. typed.missing [logical(1)] set FALSE (default), types missing values (NA, NA_integer_, NA_real_, NA_character_ NA_character_) well empty vectors allowed type-checking atomic input. Set TRUE enable strict type checking. null.ok [logical(1)] set TRUE, x may also NULL. case type check x performed, additional checks disabled.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_numeric_vector.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_try.html","id":null,"dir":"Reference","previous_headings":"","what":"Helper function to convert assert statements into checks — check_try","title":"Helper function to convert assert statements into checks — check_try","text":"Tries execute expression. Internally, used see whether assertions fail checking inputs (.e. convert assert_*() statement check). expression fails, error message returned. expression succeeds, TRUE returned.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_try.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Helper function to convert assert statements into checks — check_try","text":"","code":"check_try(expr)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_try.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Helper function to convert assert statements into checks — check_try","text":"expr expression evaluated","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_try.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Helper function to convert assert statements into checks — check_try","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/clean_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Clean forecast object — clean_forecast","title":"Clean forecast object — clean_forecast","text":"function makes possible silently validate object. addition, can return copy data remove rows missing values.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/clean_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Clean forecast object — clean_forecast","text":"","code":"clean_forecast(forecast, copy = FALSE, na.omit = FALSE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/clean_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Clean forecast object — clean_forecast","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()). copy Logical, default FALSE. TRUE, copy input data created. na.omit Logical, default FALSE. TRUE, rows missing values removed.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_forecasts.html","id":null,"dir":"Reference","previous_headings":"","what":"Compare a subset of common forecasts — compare_forecasts","title":"Compare a subset of common forecasts — compare_forecasts","text":"function compares two comparators based subset forecasts comparators made prediction. gets called pairwise_comparison_one_group(), handles comparison multiple comparators single set forecasts (subsets forecasts distinguished). pairwise_comparison_one_group() turn gets called get_pairwise_comparisons() can handle pairwise comparisons set forecasts multiple subsets, e.g. pairwise comparisons one set forecasts, done separately two different forecast targets.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_forecasts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Compare a subset of common forecasts — compare_forecasts","text":"","code":"compare_forecasts( scores, compare = \"model\", name_comparator1, name_comparator2, metric, one_sided = FALSE, test_type = c(\"non_parametric\", \"permutation\"), n_permutations = 999 )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_forecasts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Compare a subset of common forecasts — compare_forecasts","text":"scores object class scores (data.table scores additional attribute metrics produced score()). compare Character vector single colum name defines elements pairwise comparison. example, set \"model\" (default), elements \"model\" column compared. name_comparator1 Character, name first comparator name_comparator2 Character, name comparator compare metric string name metric relative skill shall computed. default either \"crps\", \"wis\" \"brier_score\" available. one_sided Boolean, default FALSE, whether two conduct one-sided instead two-sided test determine significance pairwise comparison. test_type Character, either \"non_parametric\" (default) \"permutation\". determines kind test shall conducted determine p-values. n_permutations Numeric, number permutations permutation test. Default 999.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_forecasts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Compare a subset of common forecasts — compare_forecasts","text":"list mean score ratios p-values comparison two comparators","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_forecasts.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Compare a subset of common forecasts — compare_forecasts","text":"Johannes Bracher, johannes.bracher@kit.edu Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"(Continuous) ranked probability score — crps_sample","title":"(Continuous) ranked probability score — crps_sample","text":"Wrapper around crps_sample() function scoringRules package. Can used continuous well integer valued forecasts Continuous ranked probability score (CRPS) can interpreted sum three components: overprediction, underprediction dispersion. \"Dispersion\" defined CRPS median forecast $m$. observation $y$ greater $m$ overpredictoin defined CRPS forecast $y$ minus dispersion component, underprediction zero. , hand, $y [1] 0.231225 0.329850 1.060100 0.412625 0.991550 0.793850 2.847025 1.126200 #> [9] 0.690675 0.992875 1.118300 4.911975 4.135800 1.411050 0.967825 3.425125 #> [17] 0.839600 2.404700 1.376075 1.665525 2.771375 1.168550 1.244700 2.377250 #> [25] 2.847875 2.019175 1.827975 2.074850 1.868775 3.171950"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_assert_functions.html","id":null,"dir":"Reference","previous_headings":"","what":"Documentation template for assert functions — document_assert_functions","title":"Documentation template for assert functions — document_assert_functions","text":"Documentation template assert functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_assert_functions.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Documentation template for assert functions — document_assert_functions","text":"observed Input checked. numeric vector observed values size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_assert_functions.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Documentation template for assert functions — document_assert_functions","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_check_functions.html","id":null,"dir":"Reference","previous_headings":"","what":"Documentation template for check functions — document_check_functions","title":"Documentation template for check functions — document_check_functions","text":"Documentation template check functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_check_functions.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Documentation template for check functions — document_check_functions","text":"data data.frame similar checked observed Input checked. numeric vector observed values size n. columns character vector column names check","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_check_functions.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Documentation template for check functions — document_check_functions","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_test_functions.html","id":null,"dir":"Reference","previous_headings":"","what":"Documentation template for test functions — document_test_functions","title":"Documentation template for test functions — document_test_functions","text":"Documentation template test functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_test_functions.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Documentation template for test functions — document_test_functions","text":"Returns TRUE check successful FALSE otherwise","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Dawid-Sebastiani score — dss_sample","title":"Dawid-Sebastiani score — dss_sample","text":"Wrapper around dss_sample() function scoringRules package.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Dawid-Sebastiani score — dss_sample","text":"","code":"dss_sample(observed, predicted, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Dawid-Sebastiani score — dss_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. ... Additional arguments passed dss_sample() scoringRules package.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Dawid-Sebastiani score — dss_sample","text":"Vector scores.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Dawid-Sebastiani score — dss_sample","text":"Alexander Jordan, Fabian Krüger, Sebastian Lerch, Evaluating Probabilistic Forecasts scoringRules, https://www.jstatsoft.org/article/view/v090i12","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Dawid-Sebastiani score — dss_sample","text":"","code":"observed <- rpois(30, lambda = 1:30) predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) dss_sample(observed, predicted) #> [1] -0.06179111 2.81530809 2.46742176 1.79360855 1.61740613 2.39081466 #> [7] 1.92865409 15.58374079 3.88246189 4.97070116 2.24551342 2.71528477 #> [13] 2.79485162 2.62249405 2.80770087 4.30015607 3.23096351 3.16329747 #> [19] 3.23001192 7.67628290 2.92052357 5.37555852 4.78857945 3.12865561 #> [25] 3.61999703 4.32028466 3.38593521 6.87116360 4.71995903 4.99107110"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_data.table.html","id":null,"dir":"Reference","previous_headings":"","what":"Ensure that an object is a data.table — ensure_data.table","title":"Ensure that an object is a data.table — ensure_data.table","text":"function ensures object data table. object data table, converted one. object data table, copy object returned.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_data.table.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Ensure that an object is a data.table — ensure_data.table","text":"","code":"ensure_data.table(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_data.table.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Ensure that an object is a data.table — ensure_data.table","text":"data object ensure data table.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_data.table.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Ensure that an object is a data.table — ensure_data.table","text":"data.table/copy existing data.table.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Binary forecast example data — example_binary","title":"Binary forecast example data — example_binary","text":"data set binary predictions COVID-19 cases deaths constructed data submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Binary forecast example data — example_binary","text":"","code":"example_binary"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Binary forecast example data — example_binary","text":"object class forecast_binary (see as_forecast()) following columns: location country prediction made location_name name country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed factor observed values forecast_date date prediction made model name model generated forecasts horizon forecast horizon weeks predicted predicted value","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Binary forecast example data — example_binary","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Binary forecast example data — example_binary","text":"Predictions data set constructed based continuous example data looking number samples mean prediction. outcome constructed whether actually observed value mean prediction. understood sound statistical practice, rather practical way create example data set. data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_nominal.html","id":null,"dir":"Reference","previous_headings":"","what":"Nominal example data — example_nominal","title":"Nominal example data — example_nominal","text":"data set predictions COVID-19 cases deaths submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_nominal.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Nominal example data — example_nominal","text":"","code":"example_nominal"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_nominal.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Nominal example data — example_nominal","text":"object class forecast_nominal (see as_forecast()) following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed Numeric: observed values location_name name country prediction made forecast_date date prediction made predicted_label outcome probabilty corresponds predicted predicted value model name model generated forecasts horizon forecast horizon weeks","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_nominal.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Nominal example data — example_nominal","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_nominal.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Nominal example data — example_nominal","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Point forecast example data — example_point","title":"Point forecast example data — example_point","text":"data set predictions COVID-19 cases deaths submitted European Forecast Hub. data set like quantile example data, median replaced point forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Point forecast example data — example_point","text":"","code":"example_point"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Point forecast example data — example_point","text":"object class forecast_point (see as_forecast()) following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed observed values location_name name country prediction made forecast_date date prediction made predicted predicted value model name model generated forecasts horizon forecast horizon weeks","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Point forecast example data — example_point","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Point forecast example data — example_point","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Quantile example data — example_quantile","title":"Quantile example data — example_quantile","text":"data set predictions COVID-19 cases deaths submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Quantile example data — example_quantile","text":"","code":"example_quantile"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Quantile example data — example_quantile","text":"object class forecast_quantile (see as_forecast()) following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed Numeric: observed values location_name name country prediction made forecast_date date prediction made quantile_level quantile level corresponding prediction predicted predicted value model name model generated forecasts horizon forecast horizon weeks","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Quantile example data — example_quantile","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Quantile example data — example_quantile","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":null,"dir":"Reference","previous_headings":"","what":"Continuous forecast example data — example_sample_continuous","title":"Continuous forecast example data — example_sample_continuous","text":"data set continuous predictions COVID-19 cases deaths constructed data submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Continuous forecast example data — example_sample_continuous","text":"","code":"example_sample_continuous"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Continuous forecast example data — example_sample_continuous","text":"object class forecast_sample (see as_forecast()) following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed observed values location_name name country prediction made forecast_date date prediction made model name model generated forecasts horizon forecast horizon weeks predicted predicted value sample_id id corresponding sample","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Continuous forecast example data — example_sample_continuous","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Continuous forecast example data — example_sample_continuous","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":null,"dir":"Reference","previous_headings":"","what":"Discrete forecast example data — example_sample_discrete","title":"Discrete forecast example data — example_sample_discrete","text":"data set integer predictions COVID-19 cases deaths constructed data submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Discrete forecast example data — example_sample_discrete","text":"","code":"example_sample_discrete"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Discrete forecast example data — example_sample_discrete","text":"object class forecast_sample (see as_forecast()) following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed observed values location_name name country prediction made forecast_date date prediction made model name model generated forecasts horizon forecast horizon weeks predicted predicted value sample_id id corresponding sample","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Discrete forecast example data — example_sample_discrete","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Discrete forecast example data — example_sample_discrete","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/forecast_types.html","id":null,"dir":"Reference","previous_headings":"","what":"Documentation template for forecast types — forecast_types","title":"Documentation template for forecast types — forecast_types","text":"Documentation template forecast types","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/forecast_types.html","id":"forecast-types-and-input-formats","dir":"Reference","previous_headings":"","what":"Forecast types and input formats","title":"Documentation template for forecast types — forecast_types","text":"Various different forecast types / forecast formats supported. moment, : point forecasts binary forecasts (\"soft binary classification\") nominal forecasts (\"soft classification multiple unordered classes\") Probabilistic forecasts quantile-based format (forecast represented set predictive quantiles) Probabilistic forecasts sample-based format (forecast represented set predictive samples) Forecast types determined based columns present input data. overview required format forecast type: forecast types require data.frame similar columns observed predicted, model. Point forecasts require column observed type numeric column predicted type numeric. Binary forecasts require column observed type factor exactly two levels column predicted type numeric probabilities, corresponding probability observed equal second factor level. See details information. Nominal forecasts require column observed type factor N levels, (N number possible outcomes), column predicted type numeric probabilities (sum one across possible outcomes), column predicted_label type factor N levels, denoting outcome probability given. Forecasts must complete, .e. must probability assigned every possible outcome. Quantile-based forecasts require column observed type numeric, column predicted type numeric, column quantile_level type numeric quantile-levels (0 1). Sample-based forecasts require column observed type numeric, column predicted type numeric, column sample_id type numeric sample indices. information see vignettes example data (example_quantile, example_sample_continuous, example_sample_discrete, example_point(), example_binary, example_nominal).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/forecast_types.html","id":"forecast-unit","dir":"Reference","previous_headings":"","what":"Forecast unit","title":"Documentation template for forecast types — forecast_types","text":"order score forecasts, scoringutils needs know rows data belong together jointly form single forecasts. easy e.g. point forecast, one row per forecast. quantile sample-based forecasts, however, multiple rows belong single forecast. forecast unit unit single forecast described combination columns uniquely identify single forecast. example, forecasts made different models various locations different time points, several weeks future. forecast unit described forecast_unit = c(\"model\", \"location\", \"forecast_date\", \"forecast_horizon\"). scoringutils automatically tries determine unit single forecast. uses existing columns , means columns must present unrelated forecast unit. simplistic example, additional row, \"even\", one row number even zero otherwise, mess scoring scoringutils thinks column relevant defining forecast unit. order avoid issues, recommend setting forecast unit explicitly, usually forecast_unit argument as_forecast() functions. drop unneeded columns, making sure necessary, 'protected columns' like \"predicted\" \"observed\" retained.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":null,"dir":"Reference","previous_headings":"","what":"Calculate geometric mean — geometric_mean","title":"Calculate geometric mean — geometric_mean","text":"Calculate geometric mean","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Calculate geometric mean — geometric_mean","text":"","code":"geometric_mean(x)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Calculate geometric mean — geometric_mean","text":"x Numeric vector values calculate geometric mean.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Calculate geometric mean — geometric_mean","text":"geometric mean values x. NA values ignored.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Calculate geometric mean — geometric_mean","text":"Used get_pairwise_comparisons().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":null,"dir":"Reference","previous_headings":"","what":"Calculate correlation between metrics — get_correlations","title":"Calculate correlation between metrics — get_correlations","text":"Calculate correlation different metrics data.frame scores produced score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Calculate correlation between metrics — get_correlations","text":"","code":"get_correlations(scores, metrics = get_metrics.scores(scores), ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Calculate correlation between metrics — get_correlations","text":"scores object class scores (data.table scores additional attribute metrics produced score()). metrics character vector metrics show. set NULL (default), metrics present scores shown. ... Additional arguments pass cor().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Calculate correlation between metrics — get_correlations","text":"object class scores (data.table additional attribute metrics holding names scores) correlations different metrics","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Calculate correlation between metrics — get_correlations","text":"","code":"library(magrittr) # pipe operator scores <- example_quantile %>% as_forecast_quantile() %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. get_correlations(scores) #> wis overprediction underprediction dispersion bias #> #> 1: 1.0000000 0.94297565 0.28377361 0.45566303 0.10545891 #> 2: 0.9429757 1.00000000 -0.03310356 0.32493799 0.21532161 #> 3: 0.2837736 -0.03310356 1.00000000 0.14580143 -0.35123801 #> 4: 0.4556630 0.32493799 0.14580143 1.00000000 0.11118365 #> 5: 0.1054589 0.21532161 -0.35123801 0.11118365 1.00000000 #> 6: -0.2076649 -0.14556039 -0.21392764 -0.09400664 0.01338140 #> 7: -0.4075613 -0.31824017 -0.35756699 -0.08614678 0.09802725 #> 8: 0.9886108 0.90326672 0.33589892 0.53809741 0.09578751 #> interval_coverage_50 interval_coverage_90 ae_median metric #> #> 1: -0.20766492 -0.40756133 0.98861080 wis #> 2: -0.14556039 -0.31824017 0.90326672 overprediction #> 3: -0.21392764 -0.35756699 0.33589892 underprediction #> 4: -0.09400664 -0.08614678 0.53809741 dispersion #> 5: 0.01338140 0.09802725 0.09578751 bias #> 6: 1.00000000 0.37245118 -0.24559356 interval_coverage_50 #> 7: 0.37245118 1.00000000 -0.41079097 interval_coverage_90 #> 8: -0.24559356 -0.41079097 1.00000000 ae_median"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"validated forecast object quantile-based format (see as_forecast() information), function computes: interval coverage central prediction intervals quantile coverage predictive quantiles deviation desired actual coverage (interval quantile coverage) Coverage values computed specific level grouping, specified argument. default, coverage values computed per model. Interval coverage Interval coverage given interval range defined proportion observations fall within corresponding central prediction intervals. Central prediction intervals symmetric around median formed two quantiles denote lower upper bound. example, 50% central prediction interval interval 0.25 0.75 quantiles predictive distribution. Quantile coverage Quantile coverage given quantile level defined proportion observed values smaller corresponding predictive quantile. example, 0.5 quantile coverage proportion observed values smaller 0.5 quantile predictive distribution. Just , single observation quantile single predictive distribution, value either TRUE FALSE. Coverage deviation coverage deviation difference desired coverage (can either interval quantile coverage) actual coverage. example, desired coverage 90% actual coverage 80%, coverage deviation -0.1.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"","code":"get_coverage(forecast, by = \"model\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()). character vector denotes level grouping coverage values computed. default (\"model\"), one coverage value per model returned.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"data.table columns specified additional columns coverage values described data.table columns \"interval_coverage\", \"interval_coverage_deviation\", \"quantile_coverage\", \"quantile_coverage_deviation\" columns specified .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"","code":"library(magrittr) # pipe operator example_quantile %>% as_forecast_quantile() %>% get_coverage(by = \"model\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> model quantile_level interval_range interval_coverage #> #> 1: EuroCOVIDhub-baseline 0.500 0 0.000000000 #> 2: EuroCOVIDhub-baseline 0.450 10 0.085937500 #> 3: EuroCOVIDhub-baseline 0.550 10 0.085937500 #> 4: EuroCOVIDhub-baseline 0.400 20 0.191406250 #> 5: EuroCOVIDhub-baseline 0.600 20 0.191406250 #> 6: EuroCOVIDhub-baseline 0.350 30 0.289062500 #> 7: EuroCOVIDhub-baseline 0.650 30 0.289062500 #> 8: EuroCOVIDhub-baseline 0.300 40 0.375000000 #> 9: EuroCOVIDhub-baseline 0.700 40 0.375000000 #> 10: EuroCOVIDhub-baseline 0.250 50 0.496093750 #> 11: EuroCOVIDhub-baseline 0.750 50 0.496093750 #> 12: EuroCOVIDhub-baseline 0.200 60 0.628906250 #> 13: EuroCOVIDhub-baseline 0.800 60 0.628906250 #> 14: EuroCOVIDhub-baseline 0.150 70 0.773437500 #> 15: EuroCOVIDhub-baseline 0.850 70 0.773437500 #> 16: EuroCOVIDhub-baseline 0.100 80 0.843750000 #> 17: EuroCOVIDhub-baseline 0.900 80 0.843750000 #> 18: EuroCOVIDhub-baseline 0.050 90 0.910156250 #> 19: EuroCOVIDhub-baseline 0.950 90 0.910156250 #> 20: EuroCOVIDhub-baseline 0.025 95 0.925781250 #> 21: EuroCOVIDhub-baseline 0.975 95 0.925781250 #> 22: EuroCOVIDhub-baseline 0.010 98 0.933593750 #> 23: EuroCOVIDhub-baseline 0.990 98 0.933593750 #> 24: EuroCOVIDhub-ensemble 0.500 0 0.003906250 #> 25: EuroCOVIDhub-ensemble 0.450 10 0.148437500 #> 26: EuroCOVIDhub-ensemble 0.550 10 0.148437500 #> 27: EuroCOVIDhub-ensemble 0.400 20 0.250000000 #> 28: EuroCOVIDhub-ensemble 0.600 20 0.250000000 #> 29: EuroCOVIDhub-ensemble 0.350 30 0.386718750 #> 30: EuroCOVIDhub-ensemble 0.650 30 0.386718750 #> 31: EuroCOVIDhub-ensemble 0.300 40 0.519531250 #> 32: EuroCOVIDhub-ensemble 0.700 40 0.519531250 #> 33: EuroCOVIDhub-ensemble 0.250 50 0.632812500 #> 34: EuroCOVIDhub-ensemble 0.750 50 0.632812500 #> 35: EuroCOVIDhub-ensemble 0.200 60 0.667968750 #> 36: EuroCOVIDhub-ensemble 0.800 60 0.667968750 #> 37: EuroCOVIDhub-ensemble 0.150 70 0.753906250 #> 38: EuroCOVIDhub-ensemble 0.850 70 0.753906250 #> 39: EuroCOVIDhub-ensemble 0.100 80 0.816406250 #> 40: EuroCOVIDhub-ensemble 0.900 80 0.816406250 #> 41: EuroCOVIDhub-ensemble 0.050 90 0.902343750 #> 42: EuroCOVIDhub-ensemble 0.950 90 0.902343750 #> 43: EuroCOVIDhub-ensemble 0.025 95 0.941406250 #> 44: EuroCOVIDhub-ensemble 0.975 95 0.941406250 #> 45: EuroCOVIDhub-ensemble 0.010 98 0.968750000 #> 46: EuroCOVIDhub-ensemble 0.990 98 0.968750000 #> 47: epiforecasts-EpiNow2 0.500 0 0.004048583 #> 48: epiforecasts-EpiNow2 0.450 10 0.093117409 #> 49: epiforecasts-EpiNow2 0.550 10 0.093117409 #> 50: epiforecasts-EpiNow2 0.400 20 0.165991903 #> 51: epiforecasts-EpiNow2 0.600 20 0.165991903 #> 52: epiforecasts-EpiNow2 0.350 30 0.230769231 #> 53: epiforecasts-EpiNow2 0.650 30 0.230769231 #> 54: epiforecasts-EpiNow2 0.300 40 0.319838057 #> 55: epiforecasts-EpiNow2 0.700 40 0.319838057 #> 56: epiforecasts-EpiNow2 0.250 50 0.445344130 #> 57: epiforecasts-EpiNow2 0.750 50 0.445344130 #> 58: epiforecasts-EpiNow2 0.200 60 0.538461538 #> 59: epiforecasts-EpiNow2 0.800 60 0.538461538 #> 60: epiforecasts-EpiNow2 0.150 70 0.635627530 #> 61: epiforecasts-EpiNow2 0.850 70 0.635627530 #> 62: epiforecasts-EpiNow2 0.100 80 0.732793522 #> 63: epiforecasts-EpiNow2 0.900 80 0.732793522 #> 64: epiforecasts-EpiNow2 0.050 90 0.846153846 #> 65: epiforecasts-EpiNow2 0.950 90 0.846153846 #> 66: epiforecasts-EpiNow2 0.025 95 0.874493927 #> 67: epiforecasts-EpiNow2 0.975 95 0.874493927 #> 68: epiforecasts-EpiNow2 0.010 98 0.910931174 #> 69: epiforecasts-EpiNow2 0.990 98 0.910931174 #> 70: UMass-MechBayes 0.500 0 0.015625000 #> 71: UMass-MechBayes 0.450 10 0.101562500 #> 72: UMass-MechBayes 0.550 10 0.101562500 #> 73: UMass-MechBayes 0.400 20 0.195312500 #> 74: UMass-MechBayes 0.600 20 0.195312500 #> 75: UMass-MechBayes 0.350 30 0.281250000 #> 76: UMass-MechBayes 0.650 30 0.281250000 #> 77: UMass-MechBayes 0.300 40 0.382812500 #> 78: UMass-MechBayes 0.700 40 0.382812500 #> 79: UMass-MechBayes 0.250 50 0.460937500 #> 80: UMass-MechBayes 0.750 50 0.460937500 #> 81: UMass-MechBayes 0.200 60 0.539062500 #> 82: UMass-MechBayes 0.800 60 0.539062500 #> 83: UMass-MechBayes 0.150 70 0.617187500 #> 84: UMass-MechBayes 0.850 70 0.617187500 #> 85: UMass-MechBayes 0.100 80 0.765625000 #> 86: UMass-MechBayes 0.900 80 0.765625000 #> 87: UMass-MechBayes 0.050 90 0.875000000 #> 88: UMass-MechBayes 0.950 90 0.875000000 #> 89: UMass-MechBayes 0.025 95 0.953125000 #> 90: UMass-MechBayes 0.975 95 0.953125000 #> 91: UMass-MechBayes 0.010 98 0.984375000 #> 92: UMass-MechBayes 0.990 98 0.984375000 #> model quantile_level interval_range interval_coverage #> interval_coverage_deviation quantile_coverage quantile_coverage_deviation #> #> 1: 0.000000000 0.69921875 0.199218750 #> 2: -0.014062500 0.65625000 0.206250000 #> 3: -0.014062500 0.74218750 0.192187500 #> 4: -0.008593750 0.58593750 0.185937500 #> 5: -0.008593750 0.77343750 0.173437500 #> 6: -0.010937500 0.52343750 0.173437500 #> 7: -0.010937500 0.80859375 0.158593750 #> 8: -0.025000000 0.46875000 0.168750000 #> 9: -0.025000000 0.84375000 0.143750000 #> 10: -0.003906250 0.36718750 0.117187500 #> 11: -0.003906250 0.86328125 0.113281250 #> 12: 0.028906250 0.25000000 0.050000000 #> 13: 0.028906250 0.87500000 0.075000000 #> 14: 0.073437500 0.13281250 -0.017187500 #> 15: 0.073437500 0.90625000 0.056250000 #> 16: 0.043750000 0.08203125 -0.017968750 #> 17: 0.043750000 0.92578125 0.025781250 #> 18: 0.010156250 0.04296875 -0.007031250 #> 19: 0.010156250 0.95312500 0.003125000 #> 20: -0.024218750 0.03125000 0.006250000 #> 21: -0.024218750 0.95703125 -0.017968750 #> 22: -0.046406250 0.03125000 0.021250000 #> 23: -0.046406250 0.96484375 -0.025156250 #> 24: 0.003906250 0.53125000 0.031250000 #> 25: 0.048437500 0.46484375 0.014843750 #> 26: 0.048437500 0.60156250 0.051562500 #> 27: 0.050000000 0.40625000 0.006250000 #> 28: 0.050000000 0.65625000 0.056250000 #> 29: 0.086718750 0.33984375 -0.010156250 #> 30: 0.086718750 0.72656250 0.076562500 #> 31: 0.119531250 0.26562500 -0.034375000 #> 32: 0.119531250 0.76562500 0.065625000 #> 33: 0.132812500 0.16406250 -0.085937500 #> 34: 0.132812500 0.79687500 0.046875000 #> 35: 0.067968750 0.14062500 -0.059375000 #> 36: 0.067968750 0.80468750 0.004687500 #> 37: 0.053906250 0.10156250 -0.048437500 #> 38: 0.053906250 0.85546875 0.005468750 #> 39: 0.016406250 0.07812500 -0.021875000 #> 40: 0.016406250 0.89453125 -0.005468750 #> 41: 0.002343750 0.04296875 -0.007031250 #> 42: 0.002343750 0.94531250 -0.004687500 #> 43: -0.008593750 0.03125000 0.006250000 #> 44: -0.008593750 0.97265625 -0.002343750 #> 45: -0.011250000 0.01562500 0.005625000 #> 46: -0.011250000 0.98437500 -0.005625000 #> 47: 0.004048583 0.49392713 -0.006072874 #> 48: -0.006882591 0.43724696 -0.012753036 #> 49: -0.006882591 0.53036437 -0.019635628 #> 50: -0.034008097 0.39676113 -0.003238866 #> 51: -0.034008097 0.55870445 -0.041295547 #> 52: -0.069230769 0.36437247 0.014372470 #> 53: -0.069230769 0.59514170 -0.054858300 #> 54: -0.080161943 0.31983806 0.019838057 #> 55: -0.080161943 0.63967611 -0.060323887 #> 56: -0.054655870 0.26315789 0.013157895 #> 57: -0.054655870 0.70445344 -0.045546559 #> 58: -0.061538462 0.20647773 0.006477733 #> 59: -0.061538462 0.74493927 -0.055060729 #> 60: -0.064372470 0.14574899 -0.004251012 #> 61: -0.064372470 0.78137652 -0.068623482 #> 62: -0.067206478 0.10121457 0.001214575 #> 63: -0.067206478 0.83400810 -0.065991903 #> 64: -0.053846154 0.06072874 0.010728745 #> 65: -0.053846154 0.90688259 -0.043117409 #> 66: -0.075506073 0.04858300 0.023582996 #> 67: -0.075506073 0.92307692 -0.051923077 #> 68: -0.069068826 0.02429150 0.014291498 #> 69: -0.069068826 0.93522267 -0.054777328 #> 70: 0.015625000 0.50000000 0.000000000 #> 71: 0.001562500 0.42187500 -0.028125000 #> 72: 0.001562500 0.52343750 -0.026562500 #> 73: -0.004687500 0.37500000 -0.025000000 #> 74: -0.004687500 0.57031250 -0.029687500 #> 75: -0.018750000 0.35156250 0.001562500 #> 76: -0.018750000 0.61718750 -0.032812500 #> 77: -0.017187500 0.28906250 -0.010937500 #> 78: -0.017187500 0.66406250 -0.035937500 #> 79: -0.039062500 0.27343750 0.023437500 #> 80: -0.039062500 0.71875000 -0.031250000 #> 81: -0.060937500 0.24218750 0.042187500 #> 82: -0.060937500 0.78125000 -0.018750000 #> 83: -0.082812500 0.20312500 0.053125000 #> 84: -0.082812500 0.82031250 -0.029687500 #> 85: -0.034375000 0.12500000 0.025000000 #> 86: -0.034375000 0.87500000 -0.025000000 #> 87: -0.025000000 0.06250000 0.012500000 #> 88: -0.025000000 0.93750000 -0.012500000 #> 89: 0.003125000 0.01562500 -0.009375000 #> 90: 0.003125000 0.96875000 -0.006250000 #> 91: 0.004375000 0.00781250 -0.002187500 #> 92: 0.004375000 0.99218750 0.002187500 #> interval_coverage_deviation quantile_coverage quantile_coverage_deviation"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":null,"dir":"Reference","previous_headings":"","what":"Find duplicate forecasts — get_duplicate_forecasts","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"Internal helper function identify duplicate forecasts, .e. instances one forecast prediction target.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"","code":"get_duplicate_forecasts(data, forecast_unit = NULL, counts = FALSE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. counts output show number duplicates per forecast unit instead individual duplicated rows? Default FALSE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"data.frame rows duplicate forecast found","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"","code":"example <- rbind(example_quantile, example_quantile[1000:1010]) get_duplicate_forecasts(example) #> location target_end_date target_type observed location_name forecast_date #> #> 1: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 2: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 3: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 4: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 5: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 6: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 7: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 8: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 9: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 10: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 11: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 12: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 13: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 14: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 15: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 16: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 17: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 18: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 19: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 20: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 21: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 22: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> location target_end_date target_type observed location_name forecast_date #> quantile_level predicted model horizon #> #> 1: 0.950 1464 epiforecasts-EpiNow2 1 #> 2: 0.950 1464 epiforecasts-EpiNow2 1 #> 3: 0.975 1642 epiforecasts-EpiNow2 1 #> 4: 0.975 1642 epiforecasts-EpiNow2 1 #> 5: 0.990 1951 epiforecasts-EpiNow2 1 #> 6: 0.990 1951 epiforecasts-EpiNow2 1 #> 7: 0.010 28999 EuroCOVIDhub-ensemble 3 #> 8: 0.010 28999 EuroCOVIDhub-ensemble 3 #> 9: 0.025 32612 EuroCOVIDhub-ensemble 3 #> 10: 0.025 32612 EuroCOVIDhub-ensemble 3 #> 11: 0.050 36068 EuroCOVIDhub-ensemble 3 #> 12: 0.050 36068 EuroCOVIDhub-ensemble 3 #> 13: 0.100 41484 EuroCOVIDhub-ensemble 3 #> 14: 0.100 41484 EuroCOVIDhub-ensemble 3 #> 15: 0.150 47110 EuroCOVIDhub-ensemble 3 #> 16: 0.150 47110 EuroCOVIDhub-ensemble 3 #> 17: 0.200 50929 EuroCOVIDhub-ensemble 3 #> 18: 0.200 50929 EuroCOVIDhub-ensemble 3 #> 19: 0.250 54561 EuroCOVIDhub-ensemble 3 #> 20: 0.250 54561 EuroCOVIDhub-ensemble 3 #> 21: 0.300 57739 EuroCOVIDhub-ensemble 3 #> 22: 0.300 57739 EuroCOVIDhub-ensemble 3 #> quantile_level predicted model horizon"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":null,"dir":"Reference","previous_headings":"","what":"Count number of available forecasts — get_forecast_counts","title":"Count number of available forecasts — get_forecast_counts","text":"Given data set forecasts, function counts number available forecasts. level grouping can specified using argument (e.g. count number forecasts per model, number forecasts per model location). useful determine whether missing forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Count number of available forecasts — get_forecast_counts","text":"","code":"get_forecast_counts( forecast, by = get_forecast_unit(forecast), collapse = c(\"quantile_level\", \"sample_id\") )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Count number of available forecasts — get_forecast_counts","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()). character vector NULL (default) denotes categories number forecasts counted. default unit single forecast (.e. available columns (apart \"protected\" columns 'predicted' 'observed') plus \"quantile_level\" \"sample_id\" present). collapse character vector (default: c(\"quantile_level\", \"sample_id\") names categories number rows collapsed one counting. example, single forecast usually represented set several quantiles samples collapsing one makes sure single forecast gets counted . Setting collapse = c() mean quantiles / samples counted individual forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Count number of available forecasts — get_forecast_counts","text":"data.table columns specified additional column \"count\" number forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Count number of available forecasts — get_forecast_counts","text":"","code":"library(magrittr) # pipe operator example_quantile %>% as_forecast_quantile() %>% get_forecast_counts(by = c(\"model\", \"target_type\")) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Key: #> model target_type count #> #> 1: EuroCOVIDhub-baseline Cases 128 #> 2: EuroCOVIDhub-baseline Deaths 128 #> 3: EuroCOVIDhub-ensemble Cases 128 #> 4: EuroCOVIDhub-ensemble Deaths 128 #> 5: UMass-MechBayes Cases 0 #> 6: UMass-MechBayes Deaths 128 #> 7: epiforecasts-EpiNow2 Cases 128 #> 8: epiforecasts-EpiNow2 Deaths 119"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_type.html","id":null,"dir":"Reference","previous_headings":"","what":"Get forecast type from forecast object — get_forecast_type","title":"Get forecast type from forecast object — get_forecast_type","text":"Get forecast type forecast object","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_type.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get forecast type from forecast object — get_forecast_type","text":"","code":"get_forecast_type(forecast)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_type.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get forecast type from forecast object — get_forecast_type","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_type.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get forecast type from forecast object — get_forecast_type","text":"Character vector length one forecast type.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":null,"dir":"Reference","previous_headings":"","what":"Get unit of a single forecast — get_forecast_unit","title":"Get unit of a single forecast — get_forecast_unit","text":"Helper function get unit single forecast, .e. column names define single forecast made . just takes columns available data subtracts columns protected, .e. returned get_protected_columns() well names metrics specified scoring, .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get unit of a single forecast — get_forecast_unit","text":"","code":"get_forecast_unit(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get unit of a single forecast — get_forecast_unit","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get unit of a single forecast — get_forecast_unit","text":"character vector column names define unit single forecast","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":"forecast-unit","dir":"Reference","previous_headings":"","what":"Forecast unit","title":"Get unit of a single forecast — get_forecast_unit","text":"order score forecasts, scoringutils needs know rows data belong together jointly form single forecasts. easy e.g. point forecast, one row per forecast. quantile sample-based forecasts, however, multiple rows belong single forecast. forecast unit unit single forecast described combination columns uniquely identify single forecast. example, forecasts made different models various locations different time points, several weeks future. forecast unit described forecast_unit = c(\"model\", \"location\", \"forecast_date\", \"forecast_horizon\"). scoringutils automatically tries determine unit single forecast. uses existing columns , means columns must present unrelated forecast unit. simplistic example, additional row, \"even\", one row number even zero otherwise, mess scoring scoringutils thinks column relevant defining forecast unit. order avoid issues, recommend setting forecast unit explicitly, usually forecast_unit argument as_forecast() functions. drop unneeded columns, making sure necessary, 'protected columns' like \"predicted\" \"observed\" retained.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Get default metrics for binary forecasts — get_metrics.forecast_binary","title":"Get default metrics for binary forecasts — get_metrics.forecast_binary","text":"binary forecasts, default scoring rules : \"brier_score\" = brier_score() \"log_score\" = logs_binary()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get default metrics for binary forecasts — get_metrics.forecast_binary","text":"","code":"# S3 method for class 'forecast_binary' get_metrics(x, select = NULL, exclude = NULL, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get default metrics for binary forecasts — get_metrics.forecast_binary","text":"x forecast object (validated data.table predicted observed values, see as_forecast()). select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored. ... unused","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get default metrics for binary forecasts — get_metrics.forecast_binary","text":"list scoring functions.","code":""},{"path":[]},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_binary.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Get default metrics for binary forecasts — get_metrics.forecast_binary","text":"","code":"get_metrics(example_binary) #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> #> #> #> $log_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> logs <- -log(1 - abs(observed - predicted)) #> return(logs) #> } #> #> #> get_metrics(example_binary, select = \"brier_score\") #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> #> #> get_metrics(example_binary, exclude = \"log_score\") #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> #> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_nominal.html","id":null,"dir":"Reference","previous_headings":"","what":"Get default metrics for nominal forecasts — get_metrics.forecast_nominal","title":"Get default metrics for nominal forecasts — get_metrics.forecast_nominal","text":"nominal forecasts, default scoring rule : \"log_score\" = logs_nominal()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_nominal.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get default metrics for nominal forecasts — get_metrics.forecast_nominal","text":"","code":"# S3 method for class 'forecast_nominal' get_metrics(x, select = NULL, exclude = NULL, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_nominal.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get default metrics for nominal forecasts — get_metrics.forecast_nominal","text":"x forecast object (validated data.table predicted observed values, see as_forecast()). select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored. ... unused","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_nominal.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Get default metrics for nominal forecasts — get_metrics.forecast_nominal","text":"","code":"get_metrics(example_nominal) #> $log_score #> function (observed, predicted, predicted_label) #> { #> assert_input_nominal(observed, predicted, predicted_label) #> n <- length(observed) #> if (n == 1) { #> predicted <- matrix(predicted, nrow = 1) #> } #> observed_indices <- as.numeric(observed) #> pred_for_observed <- predicted[cbind(1:n, observed_indices)] #> logs <- -log(pred_for_observed) #> return(logs) #> } #> #> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Get default metrics for point forecasts — get_metrics.forecast_point","title":"Get default metrics for point forecasts — get_metrics.forecast_point","text":"point forecasts, default scoring rules : \"ae_point\" = ae() \"se_point\" = se() \"ape\" = ape() note caution: Every scoring rule point forecast implicitly minimised specific aspect predictive distribution (see Gneiting, 2011). mean squared error, example, meaningful scoring rule forecaster actually reported mean predictive distribution point forecast. forecaster reported median, mean absolute error appropriate scoring rule. scoring rule predictive task align, results misleading. Failure respect correspondence can lead grossly misleading results! Consider example section .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get default metrics for point forecasts — get_metrics.forecast_point","text":"","code":"# S3 method for class 'forecast_point' get_metrics(x, select = NULL, exclude = NULL, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get default metrics for point forecasts — get_metrics.forecast_point","text":"x forecast object (validated data.table predicted observed values, see as_forecast()). select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored. ... unused","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_point.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Get default metrics for point forecasts — get_metrics.forecast_point","text":"Making Evaluating Point Forecasts, Gneiting, Tilmann, 2011, Journal American Statistical Association.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_point.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Get default metrics for point forecasts — get_metrics.forecast_point","text":"","code":"get_metrics(example_point, select = \"ape\") #> $ape #> function (actual, predicted) #> { #> return(ae(actual, predicted)/abs(actual)) #> } #> #> #> library(magrittr) set.seed(123) n <- 500 observed <- rnorm(n, 5, 4)^2 predicted_mu <- mean(observed) predicted_not_mu <- predicted_mu - rnorm(n, 10, 2) df <- data.frame( model = rep(c(\"perfect\", \"bad\"), each = n), predicted = c(rep(predicted_mu, n), predicted_not_mu), observed = rep(observed, 2), id = rep(1:n, 2) ) %>% as_forecast_point() score(df) %>% summarise_scores() #> model ae_point se_point ape #> #> 1: perfect 34.64686 2145.813 3543.184 #> 2: bad 32.34199 2238.566 2692.868"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Get default metrics for quantile-based forecasts — get_metrics.forecast_quantile","title":"Get default metrics for quantile-based forecasts — get_metrics.forecast_quantile","text":"quantile-based forecasts, default scoring rules : \"wis\" = wis() \"overprediction\" = overprediction_quantile() \"underprediction\" = underprediction_quantile() \"dispersion\" = dispersion_quantile() \"bias\" = bias_quantile() \"interval_coverage_50\" = interval_coverage() \"interval_coverage_90\" = purrr::partial( interval_coverage, interval_range = 90 ) \"ae_median\" = ae_median_quantile() Note: interval_coverage_90 scoring rule created modifying interval_coverage(), making use function purrr::partial(). construct allows function deal arbitrary arguments ..., making sure interval_coverage() can accept get passed . interval_range = 90 set function definition, passing argument interval_range = 90 score() mean also get passed interval_coverage_50.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get default metrics for quantile-based forecasts — get_metrics.forecast_quantile","text":"","code":"# S3 method for class 'forecast_quantile' get_metrics(x, select = NULL, exclude = NULL, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get default metrics for quantile-based forecasts — get_metrics.forecast_quantile","text":"x forecast object (validated data.table predicted observed values, see as_forecast()). select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored. ... unused","code":""},{"path":[]},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_quantile.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Get default metrics for quantile-based forecasts — get_metrics.forecast_quantile","text":"","code":"get_metrics(example_quantile, select = \"wis\") #> $wis #> function (observed, predicted, quantile_level, separate_results = FALSE, #> weigh = TRUE, count_median_twice = FALSE, na.rm = FALSE) #> { #> assert_input_quantile(observed, predicted, quantile_level) #> reformatted <- quantile_to_interval(observed, predicted, #> quantile_level) #> interval_ranges <- get_range_from_quantile(quantile_level[quantile_level != #> 0.5]) #> complete_intervals <- duplicated(interval_ranges) | duplicated(interval_ranges, #> fromLast = TRUE) #> if (!all(complete_intervals) && !isTRUE(na.rm)) { #> incomplete <- quantile_level[quantile_level != 0.5][!complete_intervals] #> cli_abort(c(`!` = \"Not all quantile levels specified form symmetric prediction\\n intervals.\\n The following quantile levels miss a corresponding lower/upper bound:\\n {.val {incomplete}}.\\n You can drop incomplete prediction intervals using `na.rm = TRUE`.\")) #> } #> assert_logical(separate_results, len = 1) #> assert_logical(weigh, len = 1) #> assert_logical(count_median_twice, len = 1) #> assert_logical(na.rm, len = 1) #> if (separate_results) { #> cols <- c(\"wis\", \"dispersion\", \"underprediction\", \"overprediction\") #> } #> else { #> cols <- \"wis\" #> } #> reformatted[, `:=`(eval(cols), do.call(interval_score, list(observed = observed, #> lower = lower, upper = upper, interval_range = interval_range, #> weigh = weigh, separate_results = separate_results)))] #> if (count_median_twice) { #> reformatted[, `:=`(weight, 1)] #> } #> else { #> reformatted[, `:=`(weight, ifelse(interval_range == 0, #> 0.5, 1))] #> } #> reformatted <- reformatted[, lapply(.SD, weighted.mean, na.rm = na.rm, #> w = weight), by = \"forecast_id\", .SDcols = colnames(reformatted) %like% #> paste(cols, collapse = \"|\")] #> if (separate_results) { #> return(list(wis = reformatted$wis, dispersion = reformatted$dispersion, #> underprediction = reformatted$underprediction, overprediction = reformatted$overprediction)) #> } #> else { #> return(reformatted$wis) #> } #> } #> #> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Get default metrics for sample-based forecasts — get_metrics.forecast_sample","title":"Get default metrics for sample-based forecasts — get_metrics.forecast_sample","text":"sample-based forecasts, default scoring rules : \"crps\" = crps_sample() \"overprediction\" = overprediction_sample() \"underprediction\" = underprediction_sample() \"dispersion\" = dispersion_sample() \"log_score\" = logs_sample() \"dss\" = dss_sample() \"mad\" = mad_sample() \"bias\" = bias_sample() \"ae_median\" = ae_median_sample() \"se_mean\" = se_mean_sample()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get default metrics for sample-based forecasts — get_metrics.forecast_sample","text":"","code":"# S3 method for class 'forecast_sample' get_metrics(x, select = NULL, exclude = NULL, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get default metrics for sample-based forecasts — get_metrics.forecast_sample","text":"x forecast object (validated data.table predicted observed values, see as_forecast()). select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored. ... unused","code":""},{"path":[]},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Get default metrics for sample-based forecasts — get_metrics.forecast_sample","text":"","code":"get_metrics(example_sample_continuous, exclude = \"mad\") #> $bias #> function (observed, predicted) #> { #> assert_input_sample(observed, predicted) #> prediction_type <- get_type(predicted) #> n_pred <- ncol(predicted) #> p_x <- rowSums(predicted <= observed)/n_pred #> if (prediction_type == \"continuous\") { #> res <- 1 - 2 * p_x #> return(res) #> } #> else { #> p_xm1 <- rowSums(predicted <= (observed - 1))/n_pred #> res <- 1 - (p_x + p_xm1) #> return(res) #> } #> } #> #> #> #> $dss #> function (observed, predicted, ...) #> { #> assert_input_sample(observed, predicted) #> scoringRules::dss_sample(y = observed, dat = predicted, ...) #> } #> #> #> #> $crps #> function (observed, predicted, separate_results = FALSE, ...) #> { #> assert_input_sample(observed, predicted) #> crps <- scoringRules::crps_sample(y = observed, dat = predicted, #> ...) #> if (separate_results) { #> medians <- apply(predicted, 1, median) #> dispersion <- scoringRules::crps_sample(y = medians, #> dat = predicted, ...) #> overprediction <- rep(0, length(observed)) #> underprediction <- rep(0, length(observed)) #> overprediction[observed < medians] <- scoringRules::crps_sample(y = observed[observed < #> medians], dat = predicted[observed < medians, , drop = FALSE], #> ...) #> underprediction[observed > medians] <- scoringRules::crps_sample(y = observed[observed > #> medians], dat = predicted[observed > medians, , drop = FALSE], #> ...) #> overprediction[overprediction > 0] <- overprediction[overprediction > #> 0] - dispersion[overprediction > 0] #> underprediction[underprediction > 0] <- underprediction[underprediction > #> 0] - dispersion[underprediction > 0] #> return(list(crps = crps, dispersion = dispersion, underprediction = underprediction, #> overprediction = overprediction)) #> } #> else { #> return(crps) #> } #> } #> #> #> #> $overprediction #> function (observed, predicted, ...) #> { #> crps <- crps_sample(observed, predicted, separate_results = TRUE, #> ...) #> return(crps$overprediction) #> } #> #> #> #> $underprediction #> function (observed, predicted, ...) #> { #> crps <- crps_sample(observed, predicted, separate_results = TRUE, #> ...) #> return(crps$underprediction) #> } #> #> #> #> $dispersion #> function (observed, predicted, ...) #> { #> crps <- crps_sample(observed, predicted, separate_results = TRUE, #> ...) #> return(crps$dispersion) #> } #> #> #> #> $log_score #> function (observed, predicted, ...) #> { #> assert_input_sample(observed, predicted) #> scoringRules::logs_sample(y = observed, dat = predicted, #> ...) #> } #> #> #> #> $ae_median #> function (observed, predicted) #> { #> assert_input_sample(observed, predicted) #> median_predictions <- apply(as.matrix(predicted), MARGIN = 1, #> FUN = median) #> ae_median <- abs(observed - median_predictions) #> return(ae_median) #> } #> #> #> #> $se_mean #> function (observed, predicted) #> { #> assert_input_sample(observed, predicted) #> mean_predictions <- rowMeans(as.matrix(predicted)) #> se_mean <- (observed - mean_predictions)^2 #> return(se_mean) #> } #> #> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Get metrics — get_metrics","title":"Get metrics — get_metrics","text":"Generic function obtain default metrics availble scoring metrics used scoring. called forecast object returns list functions can used scoring. called scores object (see score()), returns character vector names metrics used scoring. See documentation actual methods See Also section details. Alternatively call ?get_metrics. ?get_metrics.scores.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get metrics — get_metrics","text":"","code":"get_metrics(x, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get metrics — get_metrics","text":"x forecast scores object. ... Additional arguments passed method.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Get metrics — get_metrics","text":"See as_forecast() information forecast objects score() information scores objects.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Get names of the metrics that were used for scoring — get_metrics.scores","title":"Get names of the metrics that were used for scoring — get_metrics.scores","text":"applying scoring rule via score(), names scoring rules become column names resulting data.table. addition, attribute metrics added output, holding names scores vector. done functions like get_forecast_unit() summarise_scores() can still identify columns part forecast unit hold score. get_metrics() accesses returns metrics attribute. attribute, function return NULL (, error = TRUE produce error instead). addition, checks column names input consistency data stored metrics attribute. Handling missing inconsistent metrics attribute: metrics attribute missing consistent column names data.table, can either run score() , specifying names scoring rules manually, add/update attribute manually using attr(scores, \"metrics\") <- c(\"names\", \"\", \"\", \"scores\") (order matter).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get names of the metrics that were used for scoring — get_metrics.scores","text":"","code":"# S3 method for class 'scores' get_metrics(x, error = FALSE, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get names of the metrics that were used for scoring — get_metrics.scores","text":"x scores object, (data.table attribute metrics produced score()). error Throw error attribute called metrics? Default FALSE. ... unused","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.scores.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get names of the metrics that were used for scoring — get_metrics.scores","text":"Character vector names scoring rules used scoring.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":null,"dir":"Reference","previous_headings":"","what":"Obtain pairwise comparisons between models — get_pairwise_comparisons","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"Compare scores obtained different models pairwise tournament. combinations two models compared based overlapping set available forecasts common models. input scores object produced score(). Note adding additional unrelated columns can unpredictably change results, present columns taken account determining set overlapping forecasts two models. output pairwise comparisons set mean score ratios, relative skill scores p-values. following illustrates pairwise comparison process: Mean score ratios every pair two models, mean score ratio computed. simply mean score first model divided mean score second. Mean score ratios computed based set overlapping forecasts two models. means scores targets taken account models submitted forecast. (Scaled) Relative skill scores relative score model geometric mean mean score ratios involve model. baseline provided, scaled relative skill scores calculated well. Scaled relative skill scores simply relative skill score model divided relative skill score baseline model. p-values addition, function computes p-values comparison two models (based set overlapping forecasts). P-values can computed two ways: based nonparametric Wilcoxon signed-rank test (internally using wilcox.test() paired = TRUE) based permutation test. permutation test based difference mean scores two models. default null hypothesis mean score difference zero (see permutation_test()). Adjusted p-values computed calling p.adjust() raw p-values. code pairwise comparisons inspired implementation Johannes Bracher. implementation permutation test follows function permutationTest surveillance package Michael Höhle, Andrea Riebler Michaela Paul.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"","code":"get_pairwise_comparisons( scores, compare = \"model\", by = NULL, metric = intersect(c(\"wis\", \"crps\", \"brier_score\"), names(scores)), baseline = NULL, ... )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"scores object class scores (data.table scores additional attribute metrics produced score()). compare Character vector single colum name defines elements pairwise comparison. example, set \"model\" (default), elements \"model\" column compared. Character vector column names define grouping levels pairwise comparisons. default NULL one relative skill score per distinct entry column selected compare. columns given , example, = \"location\" compare = \"model\", one separate relative skill score calculated every model every location. metric string name metric relative skill shall computed. default either \"crps\", \"wis\" \"brier_score\" available. baseline string name model. baseline given, scaled relative skill respect baseline returned. default (NULL), relative skill scaled respect baseline model. ... Additional arguments comparison two models. See compare_forecasts() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"data.table results pairwise comparisons containing mean score ratios (mean_scores_ratio), unadjusted (pval) adjusted (adj_pval) p-values, relative skill values model (..._relative_skill). baseline model given scaled relative skill reported well (..._scaled_relative_skill).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"Nikos Bosse nikosbosse@gmail.com Johannes Bracher, johannes.bracher@kit.edu","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"","code":"library(magrittr) # pipe operator scores <- example_quantile %>% as_forecast_quantile() %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. pairwise <- get_pairwise_comparisons(scores, by = \"target_type\") pairwise2 <- get_pairwise_comparisons( scores, by = \"target_type\", baseline = \"EuroCOVIDhub-baseline\" ) library(ggplot2) plot_pairwise_comparisons(pairwise, type = \"mean_scores_ratio\") + facet_wrap(~target_type)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit.html","id":null,"dir":"Reference","previous_headings":"","what":"Probability integral transformation (data.frame version) — get_pit.forecast_quantile","title":"Probability integral transformation (data.frame version) — get_pit.forecast_quantile","text":"Compute Probability Integral Transformation (PIT) validated forecast objects.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Probability integral transformation (data.frame version) — get_pit.forecast_quantile","text":"","code":"# S3 method for class 'forecast_quantile' get_pit(forecast, by, ...) # S3 method for class 'forecast_sample' get_pit(forecast, by, n_replicates = 100, ...) get_pit(forecast, by, ...) # Default S3 method get_pit(forecast, by, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Probability integral transformation (data.frame version) — get_pit.forecast_quantile","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()). Character vector columns according PIT values shall grouped. e.g. columns 'model' 'location' input data want PIT histogram every model location, specify = c(\"model\", \"location\"). ... Currently unused. pass additional arguments scoring functions via .... See Customising metrics section details use purrr::partial() pass arguments individual metrics. n_replicates number draws randomised PIT discrete predictions. ignored forecasts continuous.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Probability integral transformation (data.frame version) — get_pit.forecast_quantile","text":"data.table PIT values according grouping specified .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Probability integral transformation (data.frame version) — get_pit.forecast_quantile","text":"Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind M. Eggo, W. John Edmunds (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15, doi:10.1371/journal.pcbi.1006785","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Probability integral transformation (data.frame version) — get_pit.forecast_quantile","text":"","code":"example <- as_forecast_sample(example_sample_continuous) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. result <- get_pit(example, by = \"model\") plot_pit(result) # example with quantile data example <- as_forecast_quantile(example_quantile) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. result <- get_pit(example, by = \"model\") plot_pit(result)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_protected_columns.html","id":null,"dir":"Reference","previous_headings":"","what":"Get protected columns from data — get_protected_columns","title":"Get protected columns from data — get_protected_columns","text":"Helper function get names columns data frame protected columns.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_protected_columns.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get protected columns from data — get_protected_columns","text":"","code":"get_protected_columns(data = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_protected_columns.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get protected columns from data — get_protected_columns","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_protected_columns.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get protected columns from data — get_protected_columns","text":"character vector names protected columns data. data NULL (default) returns list columns protected scoringutils.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_range_from_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Get interval range belonging to a quantile — get_range_from_quantile","title":"Get interval range belonging to a quantile — get_range_from_quantile","text":"Every quantile can thought either lower upper bound symmetric central prediction interval. helper function returns range central prediction interval quantile belongs. Due numeric instability sometimes occurred past, ranges rounded 10 decimal places. problem vast majority use cases, something aware .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_range_from_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get interval range belonging to a quantile — get_range_from_quantile","text":"","code":"get_range_from_quantile(quantile_level)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_range_from_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get interval range belonging to a quantile — get_range_from_quantile","text":"quantile_level numeric vector quantile levels size N.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_range_from_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get interval range belonging to a quantile — get_range_from_quantile","text":"numeric vector interval ranges size N","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_type.html","id":null,"dir":"Reference","previous_headings":"","what":"Get type of a vector or matrix of observed values or predictions — get_type","title":"Get type of a vector or matrix of observed values or predictions — get_type","text":"Internal helper function get type vector (usually observed predicted values). function checks whether input factor, else whether integer (can coerced integer) whether continuous.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_type.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get type of a vector or matrix of observed values or predictions — get_type","text":"","code":"get_type(x)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_type.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get type of a vector or matrix of observed values or predictions — get_type","text":"x Input type determined .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_type.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get type of a vector or matrix of observed values or predictions — get_type","text":"Character vector length one either \"classification\", \"integer\", \"continuous\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/illustration-input-metric-binary-point.html","id":null,"dir":"Reference","previous_headings":"","what":"Illustration of required inputs for binary and point forecasts — illustration-input-metric-binary-point","title":"Illustration of required inputs for binary and point forecasts — illustration-input-metric-binary-point","text":"Illustration required inputs binary point forecasts","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/illustration-input-metric-quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Illustration of required inputs for quantile-based forecasts — illustration-input-metric-quantile","title":"Illustration of required inputs for quantile-based forecasts — illustration-input-metric-quantile","text":"Illustration required inputs quantile-based forecasts","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/illustration-input-metric-sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Illustration of required inputs for sample-based forecasts — illustration-input-metric-sample","title":"Illustration of required inputs for sample-based forecasts — illustration-input-metric-sample","text":"Illustration required inputs sample-based forecasts","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interpolate_median.html","id":null,"dir":"Reference","previous_headings":"","what":"Helper function to interpolate the median prediction if it is not available — interpolate_median","title":"Helper function to interpolate the median prediction if it is not available — interpolate_median","text":"Internal function interpolate median prediction available given quantile levels. done using linear interpolation two innermost quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interpolate_median.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Helper function to interpolate the median prediction if it is not available — interpolate_median","text":"","code":"interpolate_median(predicted, quantile_level)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interpolate_median.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Helper function to interpolate the median prediction if it is not available — interpolate_median","text":"predicted Vector length N (corresponding number quantiles) holds predictions. quantile_level Vector size N quantile levels predictions made. Note contain median (0.5) median imputed mean two innermost quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interpolate_median.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Helper function to interpolate the median prediction if it is not available — interpolate_median","text":"scalar imputed median prediction","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Interval coverage (for quantile-based forecasts) — interval_coverage","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"Check whether observed value within given central prediction interval. prediction interval defined lower upper bound formed pair predictive quantiles. example, 50% prediction interval formed 0.25 0.75 quantiles predictive distribution.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"","code":"interval_coverage(observed, predicted, quantile_level, interval_range = 50)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made. interval_range single number range prediction interval percent (e.g. 50 50% prediction interval) want compute interval coverage.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"vector length n elements either TRUE, observed value within corresponding prediction interval, FALSE otherwise.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"","code":"observed <- c(1, -15, 22) predicted <- rbind( c(-1, 0, 1, 2, 3), c(-2, 1, 2, 2, 4), c(-2, 0, 3, 3, 4) ) quantile_level <- c(0.1, 0.25, 0.5, 0.75, 0.9) interval_coverage(observed, predicted, quantile_level) #> [1] TRUE FALSE FALSE"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":null,"dir":"Reference","previous_headings":"","what":"Interval score — interval_score","title":"Interval score — interval_score","text":"Proper Scoring Rule score quantile predictions, following Gneiting Raftery (2007). Smaller values better. score computed $$ \\textrm{score} = (\\textrm{upper} - \\textrm{lower}) + \\frac{2}{\\alpha}(\\textrm{lower} - \\textrm{observed}) * \\mathbf{1}(\\textrm{observed} < \\textrm{lower}) + \\frac{2}{\\alpha}(\\textrm{observed} - \\textrm{upper}) * \\mathbf{1}(\\textrm{observed} > \\textrm{upper}) $$ \\(\\mathbf{1}()\\) indicator function indicates much outside prediction interval. \\(\\alpha\\) decimal value indicates much outside prediction interval. improve usability, user asked provide interval range percentage terms, .e. interval_range = 90 (percent) 90 percent prediction interval. Correspondingly, user provide 5% 95% quantiles (corresponding alpha 0.1). specific distribution assumed, interval symmetric around median (.e use 0.1 quantile lower bound 0.7 quantile upper bound). Non-symmetric quantiles can scored using function quantile_score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Interval score — interval_score","text":"","code":"interval_score( observed, lower, upper, interval_range, weigh = TRUE, separate_results = FALSE )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Interval score — interval_score","text":"observed vector observed values size n lower Vector size n prediction lower quantile given interval range. upper Vector size n prediction upper quantile given interval range. interval_range Numeric vector (either single number vector size n) range prediction intervals. example, forecasting 0.05 0.95 quantile, interval range 90. interval range corresponds \\((100-\\alpha)/100\\), \\(\\alpha\\) decimal value indicates much outside prediction interval (see e.g. Gneiting Raftery (2007)). weigh Logical. TRUE (default), weigh score \\(\\alpha / 2\\), can averaged interval score , limit (increasing number equally spaced quantiles/prediction intervals), corresponds CRPS. \\(\\alpha\\) value corresponds (\\(\\alpha/2\\)) (\\(1 - \\alpha/2\\)), .e. decimal value represents much outside central prediction interval (E.g. 90 percent central prediction interval, alpha 0.1). separate_results Logical. TRUE (default FALSE), separate parts interval score (dispersion penalty, penalties - -prediction get returned separate elements list). want data.frame instead, simply call .data.frame() output.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Interval score — interval_score","text":"Vector scoring values, list separate entries separate_results TRUE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Interval score — interval_score","text":"Strictly Proper Scoring Rules, Prediction,Estimation, Tilmann Gneiting Adrian E. Raftery, 2007, Journal American Statistical Association, Volume 102, 2007 - Issue 477 Evaluating epidemic forecasts interval format, Johannes Bracher, Evan L. Ray, Tilmann Gneiting Nicholas G. Reich, https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1008618 # nolint","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Interval score — interval_score","text":"","code":"observed <- rnorm(30, mean = 1:30) interval_range <- rep(90, 30) alpha <- (100 - interval_range) / 100 lower <- qnorm(alpha / 2, rnorm(30, mean = 1:30)) upper <- qnorm((1 - alpha / 2), rnorm(30, mean = 11:40)) scoringutils:::interval_score( observed = observed, lower = lower, upper = upper, interval_range = interval_range ) #> [1] 0.7984288 0.6188294 0.7680146 0.7392137 2.4496739 0.6725337 0.5966822 #> [8] 0.6659581 0.6581834 1.2866256 0.7423596 0.6155816 0.6176548 0.6484069 #> [15] 0.6584252 0.6929587 0.7558694 0.6724925 0.5716195 0.6251716 0.7431658 #> [22] 0.6906088 0.6524924 0.6274762 1.8011311 0.5840689 0.6602319 0.7076748 #> [29] 0.6060337 0.5491802 # gives a warning, as the interval_range should likely be 50 instead of 0.5 scoringutils:::interval_score( observed = 4, upper = 8, lower = 2, interval_range = 0.5 ) #> Warning: ! Found interval ranges between 0 and 1. Are you sure that's right? An interval #> range of 0.5 e.g. implies a (49.75%, 50.25%) prediction interval. #> ℹ If you want to score a (25%, 75%) prediction interval, set `interval_range = #> 50`. #> This warning is displayed once per session. #> [1] 2.985 # example with missing values and separate results scoringutils:::interval_score( observed = c(observed, NA), lower = c(lower, NA), upper = c(NA, upper), separate_results = TRUE, interval_range = 90 ) #> $interval_score #> [1] NA 0.6735755 0.6315584 0.6931931 2.4596254 0.6632059 0.5523431 #> [8] 0.5386115 0.6346291 1.1441141 0.7545579 0.6109357 0.6003346 0.5762499 #> [15] 0.5621607 0.6826498 0.6576112 0.6649209 0.5101806 0.6064191 0.5613882 #> [22] 0.7192299 0.6075918 0.6736010 1.6335484 0.6182397 0.6036268 0.5671816 #> [29] 0.6245858 0.5639638 NA #> #> $dispersion #> [1] NA 0.6735755 0.6315584 0.6931931 0.6195326 0.6632059 0.5523431 #> [8] 0.5386115 0.6346291 0.5198196 0.7545579 0.6109357 0.6003346 0.5762499 #> [15] 0.5621607 0.6826498 0.6576112 0.6649209 0.5101806 0.6064191 0.5613882 #> [22] 0.7192299 0.6075918 0.6736010 0.4719967 0.6182397 0.6036268 0.5671816 #> [29] 0.6245858 0.5639638 NA #> #> $underprediction #> [1] NA 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 #> [26] 0 0 0 0 0 NA #> #> $overprediction #> [1] 0.0000000 0.0000000 0.0000000 0.0000000 1.8400928 0.0000000 0.0000000 #> [8] 0.0000000 0.0000000 0.6242945 0.0000000 0.0000000 0.0000000 0.0000000 #> [15] 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 #> [22] 0.0000000 0.0000000 0.0000000 1.1615517 0.0000000 0.0000000 0.0000000 #> [29] 0.0000000 0.0000000 NA #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Test whether an object is a forecast object — is_forecast_binary","title":"Test whether an object is a forecast object — is_forecast_binary","text":"Test whether object forecast object (see as_forecast() information). can test specific forecast_* class using appropriate is_forecast_* function.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Test whether an object is a forecast object — is_forecast_binary","text":"","code":"is_forecast_binary(x) is_forecast_nominal(x) is_forecast_point(x) is_forecast_quantile(x) is_forecast_sample(x) is_forecast(x)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Test whether an object is a forecast object — is_forecast_binary","text":"x R object.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Test whether an object is a forecast object — is_forecast_binary","text":"is_forecast: TRUE object class forecast, FALSE otherwise. is_forecast_*: TRUE object class forecast_* addition class forecast, FALSE otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Test whether an object is a forecast object — is_forecast_binary","text":"","code":"forecast_binary <- as_forecast_binary(example_binary) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. is_forecast(forecast_binary) #> [1] TRUE"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":null,"dir":"Reference","previous_headings":"","what":"Log transformation with an additive shift — log_shift","title":"Log transformation with an additive shift — log_shift","text":"Function shifts value offset applies natural logarithm .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Log transformation with an additive shift — log_shift","text":"","code":"log_shift(x, offset = 0, base = exp(1))"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Log transformation with an additive shift — log_shift","text":"x vector input values transformed offset Number add input value taking natural logarithm. base positive number: base respect logarithms computed. Defaults e = exp(1).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Log transformation with an additive shift — log_shift","text":"numeric vector transformed values","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Log transformation with an additive shift — log_shift","text":"output computed log(x + offset)","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Log transformation with an additive shift — log_shift","text":"Transformation forecasts evaluating predictive performance epidemiological context Nikos . Bosse, Sam Abbott, Anne Cori, Edwin van Leeuwen, Johannes Bracher, Sebastian Funk medRxiv 2023.01.23.23284722 doi:10.1101/2023.01.23.23284722 https://www.medrxiv.org/content/10.1101/2023.01.23.23284722v1 # nolint","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Log transformation with an additive shift — log_shift","text":"","code":"library(magrittr) # pipe operator log_shift(1:10) #> [1] 0.0000000 0.6931472 1.0986123 1.3862944 1.6094379 1.7917595 1.9459101 #> [8] 2.0794415 2.1972246 2.3025851 log_shift(0:9, offset = 1) #> [1] 0.0000000 0.6931472 1.0986123 1.3862944 1.6094379 1.7917595 1.9459101 #> [8] 2.0794415 2.1972246 2.3025851 example_quantile[observed > 0, ] %>% as_forecast_quantile() %>% transform_forecasts(fun = log_shift, offset = 1) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Forecast type: quantile #> Forecast unit: #> location, target_end_date, target_type, location_name, forecast_date, model, #> horizon, and scale #> #> location target_end_date target_type observed location_name #> #> 1: DE 2021-01-02 Cases 1.273000e+05 Germany #> 2: DE 2021-01-02 Deaths 4.534000e+03 Germany #> 3: DE 2021-01-09 Cases 1.549220e+05 Germany #> 4: DE 2021-01-09 Deaths 6.117000e+03 Germany #> 5: DE 2021-01-16 Cases 1.101830e+05 Germany #> --- #> 40672: IT 2021-07-24 Deaths 4.369448e+00 Italy #> 40673: IT 2021-07-24 Deaths 4.369448e+00 Italy #> 40674: IT 2021-07-24 Deaths 4.369448e+00 Italy #> 40675: IT 2021-07-24 Deaths 4.369448e+00 Italy #> 40676: IT 2021-07-24 Deaths 4.369448e+00 Italy #> forecast_date quantile_level predicted model horizon #> #> 1: NA NA NA #> 2: NA NA NA #> 3: NA NA NA #> 4: NA NA NA #> 5: NA NA NA #> --- #> 40672: 2021-07-12 0.850 5.866468 epiforecasts-EpiNow2 2 #> 40673: 2021-07-12 0.900 5.986452 epiforecasts-EpiNow2 2 #> 40674: 2021-07-12 0.950 6.214608 epiforecasts-EpiNow2 2 #> 40675: 2021-07-12 0.975 6.416732 epiforecasts-EpiNow2 2 #> 40676: 2021-07-12 0.990 6.579251 epiforecasts-EpiNow2 2 #> scale #> #> 1: natural #> 2: natural #> 3: natural #> 4: natural #> 5: natural #> --- #> 40672: log #> 40673: log #> 40674: log #> 40675: log #> 40676: log"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Logarithmic score (sample-based version) — logs_sample","title":"Logarithmic score (sample-based version) — logs_sample","text":"function wrapper around logs_sample() function scoringRules package. log score negative logarithm predictive density evaluated observed value. function used score continuous predictions . Log Score theory also applicable discrete forecasts, problem lies implementation: function uses kernel density estimation, well defined integer-valued Monte Carlo Samples. See scoringRules package details alternatives, e.g. calculating scores specific discrete probability distributions.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Logarithmic score (sample-based version) — logs_sample","text":"","code":"logs_sample(observed, predicted, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Logarithmic score (sample-based version) — logs_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. ... Additional arguments passed logs_sample() scoringRules package.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Logarithmic score (sample-based version) — logs_sample","text":"Vector scores.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Logarithmic score (sample-based version) — logs_sample","text":"Alexander Jordan, Fabian Krüger, Sebastian Lerch, Evaluating Probabilistic Forecasts scoringRules, https://www.jstatsoft.org/article/view/v090i12","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Logarithmic score (sample-based version) — logs_sample","text":"","code":"observed <- rpois(30, lambda = 1:30) predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) logs_sample(observed, predicted) #> [1] 2.415978 1.391162 2.876486 1.778651 1.990756 3.187023 2.735783 2.308400 #> [9] 2.217484 2.403781 2.688615 2.813455 2.178733 2.080475 2.668287 2.773460 #> [17] 2.619100 2.600304 2.610290 3.177565 3.065729 2.454865 2.860266 2.676266 #> [25] 3.299807 2.918963 2.538826 2.698623 2.858104 2.869216"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Determine dispersion of a probabilistic forecast — mad_sample","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"Sharpness ability model generate predictions within narrow range dispersion lack thereof. data-independent measure, purely feature forecasts . Dispersion predictive samples corresponding one single observed value measured normalised median absolute deviation median predictive samples. details, see mad() explanations given Funk et al. (2019)","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"","code":"mad_sample(observed = NULL, predicted, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"observed Place holder, argument ignored exists consistency scoring functions. output depend observed values. predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. ... Additional arguments passed mad().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"Vector dispersion values.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"Funk S, Camacho , Kucharski AJ, Lowe R, Eggo RM, Edmunds WJ (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15. PLoS Comput Biol 15(2): e1006785. doi:10.1371/journal.pcbi.1006785","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"","code":"predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) mad_sample(predicted = predicted) #> [1] 1.4826 1.4826 1.4826 1.4826 2.9652 2.9652 2.9652 2.9652 2.9652 2.9652 #> [11] 2.9652 2.9652 2.9652 4.4478 4.4478 3.7065 4.4478 3.7065 4.4478 4.4478 #> [21] 4.4478 4.4478 4.4478 4.4478 5.9304 5.9304 5.9304 5.9304 5.9304 5.9304"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Class constructor for forecast objects — new_forecast","title":"Class constructor for forecast objects — new_forecast","text":"Construct class based data.frame similar. constructor coerces data data.table assigns class","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Class constructor for forecast objects — new_forecast","text":"","code":"new_forecast(data, classname)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Class constructor for forecast objects — new_forecast","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. classname name class created","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Class constructor for forecast objects — new_forecast","text":"object class indicated classname","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Construct an object of class scores — new_scores","title":"Construct an object of class scores — new_scores","text":"function creates object class scores based data.table similar.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Construct an object of class scores — new_scores","text":"","code":"new_scores(scores, metrics, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Construct an object of class scores — new_scores","text":"scores data.table similar scores produced score(). metrics character vector names scores (.e. names scoring rules used scoring). ... Additional arguments data.table::.data.table()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Construct an object of class scores — new_scores","text":"object class scores","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Construct an object of class scores — new_scores","text":"","code":"if (FALSE) { # \\dontrun{ df <- data.frame( model = \"A\", wis = \"0.1\" ) new_scores(df, \"wis\") } # }"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pairwise_comparison_one_group.html","id":null,"dir":"Reference","previous_headings":"","what":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","title":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","text":"function pairwise comparison one set forecasts, multiple models involved. gets called get_pairwise_comparisons(). get_pairwise_comparisons() splits data arbitrary subgroups specified user (e.g. pairwise comparison done separately different forecast targets) actual pairwise comparison subgroup managed pairwise_comparison_one_group(). order actually comparison two models subset common forecasts calls compare_forecasts().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pairwise_comparison_one_group.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","text":"","code":"pairwise_comparison_one_group( scores, metric, baseline, compare = \"model\", by, ... )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pairwise_comparison_one_group.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","text":"scores object class scores (data.table scores additional attribute metrics produced score()). metric string name metric relative skill shall computed. default either \"crps\", \"wis\" \"brier_score\" available. baseline string name model. baseline given, scaled relative skill respect baseline returned. default (NULL), relative skill scaled respect baseline model. compare Character vector single colum name defines elements pairwise comparison. example, set \"model\" (default), elements \"model\" column compared. Character vector column names define grouping levels pairwise comparisons. default NULL one relative skill score per distinct entry column selected compare. columns given , example, = \"location\" compare = \"model\", one separate relative skill score calculated every model every location. ... Additional arguments comparison two models. See compare_forecasts() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pairwise_comparison_one_group.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","text":"data.table results pairwise comparisons containing mean score ratios (mean_scores_ratio), unadjusted (pval) adjusted (adj_pval) p-values, relative skill values model (..._relative_skill). baseline model given scaled relative skill reported well (..._scaled_relative_skill).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/permutation_test.html","id":null,"dir":"Reference","previous_headings":"","what":"Simple permutation test — permutation_test","title":"Simple permutation test — permutation_test","text":"implementation permutation test follows function permutationTest surveillance package Michael Höhle, Andrea Riebler Michaela Paul. function compares two vectors scores. computes mean vector independently takes either difference ratio two. observed difference ratio compared test statistic based permutations original data. Used get_pairwise_comparisons().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/permutation_test.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Simple permutation test — permutation_test","text":"","code":"permutation_test( scores1, scores2, n_permutation = 999, one_sided = FALSE, comparison_mode = c(\"difference\", \"ratio\") )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/permutation_test.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Simple permutation test — permutation_test","text":"scores1 Vector scores compare another vector scores. scores2 second vector scores compare first n_permutation number replications use permutation test. replications yield exact results, require computation. one_sided Whether compute one-sided test. Default FALSE. comparison_mode compute test statistic comparison two scores. either \"difference\" \"ratio\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/permutation_test.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Simple permutation test — permutation_test","text":"p-value permutation test","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Probability integral transformation for counts — pit_sample","title":"Probability integral transformation for counts — pit_sample","text":"Uses Probability integral transformation (PIT) (randomised PIT integer forecasts) assess calibration predictive Monte Carlo samples.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Probability integral transformation for counts — pit_sample","text":"","code":"pit_sample(observed, predicted, n_replicates = 100)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Probability integral transformation for counts — pit_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. n_replicates number draws randomised PIT discrete predictions. ignored forecasts continuous.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Probability integral transformation for counts — pit_sample","text":"vector PIT-values. continuous forecasts, vector correspond length observed. integer forecasts, randomised PIT returned length length(observed) * n_replicates.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Probability integral transformation for counts — pit_sample","text":"Calibration reliability forecasts ability model correctly identify uncertainty making predictions. model perfect calibration, observed data time point look came predictive probability distribution time. Equivalently, one can inspect probability integral transform predictive distribution time t, $$ u_t = F_t (x_t) $$ \\(x_t\\) observed data point time \\(t \\textrm{ } t_1, …, t_n\\), n number forecasts, \\(F_t\\) (continuous) predictive cumulative probability distribution time t. true probability distribution outcomes time t \\(G_t\\) forecasts \\(F_t\\) said ideal \\(F_t = G_t\\) times t. case, probabilities \\(u_t\\) distributed uniformly. case discrete nonnegative outcomes incidence counts, PIT longer uniform even forecasts ideal. case randomised PIT can used instead: $$ u_t = P_t(k_t) + v * (P_t(k_t) - P_t(k_t - 1) ) $$ \\(k_t\\) observed count, \\(P_t(x)\\) predictive cumulative probability observing incidence k time t, \\(P_t (-1) = 0\\) definition v standard uniform independent k. \\(P_t\\) true cumulative probability distribution, \\(u_t\\) standard uniform.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Probability integral transformation for counts — pit_sample","text":"Claudia Czado, Tilmann Gneiting Leonhard Held (2009) Predictive model assessment count data. Biometrika, 96(4), 633-648. Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind M. Eggo, W. John Edmunds (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15, doi:10.1371/journal.pcbi.1006785","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Probability integral transformation for counts — pit_sample","text":"","code":"## continuous predictions observed <- rnorm(20, mean = 1:20) predicted <- replicate(100, rnorm(n = 20, mean = 1:20)) pit <- pit_sample(observed, predicted) plot_pit(pit) ## integer predictions observed <- rpois(20, lambda = 1:20) predicted <- replicate(100, rpois(n = 20, lambda = 1:20)) pit <- pit_sample(observed, predicted, n_replicates = 30) plot_pit(pit)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot correlation between metrics — plot_correlations","title":"Plot correlation between metrics — plot_correlations","text":"Plots heatmap correlations different metrics.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot correlation between metrics — plot_correlations","text":"","code":"plot_correlations(correlations, digits = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot correlation between metrics — plot_correlations","text":"correlations data.table correlations scores produced get_correlations(). digits number indicating many decimal places correlations rounded . default (digits = NULL) rounding takes place.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot correlation between metrics — plot_correlations","text":"ggplot object showing coloured matrix correlations metrics. ggplot object visualisation correlations metrics","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot correlation between metrics — plot_correlations","text":"","code":"library(magrittr) # pipe operator scores <- example_quantile %>% as_forecast_quantile %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. correlations <- scores %>% summarise_scores() %>% get_correlations() plot_correlations(correlations, digits = 2)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":null,"dir":"Reference","previous_headings":"","what":"Visualise the number of available forecasts — plot_forecast_counts","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"Visualise Forecasts Available.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"","code":"plot_forecast_counts( forecast_counts, x, y = \"model\", x_as_factor = TRUE, show_counts = TRUE )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"forecast_counts data.table (similar) column count holding forecast counts, produced get_forecast_counts(). x Character vector length one denotes name column appear x-axis plot. y Character vector length one denotes name column appear y-axis plot. Default \"model\". x_as_factor Logical (default TRUE). Whether convert variable x-axis factor. effect e.g. dates shown x-axis. show_counts Logical (default TRUE) indicates whether show actual count numbers plot.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"ggplot object plot forecast counts","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"","code":"library(ggplot2) library(magrittr) # pipe operator forecast_counts <- example_quantile %>% as_forecast_quantile %>% get_forecast_counts(by = c(\"model\", \"target_type\", \"target_end_date\")) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_forecast_counts( forecast_counts, x = \"target_end_date\", show_counts = FALSE ) + facet_wrap(\"target_type\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a heatmap of a scoring metric — plot_heatmap","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"function can used create heatmap one metric across different groups, e.g. interval score obtained several forecasting models different locations.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"","code":"plot_heatmap(scores, y = \"model\", x, metric)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"scores data.frame scores based quantile forecasts produced score(). y variable scores want show y-Axis. default \"model\" x variable scores want show x-Axis. something like \"horizon\", \"location\" metric String, metric determines value colour shown tiles heatmap.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"ggplot object showing heatmap desired metric","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"","code":"library(magrittr) # pipe operator scores <- example_quantile %>% as_forecast_quantile %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. scores <- summarise_scores(scores, by = c(\"model\", \"target_type\")) scores <- summarise_scores( scores, by = c(\"model\", \"target_type\"), fun = signif, digits = 2 ) plot_heatmap(scores, x = \"target_type\", metric = \"bias\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot interval coverage — plot_interval_coverage","title":"Plot interval coverage — plot_interval_coverage","text":"Plot interval coverage values (see get_coverage() information).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot interval coverage — plot_interval_coverage","text":"","code":"plot_interval_coverage(coverage, colour = \"model\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot interval coverage — plot_interval_coverage","text":"coverage data frame coverage values produced get_coverage(). colour According variable shall graphs coloured? Default \"model\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot interval coverage — plot_interval_coverage","text":"ggplot object plot interval coverage","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot interval coverage — plot_interval_coverage","text":"","code":"example <- as_forecast_quantile(example_quantile) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. coverage <- get_coverage(example, by = \"model\") plot_interval_coverage(coverage)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"Creates heatmap ratios pvalues pairwise comparison models.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"","code":"plot_pairwise_comparisons( comparison_result, type = c(\"mean_scores_ratio\", \"pval\") )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"comparison_result data.frame produced get_pairwise_comparisons(). type Character vector length one either \"mean_scores_ratio\" \"pval\". denotes whether visualise ratio p-value pairwise comparison. Default \"mean_scores_ratio\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"ggplot object heatmap mean score ratios pairwise comparisons.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"","code":"library(ggplot2) library(magrittr) # pipe operator scores <- example_quantile %>% as_forecast_quantile %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. pairwise <- get_pairwise_comparisons(scores, by = \"target_type\") plot_pairwise_comparisons(pairwise, type = \"mean_scores_ratio\") + facet_wrap(~target_type)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pit.html","id":null,"dir":"Reference","previous_headings":"","what":"PIT histogram — plot_pit","title":"PIT histogram — plot_pit","text":"Make simple histogram probability integral transformed values visually check whether uniform distribution seems likely.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pit.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"PIT histogram — plot_pit","text":"","code":"plot_pit(pit, num_bins = \"auto\", breaks = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pit.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"PIT histogram — plot_pit","text":"pit Either vector PIT values, data.table produced get_pit(). num_bins number bins PIT histogram, default \"auto\". num_bins == \"auto\", plot_pit() either display 10 bins, display bin available quantile case passed data quantile-based format. can control number bins supplying number. fine sample-based pit histograms, may fail quantile-based formats. case preferred supply explicit breaks points using breaks argument. breaks Numeric vector break points bins PIT histogram. preferred creating PIT histogram based quantile-based data. Default NULL breaks determined num_bins. breaks used, num_bins ignored.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pit.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"PIT histogram — plot_pit","text":"ggplot object histogram PIT values","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pit.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"PIT histogram — plot_pit","text":"","code":"library(magrittr) # pipe operator # PIT histogram in vector based format observed <- rnorm(30, mean = 1:30) predicted <- replicate(200, rnorm(n = 30, mean = 1:30)) pit <- pit_sample(observed, predicted) plot_pit(pit) # quantile-based pit pit <- example_quantile %>% as_forecast_quantile() %>% get_pit(by = \"model\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_pit(pit, breaks = seq(0.1, 1, 0.1)) # sample-based pit pit <- example_sample_discrete %>% as_forecast_sample %>% get_pit(by = \"model\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_pit(pit)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot quantile coverage — plot_quantile_coverage","title":"Plot quantile coverage — plot_quantile_coverage","text":"Plot quantile coverage values (see get_coverage() information).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot quantile coverage — plot_quantile_coverage","text":"","code":"plot_quantile_coverage(coverage, colour = \"model\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot quantile coverage — plot_quantile_coverage","text":"coverage data frame coverage values produced get_coverage(). colour String, according variable shall graphs coloured? Default \"model\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot quantile coverage — plot_quantile_coverage","text":"ggplot object plot interval coverage","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot quantile coverage — plot_quantile_coverage","text":"","code":"example <- as_forecast_quantile(example_quantile) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. coverage <- get_coverage(example, by = \"model\") plot_quantile_coverage(coverage)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot contributions to the weighted interval score — plot_wis","title":"Plot contributions to the weighted interval score — plot_wis","text":"Visualise components weighted interval score: penalties -prediction, -prediction high dispersion (lack sharpness).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot contributions to the weighted interval score — plot_wis","text":"","code":"plot_wis(scores, x = \"model\", relative_contributions = FALSE, flip = FALSE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot contributions to the weighted interval score — plot_wis","text":"scores data.table scores based quantile forecasts produced score() summarised using summarise_scores(). x variable scores want show x-Axis. Usually \"model\". relative_contributions Logical. Show relative contributions instead absolute contributions? Default FALSE functionality available yet. flip Boolean (default FALSE), whether flip axes.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot contributions to the weighted interval score — plot_wis","text":"ggplot object showing contributions three components weighted interval score. ggplot object visualisation WIS decomposition","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Plot contributions to the weighted interval score — plot_wis","text":"Bracher J, Ray E, Gneiting T, Reich, N (2020) Evaluating epidemic forecasts interval format. https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1008618","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot contributions to the weighted interval score — plot_wis","text":"","code":"library(ggplot2) library(magrittr) # pipe operator scores <- example_quantile %>% as_forecast_quantile %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. scores <- summarise_scores(scores, by = c(\"model\", \"target_type\")) plot_wis(scores, x = \"model\", relative_contributions = TRUE ) + facet_wrap(~target_type) plot_wis(scores, x = \"model\", relative_contributions = FALSE ) + facet_wrap(~target_type, scales = \"free_x\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Print information about a forecast object — print.forecast","title":"Print information about a forecast object — print.forecast","text":"function prints information forecast object, including \"Forecast type\", \"Score columns\", \"Forecast unit\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Print information about a forecast object — print.forecast","text":"","code":"# S3 method for class 'forecast' print(x, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Print information about a forecast object — print.forecast","text":"x forecast object (validated data.table predicted observed values, see as_forecast()). ... Additional arguments print().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Print information about a forecast object — print.forecast","text":"Returns x invisibly.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Print information about a forecast object — print.forecast","text":"","code":"dat <- as_forecast_quantile(example_quantile) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. print(dat) #> Forecast type: quantile #> Forecast unit: #> location, target_end_date, target_type, location_name, forecast_date, model, #> and horizon #> #> Key: #> location target_end_date target_type observed location_name #> #> 1: DE 2021-01-02 Cases 127300 Germany #> 2: DE 2021-01-02 Deaths 4534 Germany #> 3: DE 2021-01-09 Cases 154922 Germany #> 4: DE 2021-01-09 Deaths 6117 Germany #> 5: DE 2021-01-16 Cases 110183 Germany #> --- #> 20541: IT 2021-07-24 Deaths 78 Italy #> 20542: IT 2021-07-24 Deaths 78 Italy #> 20543: IT 2021-07-24 Deaths 78 Italy #> 20544: IT 2021-07-24 Deaths 78 Italy #> 20545: IT 2021-07-24 Deaths 78 Italy #> forecast_date quantile_level predicted model horizon #> #> 1: NA NA NA #> 2: NA NA NA #> 3: NA NA NA #> 4: NA NA NA #> 5: NA NA NA #> --- #> 20541: 2021-07-12 0.850 352 epiforecasts-EpiNow2 2 #> 20542: 2021-07-12 0.900 397 epiforecasts-EpiNow2 2 #> 20543: 2021-07-12 0.950 499 epiforecasts-EpiNow2 2 #> 20544: 2021-07-12 0.975 611 epiforecasts-EpiNow2 2 #> 20545: 2021-07-12 0.990 719 epiforecasts-EpiNow2 2"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":null,"dir":"Reference","previous_headings":"","what":"Quantile score — quantile_score","title":"Quantile score — quantile_score","text":"Proper Scoring Rule score quantile predictions. Smaller values better. quantile score closely related interval score (see wis()) quantile equivalent works single quantiles instead central prediction intervals. quantile score, also called pinball loss, single quantile level \\(\\tau\\) defined $$ \\text{QS}_\\tau(F, y) = 2 \\cdot \\{ \\mathbf{1}(y \\leq q_\\tau) - \\tau\\} \\cdot (q_\\tau - y) = \\begin{cases} 2 \\cdot (1 - \\tau) * q_\\tau - y, & \\text{} y \\leq q_\\tau\\\\ 2 \\cdot \\tau * |q_\\tau - y|, & \\text{} y > q_\\tau, \\end{cases} $$ \\(q_\\tau\\) \\(\\tau\\)-quantile predictive distribution \\(F\\), \\(\\mathbf{1}(\\cdot)\\) indicator function. weighted interval score single prediction interval can obtained average quantile scores lower upper quantile prediction interval: $$ \\text{WIS}_\\alpha(F, y) = \\frac{\\text{QS}_{\\alpha/2}(F, y) + \\text{QS}_{1 - \\alpha/2}(F, y)}{2}. $$ See SI Bracher et al. (2021) details. quantile_score() returns average quantile score across quantile levels provided. set quantile levels form pairwise central prediction intervals, quantile score equivalent interval score.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Quantile score — quantile_score","text":"","code":"quantile_score(observed, predicted, quantile_level, weigh = TRUE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Quantile score — quantile_score","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made. weigh Logical. TRUE (default), weigh score \\(\\alpha / 2\\), can averaged interval score , limit (increasing number equally spaced quantiles/prediction intervals), corresponds CRPS. \\(\\alpha\\) value corresponds (\\(\\alpha/2\\)) (\\(1 - \\alpha/2\\)), .e. decimal value represents much outside central prediction interval (E.g. 90 percent central prediction interval, alpha 0.1).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Quantile score — quantile_score","text":"Numeric vector length n quantile score. scores averaged across quantile levels multiple quantile levels provided (result calling rowMeans() matrix quantile scores computed based observed predicted values).","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Quantile score — quantile_score","text":"Strictly Proper Scoring Rules, Prediction,Estimation, Tilmann Gneiting Adrian E. Raftery, 2007, Journal American Statistical Association, Volume 102, 2007 - Issue 477 Evaluating epidemic forecasts interval format, Johannes Bracher, Evan L. Ray, Tilmann Gneiting Nicholas G. Reich, 2021, https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1008618","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Quantile score — quantile_score","text":"","code":"observed <- rnorm(10, mean = 1:10) alpha <- 0.5 lower <- qnorm(alpha / 2, observed) upper <- qnorm((1 - alpha / 2), observed) qs_lower <- quantile_score(observed, predicted = matrix(lower), quantile_level = alpha / 2 ) qs_upper <- quantile_score(observed, predicted = matrix(upper), quantile_level = 1 - alpha / 2 ) interval_score <- (qs_lower + qs_upper) / 2 interval_score2 <- quantile_score( observed, predicted = cbind(lower, upper), quantile_level = c(alpha / 2, 1 - alpha / 2) ) # this is the same as the following wis( observed, predicted = cbind(lower, upper), quantile_level = c(alpha / 2, 1 - alpha / 2) ) #> [1] 0.3372449 0.3372449 0.3372449 0.3372449 0.3372449 0.3372449 0.3372449 #> [8] 0.3372449 0.3372449 0.3372449"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_to_interval.html","id":null,"dir":"Reference","previous_headings":"","what":"Transform from a quantile format to an interval format — quantile_to_interval","title":"Transform from a quantile format to an interval format — quantile_to_interval","text":"Internal helper function transform quantile format interval format (longer supported forecast format, still used internally. function mimics S3 generic, actually S3 generic, want functions internal exported.) Quantile format quantile format, prediction characterised one multiple predicted values corresponding quantile levels. example, prediction quantile format represented 0.05, 0.25, 0.5, 0.75 0.95 quantiles predictive distribution. Interval format interval format, two quantiles assumed form prediction interval. Prediction intervals need symmetric around median characterised lower upper bound. lower bound defined lower quantile upper bound defined upper quantile. 90% prediction interval, example, covers 90% probability mass defined 5% 95% quantiles. forecast therefore characterised one multiple prediction intervals, e.g. lower upper bounds 50% 90% prediction intervals (corresponding 0.25 0.75 well 0.05 0.095 quantiles).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_to_interval.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Transform from a quantile format to an interval format — quantile_to_interval","text":"","code":"quantile_to_interval(...) quantile_to_interval_dataframe( forecast, format = \"long\", keep_quantile_col = FALSE, ... ) quantile_to_interval_numeric(observed, predicted, quantile_level, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_to_interval.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Transform from a quantile format to an interval format — quantile_to_interval","text":"... Arguments forecast data.table forecasts quantile-based format (see as_forecast()). format format output. Either \"long\" \"wide\". \"long\" (default), column boundary (values either \"upper\" \"lower\" column interval_range contains range interval. \"wide\", column interval_range two columns lower upper contain lower upper bounds prediction interval, respectively. keep_quantile_col keep quantile_level column final output transformation (default FALSE). works format = \"long\". format = \"wide\", quantile_level column always dropped. observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_to_interval.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Transform from a quantile format to an interval format — quantile_to_interval","text":"data.table forecasts interval format. quantile_to_interval_dataframe: data.table interval format (either \"long\" \"wide\"), without quantile_level column. Rows reordered. quantile_to_interval.numeric: data.table wide interval format columns forecast_id, observed, lower, upper, interval_range. forecast_id column unique identifier forecast. Rows reordered according forecast_id interval_range.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/run_safely.html","id":null,"dir":"Reference","previous_headings":"","what":"Run a function safely — run_safely","title":"Run a function safely — run_safely","text":"wrapper/helper function designed run function safely completely clear arguments passed function. named arguments ... accepted fun removed. unnamed arguments passed function. case fun errors, error converted warning run_safely returns NULL. run_safely can useful constructing functions used metrics score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/run_safely.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Run a function safely — run_safely","text":"","code":"run_safely(..., fun, metric_name)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/run_safely.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Run a function safely — run_safely","text":"... Arguments pass fun. fun function execute. metric_name character string name metric. Used provide informative warning message case fun errors.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/run_safely.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Run a function safely — run_safely","text":"result fun NULL fun errors","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/run_safely.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Run a function safely — run_safely","text":"","code":"f <- function(x) {x} scoringutils:::run_safely(2, fun = f, metric_name = \"f\") #> [1] 2 scoringutils:::run_safely(2, y = 3, fun = f, metric_name = \"f\") #> [1] 2 scoringutils:::run_safely(fun = f, metric_name = \"f\") #> Warning: ! Computation for `f` failed. Error: argument \"x\" is missing, with no default. #> NULL scoringutils:::run_safely(y = 3, fun = f, metric_name = \"f\") #> Warning: ! Computation for `f` failed. Error: argument \"x\" is missing, with no default. #> NULL"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_interval_long.html","id":null,"dir":"Reference","previous_headings":"","what":"Change data from a sample-based format to a long interval range format — sample_to_interval_long","title":"Change data from a sample-based format to a long interval range format — sample_to_interval_long","text":"Transform data format based predictive samples format based interval ranges.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_interval_long.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Change data from a sample-based format to a long interval range format — sample_to_interval_long","text":"","code":"sample_to_interval_long( data, interval_range = c(0, 50, 90), type = 7, keep_quantile_col = TRUE )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_interval_long.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Change data from a sample-based format to a long interval range format — sample_to_interval_long","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. type Type argument passed quantile function. information, see quantile(). keep_quantile_col keep quantile_level column, default TRUE","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_interval_long.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Change data from a sample-based format to a long interval range format — sample_to_interval_long","text":"data.table long interval interval range format","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":null,"dir":"Reference","previous_headings":"","what":"Evaluate forecasts — score.forecast_binary","title":"Evaluate forecasts — score.forecast_binary","text":"score() applies selection scoring metrics forecast object (data.table forecasts observations) (see as_forecast()). score() generic dispatches different methods depending class input data. See Forecast types input formats section information forecast types input formats. additional help examples, check Getting Started Vignette well paper Evaluating Forecasts scoringutils R.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Evaluate forecasts — score.forecast_binary","text":"","code":"# S3 method for class 'forecast_binary' score(forecast, metrics = get_metrics(forecast), ...) # S3 method for class 'forecast_nominal' score(forecast, metrics = get_metrics(forecast), ...) # S3 method for class 'forecast_point' score(forecast, metrics = get_metrics(forecast), ...) # S3 method for class 'forecast_quantile' score(forecast, metrics = get_metrics(forecast), ...) # S3 method for class 'forecast_sample' score(forecast, metrics = get_metrics(forecast), ...) score(forecast, metrics, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Evaluate forecasts — score.forecast_binary","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()). metrics named list scoring functions. Names used column names output. See get_metrics() information default metrics used. See Customising metrics section information pass custom arguments scoring functions. ... Currently unused. pass additional arguments scoring functions via .... See Customising metrics section details use purrr::partial() pass arguments individual metrics.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Evaluate forecasts — score.forecast_binary","text":"object class scores. object data.table unsummarised scores (one score per forecast) additional attribute metrics names metrics used scoring. See summarise_scores()) information summarise scores.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Evaluate forecasts — score.forecast_binary","text":"Customising metrics want pass arguments scoring function, need change scoring function via e.g. purrr::partial() pass updated list functions custom metric metrics argument score(). example, use interval_coverage() interval_range = 90, define new function, e.g. interval_coverage_90 <- purrr::partial(interval_coverage, interval_range = 90) pass new function metrics score(). Note want pass variable argument, can unquote !! make sure value evaluated function created. Consider following example:","code":"custom_arg <- \"foo\" print1 <- purrr::partial(print, x = custom_arg) print2 <- purrr::partial(print, x = !!custom_arg) custom_arg <- \"bar\" print1() # prints 'bar' print2() # prints 'foo'"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"forecast-types-and-input-formats","dir":"Reference","previous_headings":"","what":"Forecast types and input formats","title":"Evaluate forecasts — score.forecast_binary","text":"Various different forecast types / forecast formats supported. moment, : point forecasts binary forecasts (\"soft binary classification\") nominal forecasts (\"soft classification multiple unordered classes\") Probabilistic forecasts quantile-based format (forecast represented set predictive quantiles) Probabilistic forecasts sample-based format (forecast represented set predictive samples) Forecast types determined based columns present input data. overview required format forecast type: forecast types require data.frame similar columns observed predicted, model. Point forecasts require column observed type numeric column predicted type numeric. Binary forecasts require column observed type factor exactly two levels column predicted type numeric probabilities, corresponding probability observed equal second factor level. See details information. Nominal forecasts require column observed type factor N levels, (N number possible outcomes), column predicted type numeric probabilities (sum one across possible outcomes), column predicted_label type factor N levels, denoting outcome probability given. Forecasts must complete, .e. must probability assigned every possible outcome. Quantile-based forecasts require column observed type numeric, column predicted type numeric, column quantile_level type numeric quantile-levels (0 1). Sample-based forecasts require column observed type numeric, column predicted type numeric, column sample_id type numeric sample indices. information see vignettes example data (example_quantile, example_sample_continuous, example_sample_discrete, example_point(), example_binary, example_nominal).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"forecast-unit","dir":"Reference","previous_headings":"","what":"Forecast unit","title":"Evaluate forecasts — score.forecast_binary","text":"order score forecasts, scoringutils needs know rows data belong together jointly form single forecasts. easy e.g. point forecast, one row per forecast. quantile sample-based forecasts, however, multiple rows belong single forecast. forecast unit unit single forecast described combination columns uniquely identify single forecast. example, forecasts made different models various locations different time points, several weeks future. forecast unit described forecast_unit = c(\"model\", \"location\", \"forecast_date\", \"forecast_horizon\"). scoringutils automatically tries determine unit single forecast. uses existing columns , means columns must present unrelated forecast unit. simplistic example, additional row, \"even\", one row number even zero otherwise, mess scoring scoringutils thinks column relevant defining forecast unit. order avoid issues, recommend setting forecast unit explicitly, usually forecast_unit argument as_forecast() functions. drop unneeded columns, making sure necessary, 'protected columns' like \"predicted\" \"observed\" retained.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Evaluate forecasts — score.forecast_binary","text":"Bosse NI, Gruson H, Cori , van Leeuwen E, Funk S, Abbott S (2022) Evaluating Forecasts scoringutils R. doi:10.48550/arXiv.2205.07090","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Evaluate forecasts — score.forecast_binary","text":"Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Evaluate forecasts — score.forecast_binary","text":"","code":"library(magrittr) # pipe operator validated <- as_forecast_quantile(example_quantile) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. score(validated) %>% summarise_scores(by = c(\"model\", \"target_type\")) #> model target_type wis overprediction underprediction #> #> 1: EuroCOVIDhub-ensemble Cases 17943.82383 10043.121943 4237.177310 #> 2: EuroCOVIDhub-baseline Cases 28483.57465 14096.100883 10284.972826 #> 3: epiforecasts-EpiNow2 Cases 20831.55662 11906.823030 3260.355639 #> 4: EuroCOVIDhub-ensemble Deaths 41.42249 7.138247 4.103261 #> 5: EuroCOVIDhub-baseline Deaths 159.40387 65.899117 2.098505 #> 6: UMass-MechBayes Deaths 52.65195 8.978601 16.800951 #> 7: epiforecasts-EpiNow2 Deaths 66.64282 18.892583 15.893314 #> dispersion bias interval_coverage_50 interval_coverage_90 ae_median #> #> 1: 3663.52458 -0.05640625 0.3906250 0.8046875 24101.07031 #> 2: 4102.50094 0.09796875 0.3281250 0.8203125 38473.60156 #> 3: 5664.37795 -0.07890625 0.4687500 0.7890625 27923.81250 #> 4: 30.18099 0.07265625 0.8750000 1.0000000 53.13281 #> 5: 91.40625 0.33906250 0.6640625 1.0000000 233.25781 #> 6: 26.87239 -0.02234375 0.4609375 0.8750000 78.47656 #> 7: 31.85692 -0.00512605 0.4201681 0.9075630 104.74790 # set forecast unit manually (to avoid issues with scoringutils trying to # determine the forecast unit automatically) example_quantile %>% as_forecast_quantile( forecast_unit = c( \"location\", \"target_end_date\", \"target_type\", \"horizon\", \"model\" ) ) %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> location target_end_date target_type horizon model #> #> 1: DE 2021-05-08 Cases 1 EuroCOVIDhub-ensemble #> 2: DE 2021-05-08 Cases 1 EuroCOVIDhub-baseline #> 3: DE 2021-05-08 Cases 1 epiforecasts-EpiNow2 #> 4: DE 2021-05-08 Deaths 1 EuroCOVIDhub-ensemble #> 5: DE 2021-05-08 Deaths 1 EuroCOVIDhub-baseline #> --- #> 883: IT 2021-07-24 Deaths 2 EuroCOVIDhub-baseline #> 884: IT 2021-07-24 Deaths 3 UMass-MechBayes #> 885: IT 2021-07-24 Deaths 2 UMass-MechBayes #> 886: IT 2021-07-24 Deaths 3 epiforecasts-EpiNow2 #> 887: IT 2021-07-24 Deaths 2 epiforecasts-EpiNow2 #> wis overprediction underprediction dispersion bias #> #> 1: 7990.854783 2.549870e+03 0.0000000 5440.985217 0.50 #> 2: 16925.046957 1.527583e+04 0.0000000 1649.220870 0.95 #> 3: 25395.960870 1.722226e+04 0.0000000 8173.700000 0.90 #> 4: 53.880000 0.000000e+00 0.6086957 53.271304 -0.10 #> 5: 46.793043 2.130435e+00 0.0000000 44.662609 0.30 #> --- #> 883: 80.336957 3.608696e+00 0.0000000 76.728261 0.20 #> 884: 4.881739 4.347826e-02 0.0000000 4.838261 0.10 #> 885: 25.581739 1.782609e+01 0.0000000 7.755652 0.80 #> 886: 19.762609 5.478261e+00 0.0000000 14.284348 0.50 #> 887: 66.161739 4.060870e+01 0.0000000 25.553043 0.90 #> interval_coverage_50 interval_coverage_90 ae_median #> #> 1: TRUE TRUE 12271 #> 2: FALSE FALSE 25620 #> 3: FALSE TRUE 44192 #> 4: TRUE TRUE 14 #> 5: TRUE TRUE 15 #> --- #> 883: TRUE TRUE 53 #> 884: TRUE TRUE 1 #> 885: FALSE TRUE 46 #> 886: TRUE TRUE 26 #> 887: FALSE TRUE 108 # forecast formats with different metrics if (FALSE) { # \\dontrun{ score(as_forecast_binary(example_binary)) score(as_forecast_quantile(example_quantile)) score(as_forecast_point(example_point)) score(as_forecast_sample(example_sample_discrete)) score(as_forecast_sample(example_sample_continuous)) } # }"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Metrics for binary outcomes — scoring-functions-binary","title":"Metrics for binary outcomes — scoring-functions-binary","text":"Brier score Brier Score mean squared error probabilistic prediction observed outcome. Brier score proper scoring rule. Small values better (best 0, worst 1). $$ \\textrm{Brier\\_Score} = (\\textrm{prediction} - \\textrm{outcome})^2, $$ \\(\\textrm{outcome} \\\\{0, 1\\}\\), \\(\\textrm{prediction} \\[0, 1]\\) represents probability outcome equal 1. Log score binary outcomes Log Score negative logarithm probability assigned observed value. proper scoring rule. Small values better (best zero, worst infinity).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Metrics for binary outcomes — scoring-functions-binary","text":"","code":"brier_score(observed, predicted) logs_binary(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Metrics for binary outcomes — scoring-functions-binary","text":"observed factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted numeric vector length n, holding probabilities. Values represent probability corresponding outcome equal highest level factor observed.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Metrics for binary outcomes — scoring-functions-binary","text":"numeric vector size n Brier scores numeric vector size n log scores","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Metrics for binary outcomes — scoring-functions-binary","text":"functions require users provide observed values factor order distinguish input input format required scoring point forecasts. Internally, however, factors converted numeric values. factor observed = factor(c(0, 1, 1, 0, 1) two levels (0 1) internally coerced numeric vector (case result numeric vector c(1, 2, 2, 1, 1)). subtracting 1, resulting vector (c(0, 1, 1, 0) case) used internal calculations. predictions assumed represent probability outcome equal last/highest factor level (case outcome equal 1). alternatively also provide vector like observed = factor(c(\"\", \"b\", \"b\", \"\")) (two levels, b), result exactly internal representation. Probabilities represent probability outcome equal \"b\". want predictions probabilities outcome \"\", course make observed factor levels swapped, .e. observed = factor(c(\"\", \"b\", \"b\", \"\"), levels = c(\"b\", \"\"))","code":""},{"path":[]},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Metrics for binary outcomes — scoring-functions-binary","text":"","code":"observed <- factor(sample(c(0, 1), size = 30, replace = TRUE)) predicted <- runif(n = 30, min = 0, max = 1) brier_score(observed, predicted) #> [1] 0.050968633 0.057656155 0.286927444 0.067535387 0.036218014 0.323334572 #> [7] 0.609439077 0.022618905 0.563870034 0.420850006 0.263388704 0.126243692 #> [13] 0.815464255 0.065961231 0.551322692 0.655667703 0.170750321 0.601418670 #> [19] 0.153762411 0.003327989 0.656657070 0.153676995 0.004340033 0.733833002 #> [25] 0.003680076 0.010143612 0.317605349 0.405392737 0.369087629 0.177242828 logs_binary(observed, predicted) #> [1] 0.25587639 0.27459075 0.76712981 0.30093715 0.21110420 0.84077787 #> [7] 1.51715864 0.16298472 1.38995224 1.04619879 0.71993087 0.43898241 #> [13] 2.33335976 0.29682936 1.35677856 1.65932749 0.53310439 1.49393247 #> [19] 0.49778684 0.05941962 1.66254231 0.49760766 0.06814922 1.94240004 #> [25] 0.06258165 0.10615583 0.82911550 1.01253898 0.93528454 0.54645655"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-nominal.html","id":null,"dir":"Reference","previous_headings":"","what":"Log score for nominal outcomes — logs_nominal","title":"Log score for nominal outcomes — logs_nominal","text":"Log score nominal outcomes Log Score negative logarithm probability assigned observed value. proper scoring rule. Small values better (best zero, worst infinity).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-nominal.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Log score for nominal outcomes — logs_nominal","text":"","code":"logs_nominal(observed, predicted, predicted_label)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-nominal.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Log score for nominal outcomes — logs_nominal","text":"observed factor length n N levels holding observed values. predicted nxN matrix predictive probabilities, n (number rows) number observations N (number columns) number possible outcomes. predicted_label factor length N, denoting outcome probabilities predicted correspond .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-nominal.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Log score for nominal outcomes — logs_nominal","text":"numeric vector size n log scores","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-nominal.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Log score for nominal outcomes — logs_nominal","text":"","code":"factor_levels <- c(\"one\", \"two\", \"three\") predicted_label <- factor(c(\"one\", \"two\", \"three\"), levels = factor_levels) observed <- factor(c(\"one\", \"three\", \"two\"), levels = factor_levels) predicted <- matrix(c(0.8, 0.1, 0.4, 0.1, 0.2, 0.4, 0.1, 0.7, 0.2), nrow = 3) logs_nominal(observed, predicted, predicted_label) #> [1] 0.2231436 0.3566749 0.9162907"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoringutils-package.html","id":null,"dir":"Reference","previous_headings":"","what":"scoringutils: Utilities for Scoring and Assessing Predictions — scoringutils-package","title":"scoringutils: Utilities for Scoring and Assessing Predictions — scoringutils-package","text":"scoringutils facilitates evaluation forecasts convenient framework based data.table. allows user check forecasts diagnose issues, visualise forecasts missing data, transform data scoring, handle missing forecasts, aggregate scores, visualise results evaluation. package mostly focuses evaluation probabilistic forecasts allows evaluating several different forecast types input formats. Find information package Vignettes well accompanying paper, doi:10.48550/arXiv.2205.07090 .","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoringutils-package.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"scoringutils: Utilities for Scoring and Assessing Predictions — scoringutils-package","text":"Maintainer: Nikos Bosse nikosbosse@gmail.com (ORCID) Authors: Sam Abbott contact@samabbott.co.uk (ORCID) Hugo Gruson hugo.gruson+R@normalesup.org (ORCID) Sebastian Funk sebastian.funk@lshtm.ac.uk contributors: Johannes Bracher johannes.bracher@kit.edu (ORCID) [contributor] Toshiaki Asakura toshiaki.asa9ra@gmail.com (ORCID) [contributor] James Mba Azam james.azam@lshtm.ac.uk (ORCID) [contributor] Michael Chirico michaelchirico4@gmail.com (ORCID) [contributor]","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/se_mean_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Squared error of the mean (sample-based version) — se_mean_sample","title":"Squared error of the mean (sample-based version) — se_mean_sample","text":"Squared error mean calculated $$ \\textrm{mean}(\\textrm{observed} - \\textrm{mean prediction})^2 $$ mean prediction calculated mean predictive samples.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/se_mean_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Squared error of the mean (sample-based version) — se_mean_sample","text":"","code":"se_mean_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/se_mean_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Squared error of the mean (sample-based version) — se_mean_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/se_mean_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Squared error of the mean (sample-based version) — se_mean_sample","text":"","code":"observed <- rnorm(30, mean = 1:30) predicted_values <- matrix(rnorm(30, mean = 1:30)) se_mean_sample(observed, predicted_values) #> [1] 0.014829896 0.316608439 1.142566519 0.086187944 0.058926355 #> [6] 3.073734463 0.002511154 0.008577179 3.999907031 1.048431205 #> [11] 0.281251491 2.816708187 12.593948837 1.990408824 0.123815451 #> [16] 0.569334753 1.956832571 0.052113786 1.198119654 0.285800371 #> [21] 0.113974757 8.105728298 0.019562959 0.737141759 0.375735782 #> [26] 0.062009545 0.058334996 0.492055176 0.007062469 0.978271085"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/select_metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Select metrics from a list of functions — select_metrics","title":"Select metrics from a list of functions — select_metrics","text":"Helper function return scoring rules selected user list possible functions.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/select_metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Select metrics from a list of functions — select_metrics","text":"","code":"select_metrics(metrics, select = NULL, exclude = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/select_metrics.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Select metrics from a list of functions — select_metrics","text":"metrics list scoring functions. select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/select_metrics.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Select metrics from a list of functions — select_metrics","text":"list scoring functions.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/select_metrics.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Select metrics from a list of functions — select_metrics","text":"","code":"select_metrics( metrics = get_metrics(example_binary), select = \"brier_score\" ) #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> #> #> select_metrics( metrics = get_metrics(example_binary), exclude = \"log_score\" ) #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> #> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/set_forecast_unit.html","id":null,"dir":"Reference","previous_headings":"","what":"Set unit of a single forecast manually — set_forecast_unit","title":"Set unit of a single forecast manually — set_forecast_unit","text":"Helper function set unit single forecast (.e. combination columns uniquely define single forecast) manually. simple function keeps columns specified forecast_unit (plus additional protected columns, e.g. observed values, predictions quantile levels) removes duplicate rows. set_forecast_unit() mainly called constructing forecast object (see as_forecast()) via forecast_unit argument . done explicitly, scoringutils attempts determine unit single forecast automatically simply assuming column names relevant determine forecast unit. may lead unexpected behaviour, setting forecast unit explicitly can help make code easier debug easier read.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/set_forecast_unit.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Set unit of a single forecast manually — set_forecast_unit","text":"","code":"set_forecast_unit(data, forecast_unit)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/set_forecast_unit.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Set unit of a single forecast manually — set_forecast_unit","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. forecast_unit Character vector names columns uniquely identify single forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/set_forecast_unit.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Set unit of a single forecast manually — set_forecast_unit","text":"data.table columns kept relevant scoring denote unit single forecast specified user.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/set_forecast_unit.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Set unit of a single forecast manually — set_forecast_unit","text":"","code":"library(magrittr) # pipe operator example_quantile %>% scoringutils:::set_forecast_unit( c(\"location\", \"target_end_date\", \"target_type\", \"horizon\", \"model\") ) #> Forecast type: quantile #> Forecast unit: #> location, target_end_date, target_type, horizon, and model #> #> Key: #> observed quantile_level predicted location target_end_date target_type #> #> 1: 127300 NA NA DE 2021-01-02 Cases #> 2: 4534 NA NA DE 2021-01-02 Deaths #> 3: 154922 NA NA DE 2021-01-09 Cases #> 4: 6117 NA NA DE 2021-01-09 Deaths #> 5: 110183 NA NA DE 2021-01-16 Cases #> --- #> 20541: 78 0.850 352 IT 2021-07-24 Deaths #> 20542: 78 0.900 397 IT 2021-07-24 Deaths #> 20543: 78 0.950 499 IT 2021-07-24 Deaths #> 20544: 78 0.975 611 IT 2021-07-24 Deaths #> 20545: 78 0.990 719 IT 2021-07-24 Deaths #> horizon model #> #> 1: NA #> 2: NA #> 3: NA #> 4: NA #> 5: NA #> --- #> 20541: 2 epiforecasts-EpiNow2 #> 20542: 2 epiforecasts-EpiNow2 #> 20543: 2 epiforecasts-EpiNow2 #> 20544: 2 epiforecasts-EpiNow2 #> 20545: 2 epiforecasts-EpiNow2"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/summarise_scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Summarise scores as produced by score() — summarise_scores","title":"Summarise scores as produced by score() — summarise_scores","text":"Summarise scores produced score(). summarise_scores relies way identify names scores distinguish columns denote unit single forecast. Internally, done via stored attribute, metrics stores names scores. means, however, need careful renaming scores produced score(). , also manually update attribute calling attr(scores, \"metrics\") <- new_names.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/summarise_scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Summarise scores as produced by score() — summarise_scores","text":"","code":"summarise_scores(scores, by = \"model\", fun = mean, ...) summarize_scores(scores, by = \"model\", fun = mean, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/summarise_scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Summarise scores as produced by score() — summarise_scores","text":"scores object class scores (data.table scores additional attribute metrics produced score()). Character vector column names summarise scores . Default \"model\", .e. scores summarised \"model\" column. fun function used summarising scores. Default mean(). ... Additional parameters can passed summary function provided fun. information see documentation respective function.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/summarise_scores.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Summarise scores as produced by score() — summarise_scores","text":"data.table summarised scores. Scores summarised according names columns original data specified using fun passed summarise_scores().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/summarise_scores.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Summarise scores as produced by score() — summarise_scores","text":"","code":"library(magrittr) # pipe operator scores <- example_sample_continuous %>% as_forecast_sample() %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. # get scores by model summarise_scores(scores, by = \"model\") #> model bias dss crps overprediction #> #> 1: EuroCOVIDhub-ensemble 0.009765625 16.40496 9876.95886 5281.81845 #> 2: EuroCOVIDhub-baseline 0.177734375 NaN 15309.68627 6701.83844 #> 3: epiforecasts-EpiNow2 -0.024898785 26.10137 11901.43354 6986.16572 #> 4: UMass-MechBayes -0.026953125 10.08582 60.19018 11.20505 #> underprediction dispersion log_score mad ae_median se_mean #> #> 1: 2433.20525 2161.93515 10.747811 8763.6176 12406.03100 2.103026e+09 #> 2: 5864.53649 2743.31134 Inf 9680.3792 18932.50196 2.885063e+09 #> 3: 1740.93388 3174.33393 Inf 12999.5404 14680.12285 3.152268e+09 #> 4: 19.28195 29.70318 5.941622 123.6211 79.66001 1.371418e+04 # get scores by model and target type summarise_scores(scores, by = c(\"model\", \"target_type\")) #> model target_type bias dss crps #> #> 1: EuroCOVIDhub-ensemble Cases -0.04648437 22.89997 19703.05522 #> 2: EuroCOVIDhub-baseline Cases 0.03671875 NaN 30453.58346 #> 3: epiforecasts-EpiNow2 Cases -0.03867188 40.87716 22896.51608 #> 4: EuroCOVIDhub-ensemble Deaths 0.06601562 9.90995 50.86249 #> 5: EuroCOVIDhub-baseline Deaths 0.31875000 12.99360 165.78907 #> 6: UMass-MechBayes Deaths -0.02695313 10.08582 60.19018 #> 7: epiforecasts-EpiNow2 Deaths -0.01008403 10.20807 74.79013 #> overprediction underprediction dispersion log_score mad ae_median #> #> 1: 10552.97603 4861.015121 4289.06407 15.633420 17385.2629 24749.39707 #> 2: 13346.03509 11727.330575 5380.21780 Inf 18982.2128 37648.01693 #> 3: 13462.90822 3346.583024 6087.02483 Inf 24929.3438 28233.04536 #> 4: 10.66088 5.395380 34.80623 5.862203 141.9723 62.66492 #> 5: 57.64179 1.742401 106.40489 6.977391 378.5457 216.98699 #> 6: 11.20505 19.281946 29.70318 5.941622 123.6211 79.66001 #> 7: 19.58555 13.849095 41.35549 6.024092 167.4829 102.18939 #> se_mean #> #> 1: 4.206042e+09 #> 2: 5.769964e+09 #> 3: 6.082863e+09 #> 4: 1.080233e+04 #> 5: 1.622417e+05 #> 6: 1.371418e+04 #> 7: 3.243111e+04 # get standard deviation summarise_scores(scores, by = \"model\", fun = sd) #> model bias dss crps overprediction #> #> 1: EuroCOVIDhub-ensemble 0.5468290 14.869520 39368.24836 37275.23950 #> 2: EuroCOVIDhub-baseline 0.5457971 NA 45020.82814 39070.74445 #> 3: epiforecasts-EpiNow2 0.6083410 108.130107 44957.07746 40776.81690 #> 4: UMass-MechBayes 0.6221914 2.248998 49.62465 21.34675 #> underprediction dispersion log_score mad ae_median se_mean #> #> 1: 8634.87723 5163.42293 21.510119 19799.1620 42801.64123 1.564286e+10 #> 2: 20537.03929 3664.87255 NaN 13610.4174 49458.36446 1.760651e+10 #> 3: 8096.39644 7266.11787 NaN 29616.1714 51129.54601 2.209086e+10 #> 4: 36.98584 29.60927 1.126019 123.3465 76.09471 2.994664e+04 # round digits summarise_scores(scores, by = \"model\") %>% summarise_scores(fun = signif, digits = 2) #> model bias dss crps overprediction underprediction #> #> 1: EuroCOVIDhub-ensemble 0.0098 16 9900 5300 2400 #> 2: EuroCOVIDhub-baseline 0.1800 NaN 15000 6700 5900 #> 3: epiforecasts-EpiNow2 -0.0250 26 12000 7000 1700 #> 4: UMass-MechBayes -0.0270 10 60 11 19 #> dispersion log_score mad ae_median se_mean #> #> 1: 2200 11.0 8800 12000 2.1e+09 #> 2: 2700 Inf 9700 19000 2.9e+09 #> 3: 3200 Inf 13000 15000 3.2e+09 #> 4: 30 5.9 120 80 1.4e+04"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_not_present.html","id":null,"dir":"Reference","previous_headings":"","what":"Test whether column names are NOT present in a data.frame — test_columns_not_present","title":"Test whether column names are NOT present in a data.frame — test_columns_not_present","text":"function checks whether column names present. none columns present, function returns TRUE. one columns present, function returns FALSE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_not_present.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Test whether column names are NOT present in a data.frame — test_columns_not_present","text":"","code":"test_columns_not_present(data, columns)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_not_present.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Test whether column names are NOT present in a data.frame — test_columns_not_present","text":"data data.frame similar checked columns character vector column names check","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_not_present.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Test whether column names are NOT present in a data.frame — test_columns_not_present","text":"Returns TRUE none columns present FALSE otherwise","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_present.html","id":null,"dir":"Reference","previous_headings":"","what":"Test whether all column names are present in a data.frame — test_columns_present","title":"Test whether all column names are present in a data.frame — test_columns_present","text":"function checks whether column names present. one columns missing, function returns FALSE. columns present, function returns TRUE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_present.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Test whether all column names are present in a data.frame — test_columns_present","text":"","code":"test_columns_present(data, columns)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_present.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Test whether all column names are present in a data.frame — test_columns_present","text":"data data.frame similar checked columns character vector column names check","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_present.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Test whether all column names are present in a data.frame — test_columns_present","text":"Returns TRUE columns present FALSE otherwise","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/theme_scoringutils.html","id":null,"dir":"Reference","previous_headings":"","what":"Scoringutils ggplot2 theme — theme_scoringutils","title":"Scoringutils ggplot2 theme — theme_scoringutils","text":"theme ggplot2 plots used scoringutils.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/theme_scoringutils.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Scoringutils ggplot2 theme — theme_scoringutils","text":"","code":"theme_scoringutils()"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/theme_scoringutils.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Scoringutils ggplot2 theme — theme_scoringutils","text":"ggplot2 theme","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":null,"dir":"Reference","previous_headings":"","what":"Transform forecasts and observed values — transform_forecasts","title":"Transform forecasts and observed values — transform_forecasts","text":"Function transform forecasts observed values scoring.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Transform forecasts and observed values — transform_forecasts","text":"","code":"transform_forecasts( forecast, fun = log_shift, append = TRUE, label = \"log\", ... )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Transform forecasts and observed values — transform_forecasts","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()). fun function used transform observed values predictions. default function log_shift(), custom function essentially log(), additional arguments (offset) allows add offset applying logarithm. often helpful natural log transformation defined zero. common, pragmatic solution, add small offset data applying log transformation. work often used offset 1 precise value depend application. append Logical, defaults TRUE. Whether append transformed version data currently existing data (TRUE). selected, data gets transformed appended existing data, making possible use outcome directly score(). additional column, 'scale', gets created denotes rows untransformed ('scale' value \"natural\") transformed ('scale' value passed argument label). label string newly created 'scale' column denote newly transformed values. relevant append = TRUE. ... Additional parameters pass function supplied. default option log_shift() offset argument.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Transform forecasts and observed values — transform_forecasts","text":"forecast object either transformed version data, one untransformed transformed data. includes original data well transformation original data. one additional column, `scale', present set \"natural\" untransformed forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Transform forecasts and observed values — transform_forecasts","text":"reasons, depending circumstances, might desirable (check linked reference info). epidemiology, example, may useful log-transform incidence counts evaluating forecasts using scores weighted interval score (WIS) continuous ranked probability score (CRPS). Log-transforming forecasts observations changes interpretation score measure absolute distance forecast observation score evaluates forecast exponential growth rate. Another motivation can apply variance-stabilising transformation standardise incidence counts population. Note want apply transformation, important transform forecasts observations apply score. Applying transformation score risks losing propriety proper scoring rule.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Transform forecasts and observed values — transform_forecasts","text":"Transformation forecasts evaluating predictive performance epidemiological context Nikos . Bosse, Sam Abbott, Anne Cori, Edwin van Leeuwen, Johannes Bracher, Sebastian Funk medRxiv 2023.01.23.23284722 doi:10.1101/2023.01.23.23284722 https://www.medrxiv.org/content/10.1101/2023.01.23.23284722v1","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Transform forecasts and observed values — transform_forecasts","text":"Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Transform forecasts and observed values — transform_forecasts","text":"","code":"library(magrittr) # pipe operator # transform forecasts using the natural logarithm # negative values need to be handled (here by replacing them with 0) example_quantile %>% .[, observed := ifelse(observed < 0, 0, observed)] %>% as_forecast_quantile() %>% # Here we use the default function log_shift() which is essentially the same # as log(), but has an additional arguments (offset) that allows you add an # offset before applying the logarithm. transform_forecasts(append = FALSE) %>% head() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Warning: ! Detected zeros in input values. #> ℹ Try specifying offset = 1 (or any other offset). #> Warning: ! Detected zeros in input values. #> ℹ Try specifying offset = 1 (or any other offset). #> Key: #> location target_end_date target_type observed location_name forecast_date #> #> 1: DE 2021-01-02 Cases 11.754302 Germany #> 2: DE 2021-01-02 Deaths 8.419360 Germany #> 3: DE 2021-01-09 Cases 11.950677 Germany #> 4: DE 2021-01-09 Deaths 8.718827 Germany #> 5: DE 2021-01-16 Cases 11.609898 Germany #> 6: DE 2021-01-16 Deaths 8.677099 Germany #> quantile_level predicted model horizon #> #> 1: NA NA NA #> 2: NA NA NA #> 3: NA NA NA #> 4: NA NA NA #> 5: NA NA NA #> 6: NA NA NA # alternatively, integrating the truncation in the transformation function: example_quantile %>% as_forecast_quantile() %>% transform_forecasts( fun = function(x) {log_shift(pmax(0, x))}, append = FALSE ) %>% head() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Warning: ! Detected zeros in input values. #> ℹ Try specifying offset = 1 (or any other offset). #> Warning: ! Detected zeros in input values. #> ℹ Try specifying offset = 1 (or any other offset). #> Key: #> location target_end_date target_type observed location_name forecast_date #> #> 1: DE 2021-01-02 Cases 11.754302 Germany #> 2: DE 2021-01-02 Deaths 8.419360 Germany #> 3: DE 2021-01-09 Cases 11.950677 Germany #> 4: DE 2021-01-09 Deaths 8.718827 Germany #> 5: DE 2021-01-16 Cases 11.609898 Germany #> 6: DE 2021-01-16 Deaths 8.677099 Germany #> quantile_level predicted model horizon #> #> 1: NA NA NA #> 2: NA NA NA #> 3: NA NA NA #> 4: NA NA NA #> 5: NA NA NA #> 6: NA NA NA # specifying an offset for the log transformation removes the # warning caused by zeros in the data example_quantile %>% as_forecast_quantile() %>% .[, observed := ifelse(observed < 0, 0, observed)] %>% transform_forecasts(offset = 1, append = FALSE) %>% head() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Key: #> location target_end_date target_type observed location_name forecast_date #> #> 1: DE 2021-01-02 Cases 11.754310 Germany #> 2: DE 2021-01-02 Deaths 8.419580 Germany #> 3: DE 2021-01-09 Cases 11.950683 Germany #> 4: DE 2021-01-09 Deaths 8.718991 Germany #> 5: DE 2021-01-16 Cases 11.609907 Germany #> 6: DE 2021-01-16 Deaths 8.677269 Germany #> quantile_level predicted model horizon #> #> 1: NA NA NA #> 2: NA NA NA #> 3: NA NA NA #> 4: NA NA NA #> 5: NA NA NA #> 6: NA NA NA # adding square root transformed forecasts to the original ones example_quantile %>% .[, observed := ifelse(observed < 0, 0, observed)] %>% as_forecast_quantile() %>% transform_forecasts(fun = sqrt, label = \"sqrt\") %>% score() %>% summarise_scores(by = c(\"model\", \"scale\")) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> model scale wis overprediction underprediction #> #> 1: EuroCOVIDhub-ensemble natural 5796.064569 1828.5715014 2120.6402853 #> 2: EuroCOVIDhub-baseline natural 11124.930667 3884.4414062 5143.5356658 #> 3: epiforecasts-EpiNow2 natural 7514.375476 2866.4071466 1697.2341137 #> 4: UMass-MechBayes natural 52.651946 8.9786005 16.8009511 #> 5: EuroCOVIDhub-ensemble sqrt 14.974344 5.5037665 5.1827454 #> 6: EuroCOVIDhub-baseline sqrt 27.742316 10.4190016 9.5936380 #> 7: epiforecasts-EpiNow2 sqrt 17.704899 6.5700431 5.7235785 #> 8: UMass-MechBayes sqrt 1.328653 0.3273746 0.4019195 #> dispersion bias interval_coverage_50 interval_coverage_90 #> #> 1: 1846.8527819 0.00812500 0.6328125 0.9023438 #> 2: 2096.9535954 0.21816406 0.4960938 0.9101562 #> 3: 2950.7342158 -0.04336032 0.4453441 0.8461538 #> 4: 26.8723947 -0.02234375 0.4609375 0.8750000 #> 5: 4.2878323 0.00812500 0.6328125 0.9023438 #> 6: 7.7296761 0.21816406 0.4960938 0.9101562 #> 7: 5.4112770 -0.04336032 0.4453441 0.8461538 #> 8: 0.5993586 -0.02234375 0.4609375 0.8750000 #> ae_median #> #> 1: 8880.542969 #> 2: 16156.871094 #> 3: 11208.072874 #> 4: 78.476562 #> 5: 22.458900 #> 6: 39.185406 #> 7: 25.585018 #> 8: 2.069103 # adding multiple transformations example_quantile %>% as_forecast_quantile() %>% .[, observed := ifelse(observed < 0, 0, observed)] %>% transform_forecasts(fun = log_shift, offset = 1) %>% transform_forecasts(fun = sqrt, label = \"sqrt\") %>% head() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> location target_end_date target_type observed location_name forecast_date #> #> 1: DE 2021-01-02 Cases 127300 Germany #> 2: DE 2021-01-02 Deaths 4534 Germany #> 3: DE 2021-01-09 Cases 154922 Germany #> 4: DE 2021-01-09 Deaths 6117 Germany #> 5: DE 2021-01-16 Cases 110183 Germany #> 6: DE 2021-01-16 Deaths 5867 Germany #> quantile_level predicted model horizon scale #> #> 1: NA NA NA natural #> 2: NA NA NA natural #> 3: NA NA NA natural #> 4: NA NA NA natural #> 5: NA NA NA natural #> 6: NA NA NA natural"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Validate metrics — validate_metrics","title":"Validate metrics — validate_metrics","text":"function validates whether list metrics list valid functions. function used score() make sure metrics valid functions.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Validate metrics — validate_metrics","text":"","code":"validate_metrics(metrics)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_metrics.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Validate metrics — validate_metrics","text":"metrics named list metrics. Every element scoring function applied data.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_metrics.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Validate metrics — validate_metrics","text":"named list metrics, filtered valid functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":null,"dir":"Reference","previous_headings":"","what":"Weighted interval score (WIS) — wis","title":"Weighted interval score (WIS) — wis","text":"WIS proper scoring rule used evaluate forecasts interval- / quantile-based format. See Bracher et al. (2021). Smaller values better. name suggest score assumes forecast comes form one multiple central prediction intervals. prediction interval characterised lower upper bound formed pair predictive quantiles. example, 50% central prediction interval formed 0.25 0.75 quantiles predictive distribution. Interval score interval score () sum three components: overprediction, underprediction dispersion. single prediction interval one components non-zero. single prediction interval observed value lower bound, interval score equal absolute difference lower bound observed value (\"underprediction\"). \"Overprediction\" defined analogously. observed value falls within bounds prediction interval, interval score equal width prediction interval, .e. difference upper lower bound. single interval, therefore : $$ \\textrm{} = (\\textrm{upper} - \\textrm{lower}) + \\frac{2}{\\alpha}(\\textrm{lower} - \\textrm{observed}) * \\mathbf{1}(\\textrm{observed} < \\textrm{lower}) + \\frac{2}{\\alpha}(\\textrm{observed} - \\textrm{upper}) * \\mathbf{1}(\\textrm{observed} > \\textrm{upper}) $$ \\(\\mathbf{1}()\\) indicator function indicates much outside prediction interval. \\(\\alpha\\) decimal value indicates much outside prediction interval. 90% prediction interval, example, \\(\\alpha\\) equal 0.1. specific distribution assumed, interval formed quantiles symmetric around median (.e use 0.1 quantile lower bound 0.7 quantile upper bound). Non-symmetric quantiles can scored using function quantile_score(). set \\(k = 1, \\dots, K\\) prediction intervals median \\(m\\), can compute weighted interval score (WIS) sum interval scores individual intervals: $$ \\text{WIS}_{\\alpha_{\\{0:K\\}}}(F, y) = \\frac{1}{K + 1/2} \\times \\left(w_0 \\times |y - m| + \\sum_{k=1}^{K} \\left\\{ w_k \\times \\text{}_{\\alpha_k}(F, y) \\right\\}\\right) $$ individual scores usually weighted \\(w_k = \\frac{\\alpha_k}{2}\\). weight ensures increasing number equally spaced quantiles, WIS converges continuous ranked probability score (CRPS). Quantile score addition interval score, also exists quantile score (QS) (see quantile_score()), equal -called pinball loss. quantile score can computed single quantile (whereas interval score requires two quantiles form interval). However, intuitive decomposition overprediction, underprediction dispersion exist quantile score. Two versions weighted interval score two ways conceptualise weighted interval score across several quantiles / prediction intervals median. one view, treat WIS average quantile scores (median 0.5-quantile) (default wis()). another view, treat WIS average several interval scores + difference observed value median forecast. effect contrast first view, median twice much weight (weighted like prediction interval, rather like single quantile). valid ways conceptualise WIS can control behaviour count_median_twice-argument. WIS components: WIS components can computed individually using functions overprediction, underprediction, dispersion.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Weighted interval score (WIS) — wis","text":"","code":"wis( observed, predicted, quantile_level, separate_results = FALSE, weigh = TRUE, count_median_twice = FALSE, na.rm = FALSE ) dispersion_quantile(observed, predicted, quantile_level, ...) overprediction_quantile(observed, predicted, quantile_level, ...) underprediction_quantile(observed, predicted, quantile_level, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Weighted interval score (WIS) — wis","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made. separate_results Logical. TRUE (default FALSE), separate parts interval score (dispersion penalty, penalties - -prediction get returned separate elements list). want data.frame instead, simply call .data.frame() output. weigh Logical. TRUE (default), weigh score \\(\\alpha / 2\\), can averaged interval score , limit (increasing number equally spaced quantiles/prediction intervals), corresponds CRPS. \\(\\alpha\\) value corresponds (\\(\\alpha/2\\)) (\\(1 - \\alpha/2\\)), .e. decimal value represents much outside central prediction interval (E.g. 90 percent central prediction interval, alpha 0.1). count_median_twice TRUE, count median twice score. na.rm TRUE, ignore NA values computing score. ... Additional arguments passed wis() functions overprediction_quantile(), underprediction_quantile() dispersion_quantile().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Weighted interval score (WIS) — wis","text":"wis(): numeric vector WIS values size n (one per observation), list separate entries separate_results TRUE. dispersion_quantile(): numeric vector dispersion values (one per observation). overprediction_quantile(): numeric vector overprediction values (one per observation). underprediction_quantile(): numeric vector underprediction values (one per observation)","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Weighted interval score (WIS) — wis","text":"Evaluating epidemic forecasts interval format, Johannes Bracher, Evan L. Ray, Tilmann Gneiting Nicholas G. Reich, 2021, https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1008618","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Weighted interval score (WIS) — wis","text":"","code":"observed <- c(1, -15, 22) predicted <- rbind( c(-1, 0, 1, 2, 3), c(-2, 1, 2, 2, 4), c(-2, 0, 3, 3, 4) ) quantile_level <- c(0.1, 0.25, 0.5, 0.75, 0.9) wis(observed, predicted, quantile_level) #> [1] 0.36 15.34 19.14"},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-1229000","dir":"Changelog","previous_headings":"","what":"scoringutils 1.2.2.9000","title":"scoringutils 1.2.2.9000","text":"update represents major rewrite package introduces breaking changes. want keep using older version, can download using remotes::install_github(\"epiforecasts/scoringutils@v1.2\"). update aims make package modular customisable overall cleaner easier work . particular, aimed make suggested workflows evaluating forecasts explicit easier follow (see visualisation ). , clarified input formats made consistent across functions. refactord many functions S3-methods introduced forecast objects separate classes different types forecasts. new set as_forecast_() functions introduced validate data convert inputs forecast object (data.table forecast class additional class corresponding forecast type (see )). Another major update possibility users pass scoring functions score(). updated improved function documentation added new vignettes guide users package. Internally, refactored code, improved input checks, updated notifications (now use cli package) increased test coverage. comprehensive documentation new package rewrite revised version original scoringutils paper.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"score-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"score()","title":"scoringutils 1.2.2.9000","text":"previous columns “true_value” “prediction” renamed. score() now requires columns called “observed” “predicted” “model”. column quantile renamed quantile_level sample renamed sample_id score() now generic. S3 methods classes forecast_point, forecast_binary, forecast_quantile, forecast_sample, forecast_nominal, correspond different forecast types can scored scoringutils. score() now calls na.omit() data, instead removing rows missing values columns observed predicted. NA values columns can also mess e.g. grouping forecasts according unit single forecast. score() many functions now require validated forecast object. forecast objects can created using functions as_forecast_point(), as_forecast_binary(), as_forecast_quantile(), as_forecast_sample() (replace previous check_forecast()). forecast object data.table class forecast additional class corresponding forecast type (e.g. forecast_quantile). score() now returns objects class scores stored attribute metrics holds names scoring rules used. Users can call get_metrics() access names scoring rules. score() now returns one score per forecast, instead one score per sample quantile. binary point forecasts, columns “observed” “predicted” now removed consistency forecast types. Users can now also use scoring rules (making use metrics argument, takes named list functions). Default scoring rules can accessed using function get_metrics(), generic S3 methods forecast type. returns named list scoring rules suitable respective forecast object. example, call get_metrics(example_quantile). Column names scores output score() correspond names scoring rules (.e. names functions list metrics). Instead supplying arguments score() manipulate individual scoring rules users now manipulate metric list supplied using purrr::partial() select_metric(). See ?score() information. CRPS now reported decomposition dispersion, overprediction underprediction.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"creating-a-forecast-object-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Creating a forecast object","title":"scoringutils 1.2.2.9000","text":"as_forecast_() functions create forecast object validates . also allow users rename/specify required columns specify forecast unit single step, taking functionality set_forecast_unit() cases. See ?as_forecast() information. as_forecast_() functions like e.g. as_forecast_point() as_forecast_quantile() S3 methods converting another forecast type respective forecast type. example, as_forecast_quantile() method converting forecast_sample object forecast_quantile object estimating quantiles samples.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"updated-workflows-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Updated workflows","title":"scoringutils 1.2.2.9000","text":"example workflow scoring forecast now looks like : Overall, updated suggested workflows users work package. following gives overview (see updated paper details).","code":"forecast_quantile <- as_forecast_quantile( example_quantile, observed = \"observed\", predicted = \"predicted\", model = \"model\", quantile_level = \"quantile_level\", forecast_unit = c(\"model\", \"location\", \"target_end_date\", \"forecast_date\", \"target_type\") ) scores <- score(forecast_quantile)"},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"input-formats-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Input formats","title":"scoringutils 1.2.2.9000","text":"standardised input formats score() well scoring rules exported scoreingutils. following plot gives overview expected input formats different forecast types score(). Support interval format mostly dropped (see PR #525 @nikosbosse reviewed @seabbs). co-existence quantile interval format let confusing user experience many duplicated functions providing functionality. decided simplify interface focusing exclusively quantile format. function bias_range() removed (users now use bias_quantile() instead) function interval_score() made internal function rather exported users. recommend using wis() instead.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"re-validating-forecast-objects-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"(Re-)Validating forecast objects","title":"scoringutils 1.2.2.9000","text":"create validate new forecast object, users can use as_forecast_(). revalidate existing forecast object users can call assert_forecast() (validates input returns invisible(NULL). assert_forecast() generic methods different forecast types. Alternatively, users can call `as_forecast_() re-validate forecast object. Simply printing object also provide additional information. Users can test whether object class forecast_*() using function is_forecast(). Users can also test specific forecast_* class using appropriate is_forecast.forecast_* method. example, check whether object class forecast_quantile, use use scoringutils:::is_forecast.forecast_quantile().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"pairwise-comparisons-and-relative-skill-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Pairwise comparisons and relative skill","title":"scoringutils 1.2.2.9000","text":"functionality computing pairwise comparisons now split summarise_scores(). Instead pairwise comparisons part summarising scores, new function, add_relative_skill(), introduced takes summarised scores input adds columns relative skill scores scaled relative skill scores. function pairwise_comparison() renamed get_pairwise_comparisons(), line get_-functions. Analogously, plot_pairwise_comparison() renamed plot_pairwise_comparisons(). Output columns pairwise comparisons renamed contain name metric used comparing. Replaced warnings errors get_pairwise_comparison avoid returning NULL","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"computing-coverage-values-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Computing coverage values","title":"scoringutils 1.2.2.9000","text":"add_coverage() replaced new function, get_coverage(). function comes updated workflow coverage values computed directly based original data can visualised using plot_interval_coverage() plot_quantile_coverage(). example workflow example_quantile |> as_forecast_quantile() |> get_coverage(= \"model\") |> plot_interval_coverage().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"obtaining-and-plotting-forecast-counts-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Obtaining and plotting forecast counts","title":"scoringutils 1.2.2.9000","text":"clarity, output column get_forecast_counts() renamed “Number forecasts” “count”. get_forecast_counts() now also displays combinations 0 forecasts, instead silently dropping corresponding rows. plot_avail_forecasts() renamed plot_forecast_counts() line change function name. x argument longer default value, value depend data provided user.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"renamed-functions-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Renamed functions","title":"scoringutils 1.2.2.9000","text":"function find_duplicates() renamed get_duplicate_forecasts(). Renamed interval_coverage_quantile() interval_coverage(). “range” consistently renamed “interval_range” code. “range”-format (mostly used internally) renamed “interval”-format Renamed correlation() get_correlations() plot_correlation() plot_correlations() pit() renamed get_pit() converted S3 method.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"deleted-functions-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Deleted functions","title":"scoringutils 1.2.2.9000","text":"Removed abs_error squared_error package favour Metrics::ae Metrics::se.get_duplicate_forecasts() now sorts outputs according forecast unit, making easier spot duplicates. addition, counts option allows user display number duplicates forecast unit, rather raw duplicated rows. Deleted function plot_ranges(). want continue using functionality, can find function code Deprecated-visualisations Vignette. Removed function plot_predictions(), well helper function make_NA(), favour dedicated Vignette shows different ways visualising predictions. future reference, function code can found (Issue #659) Deprecated-visualisations Vignette. Removed function plot_score_table(). can find code Deprecated-visualisations Vignette. Removed function merge_pred_and_obs() used merge two separate data frames forecasts observations. moved contents new “Deprecated functions”-vignette. Removed interval_coverage_sample() users now expected convert quantile format first scoring. Function set_forecast_unit() deleted. Instead now forecast_unit argument as_forecast_() well get_duplicate_forecasts(). Removed interval_coverage_dev_quantile(). Users can still access difference nominal actual interval coverage using get_coverage().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"function-changes-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Function changes","title":"scoringutils 1.2.2.9000","text":"bias_quantile() changed way handles forecasts median missing: median now imputed linear interpolation innermost quantiles. Previously, imputed median simply taking mean innermost quantiles. contrast previous correlation function, get_correlations doesn’t round correlations default. Instead, plot_correlations now digits argument allows users round correlations plotting . Alternatively, using dplyr, call something like mutate(correlations, across((.numeric), \\(x) signif(x, digits = 2))) output get_correlations. wis() now errors default quantile levels form valid prediction intervals returns NA missing values. Previously, na.rm set TRUE default, lead unexpected results, users aware .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"internal-package-updates-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Internal package updates","title":"scoringutils 1.2.2.9000","text":"deprecated ..density.. replaced after_stat(density) ggplot calls. Files ending “.Rda” renamed “.rds” appropriate used together saveRDS() readRDS(). Added subsetting [ operator scores, score name attribute gets preserved subsetting. Switched using cli condition handling signalling, added tests check_*() test_*() functions. See #583 @jamesmbaazam reviewed @nikosbosse @seabbs. scoringutils now requires R >= 4.0","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"documentation-and-testing-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Documentation and testing","title":"scoringutils 1.2.2.9000","text":"Updates documentation functions made sure functions documented return statements Documentation pkgdown pages now created stable dev versions. Added unit tests many functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-122","dir":"Changelog","previous_headings":"","what":"scoringutils 1.2.2","title":"scoringutils 1.2.2","text":"CRAN release: 2023-11-29","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-2-2","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.2.2","text":"scoringutils now depends R 3.6. change made since packages testthat lifecycle, used scoringutils now require R 3.6. also updated Github action CI check work R 3.6 now. Added new PR template checklist things included PRs facilitate development review process","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"bug-fixes-1-2-2","dir":"Changelog","previous_headings":"","what":"Bug fixes","title":"scoringutils 1.2.2","text":"Fixes bug set_forecast_unit() function worked data.table, data.frame input. metrics table vignette Details metrics implemented scoringutils duplicated entries. fixed removing duplicated rows.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-2-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.2.1","text":"Gets rid preferably package _pkgdown.yml. theme toggle light dark theme didn’t work properly Updates gh pages deploy action v4 also cleans files triggered Introduces gh action automatically render Readme Readme.Rmd Removes links vignettes renamed","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-120","dir":"Changelog","previous_headings":"","what":"scoringutils 1.2.0","title":"scoringutils 1.2.0","text":"major release contains range new features bug fixes introduced minor releases since 1.1.0. important changes : Documentation updated reflect changes since version 1.1.0, including new transform workflow functions. New set_forecast_unit() function allows manual setting forecast unit. summarise_scores() gains new across argument summarizing across variables. New transform_forecasts() log_shift() functions allow forecast transformations. See documentation transform_forecasts() details example use case. Input checks test coverage improved bias functions. Bug fix get_prediction_type() integer matrix input. Links scoringutils paper citation updates. Warning added interval_score() small interval ranges. Linting updates improvements. Thanks @nikosbosse, @seabbs, @sbfnk code review contributions. Thanks @bisaloo suggestion use linting GitHub Action triggers changes, @adrian-lison suggestion add warning interval_score() interval range 0 1.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-2-0","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.2.0","text":"documentation updated reflect recent changes since scoringutils 1.1.0. particular, usage functions set_forecast_unit(), check_forecasts() transform_forecasts() now documented Vignettes. introduction functions enhances overall workflow help make code readable. functions designed used together pipe operator. example, one can now use something like following: Documentation transform_forecasts() also extended. functions allows user easily add transformations forecasts, suggested paper “Scoring epidemiological forecasts transformed scales”. epidemiological context, example, may make sense apply natural logarithm first scoring forecasts, order obtain scores reflect well models able predict exponential growth rates, rather absolute values. Users can now something like following score transformed version data addition original one: use log_shift() function apply logarithmic transformation forecasts. function introduced scoringutils 1.1.2 helper function acts just like log(), additional argument offset can add number every prediction observed value applying log transformation.","code":"example_quantile |> set_forecast_unit(c(\"model\", \"location\", \"forecast_date\", \"horizon\", \"target_type\")) |> check_forecasts() |> score() data <- example_quantile[true_value > 0, ] data |> transform_forecasts(fun = log_shift, offset = 1) |> score() |> summarise_scores(by = c(\"model\", \"scale\"))"},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-1-2-0","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.2.0","text":"Made check_forecasts() score() pipeable (see issue #290). means users can now directly use output check_forecasts() input score(). score() otherwise runs check_forecasts() internally anyway simply makes step explicit helps writing clearer code.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-117","dir":"Changelog","previous_headings":"","what":"scoringutils 1.1.7","title":"scoringutils 1.1.7","text":"Release @seabbs #305. Reviewed @nikosbosse @sbfnk.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"breaking-changes-1-1-7","dir":"Changelog","previous_headings":"","what":"Breaking changes","title":"scoringutils 1.1.7","text":"prediction_type argument get_forecast_unit() changed dropped. Instead new internal function prediction_is_quantile() used detect quantile variable present. Whilst internal function may impact users accessible via `find_duplicates().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-1-7","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.7","text":"Made imputation median bias_range() bias_quantile() obvious user may cause unexpected behaviour. Simplified bias_range() uses bias_quantile() internally. Added additional input checks bias_range(), bias_quantile(), check_predictions() make sure input valid. Improve coverage unit tests bias_range(), bias_quantile(), bias_sample(). Updated pairwise comparison unit tests use realistic data.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"bug-fixes-1-1-7","dir":"Changelog","previous_headings":"","what":"Bug fixes","title":"scoringutils 1.1.7","text":"Fixed bug get_prediction_type() led unable correctly detect integer (instead categorising continuous) forecasts input matrix. issue impacted bias_sample() also score() used integer forecasts resulting lower bias scores expected.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-1-1-6","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.1.6","text":"Added new argument, across, summarise_scores(). argument allows user summarise scores across different forecast units alternative specifying . See documentation summarise_scores() details example use case.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-1-1-5","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.1.5","text":"Added new function, set_forecast_unit() allows user set forecast unit manually. function removes columns relevant uniquely identifying single forecast. done manually, scoringutils attempts determine unit single automatically simply assuming column names relevant determine forecast unit. can lead unexpected behaviour, setting forecast unit explicitly can help make code easier debug easier read (see issue #268). used part workflow, set_forecast_unit() can directly piped check_forecasts() check everything order.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-1-4","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.4","text":"Added links scoringutils paper Evaluating Forecasts scoringutils R package. Updated citation formatting comply newer standards.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-1-3","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.3","text":"Added warning interval_score() interval range 0 1. Thanks @adrian-lison (see #277) suggestion.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-1-3-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.3","text":"Switched linting GitHub Action triggers changes. Inspired @bisaloo recent contribution epinowcast package. Updated package linters extensive. Inspired @bisaloo recent contribution epinowcast package. Resolved flagged linting issues across package.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-1-1-2","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.1.2","text":"Added new function, transform_forecasts() make easy transform forecasts scoring , suggested Bosse et al. (2023), https://www.medrxiv.org/content/10.1101/2023.01.23.23284722v1. Added function, log_shift() implements default transformation function. function allows add offset applying logarithm.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-111","dir":"Changelog","previous_headings":"","what":"scoringutils 1.1.1","title":"scoringutils 1.1.1","text":"Added small change interval_score() explicitly converts logical vector numeric one. happen implicitly anyway, now done explicitly order avoid issues may come input vector type doesn’t allow implicit conversion.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-110","dir":"Changelog","previous_headings":"","what":"scoringutils 1.1.0","title":"scoringutils 1.1.0","text":"CRAN release: 2023-01-30 minor update package bug fixes minor changes.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-1-0","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.0","text":"Removed attach message warned breaking changes 1.0.0. Renamed metric argument summarise_scores() relative_skill_metric. argument now deprecated removed future version package. Please use new argument instead. Updated documentation score() related functions make soft requirement model column input data explicit. Updated documentation score(), pairwise_comparison() summarise_scores() make clearer unit single forecast required computations Simplified function plot_pairwise_comparison() now supports plotting mean score ratios p-values removed hybrid option print time.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"bug-fixes-1-1-0","dir":"Changelog","previous_headings":"","what":"Bug fixes","title":"scoringutils 1.1.0","text":"Missing baseline forecasts pairwise_comparison() now trigger explicit informative error message. requirements table getting started vignette now correct. Added support optional sample column using quantile forecast format. Previously resulted error.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-100","dir":"Changelog","previous_headings":"","what":"scoringutils 1.0.0","title":"scoringutils 1.0.0","text":"CRAN release: 2022-05-13 Major update package package functions lots breaking changes.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-1-0-0","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.0.0","text":"New updated Readme vignette. proposed scoring workflow reworked. Functions changed can easily piped simplified arguments outputs.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"new-functions-and-function-changes-1-0-0","dir":"Changelog","previous_headings":"Feature updates","what":"New functions and function changes","title":"scoringutils 1.0.0","text":"function eval_forecasts() replaced function score() much reduced set function arguments. Functionality summarise scores add relative skill scores moved function summarise_scores() New function check_forecasts() analyse input data scoring New function correlation() compute correlations different metrics New function add_coverage() add coverage specific central prediction intervals. New function avail_forecasts() allows visualise number available forecasts. New function find_duplicates() find duplicate forecasts cause error. plotting functions renamed begin plot_. Arguments simplified. function pit() now works based data.frames. old pit function renamed pit_sample(). PIT p-values removed entirely. function plot_pit() now works directly input produced pit() Many data-handling functions removed input types score() restricted sample-based, quantile-based binary forecasts. function brier_score() now returns brier scores, rather taking mean returning output. crps(), dss() logs() renamed crps_sample(), dss_sample(), logs_sample()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"bug-fixes-1-0-0","dir":"Changelog","previous_headings":"Feature updates","what":"Bug fixes","title":"scoringutils 1.0.0","text":"Testing expanded Minor bugs fixed, example bug as_forecast_quantile() function (https://github.com/epiforecasts/scoringutils/pull/223)","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-data-updated-1-0-0","dir":"Changelog","previous_headings":"Feature updates","what":"Package data updated","title":"scoringutils 1.0.0","text":"Package data now based forecasts submitted European Forecast Hub (https://covid19forecasthub.eu/). example data files renamed begin example_. new data set, summary_metrics included contains summary metrics implemented scoringutils.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"other-breaking-changes-1-0-0","dir":"Changelog","previous_headings":"","what":"Other breaking changes","title":"scoringutils 1.0.0","text":"‘sharpness’ component weighted interval score renamed dispersion. done make clear component represents maintain consistency used places.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-8","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.8","text":"Added function check_forecasts() runs basic checks input data provides feedback.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-0172","dir":"Changelog","previous_headings":"","what":"scoringutils 0.1.7.2","title":"scoringutils 0.1.7.2","text":"CRAN release: 2021-07-21","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-7-2","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.7.2","text":"Minor bug fixes (previously, ‘interval_score’ needed among selected metrics). data.tables now returned table[] rather table, don’t called twice display contents.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-017","dir":"Changelog","previous_headings":"","what":"scoringutils 0.1.7","title":"scoringutils 0.1.7","text":"CRAN release: 2021-07-14","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-7","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.7","text":"Added function, pairwise_comparison() runs pairwise comparisons models output eval_forecasts() Added functionality compute relative skill within eval_forecasts(). Added function visualise pairwise comparisons.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-7","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.7","text":"WIS definition change introduced version 0.1.5 partly corrected difference weighting introduced summarising scores different interval ranges. “sharpness” renamed ‘mad’ output [score()] sample-based forecasts.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.","text":"eval_forecasts() can now handle separate forecast truth data set input. eval_forecasts() now supports scoring point forecasts along side quantiles quantile-based format. Currently metric used absolute error.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.","text":"Many functions, especially eval_forecasts() got major rewrite. functionality unchanged, code now easier maintain data-handling functions got renamed, old names supported well now.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-5","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.5","text":"Changed default definition weighted interval score. Previously, median prediction counted twice, counted . want go back old behaviour, can call interval_score function argument count_median_twice = FALSE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-014","dir":"Changelog","previous_headings":"","what":"scoringutils 0.1.4","title":"scoringutils 0.1.4","text":"CRAN release: 2020-11-17","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-4","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.4","text":"Added basic plotting functionality visualise scores. can now easily obtain diagnostic plots based scores produced score. correlation_plot() shows correlation metrics. plot_ranges() shows contribution different prediction intervals chosen metric. plot_heatmap() visualises scores heatmap. plot_score_table() shows coloured summary table scores.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-4","dir":"Changelog","previous_headings":"","what":"package updates","title":"scoringutils 0.1.4","text":"Renamed “calibration” “coverage”. Renamed “true_values” “true_value” data.frames. Renamed “predictions” “prediction” data.frames. Renamed “is_overprediction” “overprediction”. Renamed “is_underprediction” “underprediction”.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"potentially-breaking-changes-0-1-3","dir":"Changelog","previous_headings":"","what":"(Potentially) Breaking changes","title":"scoringutils 0.1.3","text":"argument score now slightly changed meaning. now denotes lowest possible grouping unit, .e. unit one observation needs specified explicitly. default now NULL. reason change metrics need scoring observation level consistent implementation principle. pit function receives grouping now summarise_by. similar spirit, summarise_by specified explicitly e.g. doesn’t assume anymore want ‘range’ included. interval score, weigh = TRUE now default option. Renamed true_values true_value predictions prediction.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-3","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.3","text":"Updated quantile evaluation metrics score. Bias well calibration now take quantiles account. Included option summarise scores according summarise_by argument score() summary can return mean, standard deviation well arbitrary set quantiles. score() can now return pit histograms. Switched ggplot2 plotting.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"potentially-breaking-changes-0-1-2","dir":"Changelog","previous_headings":"","what":"(Potentially) Breaking changes","title":"scoringutils 0.1.2","text":"scores score consistently renamed lower case. Interval_score now interval_score, CRPS now crps etc.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-2","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.2","text":"Included support grouping scores according vector column names score(). Included support passing arguments lower-level functions score() Included support three new metrics score quantiles score(): bias, sharpness calibration","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-2","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.2","text":"Example data now horizon column illustrate use grouping. Documentation updated explain listed changes.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-1","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.1","text":"Included support long well wide input formats quantile forecasts scored score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.1","text":"Updated documentation score(). Added badges README.","code":""}] +[{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"our-pledge","dir":"","previous_headings":"","what":"Our Pledge","title":"Contributor Covenant Code of Conduct","text":"members, contributors, leaders pledge make participation community harassment-free experience everyone, regardless age, body size, visible invisible disability, ethnicity, sex characteristics, gender identity expression, level experience, education, socio-economic status, nationality, personal appearance, race, religion, sexual identity orientation. pledge act interact ways contribute open, welcoming, diverse, inclusive, healthy community.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"our-standards","dir":"","previous_headings":"","what":"Our Standards","title":"Contributor Covenant Code of Conduct","text":"Examples behavior contributes positive environment community include: Demonstrating empathy kindness toward people respectful differing opinions, viewpoints, experiences Giving gracefully accepting constructive feedback Accepting responsibility apologizing affected mistakes, learning experience Focusing best just us individuals, overall community Examples unacceptable behavior include: use sexualized language imagery, sexual attention advances kind Trolling, insulting derogatory comments, personal political attacks Public private harassment Publishing others’ private information, physical email address, without explicit permission conduct reasonably considered inappropriate professional setting","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"enforcement-responsibilities","dir":"","previous_headings":"","what":"Enforcement Responsibilities","title":"Contributor Covenant Code of Conduct","text":"Community leaders responsible clarifying enforcing standards acceptable behavior take appropriate fair corrective action response behavior deem inappropriate, threatening, offensive, harmful. Community leaders right responsibility remove, edit, reject comments, commits, code, wiki edits, issues, contributions aligned Code Conduct, communicate reasons moderation decisions appropriate.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"scope","dir":"","previous_headings":"","what":"Scope","title":"Contributor Covenant Code of Conduct","text":"Code Conduct applies within community spaces, also applies individual officially representing community public spaces. Examples representing community include using official e-mail address, posting via official social media account, acting appointed representative online offline event.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"enforcement","dir":"","previous_headings":"","what":"Enforcement","title":"Contributor Covenant Code of Conduct","text":"Instances abusive, harassing, otherwise unacceptable behavior may reported community leaders responsible enforcement Sebastian.Funk@lshtm.ac.uk. complaints reviewed investigated promptly fairly. community leaders obligated respect privacy security reporter incident.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"enforcement-guidelines","dir":"","previous_headings":"","what":"Enforcement Guidelines","title":"Contributor Covenant Code of Conduct","text":"Community leaders follow Community Impact Guidelines determining consequences action deem violation Code Conduct:","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"id_1-correction","dir":"","previous_headings":"Enforcement Guidelines","what":"1. Correction","title":"Contributor Covenant Code of Conduct","text":"Community Impact: Use inappropriate language behavior deemed unprofessional unwelcome community. Consequence: private, written warning community leaders, providing clarity around nature violation explanation behavior inappropriate. public apology may requested.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"id_2-warning","dir":"","previous_headings":"Enforcement Guidelines","what":"2. Warning","title":"Contributor Covenant Code of Conduct","text":"Community Impact: violation single incident series actions. Consequence: warning consequences continued behavior. interaction people involved, including unsolicited interaction enforcing Code Conduct, specified period time. includes avoiding interactions community spaces well external channels like social media. Violating terms may lead temporary permanent ban.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"id_3-temporary-ban","dir":"","previous_headings":"Enforcement Guidelines","what":"3. Temporary Ban","title":"Contributor Covenant Code of Conduct","text":"Community Impact: serious violation community standards, including sustained inappropriate behavior. Consequence: temporary ban sort interaction public communication community specified period time. public private interaction people involved, including unsolicited interaction enforcing Code Conduct, allowed period. Violating terms may lead permanent ban.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"id_4-permanent-ban","dir":"","previous_headings":"Enforcement Guidelines","what":"4. Permanent Ban","title":"Contributor Covenant Code of Conduct","text":"Community Impact: Demonstrating pattern violation community standards, including sustained inappropriate behavior, harassment individual, aggression toward disparagement classes individuals. Consequence: permanent ban sort public interaction within community.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CODE_OF_CONDUCT.html","id":"attribution","dir":"","previous_headings":"","what":"Attribution","title":"Contributor Covenant Code of Conduct","text":"Code Conduct adapted Contributor Covenant, version 2.0, available https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. Community Impact Guidelines inspired Mozilla’s code conduct enforcement ladder. answers common questions code conduct, see FAQ https://www.contributor-covenant.org/faq. Translations available https://www.contributor-covenant.org/translations.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":null,"dir":"","previous_headings":"","what":"Contributing to scoringutils","title":"Contributing to scoringutils","text":"outlines propose change scoringutils.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"fixing-typos","dir":"","previous_headings":"","what":"Fixing typos","title":"Contributing to scoringutils","text":"can fix typos, spelling mistakes, grammatical errors documentation directly using GitHub web interface, long changes made source file. generally means ’ll need edit roxygen2 comments .R, .Rd file. can find .R file generates .Rd reading comment first line.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"bigger-changes","dir":"","previous_headings":"","what":"Bigger changes","title":"Contributing to scoringutils","text":"want make bigger change, ’s good idea first file issue make sure someone team agrees ’s needed. ’ve found bug, please file issue illustrates bug minimal reprex (also help write unit test, needed).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"pull-request-process","dir":"","previous_headings":"Bigger changes","what":"Pull request process","title":"Contributing to scoringutils","text":"Fork package clone onto computer. haven’t done , recommend using usethis::create_from_github(\"epiforecasts/scoringutils\", fork = TRUE). Install development dependences devtools::install_dev_deps(), make sure package passes R CMD check running devtools::check(). R CMD check doesn’t pass cleanly, ’s good idea ask help continuing. Create Git branch pull request (PR). recommend using usethis::pr_init(\"brief-description--change\"). Make changes, commit git, create PR running usethis::pr_push(), following prompts browser. title PR briefly describe change. body PR contain Fixes #issue-number. user-facing changes, add bullet top NEWS.md (.e. just first header). Follow style described https://style.tidyverse.org/news.html.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"code-style","dir":"","previous_headings":"Bigger changes","what":"Code style","title":"Contributing to scoringutils","text":"New code follow tidyverse style guide. can use styler package apply styles, please don’t restyle code nothing PR. use roxygen2, Markdown syntax, documentation. use testthat unit tests. Contributions test cases included easier accept.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/CONTRIBUTING.html","id":"code-of-conduct","dir":"","previous_headings":"","what":"Code of Conduct","title":"Contributing to scoringutils","text":"Please note scoringutils project released Contributor Code Conduct. contributing project agree abide terms.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/LICENSE.html","id":null,"dir":"","previous_headings":"","what":"MIT License","title":"MIT License","text":"Copyright (c) 2020 EpiForecasts Permission hereby granted, free charge, person obtaining copy software associated documentation files (“Software”), deal Software without restriction, including without limitation rights use, copy, modify, merge, publish, distribute, sublicense, /sell copies Software, permit persons Software furnished , subject following conditions: copyright notice permission notice shall included copies substantial portions Software. SOFTWARE PROVIDED “”, WITHOUT WARRANTY KIND, EXPRESS IMPLIED, INCLUDING LIMITED WARRANTIES MERCHANTABILITY, FITNESS PARTICULAR PURPOSE NONINFRINGEMENT. EVENT SHALL AUTHORS COPYRIGHT HOLDERS LIABLE CLAIM, DAMAGES LIABILITY, WHETHER ACTION CONTRACT, TORT OTHERWISE, ARISING , CONNECTION SOFTWARE USE DEALINGS SOFTWARE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/PULL_REQUEST_TEMPLATE.html","id":"description","dir":"","previous_headings":"","what":"Description","title":"NA","text":"PR closes #. [Describe changes made pull request.]","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/PULL_REQUEST_TEMPLATE.html","id":"checklist","dir":"","previous_headings":"","what":"Checklist","title":"NA","text":"PR based package issue explicitly linked . included target issue issues PR title follows: issue-number: PR title tested changes locally. added updated unit tests necessary. updated documentation required. built package locally run rebuilt docs using roxygen2. code follows established coding standards run lintr::lint_package() check style issues introduced changes. added news item linked PR. reviewed CI checks PR addressed far able.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/Deprecated-functions.html","id":"merge_pred_and_obs","dir":"Articles","previous_headings":"","what":"merge_pred_and_obs()","title":"Deprecated functions","text":"scoringutils requires forecasts observations provided single data frame. forecasts observations two different data frames, merge_pred_and_obs() may help merge two. function mostly wrapper around merge(), additional work deal duplicated column names.","code":"#' @title Merge forecast data and observations #' #' @description #' #' The function more or less provides a wrapper around `merge` that #' aims to handle the merging well if additional columns are present #' in one or both data sets. If in doubt, you should probably merge the #' data sets manually. #' #' @param forecasts A data.frame with the forecast data (as can be passed to #' [score()]). #' @param observations A data.frame with the observations. #' @param join Character, one of `c(\"left\", \"full\", \"right\")`. Determines the #' type of the join. Usually, a left join is appropriate, but sometimes you #' may want to do a full join to keep dates for which there is a forecast, but #' no ground truth data. #' @param by Character vector that denotes the columns by which to merge. Any #' value that is not a column in observations will be removed. #' @return a data.table with forecasts and observations #' @importFrom checkmate assert_subset #' @importFrom data.table as.data.table #' @keywords data-handling #' @export merge_pred_and_obs <- function(forecasts, observations, join = c(\"left\", \"full\", \"right\"), by = NULL) { forecasts <- as.data.table(forecasts) observations <- as.data.table(observations) join <- match.arg(join) assert_subset(by, intersect(names(forecasts), names(observations))) if (is.null(by)) { protected_columns <- c( \"predicted\", \"observed\", \"sample_id\", \"quantile_level\", \"interval_range\", \"boundary\" ) by <- setdiff(colnames(forecasts), protected_columns) } obs_cols <- colnames(observations) by <- intersect(by, obs_cols) join <- match.arg(join) if (join == \"left\") { # do a left_join, where all data in the observations are kept. combined <- merge(observations, forecasts, by = by, all.x = TRUE) } else if (join == \"full\") { # do a full, where all data is kept. combined <- merge(observations, forecasts, by = by, all = TRUE) } else { combined <- merge(observations, forecasts, by = by, all.y = TRUE) } # get colnames that are the same for x and y colnames <- colnames(combined) colnames_x <- colnames[endsWith(colnames, \".x\")] colnames_y <- colnames[endsWith(colnames, \".y\")] # extract basenames basenames_x <- sub(\".x$\", \"\", colnames_x) basenames_y <- sub(\".y$\", \"\", colnames_y) # see whether the column name as well as the content is the same content_x <- as.list(combined[, ..colnames_x]) content_y <- as.list(combined[, ..colnames_y]) overlapping <- (content_x %in% content_y) & (basenames_x == basenames_y) overlap_names <- colnames_x[overlapping] basenames_overlap <- sub(\".x$\", \"\", overlap_names) # delete overlapping columns if (length(basenames_overlap) > 0) { combined[, paste0(basenames_overlap, \".x\") := NULL] combined[, paste0(basenames_overlap, \".y\") := NULL] } return(combined[]) }"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/Deprecated-visualisations.html","id":"functions-plot_predictions-and-make_na","dir":"Articles","previous_headings":"","what":"Functions plot_predictions() and make_na()","title":"Deprecated Visualisations","text":"previous versions scoringutils, forecasts observed values visualised using function plot_predictions() make_na() helper function. following shows function code first example. plot_predictions() actual work producing plot. argument needed user can facet plot correctly user needs specify columns relevant facetting. make_NA() represents form filtering, instead filtering entire rows, relevant entries columns “predicted” “observed” made NA. allows user filter observations forecasts independently. following examples using two functions create plot using scoringutils example data. Visualising median forecasts example data. truth data restricted period 2021-05-01 2021-07-22. forecast data forecast model “EuroCOVIDhub-ensemble” made “2021-06-07”. data set NA, effectively removing plot. plot, variety prediction intervals shown, instead just median. similar plot, time based continuous forecasts. predictions automatically converted quantile-based forecasts plotting. Displaying two forecasts time additional colours:","code":"#\" @title Plot Predictions vs True Values #\" #\" @description #\" Make a plot of observed and predicted values #\" #\" @param data a data.frame that follows the same specifications outlined in #\" [score()]. To customise your plotting, you can filter your data using the #\" function [make_NA()]. #\" @param by character vector with column names that denote categories by which #\" the plot should be stratified. If for example you want to have a facetted #\" plot, this should be a character vector with the columns used in facetting #\" (note that the facetting still needs to be done outside of the function call) #\" @param x character vector of length one that denotes the name of the variable #\" @param interval_range numeric vector indicating the interval ranges to plot. #\" If 0 is included in `interval_range`, the median prediction will be shown. #\" @return ggplot object with a plot of true vs predicted values #\" @importFrom ggplot2 ggplot scale_colour_manual scale_fill_manual theme_light #\" @importFrom ggplot2 facet_wrap facet_grid aes geom_line .data geom_point #\" @importFrom data.table dcast #\" @importFrom ggdist geom_lineribbon #\" @export #\" @examples #\" library(ggplot2) #\" library(magrittr) #\" #\" example_sample_continuous %>% #\" make_NA ( #\" what = \"truth\", #\" target_end_date >= \"2021-07-22\", #\" target_end_date < \"2021-05-01\" #\" ) %>% #\" make_NA ( #\" what = \"forecast\", #\" model != \"EuroCOVIDhub-ensemble\", #\" forecast_date != \"2021-06-07\" #\" ) %>% #\" plot_predictions ( #\" x = \"target_end_date\", #\" by = c(\"target_type\", \"location\"), #\" interval_range = c(0, 50, 90, 95) #\" ) + #\" facet_wrap(~ location + target_type, scales = \"free_y\") + #\" aes(fill = model, color = model) #\" #\" example_sample_continuous %>% #\" make_NA ( #\" what = \"truth\", #\" target_end_date >= \"2021-07-22\", #\" target_end_date < \"2021-05-01\" #\" ) %>% #\" make_NA ( #\" what = \"forecast\", #\" forecast_date != \"2021-06-07\" #\" ) %>% #\" plot_predictions ( #\" x = \"target_end_date\", #\" by = c(\"target_type\", \"location\"), #\" interval_range = 0 #\" ) + #\" facet_wrap(~ location + target_type, scales = \"free_y\") + #\" aes(fill = model, color = model) library(ggdist) plot_predictions <- function(data, by = NULL, x = \"date\", interval_range = c(0, 50, 90)) { # split truth data and forecasts in order to apply different filtering truth_data <- data.table::as.data.table(data)[!is.na(observed)] forecasts <- data.table::as.data.table(data)[!is.na(predicted)] del_cols <- colnames(truth_data)[!(colnames(truth_data) %in% c(by, \"observed\", x))] truth_data <- unique(suppressWarnings(truth_data[, eval(del_cols) := NULL])) # find out what type of predictions we have. convert sample based to # interval range data if (\"quantile_level\" %in% colnames(data)) { forecasts <- scoringutils:::quantile_to_interval( forecasts, keep_quantile_col = FALSE ) } else if (\"sample_id\" %in% colnames(data)) { # using a scoringutils internal function forecasts <- scoringutils:::sample_to_interval_long( as_forecast_sample(forecasts), interval_range = interval_range, keep_quantile_col = FALSE ) } # select appropriate boundaries and pivot wider select <- forecasts$interval_range %in% setdiff(interval_range, 0) intervals <- forecasts[select, ] # delete quantile column in intervals if present. This is important for # pivoting if (\"quantile_level\" %in% names(intervals)) { intervals[, quantile_level := NULL] } plot <- ggplot(data = data, aes(x = .data[[x]])) + theme_scoringutils() + ylab(\"True and predicted values\") if (nrow(intervals) != 0) { # pivot wider and convert range to a factor intervals <- data.table::dcast(intervals, ... ~ boundary, value.var = \"predicted\") # only plot interval ranges if there are interval ranges to plot plot <- plot + ggdist::geom_lineribbon( data = intervals, aes( ymin = lower, ymax = upper, # We use the fill_ramp aesthetic for this instead of the default fill # because we want to keep fill to be able to use it for other # variables fill_ramp = factor( interval_range, levels = sort(unique(interval_range), decreasing = TRUE) ) ), lwd = 0.4 ) + ggdist::scale_fill_ramp_discrete( name = \"interval_range\", # range argument was added to make sure that the line for the median # and the ribbon don\"t have the same opacity, making the line # invisible range = c(0.15, 0.75) ) } # We could treat this step as part of ggdist::geom_lineribbon() but we treat # it separately here to deal with the case when only the median is provided # (in which case ggdist::geom_lineribbon() will fail) if (0 %in% interval_range) { select_median <- forecasts$interval_range == 0 & forecasts$boundary == \"lower\" median <- forecasts[select_median] if (nrow(median) > 0) { plot <- plot + geom_line( data = median, mapping = aes(y = predicted), lwd = 0.4 ) } } # add observed values if (nrow(truth_data) > 0) { plot <- plot + geom_point( data = truth_data, show.legend = FALSE, inherit.aes = FALSE, aes(x = .data[[x]], y = observed), color = \"black\", size = 0.5 ) + geom_line( data = truth_data, inherit.aes = FALSE, show.legend = FALSE, aes(x = .data[[x]], y = observed), linetype = 1, color = \"grey40\", lwd = 0.2 ) } return(plot) } #\" @title Make Rows NA in Data for Plotting #\" #\" @description #\" Filters the data and turns values into `NA` before the data gets passed to #\" [plot_predictions()]. The reason to do this is to this is that it allows to #\" \"filter\" prediction and truth data separately. Any value that is NA will then #\" be removed in the subsequent call to [plot_predictions()]. #\" #\" @inheritParams score #\" @param what character vector that determines which values should be turned #\" into `NA`. If `what = \"truth\"`, values in the column \"observed\" will be #\" turned into `NA`. If `what = \"forecast\"`, values in the column \"prediction\" #\" will be turned into `NA`. If `what = \"both\"`, values in both column will be #\" turned into `NA`. #\" @param ... logical statements used to filter the data #\" @return A data.table #\" @importFrom rlang enexprs #\" @keywords plotting #\" @export #\" #\" @examples #\" make_NA ( #\" example_sample_continuous, #\" what = \"truth\", #\" target_end_date >= \"2021-07-22\", #\" target_end_date < \"2021-05-01\" #\" ) make_NA <- function(data = NULL, what = c(\"truth\", \"forecast\", \"both\"), ...) { stopifnot(is.data.frame(data)) data <- as.data.table(data) what <- match.arg(what) # turn ... arguments into expressions args <- enexprs(...) vars <- NULL if (what %in% c(\"forecast\", \"both\")) { vars <- c(vars, \"predicted\") } if (what %in% c(\"truth\", \"both\")) { vars <- c(vars, \"observed\") } for (expr in args) { data <- data[eval(expr), eval(vars) := NA_real_] } return(data[]) } median_forecasts <- example_quantile[quantile_level == 0.5] median_forecasts %>% make_NA(what = \"truth\", target_end_date <= \"2021-05-01\", target_end_date > \"2021-07-22\") %>% make_NA(what = \"forecast\", model != \"EuroCOVIDhub-ensemble\", forecast_date != \"2021-06-07\") %>% plot_predictions( by = c(\"location\", \"target_type\"), x = \"target_end_date\" ) + facet_wrap(location ~ target_type, scales = \"free_y\") example_quantile %>% make_NA(what = \"truth\", target_end_date <= \"2021-05-01\", target_end_date > \"2021-07-22\") %>% make_NA(what = \"forecast\", model != \"EuroCOVIDhub-ensemble\", forecast_date != \"2021-06-07\") %>% plot_predictions( by = c(\"location\", \"target_type\"), x = \"target_end_date\", interval_range = c(0, 10, 20, 30, 40, 50, 60) ) + facet_wrap(location ~ target_type, scales = \"free_y\") example_sample_continuous %>% make_NA(what = \"truth\", target_end_date <= \"2021-05-01\", target_end_date > \"2021-07-22\") %>% make_NA(what = \"forecast\", model != \"EuroCOVIDhub-ensemble\", forecast_date != \"2021-06-07\") %>% plot_predictions( by = c(\"location\", \"target_type\"), x = \"target_end_date\", interval_range = c(0, 50, 90, 95) ) + facet_wrap(location ~ target_type, scales = \"free_y\") example_quantile %>% make_NA(what = \"truth\", target_end_date > \"2021-07-15\", target_end_date <= \"2021-05-22\") %>% make_NA(what = \"forecast\", !(model %in% c(\"EuroCOVIDhub-ensemble\", \"EuroCOVIDhub-baseline\")), forecast_date != \"2021-06-28\") %>% plot_predictions(x = \"target_end_date\", by = c(\"target_type\", \"location\")) + aes(colour = model, fill = model) + facet_wrap(target_type ~ location, ncol = 4, scales = \"free_y\") + labs(x = \"Target end date\")"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/Deprecated-visualisations.html","id":"function-plot_interval_ranges-formerly-plot_ranges","dir":"Articles","previous_headings":"","what":"Function plot_interval_ranges() (formerly plot_ranges())","title":"Deprecated Visualisations","text":"functionality currently relies hack. previous versions scoringutils, scores computed per interval range/per quantile. Now, scoringutils returns one score per forecast, per interval range/quantile. therefore need add range column, using internal function get_range_from_quantile(). column interpreted one defines unit single forecast scoringutils. also means get warning different number quantile levels different forecasts (0% prediction interval one median forecast, prediction intervals two (lower upper bound)). Plotting dispersion instead WIS:","code":"#\" @title Plot Metrics by Range of the Prediction Interval #\" #\" @description #\" Visualise the metrics by range, e.g. if you are interested how different #\" interval ranges contribute to the overall interval score, or how #\" sharpness / dispersion changes by range. #\" #\" @param scores A data.frame of scores based on quantile forecasts as #\" produced by [score()] or [summarise_scores()]. Note that \"range\" must be included #\" in the `by` argument when running [summarise_scores()] #\" @param y The variable from the scores you want to show on the y-Axis. #\" This could be something like \"wis\" (the default) or \"dispersion\" #\" @param x The variable from the scores you want to show on the x-Axis. #\" Usually this will be \"model\" #\" @param colour Character vector of length one used to determine a variable #\" for colouring dots. The Default is \"range\". #\" @return A ggplot2 object showing a contributions from the three components of #\" the weighted interval score #\" @importFrom ggplot2 ggplot aes aes geom_point geom_line #\" expand_limits theme theme_light element_text scale_color_continuous labs #\" @export #\" @examples #\" library(ggplot2) #\" ex <- data.table::copy(example_quantile) #\" ex$range <- scoringutils:::get_range_from_quantile(ex$quantile) #\" scores <- suppressWarnings(score(as_forecast_quantile(ex), metrics = list(\"wis\" = wis))) #\" summarised <- summarise_scores( #\" scores, #\" by = c(\"model\", \"target_type\", \"range\") #\" ) #\" plot_interval_ranges(summarised, x = \"model\") + #\" facet_wrap(~target_type, scales = \"free\") plot_interval_ranges <- function(scores, y = \"wis\", x = \"model\", colour = \"range\") { plot <- ggplot( scores, aes( x = .data[[x]], y = .data[[y]], colour = .data[[colour]] ) ) + geom_point(size = 2) + geom_line(aes(group = range), colour = \"black\", linewidth = 0.01 ) + scale_color_continuous(low = \"steelblue\", high = \"salmon\") + theme_scoringutils() + expand_limits(y = 0) + theme( legend.position = \"right\", axis.text.x = element_text( angle = 90, vjust = 1, hjust = 1 ) ) return(plot) } range_example <- copy(example_quantile) %>% na.omit() %>% .[, range := scoringutils:::get_range_from_quantile(quantile_level)] sum_scores <- range_example %>% as_forecast_quantile() %>% score(metrics = list(wis = wis, dispersion = dispersion_quantile)) %>% summarise_scores(by = c(\"model\", \"target_type\", \"range\")) %>% suppressWarnings() plot_interval_ranges(sum_scores, x = \"model\") + facet_wrap(~target_type, scales = \"free\") plot_interval_ranges(sum_scores, y = \"dispersion\", x = \"model\") + facet_wrap(~target_type, scales = \"free_y\")"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/Deprecated-visualisations.html","id":"function-plot_score_table","dir":"Articles","previous_headings":"","what":"Function plot_score_table()","title":"Deprecated Visualisations","text":"function allowed users turn table (summarised) scores coloured table. function hard-coded information colour scale pick metric. scoringutils 2.0.0, allowed users assign names metrics use custom scoring functions. stick default names provided scoringutils, function still work. However, functionality easily generalisable, decided deprecate function. main functionality old function provided, scaling scores order obtain reasonable colour shades. per metric, one also pass additional grouping variables argument. allowed users achieve faceting table (note course scores also needed summarised according grouping). function also allowed users combine different facets one, creating combined y-variable. done passing vector column names y argument.","code":"#' @title Plot Coloured Score Table #' #' @description #' Plots a coloured table of summarised scores obtained using #' [score()]. #' #' @param y the variable to be shown on the y-axis. Instead of a single character string, #' you can also specify a vector with column names, e.g. #' `y = c(\"model\", \"location\")`. These column names will be concatenated #' to create a unique row identifier (e.g. \"model1_location1\"). #' @param by A character vector that determines how the colour shading for the #' plot gets computed. By default (`NULL`), shading will be determined per #' metric, but you can provide additional column names (see examples). #' @param metrics A character vector with the metrics to show. If set to #' `NULL` (default), all metrics present in `scores` will be shown. #' #' @return A ggplot object with a coloured table of summarised scores #' @inheritParams get_pairwise_comparisons #' @importFrom ggplot2 ggplot aes element_blank element_text labs coord_cartesian coord_flip #' @importFrom data.table setDT melt #' @importFrom stats sd #' @export #' #' @examples #' library(ggplot2) #' library(magrittr) # pipe operator #' \\dontshow{ #' data.table::setDTthreads(2) # restricts number of cores used on CRAN #' } #' #' scores <- score(as_forecast_quantile(example_quantile)) %>% #' summarise_scores(by = c(\"model\", \"target_type\")) %>% #' summarise_scores(by = c(\"model\", \"target_type\"), fun = signif, digits = 2) #' #' plot_score_table(scores, y = \"model\", by = \"target_type\") + #' facet_wrap(~target_type, ncol = 1) #' #' # can also put target description on the y-axis #' plot_score_table(scores, #' y = c(\"model\", \"target_type\"), #' by = \"target_type\") plot_score_table <- function(scores, y = \"model\", by = NULL, metrics = NULL) { # identify metrics ----------------------------------------------------------- id_vars <- get_forecast_unit(scores) metrics <- get_metrics(scores) cols_to_delete <- names(scores)[!(names(scores) %in% c(metrics, id_vars))] suppressWarnings(scores[, eval(cols_to_delete) := NULL]) # compute scaled values ------------------------------------------------------ # scaling is done in order to colour the different scores # for most metrics larger is worse, but others like bias are better if they # are close to zero and deviations in both directions are bad # define which metrics are scaled using min (larger is worse) and # which not (metrics like bias where deviations in both directions are bad) metrics_zero_good <- c(\"bias\", \"interval_coverage_deviation\") metrics_no_color <- \"coverage\" metrics_min_good <- setdiff(metrics, c( metrics_zero_good, metrics_no_color )) # write scale functions that can be used in data.table scale <- function(x) { scaled <- x / sd(x, na.rm = TRUE) return(scaled) } scale_min_good <- function(x) { scaled <- (x - min(x)) / sd(x, na.rm = TRUE) return(scaled) } # pivot longer and add scaled values df <- data.table::melt(scores, value.vars = metrics, id.vars = id_vars, variable.name = \"metric\" ) df[metric %in% metrics_min_good, value_scaled := scale_min_good(value), by = c(\"metric\", by) ] df[metric %in% metrics_zero_good, value_scaled := scale(value), by = c(\"metric\", by) ] df[metric %in% metrics_no_color, value_scaled := 0, by = c(\"metric\", by) ] # create identifier column for plot ------------------------------------------ # if there is only one column, leave column as is. Reason to do that is that # users can then pass in a factor and keep the ordering of that column intact if (length(y) > 1) { df[, identifCol := do.call(paste, c(.SD, sep = \"_\")), .SDcols = y[y %in% names(df)]] } else { setnames(df, old = eval(y), new = \"identifCol\") } # plot ----------------------------------------------------------------------- # make plot with all metrics that are not NA plot <- ggplot( df[!is.na(value), ], aes(y = identifCol, x = metric) ) + geom_tile(aes(fill = value_scaled), colour = \"white\", show.legend = FALSE) + geom_text(aes(y = identifCol, label = value)) + scale_fill_gradient2(low = \"steelblue\", high = \"salmon\") + theme_scoringutils() + theme( legend.title = element_blank(), legend.position = \"none\", axis.text.x = element_text( angle = 90, vjust = 1, hjust = 1 ) ) + labs(x = \"\", y = \"\") + coord_cartesian(expand = FALSE) return(plot) } scores <- score(as_forecast_quantile(example_quantile)) %>% summarise_scores(by = c(\"model\", \"target_type\")) %>% summarise_scores(by = c(\"model\", \"target_type\"), fun = signif, digits = 2) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_score_table(scores, y = \"model\", by = \"target_type\") + facet_wrap(~target_type, ncol = 1) # can also put target description on the y-axis plot_score_table(scores, y = c(\"model\", \"target_type\"), by = \"target_type\")"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"introduction","dir":"Articles","previous_headings":"","what":"Introduction","title":"Scoring rules in `scoringutils`","text":"vignette gives overview default scoring rules made available scoringutils package. can, course, also use scoring rules, provided follow format. want obtain detailed information pacakge works, look revised version scoringutils paper. can distinguish two types forecasts: point forecasts probabilistic forecasts. point forecast single number representing single outcome. probabilistic forecast full predictive probability distribution multiple possible outcomes. contrast point forecasts, probabilistic forecasts incorporate uncertainty different possible outcomes. Scoring rules functions take forecast observation input return single numeric value. point forecasts, take form S(ŷ,y)S(\\hat{y}, y), ŷ\\hat{y} forecast yy observation. probabilistic forecasts, usually take form S(F,y)S(F, y), FF cumulative density function (CDF) predictive distribution yy observation. convention, scoring rules usually negatively oriented, meaning smaller values better (best possible score usually zero). sense, score can understood penalty. Many scoring rules probabilistic forecasts -called (strictly) proper scoring rules. Essentially, means “cheated”: forecaster evaluated strictly proper scoring rule always incentivised report honest best belief future , expectation, improve score reporting something else. formal definition following: Let GG true, unobserved data-generating distribution. scoring rule said proper, GG ideal forecast F=GF = G, forecast F′≠FF' \\neq F expectation receives better score FF. scoring rule considered strictly proper , GG, forecast F′F' expectation receives score better FF.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"metrics-for-point-forecasts","dir":"Articles","previous_headings":"","what":"Metrics for point forecasts","title":"Scoring rules in `scoringutils`","text":"See list default metrics point forecasts calling get_metrics(example_point). overview input output formats point forecasts: Input output formats: metrics point.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"a-note-of-caution","dir":"Articles","previous_headings":"Metrics for point forecasts","what":"A note of caution","title":"Scoring rules in `scoringutils`","text":"Scoring point forecasts can tricky business. Depending choice scoring rule, forecaster clearly worse another, might consistently receive better scores (see Gneiting (2011) illustrative example). Every scoring rule point forecast implicitly minimised specific aspect predictive distribution. mean squared error, example, meaningful scoring rule forecaster actually reported mean predictive distribution point forecast. forecaster reported median, mean absolute error appropriate scoring rule. scoring rule predictive task align, misleading results ensue. Consider following example:","code":"set.seed(123) n <- 1000 observed <- rnorm(n, 5, 4)^2 predicted_mu <- mean(observed) predicted_not_mu <- predicted_mu - rnorm(n, 10, 2) mean(Metrics::ae(observed, predicted_mu)) #> [1] 34.45981 mean(Metrics::ae(observed, predicted_not_mu)) #> [1] 32.54821 mean(Metrics::se(observed, predicted_mu)) #> [1] 2171.089 mean(Metrics::se(observed, predicted_not_mu)) #> [1] 2290.155"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"absolute-error","dir":"Articles","previous_headings":"Metrics for point forecasts","what":"Absolute error","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: ŷ\\hat{y}, real number, median forecaster’s predictive distribution. absolute error absolute difference predicted observed values. See ?Metrics::ae. ae=|y−ŷ|\\text{ae} = |y - \\hat{y}| absolute error appropriate rule ŷ\\hat{y} corresponds median forecaster’s predictive distribution. Otherwise, results misleading (see Gneiting (2011)).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"squared-error","dir":"Articles","previous_headings":"Metrics for point forecasts","what":"Squared error","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: ŷ\\hat{y}, real number, mean forecaster’s predictive distribution. squared error squared difference predicted observed values. See ?Metrics::se. se=(y−ŷ)2\\text{se} = (y - \\hat{y})^2 squared error appropriate rule ŷ\\hat{y} corresponds mean forecaster’s predictive distribution. Otherwise, results misleading (see Gneiting (2011)).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"absolute-percentage-error","dir":"Articles","previous_headings":"Metrics for point forecasts","what":"Absolute percentage error","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: ŷ\\hat{y}, real number absolute percentage error absolute percent difference predicted observed values. See ?Metrics::ape. ape=|y−ŷ||y|\\text{ape} = \\frac{|y - \\hat{y}|}{|y|} absolute percentage error appropriate rule ŷ\\hat{y} corresponds β\\beta-median forecaster’s predictive distribution β=−1\\beta = -1. β\\beta-median, med(β)(F)\\text{med}^{(\\beta)}(F), median random variable whose density proportional yβf(y)y^\\beta f(y). specific β\\beta-median corresponds absolute percentage error med(−1)(F)\\text{med}^{(-1)}(F). Otherwise, results misleading (see Gneiting (2011)).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"binary-forecasts","dir":"Articles","previous_headings":"","what":"Binary forecasts","title":"Scoring rules in `scoringutils`","text":"See list default metrics point forecasts calling ?get_metrics(example_binary). overview input output formats point forecasts: Input output formats: metrics binary forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"brier-score","dir":"Articles","previous_headings":"Binary forecasts","what":"Brier score","title":"Scoring rules in `scoringutils`","text":"Observation: yy, either 0 1 Forecast: pp, probability observed outcome 1. Brier score strictly proper scoring rule. computed mean squared error probabilistic prediction observed outcome. BS(p,y)=(p−y)2={p2,y=1(1−p)2,y=0\\begin{equation} \\text{BS}(p, y) = (p - y)^2 = \\begin{cases} p^2, & \\text{} y = 1\\\\ (1 - p)^2, & \\text{} y = 0 \\end{cases} \\end{equation} Brier score logarithmic score (see ) differ penalise - underconfidence (see Machete (2012)). Brier score penalises overconfidence underconfidence probability space . Consider following example: See ?brier_score() information.","code":"n <- 1e6 p_true <- 0.7 observed <- factor(rbinom(n = n, size = 1, prob = p_true), levels = c(0, 1)) p_over <- p_true + 0.15 p_under <- p_true - 0.15 abs(mean(brier_score(observed, p_true)) - mean(brier_score(observed, p_over))) #> [1] 0.0223866 abs(mean(brier_score(observed, p_true)) - mean(brier_score(observed, p_under))) #> [1] 0.0226134"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"logarithmic-score","dir":"Articles","previous_headings":"Binary forecasts","what":"Logarithmic score","title":"Scoring rules in `scoringutils`","text":"Observation: yy, either 0 1 Forecast: pp, probability observed outcome 1. logarithmic score (log score) strictly proper scoring rule. computed negative logarithm probability assigned observed outcome. Log score(p,y)=−log(1−|y−p|)={−log(p),y=1−log(1−p),y=0\\begin{equation} \\text{Log score}(p, y) = - \\log(1 - |y - p|) = \\begin{cases} -\\log (p), & \\text{} y = 1\\\\ -\\log (1 - p), & \\text{} y = 0 \\end{cases} \\end{equation} log score penalises overconfidence strongly underconfidence (probability space). Consider following example: See ?logs_binary() information.","code":"abs(mean(logs_binary(observed, p_true)) - mean(logs_binary(observed, p_over))) #> [1] 0.07169954 abs(mean(logs_binary(observed, p_true)) - mean(logs_binary(observed, p_under))) #> [1] 0.04741833"},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"sample-based-forecasts","dir":"Articles","previous_headings":"","what":"Sample-based forecasts","title":"Scoring rules in `scoringutils`","text":"See list default metrics sample-based forecasts calling get_metrics(example_sample_continuous). overview input output formats quantile forecasts: Input output formats: metrics sample-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"crps","dir":"Articles","previous_headings":"Sample-based forecasts","what":"CRPS","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number (discrete number). Forecast: continuous (FF) discrete (PP) forecast. continuous ranked probability score (CRPS) popular fields meteorology epidemiology. CRPS defined CRPS(F,y)=∫−∞∞(F(x)−1(x≥y))2dx,\\text{CRPS}(F, y) = \\int_{-\\infty}^\\infty \\left( F(x) - 1(x \\geq y) \\right)^2 dx, yy observed value FF CDF predictive distribution. discrete forecasts, example count data, ranked probability score (RPS) can used instead commonly defined : RPS(P,y)=∑x=0∞(P(x)−1(x≥y))2, \\text{RPS}(P, y) = \\sum_{x = 0}^\\infty (P(x) - 1(x \\geq y))^2, PP cumulative probability mass function (PMF) predictive distribution. CRPS can understood generalisation absolute error predictive distributions (Gneiting Raftery 2007). can also understood integral Brier score binary probability forecasts implied CDF possible observed values. CRPS also related Cramér-distance two distributions equals special case one distributions concentrated single point (see e.g. Ziel (2021)). CRPS global scoring rule, meaning entire predictive distribution taken account determining quality forecast. scoringutils re-exports crps_sample() function scoringRules package, assumes forecast represented set samples predictive distribution. See ?crps_sample() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"overprediction-underprediction-and-dispersion","dir":"Articles","previous_headings":"Sample-based forecasts > CRPS","what":"Overprediction, underprediction and dispersion","title":"Scoring rules in `scoringutils`","text":"CRPS can interpreted sum dispersion, overprediction underprediction component. mm median forecast dispersion component CRPS(F,m),\\text{CRPS}(F, m), overprediction component {m>yCRPS(F,y)−CRPS(F,m)m≤y0 \\begin{cases} m > y & CRPS(F, y) - CRPS(F, m)\\\\ m \\leq y & 0\\\\ \\end{cases} underprediction component {m Weighted interval score (WIS)","what":"Overprediction, underprediction and dispersion","title":"Scoring rules in `scoringutils`","text":"individual components WIS. See ?overprediction_quantile(), ?underprediction_quantile() ?dispersion_quantile() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"bias-1","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Bias","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: FF. CDF predictive distribution represented set quantiles, QQ. Bias can measured B(F,y)={(1−2⋅max{α|qα∈Q∧qα≤y}),yq0.5(underprediction)0,y=q0.5,\\begin{equation} \\text{B}(F, y) = \\begin{cases} (1 - 2 \\cdot \\max \\{\\alpha | q_\\alpha \\Q \\land q_\\alpha \\leq y\\}), & \\text{} y < q_{0.5} \\quad \\text{(overprediction)}\\\\ (1 - 2 \\cdot \\min \\{\\alpha | q_\\alpha \\Q_t \\land q_\\alpha \\geq y\\}, & \\text{} y > q_{0.5} \\quad \\text{(underprediction)}\\\\ 0, & \\text{} y = q_{0.5}, \\\\ \\end{cases} \\end{equation} qαq_\\alpha α\\alpha-quantile predictive distribution. consistency, define QQ (set quantiles form predictive distribution FF) always includes element q0=−∞q_0 = -\\infty q1=∞q_1 = \\infty. clearer terms, bias : 1−(2×1 - (2 \\times maximum percentile rank corresponding quantile still observed value), observed value smaller median predictive distribution. 1−(2×1 - (2 \\times minimum percentile rank corresponding quantile still larger observed value) observed value larger median predictive distribution.. 00if observed value exactly median. Bias can assume values -1 (underprediction) 1 (overpredictin) 0 ideally (.e. unbiased). increasing number quantiles, percentile rank equal proportion predictive samples observed value, bias metric coincides one continuous forecasts (see ). See ?bias_quantile() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"interval-coverage","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Interval coverage","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: FF. CDF predictive distribution represented set quantiles. quantiles form central prediction intervals. Interval coverage given interval range defined proportion observations fall within corresponding central prediction intervals. Central prediction intervals symmetric around median formed two quantiles denote lower upper bound. example, 50% central prediction interval interval 0.25 0.75 quantiles predictive distribution.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"interval-coverage-deviation","dir":"Articles","previous_headings":"Quantile-based forecasts > Interval coverage","what":"Interval coverage deviation","title":"Scoring rules in `scoringutils`","text":"interval coverage deviation difference observed interval coverage nominal interval coverage. example, observed interval coverage 50% central prediction interval 0.6, interval coverage deviation 0.6=−0.5=0.1.0.6 = - 0.5 = 0.1. interval coverage deviation=observed interval coverage−nominal interval coverage\\text{interval coverage deviation} = \\text{observed interval coverage} - \\text{nominal interval coverage}","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"absolute-error-of-the-median-1","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Absolute error of the median","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: FF. CDF predictive distribution represented set quantiles. absolute error median absolute difference median predictive distribution observed value. aemedian=|median(F)−y|\\text{ae}_\\text{median} = |\\text{median}(F) - y| See section note caution Gneiting (2011) discussion correspondence absolute error median.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"quantile-score","dir":"Articles","previous_headings":"Quantile-based forecasts","what":"Quantile score","title":"Scoring rules in `scoringutils`","text":"Observation: yy, real number Forecast: FF. CDF predictive distribution represented set quantiles. quantile score, also called pinball loss, single quantile level τ\\tau defined QSτ(F,y)=2⋅{𝟏(y≤qτ)−τ}⋅(qτ−y)={2⋅(1−τ)*qτ−y,y≤qτ2⋅τ*|qτ−y|,y>qτ,\\begin{equation} \\text{QS}_\\tau(F, y) = 2 \\cdot \\{ \\mathbf{1}(y \\leq q_\\tau) - \\tau\\} \\cdot (q_\\tau - y) = \\begin{cases} 2 \\cdot (1 - \\tau) * q_\\tau - y, & \\text{} y \\leq q_\\tau\\\\ 2 \\cdot \\tau * |q_\\tau - y|, & \\text{} y > q_\\tau, \\end{cases} \\end{equation} qτq_\\tau τ\\tau-quantile predictive distribution FF, 𝟏(⋅)\\mathbf{1}(\\cdot) indicator function. (unweighted) interval score (see ) 1−α1 - \\alpha prediction interval can computed quantile scores levels α/2\\alpha/2 1−α/21 - \\alpha/2 ISα(F,y)=QSα/2(F,y)+QS1−α/2(F,y)α\\text{}_\\alpha(F, y) = \\frac{\\text{QS}_{\\alpha/2}(F, y) + \\text{QS}_{1 - \\alpha/2}(F, y)}{\\alpha}. weighted interval score can obtained simple average quantile scores: WISα(F,y)=QSα/2(F,y)+QS1−α/2(F,y)2\\text{WIS}_\\alpha(F, y) = \\frac{\\text{QS}_{\\alpha/2}(F, y) + \\text{QS}_{1 - \\alpha/2}(F, y)}{2}. See ?quantile_score Bracher et al. (2021) details.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/articles/scoring-rules.html","id":"quantile-coverage","dir":"Articles","previous_headings":"Additional metrics","what":"Quantile coverage","title":"Scoring rules in `scoringutils`","text":"Quantile coverage given quantile level defined proportion observed values smaller corresponding predictive quantiles. example, 0.5 quantile coverage proportion observed values smaller 0.5-quantiles predictive distribution.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/authors.html","id":null,"dir":"","previous_headings":"","what":"Authors","title":"Authors and Citation","text":"Nikos Bosse. Author, maintainer. Sam Abbott. Author. Hugo Gruson. Author. Johannes Bracher. Contributor. Toshiaki Asakura. Contributor. James Mba Azam. Contributor. Sebastian Funk. Author. Michael Chirico. Contributor.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/authors.html","id":"citation","dir":"","previous_headings":"","what":"Citation","title":"Authors and Citation","text":"Nikos . Bosse, Hugo Gruson, Sebastian Funk, Anne Cori, Edwin van Leeuwen, Sam Abbott (2022). Evaluating Forecasts scoringutils R, arXiv. DOI: 10.48550/ARXIV.2205.07090 Alexander Jordan, Fabian Krueger, Sebastian Lerch (2019). Evaluating Probabilistic Forecasts scoringRules. Journal Statistical Software, 90(12), 1-37. DOI 10.18637/jss.v090.i12","code":"@Article{, title = {Evaluating Forecasts with scoringutils in R}, author = {Nikos I. Bosse and Hugo Gruson and Anne Cori and Edwin {van Leeuwen} and Sebastian Funk and Sam Abbott}, journal = {arXiv}, url = {https://arxiv.org/abs/2205.07090}, year = {2022}, doi = {10.48550/ARXIV.2205.07090}, } @Article{, title = {Evaluating Probabilistic Forecasts with {scoringRules}}, author = {Alexander Jordan and Fabian Kr\\\"uger and Sebastian Lerch}, journal = {Journal of Statistical Software}, year = {2019}, volume = {90}, number = {12}, pages = {1--37}, doi = {10.18637/jss.v090.i12}, }"},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"scoringutils-utilities-for-scoring-and-assessing-predictions","dir":"","previous_headings":"","what":"Utilities for Scoring and Assessing Predictions","title":"Utilities for Scoring and Assessing Predictions","text":"Note: documentation refers development version scoringutils. can also view documentation stable version. scoringutils package facilitates process evaluating forecasts R, using convenient flexible data.table-based framework. provides broad functionality check input data diagnose issues, visualise forecasts missing data, transform data scoring, handle missing forecasts, aggregate scores, visualise results evaluation. package easily extendable, meaning users can supply scoring rules extend existing classes handle new types forecasts. package underwent major re-write. comprehensive documentation updated package revised version original scoringutils paper. Another good starting point vignettes Getting started, Details metrics implemented Scoring forecasts directly. details specific issue transforming forecasts scoring see: Nikos . Bosse, Sam Abbott, Anne Cori, Edwin van Leeuwen, Johannes Bracher* Sebastian Funk* (*: equal contribution) (2023). Scoring epidemiological forecasts transformed scales, PLoS Comput Biol 19(8): e1011393 https://doi.org/10.1371/journal.pcbi.1011393","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"installation","dir":"","previous_headings":"","what":"Installation","title":"Utilities for Scoring and Assessing Predictions","text":"Install CRAN version package using Install unstable development version GitHub using","code":"install.packages(\"scoringutils\") remotes::install_github(\"epiforecasts/scoringutils\", dependencies = TRUE)"},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"forecast-types","dir":"","previous_headings":"Quick start","what":"Forecast types","title":"Utilities for Scoring and Assessing Predictions","text":"scoringutils currently supports scoring following forecast types: - binary: probability binary (yes/) outcome variable. - point: forecast continuous discrete outcome variable represented single number. - quantile: probabilistic forecast continuous discrete outcome variable, forecast distribution represented set predictive quantiles. - sample: probabilistic forecast continuous discrete outcome variable, forecast represented finite set samples drawn predictive distribution.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"input-formats-and-input-validation","dir":"","previous_headings":"Quick start","what":"Input formats and input validation","title":"Utilities for Scoring and Assessing Predictions","text":"expected input format generally data.frame (similar) required columns observed, predicted, model holds forecasts observed values. Exact requirements depend forecast type. information, look paper, call ?as_forecast(), look example data provided package (example_binary, example_point, example_quantile, example_sample_continuous, example_sample_discrete). scoring, input data needs validated transformed forecast object using function as_forecast().","code":"forecast_quantile <- example_quantile |> as_forecast_quantile( forecast_unit = c( \"location\", \"forecast_date\", \"target_end_date\", \"target_type\", \"model\", \"horizon\" ) ) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. print(forecast_quantile, 2) #> Forecast type: quantile #> Forecast unit: #> location, forecast_date, target_end_date, target_type, model, and horizon #> #> Key: #> observed quantile_level predicted location forecast_date target_end_date #> #> 1: 127300 NA NA DE 2021-01-02 #> 2: 4534 NA NA DE 2021-01-02 #> --- #> 20544: 78 0.975 611 IT 2021-07-12 2021-07-24 #> 20545: 78 0.990 719 IT 2021-07-12 2021-07-24 #> target_type model horizon #> #> 1: Cases NA #> 2: Deaths NA #> --- #> 20544: Deaths epiforecasts-EpiNow2 2 #> 20545: Deaths epiforecasts-EpiNow2 2"},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"the-forecast-unit","dir":"","previous_headings":"Quick start","what":"The forecast unit","title":"Utilities for Scoring and Assessing Predictions","text":"quantile-based sample-based forecasts, single prediction represented set several quantiles (samples) predictive distribution, .e. several rows input data. scoringutils therefore needs group rows together form single forecast. scoringutils uses existing columns input data achieve - values columns uniquely identify single forecast. Additional columns unrelated forecast unit can mess . forecast_unit argument as_forecast_() makes sure columns retained relevant defining unit single forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"scoring-forecasts","dir":"","previous_headings":"Quick start","what":"Scoring forecasts","title":"Utilities for Scoring and Assessing Predictions","text":"Forecasts can scored calling score() validated forecast object. score() takes additional argument, metrics, list scoring rules. Every forecast type default list metrics. can easily add scoring functions, long conform format forecast type. See paper information. can summarise scores using function summarise_scores(). argument used specify desired level summary. fun let’s specify summary function, although recommended stick mean primary summary function, functions can lead improper scores.","code":"scores <- forecast_quantile |> score() scores |> summarise_scores(by = c(\"model\", \"target_type\")) |> summarise_scores(by = c(\"model\", \"target_type\"), fun = signif, digits = 3) #> model target_type wis overprediction underprediction #> #> 1: EuroCOVIDhub-ensemble Cases 17900.0 10000.00 4240.0 #> 2: EuroCOVIDhub-baseline Cases 28500.0 14100.00 10300.0 #> 3: epiforecasts-EpiNow2 Cases 20800.0 11900.00 3260.0 #> 4: EuroCOVIDhub-ensemble Deaths 41.4 7.14 4.1 #> 5: EuroCOVIDhub-baseline Deaths 159.0 65.90 2.1 #> 6: UMass-MechBayes Deaths 52.7 8.98 16.8 #> 7: epiforecasts-EpiNow2 Deaths 66.6 18.90 15.9 #> dispersion bias interval_coverage_50 interval_coverage_90 ae_median #> #> 1: 3660.0 -0.05640 0.391 0.805 24100.0 #> 2: 4100.0 0.09800 0.328 0.820 38500.0 #> 3: 5660.0 -0.07890 0.469 0.789 27900.0 #> 4: 30.2 0.07270 0.875 1.000 53.1 #> 5: 91.4 0.33900 0.664 1.000 233.0 #> 6: 26.9 -0.02230 0.461 0.875 78.5 #> 7: 31.9 -0.00513 0.420 0.908 105.0"},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"package-workflow","dir":"","previous_headings":"","what":"Package workflow","title":"Utilities for Scoring and Assessing Predictions","text":"following depicts suggested workflow evaluating forecasts scoringutils (sections refer paper). Please find information paper, function documentation vignettes.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"citation","dir":"","previous_headings":"","what":"Citation","title":"Utilities for Scoring and Assessing Predictions","text":"using scoringutils work please consider citing using output citation(\"scoringutils\") (print(citation(\"scoringutils\"), bibtex = TRUE)):","code":"#> To cite scoringutils in publications use the following. If you use the #> CRPS, DSS, or Log Score, please also cite scoringRules. #> #> Nikos I. Bosse, Hugo Gruson, Sebastian Funk, Anne Cori, Edwin van #> Leeuwen, and Sam Abbott (2022). Evaluating Forecasts with #> scoringutils in R, arXiv. DOI: 10.48550/ARXIV.2205.07090 #> #> To cite scoringRules in publications use: #> #> Alexander Jordan, Fabian Krueger, Sebastian Lerch (2019). Evaluating #> Probabilistic Forecasts with scoringRules. Journal of Statistical #> Software, 90(12), 1-37. DOI 10.18637/jss.v090.i12 #> #> To see these entries in BibTeX format, use 'print(, #> bibtex=TRUE)', 'toBibtex(.)', or set #> 'options(citation.bibtex.max=999)'."},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"how-to-make-a-bug-report-or-feature-request","dir":"","previous_headings":"","what":"How to make a bug report or feature request","title":"Utilities for Scoring and Assessing Predictions","text":"Please briefly describe problem output expect issue. question, please don’t open issue. Instead, ask Q page.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"contributing","dir":"","previous_headings":"","what":"Contributing","title":"Utilities for Scoring and Assessing Predictions","text":"welcome contributions new contributors! particularly appreciate help priority problems issues. Please check add issues, /add pull request.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"code-of-conduct","dir":"","previous_headings":"","what":"Code of Conduct","title":"Utilities for Scoring and Assessing Predictions","text":"Please note scoringutils project released Contributor Code Conduct. contributing project, agree abide terms.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"funding","dir":"","previous_headings":"","what":"Funding","title":"Utilities for Scoring and Assessing Predictions","text":"development scoringutils funded via Health Protection Research Unit (grant code NIHR200908) Wellcome Trust (grant: 210758/Z/18/Z). work also supported US National Institutes General Medical Sciences (R35GM119582). content solely responsibility authors necessarily represent official views NIGMS, National Institutes Health.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"contributors","dir":"","previous_headings":"","what":"Contributors","title":"Utilities for Scoring and Assessing Predictions","text":"contributions project gratefully acknowledged using allcontributors package following -contributors specification. Contributions kind welcome!","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"code","dir":"","previous_headings":"Contributors","what":"Code","title":"Utilities for Scoring and Assessing Predictions","text":"nikosbosse, seabbs, jamesmbaazam, Bisaloo, sbfnk, actions-user, toshiakiasakura, MichaelChirico, nickreich, jhellewell14, damonbayer","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"issue-authors","dir":"","previous_headings":"Contributors","what":"Issue Authors","title":"Utilities for Scoring and Assessing Predictions","text":"DavideMagno, mbojan, dshemetov, elray1","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/index.html","id":"issue-contributors","dir":"","previous_headings":"Contributors","what":"Issue Contributors","title":"Utilities for Scoring and Assessing Predictions","text":"jbracher, dylanhmorris, jonathonmellor, kathsherratt","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/add_relative_skill.html","id":null,"dir":"Reference","previous_headings":"","what":"Add relative skill scores based on pairwise comparisons — add_relative_skill","title":"Add relative skill scores based on pairwise comparisons — add_relative_skill","text":"Adds columns relative skills computed running pairwise comparisons scores. information computation relative skill, see get_pairwise_comparisons(). Relative skill calculated aggregation level specified .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/add_relative_skill.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Add relative skill scores based on pairwise comparisons — add_relative_skill","text":"","code":"add_relative_skill( scores, compare = \"model\", by = NULL, metric = intersect(c(\"wis\", \"crps\", \"brier_score\"), names(scores)), baseline = NULL )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/add_relative_skill.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Add relative skill scores based on pairwise comparisons — add_relative_skill","text":"scores object class scores (data.table scores additional attribute metrics produced score()). compare Character vector single colum name defines elements pairwise comparison. example, set \"model\" (default), elements \"model\" column compared. Character vector column names define grouping levels pairwise comparisons. default NULL one relative skill score per distinct entry column selected compare. columns given , example, = \"location\" compare = \"model\", one separate relative skill score calculated every model every location. metric string name metric relative skill shall computed. default either \"crps\", \"wis\" \"brier_score\" available. baseline string name model. baseline given, scaled relative skill respect baseline returned. default (NULL), relative skill scaled respect baseline model.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Absolute error of the median (quantile-based version) — ae_median_quantile","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"Compute absolute error median calculated $$ \\textrm{abs}(\\textrm{observed} - \\textrm{median prediction}) $$ median prediction predicted value quantile_level == 0.5, function therefore requires 0.5 among quantile levels quantile_level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"","code":"ae_median_quantile(observed, predicted, quantile_level)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"Numeric vector length N absolute error median.","code":""},{"path":[]},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_quantile.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Absolute error of the median (quantile-based version) — ae_median_quantile","text":"","code":"observed <- rnorm(30, mean = 1:30) predicted_values <- replicate(3, rnorm(30, mean = 1:30)) ae_median_quantile( observed, predicted_values, quantile_level = c(0.2, 0.5, 0.8) ) #> [1] 2.47438940 0.92040530 3.55121603 0.24032512 1.79911603 2.12426222 #> [7] 2.88687498 0.37899594 0.73282842 1.41674512 0.91703692 0.34483170 #> [13] 0.72770448 1.86768569 0.80586643 2.38692128 1.12876056 0.05733376 #> [19] 0.37081463 0.82374754 1.45618892 0.93544150 2.05333481 0.18155199 #> [25] 2.43676219 1.20798000 1.67648698 0.13974346 1.26067874 1.13044854"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Absolute error of the median (sample-based version) — ae_median_sample","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"Absolute error median calculated $$% \\textrm{abs}(\\textrm{observevd} - \\textrm{median\\_prediction}) $$","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"","code":"ae_median_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"vector scoring values","code":""},{"path":[]},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ae_median_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Absolute error of the median (sample-based version) — ae_median_sample","text":"","code":"observed <- rnorm(30, mean = 1:30) predicted_values <- matrix(rnorm(30, mean = 1:30)) ae_median_sample(observed, predicted_values) #> [1] 1.61022189 0.32735036 2.52982645 0.98458168 0.94495454 0.65538891 #> [7] 0.56511146 0.09373061 1.31110818 0.61226219 0.75386115 0.08959962 #> [13] 0.39077113 1.56818369 0.84567980 1.24260044 0.27781917 0.65054779 #> [19] 1.18084954 0.45036469 0.05976767 0.14675942 0.60583332 0.19442459 #> [25] 0.21123533 0.28585022 0.64582375 1.78993469 1.20347916 0.67902801"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/apply_metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Apply a list of functions to a data table of forecasts — apply_metrics","title":"Apply a list of functions to a data table of forecasts — apply_metrics","text":"helper function applies scoring rules (stored list functions) data table forecasts. apply_metrics used within score() apply scoring rules data. Scoring rules wrapped run_safely() catch errors make sure arguments passed scoring rule actually accepted .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/apply_metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Apply a list of functions to a data table of forecasts — apply_metrics","text":"","code":"apply_metrics(forecast, metrics, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/apply_metrics.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Apply a list of functions to a data table of forecasts — apply_metrics","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()). metrics named list scoring functions. Names used column names output. See get_metrics() information default metrics used. See Customising metrics section information pass custom arguments scoring functions. ... Additional arguments passed scoring rules. Note currently used, calls apply_scores currently avoid passing arguments via ... instead expect metrics directly modified using purrr::partial().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/apply_metrics.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Apply a list of functions to a data table of forecasts — apply_metrics","text":"data table forecasts calculated metrics.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"General information on creating a forecast object — as_forecast","title":"General information on creating a forecast object — as_forecast","text":"several as_forecast_() functions process validate data.frame (similar) similar forecasts observations. input passes input checks, functions converted forecast object. forecast object data.table class forecast additional class depends forecast type. Every forecast type as_forecast_() function. See details section information expected input formats. as_forecast_() functions give users control data parsed. Using arguments observed, predicted, etc. users can rename existing columns input data match required columns forecast object. Using argument forecast_unit, users can specify columns uniquely identify single forecast (remove others, see docs internal set_forecast_unit() details). following functions available: as_forecast_point() as_forecast_binary() as_forecast_sample() as_forecast_quantile()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"General information on creating a forecast object — as_forecast","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. observed (optional) Name column data contains observed values. column renamed \"observed\". predicted (optional) Name column data contains predicted values. column renamed \"predicted\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"General information on creating a forecast object — as_forecast","text":"Depending forecast type, object following class returned: forecast_binary binary forecasts forecast_point point forecasts forecast_sample sample-based forecasts forecast_quantile quantile-based forecasts","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"forecast-types-and-input-formats","dir":"Reference","previous_headings":"","what":"Forecast types and input formats","title":"General information on creating a forecast object — as_forecast","text":"Various different forecast types / forecast formats supported. moment, : point forecasts binary forecasts (\"soft binary classification\") nominal forecasts (\"soft classification multiple unordered classes\") Probabilistic forecasts quantile-based format (forecast represented set predictive quantiles) Probabilistic forecasts sample-based format (forecast represented set predictive samples) Forecast types determined based columns present input data. overview required format forecast type: forecast types require data.frame similar columns observed predicted, model. Point forecasts require column observed type numeric column predicted type numeric. Binary forecasts require column observed type factor exactly two levels column predicted type numeric probabilities, corresponding probability observed equal second factor level. See details information. Nominal forecasts require column observed type factor N levels, (N number possible outcomes), column predicted type numeric probabilities (sum one across possible outcomes), column predicted_label type factor N levels, denoting outcome probability given. Forecasts must complete, .e. must probability assigned every possible outcome. Quantile-based forecasts require column observed type numeric, column predicted type numeric, column quantile_level type numeric quantile-levels (0 1). Sample-based forecasts require column observed type numeric, column predicted type numeric, column sample_id type numeric sample indices. information see vignettes example data (example_quantile, example_sample_continuous, example_sample_discrete, example_point(), example_binary, example_nominal).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"forecast-unit","dir":"Reference","previous_headings":"","what":"Forecast unit","title":"General information on creating a forecast object — as_forecast","text":"order score forecasts, scoringutils needs know rows data belong together jointly form single forecasts. easy e.g. point forecast, one row per forecast. quantile sample-based forecasts, however, multiple rows belong single forecast. forecast unit unit single forecast described combination columns uniquely identify single forecast. example, forecasts made different models various locations different time points, several weeks future. forecast unit described forecast_unit = c(\"model\", \"location\", \"forecast_date\", \"forecast_horizon\"). scoringutils automatically tries determine unit single forecast. uses existing columns , means columns must present unrelated forecast unit. simplistic example, additional row, \"even\", one row number even zero otherwise, mess scoring scoringutils thinks column relevant defining forecast unit. order avoid issues, recommend setting forecast unit explicitly, usually forecast_unit argument as_forecast() functions. drop unneeded columns, making sure necessary, 'protected columns' like \"predicted\" \"observed\" retained.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"General information on creating a forecast object — as_forecast","text":"","code":"as_forecast_binary(example_binary) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Forecast type: binary #> Forecast unit: #> location, location_name, target_end_date, target_type, forecast_date, model, #> and horizon #> #> location location_name target_end_date target_type forecast_date #> #> 1: DE Germany 2021-01-02 Cases #> 2: DE Germany 2021-01-02 Deaths #> 3: DE Germany 2021-01-09 Cases #> 4: DE Germany 2021-01-09 Deaths #> 5: DE Germany 2021-01-16 Cases #> --- #> 1027: IT Italy 2021-07-24 Deaths 2021-07-12 #> 1028: IT Italy 2021-07-24 Deaths 2021-07-05 #> 1029: IT Italy 2021-07-24 Deaths 2021-07-12 #> 1030: IT Italy 2021-07-24 Deaths 2021-07-05 #> 1031: IT Italy 2021-07-24 Deaths 2021-07-12 #> model horizon predicted observed #> #> 1: NA NA #> 2: NA NA #> 3: NA NA #> 4: NA NA #> 5: NA NA #> --- #> 1027: EuroCOVIDhub-baseline 2 0.250 0 #> 1028: UMass-MechBayes 3 0.475 0 #> 1029: UMass-MechBayes 2 0.450 0 #> 1030: epiforecasts-EpiNow2 3 0.375 0 #> 1031: epiforecasts-EpiNow2 2 0.300 0 as_forecast_quantile( example_quantile, forecast_unit = c(\"model\", \"target_type\", \"target_end_date\", \"horizon\", \"location\") ) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Forecast type: quantile #> Forecast unit: #> model, target_type, target_end_date, horizon, and location #> #> Key: #> observed quantile_level predicted model target_type #> #> 1: 127300 NA NA Cases #> 2: 4534 NA NA Deaths #> 3: 154922 NA NA Cases #> 4: 6117 NA NA Deaths #> 5: 110183 NA NA Cases #> --- #> 20541: 78 0.850 352 epiforecasts-EpiNow2 Deaths #> 20542: 78 0.900 397 epiforecasts-EpiNow2 Deaths #> 20543: 78 0.950 499 epiforecasts-EpiNow2 Deaths #> 20544: 78 0.975 611 epiforecasts-EpiNow2 Deaths #> 20545: 78 0.990 719 epiforecasts-EpiNow2 Deaths #> target_end_date horizon location #> #> 1: 2021-01-02 NA DE #> 2: 2021-01-02 NA DE #> 3: 2021-01-09 NA DE #> 4: 2021-01-09 NA DE #> 5: 2021-01-16 NA DE #> --- #> 20541: 2021-07-24 2 IT #> 20542: 2021-07-24 2 IT #> 20543: 2021-07-24 2 IT #> 20544: 2021-07-24 2 IT #> 20545: 2021-07-24 2 IT"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a forecast object for binary forecasts — as_forecast_binary","title":"Create a forecast object for binary forecasts — as_forecast_binary","text":"Create forecast object binary forecasts. See information forecast types expected input formats calling ?as_forecast().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a forecast object for binary forecasts — as_forecast_binary","text":"","code":"as_forecast_binary( data, forecast_unit = NULL, observed = NULL, predicted = NULL )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a forecast object for binary forecasts — as_forecast_binary","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. observed (optional) Name column data contains observed values. column renamed \"observed\". predicted (optional) Name column data contains predicted values. column renamed \"predicted\".","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_generic.html","id":null,"dir":"Reference","previous_headings":"","what":"Common functionality for as_forecast_ functions — as_forecast_generic","title":"Common functionality for as_forecast_ functions — as_forecast_generic","text":"Common functionality as_forecast_ functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_generic.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Common functionality for as_forecast_ functions — as_forecast_generic","text":"","code":"as_forecast_generic( data, forecast_unit = NULL, observed = NULL, predicted = NULL )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_generic.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Common functionality for as_forecast_ functions — as_forecast_generic","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. observed (optional) Name column data contains observed values. column renamed \"observed\". predicted (optional) Name column data contains predicted values. column renamed \"predicted\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_generic.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Common functionality for as_forecast_ functions — as_forecast_generic","text":"function splits part functionality as_forecast_ as_forecast_ functions. renames required columns, appropriate, sets forecast unit.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_nominal.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a forecast object for nominal forecasts — as_forecast_nominal","title":"Create a forecast object for nominal forecasts — as_forecast_nominal","text":"Nominal forecasts form categorical forecasts possible outcomes observed values can assume ordered. sense, Nominal forecasts represent generalisation binary forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_nominal.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a forecast object for nominal forecasts — as_forecast_nominal","text":"","code":"as_forecast_nominal( data, forecast_unit = NULL, observed = NULL, predicted = NULL, predicted_label = NULL )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_nominal.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a forecast object for nominal forecasts — as_forecast_nominal","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. observed (optional) Name column data contains observed values. column renamed \"observed\". predicted (optional) Name column data contains predicted values. column renamed \"predicted\". predicted_label (optional) Name column data denotes outcome predicted probability corresponds . column renamed \"predicted_label\". applicable nominal forecasts.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a forecast object for point forecasts — as_forecast_point","title":"Create a forecast object for point forecasts — as_forecast_point","text":"Create forecast object point forecasts. See information forecast types expected input formats calling ?as_forecast(). converting forecast_quantile object forecast_point object, 0.5 quantile extracted returned point forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a forecast object for point forecasts — as_forecast_point","text":"","code":"as_forecast_point(data, ...) # Default S3 method as_forecast_point( data, forecast_unit = NULL, observed = NULL, predicted = NULL, ... ) # S3 method for class 'forecast_quantile' as_forecast_point(data, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a forecast object for point forecasts — as_forecast_point","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. ... Unused forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. observed (optional) Name column data contains observed values. column renamed \"observed\". predicted (optional) Name column data contains predicted values. column renamed \"predicted\".","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a forecast object for quantile-based forecasts — as_forecast_quantile","title":"Create a forecast object for quantile-based forecasts — as_forecast_quantile","text":"Create forecast object quantile-based forecasts. See information forecast types expected input formats calling ?as_forecast(). creating forecast_quantile object forecast_sample object, quantiles estimated computing empircal quantiles samples via quantile(). Note empirical quantiles biased estimator true quantiles particular tails distribution number available samples low.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a forecast object for quantile-based forecasts — as_forecast_quantile","text":"","code":"as_forecast_quantile(data, ...) # Default S3 method as_forecast_quantile( data, forecast_unit = NULL, observed = NULL, predicted = NULL, quantile_level = NULL, ... ) # S3 method for class 'forecast_sample' as_forecast_quantile( data, probs = c(0.05, 0.25, 0.5, 0.75, 0.95), type = 7, ... )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a forecast object for quantile-based forecasts — as_forecast_quantile","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. ... Unused forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. observed (optional) Name column data contains observed values. column renamed \"observed\". predicted (optional) Name column data contains predicted values. column renamed \"predicted\". quantile_level (optional) Name column data contains quantile level predicted values. column renamed \"quantile_level\". applicable quantile-based forecasts. probs numeric vector quantile levels quantiles computed. Corresponds probs argument quantile(). type Type argument passed quantile function. information, see quantile().","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a forecast object for sample-based forecasts — as_forecast_sample","title":"Create a forecast object for sample-based forecasts — as_forecast_sample","text":"Create forecast object sample-based forecasts","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a forecast object for sample-based forecasts — as_forecast_sample","text":"","code":"as_forecast_sample( data, forecast_unit = NULL, observed = NULL, predicted = NULL, sample_id = NULL )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_forecast_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a forecast object for sample-based forecasts — as_forecast_sample","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. observed (optional) Name column data contains observed values. column renamed \"observed\". predicted (optional) Name column data contains predicted values. column renamed \"predicted\". sample_id (optional) Name column data contains sample id. column renamed \"sample_id\". applicable sample-based forecasts.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Create an object of class scores from data — as_scores","title":"Create an object of class scores from data — as_scores","text":"convenience function wraps new_scores() validates scores object.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create an object of class scores from data — as_scores","text":"","code":"as_scores(scores, metrics)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create an object of class scores from data — as_scores","text":"scores data.table similar scores produced score(). metrics character vector names scores (.e. names scoring rules used scoring).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/as_scores.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Create an object of class scores from data — as_scores","text":"object class scores","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_dims_ok_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","title":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","text":"Function assesses whether input dimensions match. following, n number observations / forecasts. Scalar values may repeated match length input. Allowed options therefore: observed vector length 1 length n predicted : vector length 1 length n matrix n rows 1 column","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_dims_ok_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","text":"","code":"assert_dims_ok_point(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_dims_ok_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","text":"observed Input checked. factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_dims_ok_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert Inputs Have Matching Dimensions — assert_dims_ok_point","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that input is a forecast object and passes validations — assert_forecast.forecast_binary","title":"Assert that input is a forecast object and passes validations — assert_forecast.forecast_binary","text":"Assert object forecast object (.e. data.table class forecast additional class forecast_* corresponding forecast type).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that input is a forecast object and passes validations — assert_forecast.forecast_binary","text":"","code":"# S3 method for class 'forecast_binary' assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) # S3 method for class 'forecast_point' assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) # S3 method for class 'forecast_quantile' assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) # S3 method for class 'forecast_sample' assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...) # Default S3 method assert_forecast(forecast, forecast_type = NULL, verbose = TRUE, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that input is a forecast object and passes validations — assert_forecast.forecast_binary","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()). forecast_type (optional) forecast type expect forecasts . forecast type determined scoringutils based input match , error thrown. NULL (default), forecast type inferred data. verbose Logical. FALSE (default TRUE), messages warnings created. ... Currently unused. pass additional arguments scoring functions via .... See Customising metrics section details use purrr::partial() pass arguments individual metrics.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that input is a forecast object and passes validations — assert_forecast.forecast_binary","text":"Returns NULL invisibly.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"forecast-types-and-input-formats","dir":"Reference","previous_headings":"","what":"Forecast types and input formats","title":"Assert that input is a forecast object and passes validations — assert_forecast.forecast_binary","text":"Various different forecast types / forecast formats supported. moment, : point forecasts binary forecasts (\"soft binary classification\") nominal forecasts (\"soft classification multiple unordered classes\") Probabilistic forecasts quantile-based format (forecast represented set predictive quantiles) Probabilistic forecasts sample-based format (forecast represented set predictive samples) Forecast types determined based columns present input data. overview required format forecast type: forecast types require data.frame similar columns observed predicted, model. Point forecasts require column observed type numeric column predicted type numeric. Binary forecasts require column observed type factor exactly two levels column predicted type numeric probabilities, corresponding probability observed equal second factor level. See details information. Nominal forecasts require column observed type factor N levels, (N number possible outcomes), column predicted type numeric probabilities (sum one across possible outcomes), column predicted_label type factor N levels, denoting outcome probability given. Forecasts must complete, .e. must probability assigned every possible outcome. Quantile-based forecasts require column observed type numeric, column predicted type numeric, column quantile_level type numeric quantile-levels (0 1). Sample-based forecasts require column observed type numeric, column predicted type numeric, column sample_id type numeric sample indices. information see vignettes example data (example_quantile, example_sample_continuous, example_sample_discrete, example_point(), example_binary, example_nominal).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Assert that input is a forecast object and passes validations — assert_forecast.forecast_binary","text":"","code":"forecast <- as_forecast_binary(example_binary) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. assert_forecast(forecast) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected."},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_generic.html","id":null,"dir":"Reference","previous_headings":"","what":"Validation common to all forecast types — assert_forecast_generic","title":"Validation common to all forecast types — assert_forecast_generic","text":"function runs input checks apply input data, regardless forecast type. function asserts forecast data.table columns observed predicted checks forecast type forecast unit checks duplicate forecasts appropriate, checks number samples / quantiles forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_generic.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Validation common to all forecast types — assert_forecast_generic","text":"","code":"assert_forecast_generic(data, verbose = TRUE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_generic.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Validation common to all forecast types — assert_forecast_generic","text":"data data.table forecasts observed values validated. verbose Logical. FALSE (default TRUE), messages warnings created.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_generic.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Validation common to all forecast types — assert_forecast_generic","text":"returns input","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_type.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that forecast type is as expected — assert_forecast_type","title":"Assert that forecast type is as expected — assert_forecast_type","text":"Assert forecast type expected","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_type.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that forecast type is as expected — assert_forecast_type","text":"","code":"assert_forecast_type(data, actual = get_forecast_type(data), desired = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_type.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that forecast type is as expected — assert_forecast_type","text":"data forecast object (see as_forecast()). actual actual forecast type data desired desired forecast type data","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_forecast_type.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that forecast type is as expected — assert_forecast_type","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for binary forecast — assert_input_binary","title":"Assert that inputs are correct for binary forecast — assert_input_binary","text":"Function assesses whether inputs correspond requirements scoring binary forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for binary forecast — assert_input_binary","text":"","code":"assert_input_binary(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for binary forecast — assert_input_binary","text":"observed Input checked. factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for binary forecast — assert_input_binary","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_interval.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for interval-based forecast — assert_input_interval","title":"Assert that inputs are correct for interval-based forecast — assert_input_interval","text":"Function assesses whether inputs correspond requirements scoring interval-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_interval.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for interval-based forecast — assert_input_interval","text":"","code":"assert_input_interval(observed, lower, upper, interval_range)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_interval.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for interval-based forecast — assert_input_interval","text":"observed Input checked. numeric vector observed values size n. lower Input checked. numeric vector size n holds predicted value lower bounds prediction intervals. upper Input checked. numeric vector size n holds predicted value upper bounds prediction intervals. interval_range Input checked. vector size n denotes interval range percent. E.g. value 50 denotes (25%, 75%) prediction interval.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_interval.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for interval-based forecast — assert_input_interval","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_nominal.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for nominal forecasts — assert_input_nominal","title":"Assert that inputs are correct for nominal forecasts — assert_input_nominal","text":"Function assesses whether inputs correspond requirements scoring nominal forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_nominal.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for nominal forecasts — assert_input_nominal","text":"","code":"assert_input_nominal(observed, predicted, predicted_label)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_nominal.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for nominal forecasts — assert_input_nominal","text":"observed Input checked. factor length n N levels holding observed values. n number observations N number possible outcomes observed values can assume. output) predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level. predicted_label Factor length N N levels, N number possible outcomes observed values can assume.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_nominal.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for nominal forecasts — assert_input_nominal","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for point forecast — assert_input_point","title":"Assert that inputs are correct for point forecast — assert_input_point","text":"Function assesses whether inputs correspond requirements scoring point forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for point forecast — assert_input_point","text":"","code":"assert_input_point(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for point forecast — assert_input_point","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. numeric vector predicted values size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for point forecast — assert_input_point","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","title":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","text":"Function assesses whether inputs correspond requirements scoring quantile-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","text":"","code":"assert_input_quantile( observed, predicted, quantile_level, unique_quantile_levels = TRUE )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. nxN matrix predictive quantiles, n (number rows) number data points N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Input checked. vector size N denotes quantile levels corresponding columns prediction matrix. unique_quantile_levels Whether quantile levels required unique (TRUE, default) (FALSE).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for quantile-based forecast — assert_input_quantile","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Assert that inputs are correct for sample-based forecast — assert_input_sample","title":"Assert that inputs are correct for sample-based forecast — assert_input_sample","text":"Function assesses whether inputs correspond requirements scoring sample-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Assert that inputs are correct for sample-based forecast — assert_input_sample","text":"","code":"assert_input_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Assert that inputs are correct for sample-based forecast — assert_input_sample","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. numeric nxN matrix predictive samples, n (number rows) number data points N (number columns) number samples per forecast. observed just single number, predicted values can just vector size N.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_input_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Assert that inputs are correct for sample-based forecast — assert_input_sample","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Validate an object of class scores — assert_scores","title":"Validate an object of class scores — assert_scores","text":"function validates object class scores, checking correct class metrics attribute.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Validate an object of class scores — assert_scores","text":"","code":"assert_scores(scores)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Validate an object of class scores — assert_scores","text":"scores data.table similar scores produced score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/assert_scores.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Validate an object of class scores — assert_scores","text":"Returns NULL invisibly","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Determines bias of quantile forecasts — bias_quantile","title":"Determines bias of quantile forecasts — bias_quantile","text":"Determines bias quantile forecasts. increasing number quantiles measure converges sample based bias version integer continuous forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Determines bias of quantile forecasts — bias_quantile","text":"","code":"bias_quantile(observed, predicted, quantile_level, na.rm = TRUE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Determines bias of quantile forecasts — bias_quantile","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made. Note contain median (0.5) median imputed mean two innermost quantiles. na.rm Logical. missing values removed?","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Determines bias of quantile forecasts — bias_quantile","text":"scalar quantile bias single quantile prediction","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Determines bias of quantile forecasts — bias_quantile","text":"quantile forecasts, bias measured $$ B_t = (1 - 2 \\cdot \\max \\{| q_{t,} \\Q_t \\land q_{t,} \\leq x_t\\}) \\mathbf{1}( x_t \\leq q_{t, 0.5}) \\\\ + (1 - 2 \\cdot \\min \\{| q_{t,} \\Q_t \\land q_{t,} \\geq x_t\\}) 1( x_t \\geq q_{t, 0.5}),$$ \\(Q_t\\) set quantiles form predictive distribution time \\(t\\) \\(x_t\\) observed value. consistency, define \\(Q_t\\) always includes element \\(q_{t, 0} = - \\infty\\) \\(q_{t,1} = \\infty\\). \\(1()\\) indicator function \\(1\\) condition satisfied \\(0\\) otherwise. clearer terms, bias \\(B_t\\) : \\(1 - 2 \\cdot\\) maximum percentile rank corresponding quantile still smaller equal observed value, observed value smaller median predictive distribution. \\(1 - 2 \\cdot\\) minimum percentile rank corresponding quantile still larger equal observed value observed value larger median predictive distribution.. \\(0\\) observed value exactly median (terms cancel ) Bias can assume values -1 1 0 ideally (.e. unbiased). Note given quantiles contain median, median imputed linear interpolation two innermost quantiles. median available imputed, error thrown. Note order compute bias, quantiles must non-decreasing increasing quantile levels. large enough number quantiles, percentile rank equal proportion predictive samples observed value, bias metric coincides one continuous forecasts (see bias_sample()).","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Determines bias of quantile forecasts — bias_quantile","text":"","code":"predicted <- matrix(c(1.5:23.5, 3.3:25.3), nrow = 2, byrow = TRUE) quantile_level <- c(0.01, 0.025, seq(0.05, 0.95, 0.05), 0.975, 0.99) observed <- c(15, 12.4) bias_quantile(observed, predicted, quantile_level) #> [1] -0.3 0.2"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile_single_vector.html","id":null,"dir":"Reference","previous_headings":"","what":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","title":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","text":"Internal function compute bias single observed value, vector predicted values vector quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile_single_vector.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","text":"","code":"bias_quantile_single_vector(observed, predicted, quantile_level, na.rm)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile_single_vector.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","text":"observed Scalar observed value. predicted Vector length N (corresponding number quantiles) holds predictions. quantile_level Vector size N quantile levels predictions made. Note contain median (0.5) median imputed mean two innermost quantiles. na.rm Logical. missing values removed?","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_quantile_single_vector.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Compute bias for a single vector of quantile predictions — bias_quantile_single_vector","text":"scalar quantile bias single quantile prediction","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Determine bias of forecasts — bias_sample","title":"Determine bias of forecasts — bias_sample","text":"Determines bias predictive Monte-Carlo samples. function automatically recognises whether forecasts continuous integer valued adapts Bias function accordingly.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Determine bias of forecasts — bias_sample","text":"","code":"bias_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Determine bias of forecasts — bias_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Determine bias of forecasts — bias_sample","text":"Numeric vector length n biases predictive samples respect observed values.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Determine bias of forecasts — bias_sample","text":"continuous forecasts, Bias measured $$ B_t (P_t, x_t) = 1 - 2 * (P_t (x_t)) $$ \\(P_t\\) empirical cumulative distribution function prediction observed value \\(x_t\\). Computationally, \\(P_t (x_t)\\) just calculated fraction predictive samples \\(x_t\\) smaller \\(x_t\\). integer valued forecasts, Bias measured $$ B_t (P_t, x_t) = 1 - (P_t (x_t) + P_t (x_t + 1)) $$ adjust integer nature forecasts. cases, Bias can assume values -1 1 0 ideally.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Determine bias of forecasts — bias_sample","text":"integer valued Bias function discussed Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15 Funk S, Camacho , Kucharski AJ, Lowe R, Eggo RM, et al. (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15. PLOS Computational Biology 15(2): e1006785. doi:10.1371/journal.pcbi.1006785","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/bias_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Determine bias of forecasts — bias_sample","text":"","code":"## integer valued forecasts observed <- rpois(30, lambda = 1:30) predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) bias_sample(observed, predicted) #> [1] 0.660 -0.025 0.710 -0.930 0.870 0.435 -0.885 -0.940 -0.515 -0.790 #> [11] 0.975 -0.975 -0.620 -0.740 -0.640 0.395 0.695 -0.765 -0.935 -0.680 #> [21] -0.725 -0.320 0.355 0.730 0.250 0.995 -0.650 0.235 0.250 0.850 ## continuous forecasts observed <- rnorm(30, mean = 1:30) predicted <- replicate(200, rnorm(30, mean = 1:30)) bias_sample(observed, predicted) #> [1] -0.46 0.02 0.02 0.12 -0.18 -0.07 0.96 -0.60 0.16 -0.31 0.79 0.49 #> [13] -0.74 -0.48 0.26 -0.56 0.82 0.89 0.41 -0.31 0.19 0.47 -0.85 0.32 #> [25] 0.15 -0.16 0.34 -0.30 0.80 0.27"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_columns_present.html","id":null,"dir":"Reference","previous_headings":"","what":"Check column names are present in a data.frame — check_columns_present","title":"Check column names are present in a data.frame — check_columns_present","text":"functions loops column names checks whether present. issue encountered, function immediately stops returns message first issue encountered.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_columns_present.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check column names are present in a data.frame — check_columns_present","text":"","code":"check_columns_present(data, columns)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_columns_present.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check column names are present in a data.frame — check_columns_present","text":"data data.frame similar checked columns character vector column names check","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_columns_present.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check column names are present in a data.frame — check_columns_present","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_dims_ok_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Check Inputs Have Matching Dimensions — check_dims_ok_point","title":"Check Inputs Have Matching Dimensions — check_dims_ok_point","text":"Function assesses whether input dimensions match. following, n number observations / forecasts. Scalar values may repeated match length input. Allowed options therefore: observed vector length 1 length n predicted : vector length 1 length n matrix n rows 1 column","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_dims_ok_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check Inputs Have Matching Dimensions — check_dims_ok_point","text":"","code":"check_dims_ok_point(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_dims_ok_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check Inputs Have Matching Dimensions — check_dims_ok_point","text":"observed Input checked. factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_dims_ok_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check Inputs Have Matching Dimensions — check_dims_ok_point","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_duplicates.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that there are no duplicate forecasts — check_duplicates","title":"Check that there are no duplicate forecasts — check_duplicates","text":"Runs get_duplicate_forecasts() returns message issue encountered","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_duplicates.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that there are no duplicate forecasts — check_duplicates","text":"","code":"check_duplicates(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_duplicates.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that there are no duplicate forecasts — check_duplicates","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_duplicates.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that there are no duplicate forecasts — check_duplicates","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for binary forecast — check_input_binary","title":"Check that inputs are correct for binary forecast — check_input_binary","text":"Function assesses whether inputs correspond requirements scoring binary forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for binary forecast — check_input_binary","text":"","code":"check_input_binary(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for binary forecast — check_input_binary","text":"observed Input checked. factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted Input checked. predicted vector length n, holding probabilities. Alternatively, predicted can matrix size n x 1. Values represent probability corresponding value observed equal highest available factor level.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for binary forecast — check_input_binary","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_interval.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for interval-based forecast — check_input_interval","title":"Check that inputs are correct for interval-based forecast — check_input_interval","text":"Function assesses whether inputs correspond requirements scoring interval-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_interval.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for interval-based forecast — check_input_interval","text":"","code":"check_input_interval(observed, lower, upper, interval_range)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_interval.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for interval-based forecast — check_input_interval","text":"observed Input checked. numeric vector observed values size n. lower Input checked. numeric vector size n holds predicted value lower bounds prediction intervals. upper Input checked. numeric vector size n holds predicted value upper bounds prediction intervals. interval_range Input checked. vector size n denotes interval range percent. E.g. value 50 denotes (25%, 75%) prediction interval.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_interval.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for interval-based forecast — check_input_interval","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for point forecast — check_input_point","title":"Check that inputs are correct for point forecast — check_input_point","text":"Function assesses whether inputs correspond requirements scoring point forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for point forecast — check_input_point","text":"","code":"check_input_point(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for point forecast — check_input_point","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. numeric vector predicted values size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_point.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for point forecast — check_input_point","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for quantile-based forecast — check_input_quantile","title":"Check that inputs are correct for quantile-based forecast — check_input_quantile","text":"Function assesses whether inputs correspond requirements scoring quantile-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for quantile-based forecast — check_input_quantile","text":"","code":"check_input_quantile(observed, predicted, quantile_level)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for quantile-based forecast — check_input_quantile","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. nxN matrix predictive quantiles, n (number rows) number data points N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Input checked. vector size N denotes quantile levels corresponding columns prediction matrix.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for quantile-based forecast — check_input_quantile","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that inputs are correct for sample-based forecast — check_input_sample","title":"Check that inputs are correct for sample-based forecast — check_input_sample","text":"Function assesses whether inputs correspond requirements scoring sample-based forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that inputs are correct for sample-based forecast — check_input_sample","text":"","code":"check_input_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that inputs are correct for sample-based forecast — check_input_sample","text":"observed Input checked. numeric vector observed values size n. predicted Input checked. numeric nxN matrix predictive samples, n (number rows) number data points N (number columns) number samples per forecast. observed just single number, predicted values can just vector size N.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_input_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that inputs are correct for sample-based forecast — check_input_sample","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_number_per_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Check that all forecasts have the same number of rows — check_number_per_forecast","title":"Check that all forecasts have the same number of rows — check_number_per_forecast","text":"Helper function checks number rows (corresponding e.g quantiles samples) per forecast. number quantiles samples forecasts, returns TRUE string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_number_per_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check that all forecasts have the same number of rows — check_number_per_forecast","text":"","code":"check_number_per_forecast(data, forecast_unit)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_number_per_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check that all forecasts have the same number of rows — check_number_per_forecast","text":"data data.frame similar checked forecast_unit Character vector denoting unit single forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_number_per_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check that all forecasts have the same number of rows — check_number_per_forecast","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_numeric_vector.html","id":null,"dir":"Reference","previous_headings":"","what":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","title":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","text":"Helper function check whether input numeric vector.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_numeric_vector.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","text":"","code":"check_numeric_vector(x, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_numeric_vector.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","text":"x input check ... Arguments passed checkmate::check_numeric lower [numeric(1)] Lower value elements x must greater equal . upper [numeric(1)] Upper value elements x must lower equal . finite [logical(1)] Check finite values? Default FALSE. .missing [logical(1)] vectors missing values allowed? Default TRUE. .missing [logical(1)] vectors non-missing values allowed? Default TRUE. Note empty vectors non-missing values. len [integer(1)] Exact expected length x. min.len [integer(1)] Minimal length x. max.len [integer(1)] Maximal length x. unique [logical(1)] Must values unique? Default FALSE. sorted [logical(1)] Elements must sorted ascending order. Missing values ignored. names [character(1)] Check names. See checkNamed possible values. Default “” performs check . Note can use checkSubset check specific set names. typed.missing [logical(1)] set FALSE (default), types missing values (NA, NA_integer_, NA_real_, NA_character_ NA_character_) well empty vectors allowed type-checking atomic input. Set TRUE enable strict type checking. null.ok [logical(1)] set TRUE, x may also NULL. case type check x performed, additional checks disabled.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_numeric_vector.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Check whether an input is an atomic vector of mode 'numeric' — check_numeric_vector","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_try.html","id":null,"dir":"Reference","previous_headings":"","what":"Helper function to convert assert statements into checks — check_try","title":"Helper function to convert assert statements into checks — check_try","text":"Tries execute expression. Internally, used see whether assertions fail checking inputs (.e. convert assert_*() statement check). expression fails, error message returned. expression succeeds, TRUE returned.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_try.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Helper function to convert assert statements into checks — check_try","text":"","code":"check_try(expr)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_try.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Helper function to convert assert statements into checks — check_try","text":"expr expression evaluated","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/check_try.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Helper function to convert assert statements into checks — check_try","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/clean_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Clean forecast object — clean_forecast","title":"Clean forecast object — clean_forecast","text":"function makes possible silently validate object. addition, can return copy data remove rows missing values.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/clean_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Clean forecast object — clean_forecast","text":"","code":"clean_forecast(forecast, copy = FALSE, na.omit = FALSE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/clean_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Clean forecast object — clean_forecast","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()). copy Logical, default FALSE. TRUE, copy input data created. na.omit Logical, default FALSE. TRUE, rows missing values removed.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_forecasts.html","id":null,"dir":"Reference","previous_headings":"","what":"Compare a subset of common forecasts — compare_forecasts","title":"Compare a subset of common forecasts — compare_forecasts","text":"function compares two comparators based subset forecasts comparators made prediction. gets called pairwise_comparison_one_group(), handles comparison multiple comparators single set forecasts (subsets forecasts distinguished). pairwise_comparison_one_group() turn gets called get_pairwise_comparisons() can handle pairwise comparisons set forecasts multiple subsets, e.g. pairwise comparisons one set forecasts, done separately two different forecast targets.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_forecasts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Compare a subset of common forecasts — compare_forecasts","text":"","code":"compare_forecasts( scores, compare = \"model\", name_comparator1, name_comparator2, metric, one_sided = FALSE, test_type = c(\"non_parametric\", \"permutation\"), n_permutations = 999 )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_forecasts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Compare a subset of common forecasts — compare_forecasts","text":"scores object class scores (data.table scores additional attribute metrics produced score()). compare Character vector single colum name defines elements pairwise comparison. example, set \"model\" (default), elements \"model\" column compared. name_comparator1 Character, name first comparator name_comparator2 Character, name comparator compare metric string name metric relative skill shall computed. default either \"crps\", \"wis\" \"brier_score\" available. one_sided Boolean, default FALSE, whether two conduct one-sided instead two-sided test determine significance pairwise comparison. test_type Character, either \"non_parametric\" (default) \"permutation\". determines kind test shall conducted determine p-values. n_permutations Numeric, number permutations permutation test. Default 999.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_forecasts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Compare a subset of common forecasts — compare_forecasts","text":"list mean score ratios p-values comparison two comparators","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/compare_forecasts.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Compare a subset of common forecasts — compare_forecasts","text":"Johannes Bracher, johannes.bracher@kit.edu Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/crps_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"(Continuous) ranked probability score — crps_sample","title":"(Continuous) ranked probability score — crps_sample","text":"Wrapper around crps_sample() function scoringRules package. Can used continuous well integer valued forecasts Continuous ranked probability score (CRPS) can interpreted sum three components: overprediction, underprediction dispersion. \"Dispersion\" defined CRPS median forecast $m$. observation $y$ greater $m$ overpredictoin defined CRPS forecast $y$ minus dispersion component, underprediction zero. , hand, $y [1] 0.231225 0.329850 1.060100 0.412625 0.991550 0.793850 2.847025 1.126200 #> [9] 0.690675 0.992875 1.118300 4.911975 4.135800 1.411050 0.967825 3.425125 #> [17] 0.839600 2.404700 1.376075 1.665525 2.771375 1.168550 1.244700 2.377250 #> [25] 2.847875 2.019175 1.827975 2.074850 1.868775 3.171950"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_assert_functions.html","id":null,"dir":"Reference","previous_headings":"","what":"Documentation template for assert functions — document_assert_functions","title":"Documentation template for assert functions — document_assert_functions","text":"Documentation template assert functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_assert_functions.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Documentation template for assert functions — document_assert_functions","text":"observed Input checked. numeric vector observed values size n.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_assert_functions.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Documentation template for assert functions — document_assert_functions","text":"Returns NULL invisibly assertion successful throws error otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_check_functions.html","id":null,"dir":"Reference","previous_headings":"","what":"Documentation template for check functions — document_check_functions","title":"Documentation template for check functions — document_check_functions","text":"Documentation template check functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_check_functions.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Documentation template for check functions — document_check_functions","text":"data data.frame similar checked observed Input checked. numeric vector observed values size n. columns character vector column names check","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_check_functions.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Documentation template for check functions — document_check_functions","text":"Returns TRUE check successful string error message otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_test_functions.html","id":null,"dir":"Reference","previous_headings":"","what":"Documentation template for test functions — document_test_functions","title":"Documentation template for test functions — document_test_functions","text":"Documentation template test functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/document_test_functions.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Documentation template for test functions — document_test_functions","text":"Returns TRUE check successful FALSE otherwise","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Dawid-Sebastiani score — dss_sample","title":"Dawid-Sebastiani score — dss_sample","text":"Wrapper around dss_sample() function scoringRules package.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Dawid-Sebastiani score — dss_sample","text":"","code":"dss_sample(observed, predicted, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Dawid-Sebastiani score — dss_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. ... Additional arguments passed dss_sample() scoringRules package.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Dawid-Sebastiani score — dss_sample","text":"Vector scores.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Dawid-Sebastiani score — dss_sample","text":"Alexander Jordan, Fabian Krüger, Sebastian Lerch, Evaluating Probabilistic Forecasts scoringRules, https://www.jstatsoft.org/article/view/v090i12","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/dss_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Dawid-Sebastiani score — dss_sample","text":"","code":"observed <- rpois(30, lambda = 1:30) predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) dss_sample(observed, predicted) #> [1] -0.06179111 2.81530809 2.46742176 1.79360855 1.61740613 2.39081466 #> [7] 1.92865409 15.58374079 3.88246189 4.97070116 2.24551342 2.71528477 #> [13] 2.79485162 2.62249405 2.80770087 4.30015607 3.23096351 3.16329747 #> [19] 3.23001192 7.67628290 2.92052357 5.37555852 4.78857945 3.12865561 #> [25] 3.61999703 4.32028466 3.38593521 6.87116360 4.71995903 4.99107110"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_data.table.html","id":null,"dir":"Reference","previous_headings":"","what":"Ensure that an object is a data.table — ensure_data.table","title":"Ensure that an object is a data.table — ensure_data.table","text":"function ensures object data table. object data table, converted one. object data table, copy object returned.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_data.table.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Ensure that an object is a data.table — ensure_data.table","text":"","code":"ensure_data.table(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_data.table.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Ensure that an object is a data.table — ensure_data.table","text":"data object ensure data table.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/ensure_data.table.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Ensure that an object is a data.table — ensure_data.table","text":"data.table/copy existing data.table.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Binary forecast example data — example_binary","title":"Binary forecast example data — example_binary","text":"data set binary predictions COVID-19 cases deaths constructed data submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Binary forecast example data — example_binary","text":"","code":"example_binary"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Binary forecast example data — example_binary","text":"object class forecast_binary (see as_forecast()) following columns: location country prediction made location_name name country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed factor observed values forecast_date date prediction made model name model generated forecasts horizon forecast horizon weeks predicted predicted value","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Binary forecast example data — example_binary","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_binary.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Binary forecast example data — example_binary","text":"Predictions data set constructed based continuous example data looking number samples mean prediction. outcome constructed whether actually observed value mean prediction. understood sound statistical practice, rather practical way create example data set. data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_nominal.html","id":null,"dir":"Reference","previous_headings":"","what":"Nominal example data — example_nominal","title":"Nominal example data — example_nominal","text":"data set predictions COVID-19 cases deaths submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_nominal.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Nominal example data — example_nominal","text":"","code":"example_nominal"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_nominal.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Nominal example data — example_nominal","text":"object class forecast_nominal (see as_forecast()) following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed Numeric: observed values location_name name country prediction made forecast_date date prediction made predicted_label outcome probabilty corresponds predicted predicted value model name model generated forecasts horizon forecast horizon weeks","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_nominal.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Nominal example data — example_nominal","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_nominal.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Nominal example data — example_nominal","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Point forecast example data — example_point","title":"Point forecast example data — example_point","text":"data set predictions COVID-19 cases deaths submitted European Forecast Hub. data set like quantile example data, median replaced point forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Point forecast example data — example_point","text":"","code":"example_point"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Point forecast example data — example_point","text":"object class forecast_point (see as_forecast()) following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed observed values location_name name country prediction made forecast_date date prediction made predicted predicted value model name model generated forecasts horizon forecast horizon weeks","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Point forecast example data — example_point","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_point.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Point forecast example data — example_point","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Quantile example data — example_quantile","title":"Quantile example data — example_quantile","text":"data set predictions COVID-19 cases deaths submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Quantile example data — example_quantile","text":"","code":"example_quantile"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Quantile example data — example_quantile","text":"object class forecast_quantile (see as_forecast()) following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed Numeric: observed values location_name name country prediction made forecast_date date prediction made quantile_level quantile level corresponding prediction predicted predicted value model name model generated forecasts horizon forecast horizon weeks","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Quantile example data — example_quantile","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_quantile.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Quantile example data — example_quantile","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":null,"dir":"Reference","previous_headings":"","what":"Continuous forecast example data — example_sample_continuous","title":"Continuous forecast example data — example_sample_continuous","text":"data set continuous predictions COVID-19 cases deaths constructed data submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Continuous forecast example data — example_sample_continuous","text":"","code":"example_sample_continuous"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Continuous forecast example data — example_sample_continuous","text":"object class forecast_sample (see as_forecast()) following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed observed values location_name name country prediction made forecast_date date prediction made model name model generated forecasts horizon forecast horizon weeks predicted predicted value sample_id id corresponding sample","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Continuous forecast example data — example_sample_continuous","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_continuous.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Continuous forecast example data — example_sample_continuous","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":null,"dir":"Reference","previous_headings":"","what":"Discrete forecast example data — example_sample_discrete","title":"Discrete forecast example data — example_sample_discrete","text":"data set integer predictions COVID-19 cases deaths constructed data submitted European Forecast Hub.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Discrete forecast example data — example_sample_discrete","text":"","code":"example_sample_discrete"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":"format","dir":"Reference","previous_headings":"","what":"Format","title":"Discrete forecast example data — example_sample_discrete","text":"object class forecast_sample (see as_forecast()) following columns: location country prediction made target_end_date date prediction made target_type target predicted (cases deaths) observed observed values location_name name country prediction made forecast_date date prediction made model name model generated forecasts horizon forecast horizon weeks predicted predicted value sample_id id corresponding sample","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Discrete forecast example data — example_sample_discrete","text":"https://github.com/european-modelling-hubs/covid19-forecast-hub-europe/commit/a42867b1ea152c57e25b04f9faa26cfd4bfd8fa6/","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/example_sample_discrete.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Discrete forecast example data — example_sample_discrete","text":"data created using script create-example-data.R inst/ folder (top level folder compiled package).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/forecast_types.html","id":null,"dir":"Reference","previous_headings":"","what":"Documentation template for forecast types — forecast_types","title":"Documentation template for forecast types — forecast_types","text":"Documentation template forecast types","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/forecast_types.html","id":"forecast-types-and-input-formats","dir":"Reference","previous_headings":"","what":"Forecast types and input formats","title":"Documentation template for forecast types — forecast_types","text":"Various different forecast types / forecast formats supported. moment, : point forecasts binary forecasts (\"soft binary classification\") nominal forecasts (\"soft classification multiple unordered classes\") Probabilistic forecasts quantile-based format (forecast represented set predictive quantiles) Probabilistic forecasts sample-based format (forecast represented set predictive samples) Forecast types determined based columns present input data. overview required format forecast type: forecast types require data.frame similar columns observed predicted, model. Point forecasts require column observed type numeric column predicted type numeric. Binary forecasts require column observed type factor exactly two levels column predicted type numeric probabilities, corresponding probability observed equal second factor level. See details information. Nominal forecasts require column observed type factor N levels, (N number possible outcomes), column predicted type numeric probabilities (sum one across possible outcomes), column predicted_label type factor N levels, denoting outcome probability given. Forecasts must complete, .e. must probability assigned every possible outcome. Quantile-based forecasts require column observed type numeric, column predicted type numeric, column quantile_level type numeric quantile-levels (0 1). Sample-based forecasts require column observed type numeric, column predicted type numeric, column sample_id type numeric sample indices. information see vignettes example data (example_quantile, example_sample_continuous, example_sample_discrete, example_point(), example_binary, example_nominal).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/forecast_types.html","id":"forecast-unit","dir":"Reference","previous_headings":"","what":"Forecast unit","title":"Documentation template for forecast types — forecast_types","text":"order score forecasts, scoringutils needs know rows data belong together jointly form single forecasts. easy e.g. point forecast, one row per forecast. quantile sample-based forecasts, however, multiple rows belong single forecast. forecast unit unit single forecast described combination columns uniquely identify single forecast. example, forecasts made different models various locations different time points, several weeks future. forecast unit described forecast_unit = c(\"model\", \"location\", \"forecast_date\", \"forecast_horizon\"). scoringutils automatically tries determine unit single forecast. uses existing columns , means columns must present unrelated forecast unit. simplistic example, additional row, \"even\", one row number even zero otherwise, mess scoring scoringutils thinks column relevant defining forecast unit. order avoid issues, recommend setting forecast unit explicitly, usually forecast_unit argument as_forecast() functions. drop unneeded columns, making sure necessary, 'protected columns' like \"predicted\" \"observed\" retained.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":null,"dir":"Reference","previous_headings":"","what":"Calculate geometric mean — geometric_mean","title":"Calculate geometric mean — geometric_mean","text":"Calculate geometric mean","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Calculate geometric mean — geometric_mean","text":"","code":"geometric_mean(x)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Calculate geometric mean — geometric_mean","text":"x Numeric vector values calculate geometric mean.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Calculate geometric mean — geometric_mean","text":"geometric mean values x. NA values ignored.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/geometric_mean.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Calculate geometric mean — geometric_mean","text":"Used get_pairwise_comparisons().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":null,"dir":"Reference","previous_headings":"","what":"Calculate correlation between metrics — get_correlations","title":"Calculate correlation between metrics — get_correlations","text":"Calculate correlation different metrics data.frame scores produced score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Calculate correlation between metrics — get_correlations","text":"","code":"get_correlations(scores, metrics = get_metrics.scores(scores), ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Calculate correlation between metrics — get_correlations","text":"scores object class scores (data.table scores additional attribute metrics produced score()). metrics character vector metrics show. set NULL (default), metrics present scores shown. ... Additional arguments pass cor().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Calculate correlation between metrics — get_correlations","text":"object class scores (data.table additional attribute metrics holding names scores) correlations different metrics","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_correlations.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Calculate correlation between metrics — get_correlations","text":"","code":"library(magrittr) # pipe operator scores <- example_quantile %>% as_forecast_quantile() %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. get_correlations(scores) #> wis overprediction underprediction dispersion bias #> #> 1: 1.0000000 0.94297565 0.28377361 0.45566303 0.10545891 #> 2: 0.9429757 1.00000000 -0.03310356 0.32493799 0.21532161 #> 3: 0.2837736 -0.03310356 1.00000000 0.14580143 -0.35123801 #> 4: 0.4556630 0.32493799 0.14580143 1.00000000 0.11118365 #> 5: 0.1054589 0.21532161 -0.35123801 0.11118365 1.00000000 #> 6: -0.2076649 -0.14556039 -0.21392764 -0.09400664 0.01338140 #> 7: -0.4075613 -0.31824017 -0.35756699 -0.08614678 0.09802725 #> 8: 0.9886108 0.90326672 0.33589892 0.53809741 0.09578751 #> interval_coverage_50 interval_coverage_90 ae_median metric #> #> 1: -0.20766492 -0.40756133 0.98861080 wis #> 2: -0.14556039 -0.31824017 0.90326672 overprediction #> 3: -0.21392764 -0.35756699 0.33589892 underprediction #> 4: -0.09400664 -0.08614678 0.53809741 dispersion #> 5: 0.01338140 0.09802725 0.09578751 bias #> 6: 1.00000000 0.37245118 -0.24559356 interval_coverage_50 #> 7: 0.37245118 1.00000000 -0.41079097 interval_coverage_90 #> 8: -0.24559356 -0.41079097 1.00000000 ae_median"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"validated forecast object quantile-based format (see as_forecast() information), function computes: interval coverage central prediction intervals quantile coverage predictive quantiles deviation desired actual coverage (interval quantile coverage) Coverage values computed specific level grouping, specified argument. default, coverage values computed per model. Interval coverage Interval coverage given interval range defined proportion observations fall within corresponding central prediction intervals. Central prediction intervals symmetric around median formed two quantiles denote lower upper bound. example, 50% central prediction interval interval 0.25 0.75 quantiles predictive distribution. Quantile coverage Quantile coverage given quantile level defined proportion observed values smaller corresponding predictive quantile. example, 0.5 quantile coverage proportion observed values smaller 0.5 quantile predictive distribution. Just , single observation quantile single predictive distribution, value either TRUE FALSE. Coverage deviation coverage deviation difference desired coverage (can either interval quantile coverage) actual coverage. example, desired coverage 90% actual coverage 80%, coverage deviation -0.1.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"","code":"get_coverage(forecast, by = \"model\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()). character vector denotes level grouping coverage values computed. default (\"model\"), one coverage value per model returned.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"data.table columns specified additional columns coverage values described data.table columns \"interval_coverage\", \"interval_coverage_deviation\", \"quantile_coverage\", \"quantile_coverage_deviation\" columns specified .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Get quantile and interval coverage values for quantile-based forecasts — get_coverage","text":"","code":"library(magrittr) # pipe operator example_quantile %>% as_forecast_quantile() %>% get_coverage(by = \"model\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> model quantile_level interval_range interval_coverage #> #> 1: EuroCOVIDhub-baseline 0.500 0 0.000000000 #> 2: EuroCOVIDhub-baseline 0.450 10 0.085937500 #> 3: EuroCOVIDhub-baseline 0.550 10 0.085937500 #> 4: EuroCOVIDhub-baseline 0.400 20 0.191406250 #> 5: EuroCOVIDhub-baseline 0.600 20 0.191406250 #> 6: EuroCOVIDhub-baseline 0.350 30 0.289062500 #> 7: EuroCOVIDhub-baseline 0.650 30 0.289062500 #> 8: EuroCOVIDhub-baseline 0.300 40 0.375000000 #> 9: EuroCOVIDhub-baseline 0.700 40 0.375000000 #> 10: EuroCOVIDhub-baseline 0.250 50 0.496093750 #> 11: EuroCOVIDhub-baseline 0.750 50 0.496093750 #> 12: EuroCOVIDhub-baseline 0.200 60 0.628906250 #> 13: EuroCOVIDhub-baseline 0.800 60 0.628906250 #> 14: EuroCOVIDhub-baseline 0.150 70 0.773437500 #> 15: EuroCOVIDhub-baseline 0.850 70 0.773437500 #> 16: EuroCOVIDhub-baseline 0.100 80 0.843750000 #> 17: EuroCOVIDhub-baseline 0.900 80 0.843750000 #> 18: EuroCOVIDhub-baseline 0.050 90 0.910156250 #> 19: EuroCOVIDhub-baseline 0.950 90 0.910156250 #> 20: EuroCOVIDhub-baseline 0.025 95 0.925781250 #> 21: EuroCOVIDhub-baseline 0.975 95 0.925781250 #> 22: EuroCOVIDhub-baseline 0.010 98 0.933593750 #> 23: EuroCOVIDhub-baseline 0.990 98 0.933593750 #> 24: EuroCOVIDhub-ensemble 0.500 0 0.003906250 #> 25: EuroCOVIDhub-ensemble 0.450 10 0.148437500 #> 26: EuroCOVIDhub-ensemble 0.550 10 0.148437500 #> 27: EuroCOVIDhub-ensemble 0.400 20 0.250000000 #> 28: EuroCOVIDhub-ensemble 0.600 20 0.250000000 #> 29: EuroCOVIDhub-ensemble 0.350 30 0.386718750 #> 30: EuroCOVIDhub-ensemble 0.650 30 0.386718750 #> 31: EuroCOVIDhub-ensemble 0.300 40 0.519531250 #> 32: EuroCOVIDhub-ensemble 0.700 40 0.519531250 #> 33: EuroCOVIDhub-ensemble 0.250 50 0.632812500 #> 34: EuroCOVIDhub-ensemble 0.750 50 0.632812500 #> 35: EuroCOVIDhub-ensemble 0.200 60 0.667968750 #> 36: EuroCOVIDhub-ensemble 0.800 60 0.667968750 #> 37: EuroCOVIDhub-ensemble 0.150 70 0.753906250 #> 38: EuroCOVIDhub-ensemble 0.850 70 0.753906250 #> 39: EuroCOVIDhub-ensemble 0.100 80 0.816406250 #> 40: EuroCOVIDhub-ensemble 0.900 80 0.816406250 #> 41: EuroCOVIDhub-ensemble 0.050 90 0.902343750 #> 42: EuroCOVIDhub-ensemble 0.950 90 0.902343750 #> 43: EuroCOVIDhub-ensemble 0.025 95 0.941406250 #> 44: EuroCOVIDhub-ensemble 0.975 95 0.941406250 #> 45: EuroCOVIDhub-ensemble 0.010 98 0.968750000 #> 46: EuroCOVIDhub-ensemble 0.990 98 0.968750000 #> 47: epiforecasts-EpiNow2 0.500 0 0.004048583 #> 48: epiforecasts-EpiNow2 0.450 10 0.093117409 #> 49: epiforecasts-EpiNow2 0.550 10 0.093117409 #> 50: epiforecasts-EpiNow2 0.400 20 0.165991903 #> 51: epiforecasts-EpiNow2 0.600 20 0.165991903 #> 52: epiforecasts-EpiNow2 0.350 30 0.230769231 #> 53: epiforecasts-EpiNow2 0.650 30 0.230769231 #> 54: epiforecasts-EpiNow2 0.300 40 0.319838057 #> 55: epiforecasts-EpiNow2 0.700 40 0.319838057 #> 56: epiforecasts-EpiNow2 0.250 50 0.445344130 #> 57: epiforecasts-EpiNow2 0.750 50 0.445344130 #> 58: epiforecasts-EpiNow2 0.200 60 0.538461538 #> 59: epiforecasts-EpiNow2 0.800 60 0.538461538 #> 60: epiforecasts-EpiNow2 0.150 70 0.635627530 #> 61: epiforecasts-EpiNow2 0.850 70 0.635627530 #> 62: epiforecasts-EpiNow2 0.100 80 0.732793522 #> 63: epiforecasts-EpiNow2 0.900 80 0.732793522 #> 64: epiforecasts-EpiNow2 0.050 90 0.846153846 #> 65: epiforecasts-EpiNow2 0.950 90 0.846153846 #> 66: epiforecasts-EpiNow2 0.025 95 0.874493927 #> 67: epiforecasts-EpiNow2 0.975 95 0.874493927 #> 68: epiforecasts-EpiNow2 0.010 98 0.910931174 #> 69: epiforecasts-EpiNow2 0.990 98 0.910931174 #> 70: UMass-MechBayes 0.500 0 0.015625000 #> 71: UMass-MechBayes 0.450 10 0.101562500 #> 72: UMass-MechBayes 0.550 10 0.101562500 #> 73: UMass-MechBayes 0.400 20 0.195312500 #> 74: UMass-MechBayes 0.600 20 0.195312500 #> 75: UMass-MechBayes 0.350 30 0.281250000 #> 76: UMass-MechBayes 0.650 30 0.281250000 #> 77: UMass-MechBayes 0.300 40 0.382812500 #> 78: UMass-MechBayes 0.700 40 0.382812500 #> 79: UMass-MechBayes 0.250 50 0.460937500 #> 80: UMass-MechBayes 0.750 50 0.460937500 #> 81: UMass-MechBayes 0.200 60 0.539062500 #> 82: UMass-MechBayes 0.800 60 0.539062500 #> 83: UMass-MechBayes 0.150 70 0.617187500 #> 84: UMass-MechBayes 0.850 70 0.617187500 #> 85: UMass-MechBayes 0.100 80 0.765625000 #> 86: UMass-MechBayes 0.900 80 0.765625000 #> 87: UMass-MechBayes 0.050 90 0.875000000 #> 88: UMass-MechBayes 0.950 90 0.875000000 #> 89: UMass-MechBayes 0.025 95 0.953125000 #> 90: UMass-MechBayes 0.975 95 0.953125000 #> 91: UMass-MechBayes 0.010 98 0.984375000 #> 92: UMass-MechBayes 0.990 98 0.984375000 #> model quantile_level interval_range interval_coverage #> interval_coverage_deviation quantile_coverage quantile_coverage_deviation #> #> 1: 0.000000000 0.69921875 0.199218750 #> 2: -0.014062500 0.65625000 0.206250000 #> 3: -0.014062500 0.74218750 0.192187500 #> 4: -0.008593750 0.58593750 0.185937500 #> 5: -0.008593750 0.77343750 0.173437500 #> 6: -0.010937500 0.52343750 0.173437500 #> 7: -0.010937500 0.80859375 0.158593750 #> 8: -0.025000000 0.46875000 0.168750000 #> 9: -0.025000000 0.84375000 0.143750000 #> 10: -0.003906250 0.36718750 0.117187500 #> 11: -0.003906250 0.86328125 0.113281250 #> 12: 0.028906250 0.25000000 0.050000000 #> 13: 0.028906250 0.87500000 0.075000000 #> 14: 0.073437500 0.13281250 -0.017187500 #> 15: 0.073437500 0.90625000 0.056250000 #> 16: 0.043750000 0.08203125 -0.017968750 #> 17: 0.043750000 0.92578125 0.025781250 #> 18: 0.010156250 0.04296875 -0.007031250 #> 19: 0.010156250 0.95312500 0.003125000 #> 20: -0.024218750 0.03125000 0.006250000 #> 21: -0.024218750 0.95703125 -0.017968750 #> 22: -0.046406250 0.03125000 0.021250000 #> 23: -0.046406250 0.96484375 -0.025156250 #> 24: 0.003906250 0.53125000 0.031250000 #> 25: 0.048437500 0.46484375 0.014843750 #> 26: 0.048437500 0.60156250 0.051562500 #> 27: 0.050000000 0.40625000 0.006250000 #> 28: 0.050000000 0.65625000 0.056250000 #> 29: 0.086718750 0.33984375 -0.010156250 #> 30: 0.086718750 0.72656250 0.076562500 #> 31: 0.119531250 0.26562500 -0.034375000 #> 32: 0.119531250 0.76562500 0.065625000 #> 33: 0.132812500 0.16406250 -0.085937500 #> 34: 0.132812500 0.79687500 0.046875000 #> 35: 0.067968750 0.14062500 -0.059375000 #> 36: 0.067968750 0.80468750 0.004687500 #> 37: 0.053906250 0.10156250 -0.048437500 #> 38: 0.053906250 0.85546875 0.005468750 #> 39: 0.016406250 0.07812500 -0.021875000 #> 40: 0.016406250 0.89453125 -0.005468750 #> 41: 0.002343750 0.04296875 -0.007031250 #> 42: 0.002343750 0.94531250 -0.004687500 #> 43: -0.008593750 0.03125000 0.006250000 #> 44: -0.008593750 0.97265625 -0.002343750 #> 45: -0.011250000 0.01562500 0.005625000 #> 46: -0.011250000 0.98437500 -0.005625000 #> 47: 0.004048583 0.49392713 -0.006072874 #> 48: -0.006882591 0.43724696 -0.012753036 #> 49: -0.006882591 0.53036437 -0.019635628 #> 50: -0.034008097 0.39676113 -0.003238866 #> 51: -0.034008097 0.55870445 -0.041295547 #> 52: -0.069230769 0.36437247 0.014372470 #> 53: -0.069230769 0.59514170 -0.054858300 #> 54: -0.080161943 0.31983806 0.019838057 #> 55: -0.080161943 0.63967611 -0.060323887 #> 56: -0.054655870 0.26315789 0.013157895 #> 57: -0.054655870 0.70445344 -0.045546559 #> 58: -0.061538462 0.20647773 0.006477733 #> 59: -0.061538462 0.74493927 -0.055060729 #> 60: -0.064372470 0.14574899 -0.004251012 #> 61: -0.064372470 0.78137652 -0.068623482 #> 62: -0.067206478 0.10121457 0.001214575 #> 63: -0.067206478 0.83400810 -0.065991903 #> 64: -0.053846154 0.06072874 0.010728745 #> 65: -0.053846154 0.90688259 -0.043117409 #> 66: -0.075506073 0.04858300 0.023582996 #> 67: -0.075506073 0.92307692 -0.051923077 #> 68: -0.069068826 0.02429150 0.014291498 #> 69: -0.069068826 0.93522267 -0.054777328 #> 70: 0.015625000 0.50000000 0.000000000 #> 71: 0.001562500 0.42187500 -0.028125000 #> 72: 0.001562500 0.52343750 -0.026562500 #> 73: -0.004687500 0.37500000 -0.025000000 #> 74: -0.004687500 0.57031250 -0.029687500 #> 75: -0.018750000 0.35156250 0.001562500 #> 76: -0.018750000 0.61718750 -0.032812500 #> 77: -0.017187500 0.28906250 -0.010937500 #> 78: -0.017187500 0.66406250 -0.035937500 #> 79: -0.039062500 0.27343750 0.023437500 #> 80: -0.039062500 0.71875000 -0.031250000 #> 81: -0.060937500 0.24218750 0.042187500 #> 82: -0.060937500 0.78125000 -0.018750000 #> 83: -0.082812500 0.20312500 0.053125000 #> 84: -0.082812500 0.82031250 -0.029687500 #> 85: -0.034375000 0.12500000 0.025000000 #> 86: -0.034375000 0.87500000 -0.025000000 #> 87: -0.025000000 0.06250000 0.012500000 #> 88: -0.025000000 0.93750000 -0.012500000 #> 89: 0.003125000 0.01562500 -0.009375000 #> 90: 0.003125000 0.96875000 -0.006250000 #> 91: 0.004375000 0.00781250 -0.002187500 #> 92: 0.004375000 0.99218750 0.002187500 #> interval_coverage_deviation quantile_coverage quantile_coverage_deviation"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":null,"dir":"Reference","previous_headings":"","what":"Find duplicate forecasts — get_duplicate_forecasts","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"Internal helper function identify duplicate forecasts, .e. instances one forecast prediction target.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"","code":"get_duplicate_forecasts(data, forecast_unit = NULL, counts = FALSE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. forecast_unit (optional) Name columns data (renaming columns) denote unit single forecast. See get_forecast_unit() details. NULL (default), columns required columns assumed form unit single forecast. specified, columns part forecast unit (required columns) removed. counts output show number duplicates per forecast unit instead individual duplicated rows? Default FALSE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"data.frame rows duplicate forecast found","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_duplicate_forecasts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Find duplicate forecasts — get_duplicate_forecasts","text":"","code":"example <- rbind(example_quantile, example_quantile[1000:1010]) get_duplicate_forecasts(example) #> location target_end_date target_type observed location_name forecast_date #> #> 1: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 2: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 3: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 4: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 5: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 6: DE 2021-05-22 Deaths 1285 Germany 2021-05-17 #> 7: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 8: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 9: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 10: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 11: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 12: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 13: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 14: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 15: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 16: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 17: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 18: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 19: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 20: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 21: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> 22: DE 2021-05-29 Cases 31653 Germany 2021-05-10 #> location target_end_date target_type observed location_name forecast_date #> quantile_level predicted model horizon #> #> 1: 0.950 1464 epiforecasts-EpiNow2 1 #> 2: 0.950 1464 epiforecasts-EpiNow2 1 #> 3: 0.975 1642 epiforecasts-EpiNow2 1 #> 4: 0.975 1642 epiforecasts-EpiNow2 1 #> 5: 0.990 1951 epiforecasts-EpiNow2 1 #> 6: 0.990 1951 epiforecasts-EpiNow2 1 #> 7: 0.010 28999 EuroCOVIDhub-ensemble 3 #> 8: 0.010 28999 EuroCOVIDhub-ensemble 3 #> 9: 0.025 32612 EuroCOVIDhub-ensemble 3 #> 10: 0.025 32612 EuroCOVIDhub-ensemble 3 #> 11: 0.050 36068 EuroCOVIDhub-ensemble 3 #> 12: 0.050 36068 EuroCOVIDhub-ensemble 3 #> 13: 0.100 41484 EuroCOVIDhub-ensemble 3 #> 14: 0.100 41484 EuroCOVIDhub-ensemble 3 #> 15: 0.150 47110 EuroCOVIDhub-ensemble 3 #> 16: 0.150 47110 EuroCOVIDhub-ensemble 3 #> 17: 0.200 50929 EuroCOVIDhub-ensemble 3 #> 18: 0.200 50929 EuroCOVIDhub-ensemble 3 #> 19: 0.250 54561 EuroCOVIDhub-ensemble 3 #> 20: 0.250 54561 EuroCOVIDhub-ensemble 3 #> 21: 0.300 57739 EuroCOVIDhub-ensemble 3 #> 22: 0.300 57739 EuroCOVIDhub-ensemble 3 #> quantile_level predicted model horizon"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":null,"dir":"Reference","previous_headings":"","what":"Count number of available forecasts — get_forecast_counts","title":"Count number of available forecasts — get_forecast_counts","text":"Given data set forecasts, function counts number available forecasts. level grouping can specified using argument (e.g. count number forecasts per model, number forecasts per model location). useful determine whether missing forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Count number of available forecasts — get_forecast_counts","text":"","code":"get_forecast_counts( forecast, by = get_forecast_unit(forecast), collapse = c(\"quantile_level\", \"sample_id\") )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Count number of available forecasts — get_forecast_counts","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()). character vector NULL (default) denotes categories number forecasts counted. default unit single forecast (.e. available columns (apart \"protected\" columns 'predicted' 'observed') plus \"quantile_level\" \"sample_id\" present). collapse character vector (default: c(\"quantile_level\", \"sample_id\") names categories number rows collapsed one counting. example, single forecast usually represented set several quantiles samples collapsing one makes sure single forecast gets counted . Setting collapse = c() mean quantiles / samples counted individual forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Count number of available forecasts — get_forecast_counts","text":"data.table columns specified additional column \"count\" number forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_counts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Count number of available forecasts — get_forecast_counts","text":"","code":"library(magrittr) # pipe operator example_quantile %>% as_forecast_quantile() %>% get_forecast_counts(by = c(\"model\", \"target_type\")) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Key: #> model target_type count #> #> 1: EuroCOVIDhub-baseline Cases 128 #> 2: EuroCOVIDhub-baseline Deaths 128 #> 3: EuroCOVIDhub-ensemble Cases 128 #> 4: EuroCOVIDhub-ensemble Deaths 128 #> 5: UMass-MechBayes Cases 0 #> 6: UMass-MechBayes Deaths 128 #> 7: epiforecasts-EpiNow2 Cases 128 #> 8: epiforecasts-EpiNow2 Deaths 119"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_type.html","id":null,"dir":"Reference","previous_headings":"","what":"Get forecast type from forecast object — get_forecast_type","title":"Get forecast type from forecast object — get_forecast_type","text":"Get forecast type forecast object","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_type.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get forecast type from forecast object — get_forecast_type","text":"","code":"get_forecast_type(forecast)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_type.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get forecast type from forecast object — get_forecast_type","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_type.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get forecast type from forecast object — get_forecast_type","text":"Character vector length one forecast type.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":null,"dir":"Reference","previous_headings":"","what":"Get unit of a single forecast — get_forecast_unit","title":"Get unit of a single forecast — get_forecast_unit","text":"Helper function get unit single forecast, .e. column names define single forecast made . just takes columns available data subtracts columns protected, .e. returned get_protected_columns() well names metrics specified scoring, .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get unit of a single forecast — get_forecast_unit","text":"","code":"get_forecast_unit(data)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get unit of a single forecast — get_forecast_unit","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get unit of a single forecast — get_forecast_unit","text":"character vector column names define unit single forecast","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_forecast_unit.html","id":"forecast-unit","dir":"Reference","previous_headings":"","what":"Forecast unit","title":"Get unit of a single forecast — get_forecast_unit","text":"order score forecasts, scoringutils needs know rows data belong together jointly form single forecasts. easy e.g. point forecast, one row per forecast. quantile sample-based forecasts, however, multiple rows belong single forecast. forecast unit unit single forecast described combination columns uniquely identify single forecast. example, forecasts made different models various locations different time points, several weeks future. forecast unit described forecast_unit = c(\"model\", \"location\", \"forecast_date\", \"forecast_horizon\"). scoringutils automatically tries determine unit single forecast. uses existing columns , means columns must present unrelated forecast unit. simplistic example, additional row, \"even\", one row number even zero otherwise, mess scoring scoringutils thinks column relevant defining forecast unit. order avoid issues, recommend setting forecast unit explicitly, usually forecast_unit argument as_forecast() functions. drop unneeded columns, making sure necessary, 'protected columns' like \"predicted\" \"observed\" retained.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Get default metrics for binary forecasts — get_metrics.forecast_binary","title":"Get default metrics for binary forecasts — get_metrics.forecast_binary","text":"binary forecasts, default scoring rules : \"brier_score\" = brier_score() \"log_score\" = logs_binary()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get default metrics for binary forecasts — get_metrics.forecast_binary","text":"","code":"# S3 method for class 'forecast_binary' get_metrics(x, select = NULL, exclude = NULL, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get default metrics for binary forecasts — get_metrics.forecast_binary","text":"x forecast object (validated data.table predicted observed values, see as_forecast()). select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored. ... unused","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get default metrics for binary forecasts — get_metrics.forecast_binary","text":"list scoring functions.","code":""},{"path":[]},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_binary.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Get default metrics for binary forecasts — get_metrics.forecast_binary","text":"","code":"get_metrics(example_binary) #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> #> #> #> $log_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> logs <- -log(1 - abs(observed - predicted)) #> return(logs) #> } #> #> #> get_metrics(example_binary, select = \"brier_score\") #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> #> #> get_metrics(example_binary, exclude = \"log_score\") #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> #> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_nominal.html","id":null,"dir":"Reference","previous_headings":"","what":"Get default metrics for nominal forecasts — get_metrics.forecast_nominal","title":"Get default metrics for nominal forecasts — get_metrics.forecast_nominal","text":"nominal forecasts, default scoring rule : \"log_score\" = logs_nominal()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_nominal.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get default metrics for nominal forecasts — get_metrics.forecast_nominal","text":"","code":"# S3 method for class 'forecast_nominal' get_metrics(x, select = NULL, exclude = NULL, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_nominal.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get default metrics for nominal forecasts — get_metrics.forecast_nominal","text":"x forecast object (validated data.table predicted observed values, see as_forecast()). select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored. ... unused","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_nominal.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Get default metrics for nominal forecasts — get_metrics.forecast_nominal","text":"","code":"get_metrics(example_nominal) #> $log_score #> function (observed, predicted, predicted_label) #> { #> assert_input_nominal(observed, predicted, predicted_label) #> n <- length(observed) #> if (n == 1) { #> predicted <- matrix(predicted, nrow = 1) #> } #> observed_indices <- as.numeric(observed) #> pred_for_observed <- predicted[cbind(1:n, observed_indices)] #> logs <- -log(pred_for_observed) #> return(logs) #> } #> #> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_point.html","id":null,"dir":"Reference","previous_headings":"","what":"Get default metrics for point forecasts — get_metrics.forecast_point","title":"Get default metrics for point forecasts — get_metrics.forecast_point","text":"point forecasts, default scoring rules : \"ae_point\" = ae() \"se_point\" = se() \"ape\" = ape() note caution: Every scoring rule point forecast implicitly minimised specific aspect predictive distribution (see Gneiting, 2011). mean squared error, example, meaningful scoring rule forecaster actually reported mean predictive distribution point forecast. forecaster reported median, mean absolute error appropriate scoring rule. scoring rule predictive task align, results misleading. Failure respect correspondence can lead grossly misleading results! Consider example section .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_point.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get default metrics for point forecasts — get_metrics.forecast_point","text":"","code":"# S3 method for class 'forecast_point' get_metrics(x, select = NULL, exclude = NULL, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_point.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get default metrics for point forecasts — get_metrics.forecast_point","text":"x forecast object (validated data.table predicted observed values, see as_forecast()). select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored. ... unused","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_point.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Get default metrics for point forecasts — get_metrics.forecast_point","text":"Making Evaluating Point Forecasts, Gneiting, Tilmann, 2011, Journal American Statistical Association.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_point.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Get default metrics for point forecasts — get_metrics.forecast_point","text":"","code":"get_metrics(example_point, select = \"ape\") #> $ape #> function (actual, predicted) #> { #> return(ae(actual, predicted)/abs(actual)) #> } #> #> #> library(magrittr) set.seed(123) n <- 500 observed <- rnorm(n, 5, 4)^2 predicted_mu <- mean(observed) predicted_not_mu <- predicted_mu - rnorm(n, 10, 2) df <- data.frame( model = rep(c(\"perfect\", \"bad\"), each = n), predicted = c(rep(predicted_mu, n), predicted_not_mu), observed = rep(observed, 2), id = rep(1:n, 2) ) %>% as_forecast_point() score(df) %>% summarise_scores() #> model ae_point se_point ape #> #> 1: perfect 34.64686 2145.813 3543.184 #> 2: bad 32.34199 2238.566 2692.868"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Get default metrics for quantile-based forecasts — get_metrics.forecast_quantile","title":"Get default metrics for quantile-based forecasts — get_metrics.forecast_quantile","text":"quantile-based forecasts, default scoring rules : \"wis\" = wis() \"overprediction\" = overprediction_quantile() \"underprediction\" = underprediction_quantile() \"dispersion\" = dispersion_quantile() \"bias\" = bias_quantile() \"interval_coverage_50\" = interval_coverage() \"interval_coverage_90\" = purrr::partial( interval_coverage, interval_range = 90 ) \"ae_median\" = ae_median_quantile() Note: interval_coverage_90 scoring rule created modifying interval_coverage(), making use function purrr::partial(). construct allows function deal arbitrary arguments ..., making sure interval_coverage() can accept get passed . interval_range = 90 set function definition, passing argument interval_range = 90 score() mean also get passed interval_coverage_50.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get default metrics for quantile-based forecasts — get_metrics.forecast_quantile","text":"","code":"# S3 method for class 'forecast_quantile' get_metrics(x, select = NULL, exclude = NULL, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get default metrics for quantile-based forecasts — get_metrics.forecast_quantile","text":"x forecast object (validated data.table predicted observed values, see as_forecast()). select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored. ... unused","code":""},{"path":[]},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_quantile.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Get default metrics for quantile-based forecasts — get_metrics.forecast_quantile","text":"","code":"get_metrics(example_quantile, select = \"wis\") #> $wis #> function (observed, predicted, quantile_level, separate_results = FALSE, #> weigh = TRUE, count_median_twice = FALSE, na.rm = FALSE) #> { #> assert_input_quantile(observed, predicted, quantile_level) #> reformatted <- quantile_to_interval(observed, predicted, #> quantile_level) #> interval_ranges <- get_range_from_quantile(quantile_level[quantile_level != #> 0.5]) #> complete_intervals <- duplicated(interval_ranges) | duplicated(interval_ranges, #> fromLast = TRUE) #> if (!all(complete_intervals) && !isTRUE(na.rm)) { #> incomplete <- quantile_level[quantile_level != 0.5][!complete_intervals] #> cli_abort(c(`!` = \"Not all quantile levels specified form symmetric prediction\\n intervals.\\n The following quantile levels miss a corresponding lower/upper bound:\\n {.val {incomplete}}.\\n You can drop incomplete prediction intervals using `na.rm = TRUE`.\")) #> } #> assert_logical(separate_results, len = 1) #> assert_logical(weigh, len = 1) #> assert_logical(count_median_twice, len = 1) #> assert_logical(na.rm, len = 1) #> if (separate_results) { #> cols <- c(\"wis\", \"dispersion\", \"underprediction\", \"overprediction\") #> } #> else { #> cols <- \"wis\" #> } #> reformatted[, `:=`(eval(cols), do.call(interval_score, list(observed = observed, #> lower = lower, upper = upper, interval_range = interval_range, #> weigh = weigh, separate_results = separate_results)))] #> if (count_median_twice) { #> reformatted[, `:=`(weight, 1)] #> } #> else { #> reformatted[, `:=`(weight, ifelse(interval_range == 0, #> 0.5, 1))] #> } #> reformatted <- reformatted[, lapply(.SD, weighted.mean, na.rm = na.rm, #> w = weight), by = \"forecast_id\", .SDcols = colnames(reformatted) %like% #> paste(cols, collapse = \"|\")] #> if (separate_results) { #> return(list(wis = reformatted$wis, dispersion = reformatted$dispersion, #> underprediction = reformatted$underprediction, overprediction = reformatted$overprediction)) #> } #> else { #> return(reformatted$wis) #> } #> } #> #> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Get default metrics for sample-based forecasts — get_metrics.forecast_sample","title":"Get default metrics for sample-based forecasts — get_metrics.forecast_sample","text":"sample-based forecasts, default scoring rules : \"crps\" = crps_sample() \"overprediction\" = overprediction_sample() \"underprediction\" = underprediction_sample() \"dispersion\" = dispersion_sample() \"log_score\" = logs_sample() \"dss\" = dss_sample() \"mad\" = mad_sample() \"bias\" = bias_sample() \"ae_median\" = ae_median_sample() \"se_mean\" = se_mean_sample()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get default metrics for sample-based forecasts — get_metrics.forecast_sample","text":"","code":"# S3 method for class 'forecast_sample' get_metrics(x, select = NULL, exclude = NULL, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get default metrics for sample-based forecasts — get_metrics.forecast_sample","text":"x forecast object (validated data.table predicted observed values, see as_forecast()). select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored. ... unused","code":""},{"path":[]},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.forecast_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Get default metrics for sample-based forecasts — get_metrics.forecast_sample","text":"","code":"get_metrics(example_sample_continuous, exclude = \"mad\") #> $bias #> function (observed, predicted) #> { #> assert_input_sample(observed, predicted) #> prediction_type <- get_type(predicted) #> n_pred <- ncol(predicted) #> p_x <- rowSums(predicted <= observed)/n_pred #> if (prediction_type == \"continuous\") { #> res <- 1 - 2 * p_x #> return(res) #> } #> else { #> p_xm1 <- rowSums(predicted <= (observed - 1))/n_pred #> res <- 1 - (p_x + p_xm1) #> return(res) #> } #> } #> #> #> #> $dss #> function (observed, predicted, ...) #> { #> assert_input_sample(observed, predicted) #> scoringRules::dss_sample(y = observed, dat = predicted, ...) #> } #> #> #> #> $crps #> function (observed, predicted, separate_results = FALSE, ...) #> { #> assert_input_sample(observed, predicted) #> crps <- scoringRules::crps_sample(y = observed, dat = predicted, #> ...) #> if (separate_results) { #> medians <- apply(predicted, 1, median) #> dispersion <- scoringRules::crps_sample(y = medians, #> dat = predicted, ...) #> overprediction <- rep(0, length(observed)) #> underprediction <- rep(0, length(observed)) #> overprediction[observed < medians] <- scoringRules::crps_sample(y = observed[observed < #> medians], dat = predicted[observed < medians, , drop = FALSE], #> ...) #> underprediction[observed > medians] <- scoringRules::crps_sample(y = observed[observed > #> medians], dat = predicted[observed > medians, , drop = FALSE], #> ...) #> overprediction[overprediction > 0] <- overprediction[overprediction > #> 0] - dispersion[overprediction > 0] #> underprediction[underprediction > 0] <- underprediction[underprediction > #> 0] - dispersion[underprediction > 0] #> return(list(crps = crps, dispersion = dispersion, underprediction = underprediction, #> overprediction = overprediction)) #> } #> else { #> return(crps) #> } #> } #> #> #> #> $overprediction #> function (observed, predicted, ...) #> { #> crps <- crps_sample(observed, predicted, separate_results = TRUE, #> ...) #> return(crps$overprediction) #> } #> #> #> #> $underprediction #> function (observed, predicted, ...) #> { #> crps <- crps_sample(observed, predicted, separate_results = TRUE, #> ...) #> return(crps$underprediction) #> } #> #> #> #> $dispersion #> function (observed, predicted, ...) #> { #> crps <- crps_sample(observed, predicted, separate_results = TRUE, #> ...) #> return(crps$dispersion) #> } #> #> #> #> $log_score #> function (observed, predicted, ...) #> { #> assert_input_sample(observed, predicted) #> scoringRules::logs_sample(y = observed, dat = predicted, #> ...) #> } #> #> #> #> $ae_median #> function (observed, predicted) #> { #> assert_input_sample(observed, predicted) #> median_predictions <- apply(as.matrix(predicted), MARGIN = 1, #> FUN = median) #> ae_median <- abs(observed - median_predictions) #> return(ae_median) #> } #> #> #> #> $se_mean #> function (observed, predicted) #> { #> assert_input_sample(observed, predicted) #> mean_predictions <- rowMeans(as.matrix(predicted)) #> se_mean <- (observed - mean_predictions)^2 #> return(se_mean) #> } #> #> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Get metrics — get_metrics","title":"Get metrics — get_metrics","text":"Generic function obtain default metrics availble scoring metrics used scoring. called forecast object returns list functions can used scoring. called scores object (see score()), returns character vector names metrics used scoring. See documentation actual methods See Also section details. Alternatively call ?get_metrics. ?get_metrics.scores.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get metrics — get_metrics","text":"","code":"get_metrics(x, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get metrics — get_metrics","text":"x forecast scores object. ... Additional arguments passed method.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Get metrics — get_metrics","text":"See as_forecast() information forecast objects score() information scores objects.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Get names of the metrics that were used for scoring — get_metrics.scores","title":"Get names of the metrics that were used for scoring — get_metrics.scores","text":"applying scoring rule via score(), names scoring rules become column names resulting data.table. addition, attribute metrics added output, holding names scores vector. done functions like get_forecast_unit() summarise_scores() can still identify columns part forecast unit hold score. get_metrics() accesses returns metrics attribute. attribute, function return NULL (, error = TRUE produce error instead). addition, checks column names input consistency data stored metrics attribute. Handling missing inconsistent metrics attribute: metrics attribute missing consistent column names data.table, can either run score() , specifying names scoring rules manually, add/update attribute manually using attr(scores, \"metrics\") <- c(\"names\", \"\", \"\", \"scores\") (order matter).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get names of the metrics that were used for scoring — get_metrics.scores","text":"","code":"# S3 method for class 'scores' get_metrics(x, error = FALSE, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get names of the metrics that were used for scoring — get_metrics.scores","text":"x scores object, (data.table attribute metrics produced score()). error Throw error attribute called metrics? Default FALSE. ... unused","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_metrics.scores.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get names of the metrics that were used for scoring — get_metrics.scores","text":"Character vector names scoring rules used scoring.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":null,"dir":"Reference","previous_headings":"","what":"Obtain pairwise comparisons between models — get_pairwise_comparisons","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"Compare scores obtained different models pairwise tournament. combinations two models compared based overlapping set available forecasts common models. input scores object produced score(). Note adding additional unrelated columns can unpredictably change results, present columns taken account determining set overlapping forecasts two models. output pairwise comparisons set mean score ratios, relative skill scores p-values. following illustrates pairwise comparison process: Mean score ratios every pair two models, mean score ratio computed. simply mean score first model divided mean score second. Mean score ratios computed based set overlapping forecasts two models. means scores targets taken account models submitted forecast. (Scaled) Relative skill scores relative score model geometric mean mean score ratios involve model. baseline provided, scaled relative skill scores calculated well. Scaled relative skill scores simply relative skill score model divided relative skill score baseline model. p-values addition, function computes p-values comparison two models (based set overlapping forecasts). P-values can computed two ways: based nonparametric Wilcoxon signed-rank test (internally using wilcox.test() paired = TRUE) based permutation test. permutation test based difference mean scores two models. default null hypothesis mean score difference zero (see permutation_test()). Adjusted p-values computed calling p.adjust() raw p-values. code pairwise comparisons inspired implementation Johannes Bracher. implementation permutation test follows function permutationTest surveillance package Michael Höhle, Andrea Riebler Michaela Paul.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"","code":"get_pairwise_comparisons( scores, compare = \"model\", by = NULL, metric = intersect(c(\"wis\", \"crps\", \"brier_score\"), names(scores)), baseline = NULL, ... )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"scores object class scores (data.table scores additional attribute metrics produced score()). compare Character vector single colum name defines elements pairwise comparison. example, set \"model\" (default), elements \"model\" column compared. Character vector column names define grouping levels pairwise comparisons. default NULL one relative skill score per distinct entry column selected compare. columns given , example, = \"location\" compare = \"model\", one separate relative skill score calculated every model every location. metric string name metric relative skill shall computed. default either \"crps\", \"wis\" \"brier_score\" available. baseline string name model. baseline given, scaled relative skill respect baseline returned. default (NULL), relative skill scaled respect baseline model. ... Additional arguments comparison two models. See compare_forecasts() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"data.table results pairwise comparisons containing mean score ratios (mean_scores_ratio), unadjusted (pval) adjusted (adj_pval) p-values, relative skill values model (..._relative_skill). baseline model given scaled relative skill reported well (..._scaled_relative_skill).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"Nikos Bosse nikosbosse@gmail.com Johannes Bracher, johannes.bracher@kit.edu","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pairwise_comparisons.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Obtain pairwise comparisons between models — get_pairwise_comparisons","text":"","code":"library(magrittr) # pipe operator scores <- example_quantile %>% as_forecast_quantile() %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. pairwise <- get_pairwise_comparisons(scores, by = \"target_type\") pairwise2 <- get_pairwise_comparisons( scores, by = \"target_type\", baseline = \"EuroCOVIDhub-baseline\" ) library(ggplot2) plot_pairwise_comparisons(pairwise, type = \"mean_scores_ratio\") + facet_wrap(~target_type)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit.html","id":null,"dir":"Reference","previous_headings":"","what":"Probability integral transformation (data.frame version) — get_pit.forecast_quantile","title":"Probability integral transformation (data.frame version) — get_pit.forecast_quantile","text":"Compute Probability Integral Transformation (PIT) validated forecast objects.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Probability integral transformation (data.frame version) — get_pit.forecast_quantile","text":"","code":"# S3 method for class 'forecast_quantile' get_pit(forecast, by, ...) # S3 method for class 'forecast_sample' get_pit(forecast, by, n_replicates = 100, ...) get_pit(forecast, by, ...) # Default S3 method get_pit(forecast, by, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Probability integral transformation (data.frame version) — get_pit.forecast_quantile","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()). Character vector columns according PIT values shall grouped. e.g. columns 'model' 'location' input data want PIT histogram every model location, specify = c(\"model\", \"location\"). ... Currently unused. pass additional arguments scoring functions via .... See Customising metrics section details use purrr::partial() pass arguments individual metrics. n_replicates number draws randomised PIT discrete predictions. ignored forecasts continuous.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Probability integral transformation (data.frame version) — get_pit.forecast_quantile","text":"data.table PIT values according grouping specified .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Probability integral transformation (data.frame version) — get_pit.forecast_quantile","text":"Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind M. Eggo, W. John Edmunds (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15, doi:10.1371/journal.pcbi.1006785","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_pit.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Probability integral transformation (data.frame version) — get_pit.forecast_quantile","text":"","code":"example <- as_forecast_sample(example_sample_continuous) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. result <- get_pit(example, by = \"model\") plot_pit(result) # example with quantile data example <- as_forecast_quantile(example_quantile) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. result <- get_pit(example, by = \"model\") plot_pit(result)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_protected_columns.html","id":null,"dir":"Reference","previous_headings":"","what":"Get protected columns from data — get_protected_columns","title":"Get protected columns from data — get_protected_columns","text":"Helper function get names columns data frame protected columns.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_protected_columns.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get protected columns from data — get_protected_columns","text":"","code":"get_protected_columns(data = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_protected_columns.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get protected columns from data — get_protected_columns","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_protected_columns.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get protected columns from data — get_protected_columns","text":"character vector names protected columns data. data NULL (default) returns list columns protected scoringutils.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_range_from_quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Get interval range belonging to a quantile — get_range_from_quantile","title":"Get interval range belonging to a quantile — get_range_from_quantile","text":"Every quantile can thought either lower upper bound symmetric central prediction interval. helper function returns range central prediction interval quantile belongs. Due numeric instability sometimes occurred past, ranges rounded 10 decimal places. problem vast majority use cases, something aware .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_range_from_quantile.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get interval range belonging to a quantile — get_range_from_quantile","text":"","code":"get_range_from_quantile(quantile_level)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_range_from_quantile.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get interval range belonging to a quantile — get_range_from_quantile","text":"quantile_level numeric vector quantile levels size N.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_range_from_quantile.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get interval range belonging to a quantile — get_range_from_quantile","text":"numeric vector interval ranges size N","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_type.html","id":null,"dir":"Reference","previous_headings":"","what":"Get type of a vector or matrix of observed values or predictions — get_type","title":"Get type of a vector or matrix of observed values or predictions — get_type","text":"Internal helper function get type vector (usually observed predicted values). function checks whether input factor, else whether integer (can coerced integer) whether continuous.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_type.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Get type of a vector or matrix of observed values or predictions — get_type","text":"","code":"get_type(x)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_type.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Get type of a vector or matrix of observed values or predictions — get_type","text":"x Input type determined .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/get_type.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Get type of a vector or matrix of observed values or predictions — get_type","text":"Character vector length one either \"classification\", \"integer\", \"continuous\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/illustration-input-metric-binary-point.html","id":null,"dir":"Reference","previous_headings":"","what":"Illustration of required inputs for binary and point forecasts — illustration-input-metric-binary-point","title":"Illustration of required inputs for binary and point forecasts — illustration-input-metric-binary-point","text":"Illustration required inputs binary point forecasts","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/illustration-input-metric-quantile.html","id":null,"dir":"Reference","previous_headings":"","what":"Illustration of required inputs for quantile-based forecasts — illustration-input-metric-quantile","title":"Illustration of required inputs for quantile-based forecasts — illustration-input-metric-quantile","text":"Illustration required inputs quantile-based forecasts","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/illustration-input-metric-sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Illustration of required inputs for sample-based forecasts — illustration-input-metric-sample","title":"Illustration of required inputs for sample-based forecasts — illustration-input-metric-sample","text":"Illustration required inputs sample-based forecasts","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interpolate_median.html","id":null,"dir":"Reference","previous_headings":"","what":"Helper function to interpolate the median prediction if it is not available — interpolate_median","title":"Helper function to interpolate the median prediction if it is not available — interpolate_median","text":"Internal function interpolate median prediction available given quantile levels. done using linear interpolation two innermost quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interpolate_median.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Helper function to interpolate the median prediction if it is not available — interpolate_median","text":"","code":"interpolate_median(predicted, quantile_level)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interpolate_median.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Helper function to interpolate the median prediction if it is not available — interpolate_median","text":"predicted Vector length N (corresponding number quantiles) holds predictions. quantile_level Vector size N quantile levels predictions made. Note contain median (0.5) median imputed mean two innermost quantiles.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interpolate_median.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Helper function to interpolate the median prediction if it is not available — interpolate_median","text":"scalar imputed median prediction","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Interval coverage (for quantile-based forecasts) — interval_coverage","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"Check whether observed value within given central prediction interval. prediction interval defined lower upper bound formed pair predictive quantiles. example, 50% prediction interval formed 0.25 0.75 quantiles predictive distribution.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"","code":"interval_coverage(observed, predicted, quantile_level, interval_range = 50)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made. interval_range single number range prediction interval percent (e.g. 50 50% prediction interval) want compute interval coverage.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"vector length n elements either TRUE, observed value within corresponding prediction interval, FALSE otherwise.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Interval coverage (for quantile-based forecasts) — interval_coverage","text":"","code":"observed <- c(1, -15, 22) predicted <- rbind( c(-1, 0, 1, 2, 3), c(-2, 1, 2, 2, 4), c(-2, 0, 3, 3, 4) ) quantile_level <- c(0.1, 0.25, 0.5, 0.75, 0.9) interval_coverage(observed, predicted, quantile_level) #> [1] TRUE FALSE FALSE"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":null,"dir":"Reference","previous_headings":"","what":"Interval score — interval_score","title":"Interval score — interval_score","text":"Proper Scoring Rule score quantile predictions, following Gneiting Raftery (2007). Smaller values better. score computed $$ \\textrm{score} = (\\textrm{upper} - \\textrm{lower}) + \\frac{2}{\\alpha}(\\textrm{lower} - \\textrm{observed}) * \\mathbf{1}(\\textrm{observed} < \\textrm{lower}) + \\frac{2}{\\alpha}(\\textrm{observed} - \\textrm{upper}) * \\mathbf{1}(\\textrm{observed} > \\textrm{upper}) $$ \\(\\mathbf{1}()\\) indicator function indicates much outside prediction interval. \\(\\alpha\\) decimal value indicates much outside prediction interval. improve usability, user asked provide interval range percentage terms, .e. interval_range = 90 (percent) 90 percent prediction interval. Correspondingly, user provide 5% 95% quantiles (corresponding alpha 0.1). specific distribution assumed, interval symmetric around median (.e use 0.1 quantile lower bound 0.7 quantile upper bound). Non-symmetric quantiles can scored using function quantile_score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Interval score — interval_score","text":"","code":"interval_score( observed, lower, upper, interval_range, weigh = TRUE, separate_results = FALSE )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Interval score — interval_score","text":"observed vector observed values size n lower Vector size n prediction lower quantile given interval range. upper Vector size n prediction upper quantile given interval range. interval_range Numeric vector (either single number vector size n) range prediction intervals. example, forecasting 0.05 0.95 quantile, interval range 90. interval range corresponds \\((100-\\alpha)/100\\), \\(\\alpha\\) decimal value indicates much outside prediction interval (see e.g. Gneiting Raftery (2007)). weigh Logical. TRUE (default), weigh score \\(\\alpha / 2\\), can averaged interval score , limit (increasing number equally spaced quantiles/prediction intervals), corresponds CRPS. \\(\\alpha\\) value corresponds (\\(\\alpha/2\\)) (\\(1 - \\alpha/2\\)), .e. decimal value represents much outside central prediction interval (E.g. 90 percent central prediction interval, alpha 0.1). separate_results Logical. TRUE (default FALSE), separate parts interval score (dispersion penalty, penalties - -prediction get returned separate elements list). want data.frame instead, simply call .data.frame() output.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Interval score — interval_score","text":"Vector scoring values, list separate entries separate_results TRUE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Interval score — interval_score","text":"Strictly Proper Scoring Rules, Prediction,Estimation, Tilmann Gneiting Adrian E. Raftery, 2007, Journal American Statistical Association, Volume 102, 2007 - Issue 477 Evaluating epidemic forecasts interval format, Johannes Bracher, Evan L. Ray, Tilmann Gneiting Nicholas G. Reich, https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1008618 # nolint","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/interval_score.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Interval score — interval_score","text":"","code":"observed <- rnorm(30, mean = 1:30) interval_range <- rep(90, 30) alpha <- (100 - interval_range) / 100 lower <- qnorm(alpha / 2, rnorm(30, mean = 1:30)) upper <- qnorm((1 - alpha / 2), rnorm(30, mean = 11:40)) scoringutils:::interval_score( observed = observed, lower = lower, upper = upper, interval_range = interval_range ) #> [1] 0.7984288 0.6188294 0.7680146 0.7392137 2.4496739 0.6725337 0.5966822 #> [8] 0.6659581 0.6581834 1.2866256 0.7423596 0.6155816 0.6176548 0.6484069 #> [15] 0.6584252 0.6929587 0.7558694 0.6724925 0.5716195 0.6251716 0.7431658 #> [22] 0.6906088 0.6524924 0.6274762 1.8011311 0.5840689 0.6602319 0.7076748 #> [29] 0.6060337 0.5491802 # gives a warning, as the interval_range should likely be 50 instead of 0.5 scoringutils:::interval_score( observed = 4, upper = 8, lower = 2, interval_range = 0.5 ) #> Warning: ! Found interval ranges between 0 and 1. Are you sure that's right? An interval #> range of 0.5 e.g. implies a (49.75%, 50.25%) prediction interval. #> ℹ If you want to score a (25%, 75%) prediction interval, set `interval_range = #> 50`. #> This warning is displayed once per session. #> [1] 2.985 # example with missing values and separate results scoringutils:::interval_score( observed = c(observed, NA), lower = c(lower, NA), upper = c(NA, upper), separate_results = TRUE, interval_range = 90 ) #> $interval_score #> [1] NA 0.6735755 0.6315584 0.6931931 2.4596254 0.6632059 0.5523431 #> [8] 0.5386115 0.6346291 1.1441141 0.7545579 0.6109357 0.6003346 0.5762499 #> [15] 0.5621607 0.6826498 0.6576112 0.6649209 0.5101806 0.6064191 0.5613882 #> [22] 0.7192299 0.6075918 0.6736010 1.6335484 0.6182397 0.6036268 0.5671816 #> [29] 0.6245858 0.5639638 NA #> #> $dispersion #> [1] NA 0.6735755 0.6315584 0.6931931 0.6195326 0.6632059 0.5523431 #> [8] 0.5386115 0.6346291 0.5198196 0.7545579 0.6109357 0.6003346 0.5762499 #> [15] 0.5621607 0.6826498 0.6576112 0.6649209 0.5101806 0.6064191 0.5613882 #> [22] 0.7192299 0.6075918 0.6736010 0.4719967 0.6182397 0.6036268 0.5671816 #> [29] 0.6245858 0.5639638 NA #> #> $underprediction #> [1] NA 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 #> [26] 0 0 0 0 0 NA #> #> $overprediction #> [1] 0.0000000 0.0000000 0.0000000 0.0000000 1.8400928 0.0000000 0.0000000 #> [8] 0.0000000 0.0000000 0.6242945 0.0000000 0.0000000 0.0000000 0.0000000 #> [15] 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 0.0000000 #> [22] 0.0000000 0.0000000 0.0000000 1.1615517 0.0000000 0.0000000 0.0000000 #> [29] 0.0000000 0.0000000 NA #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Test whether an object is a forecast object — is_forecast_binary","title":"Test whether an object is a forecast object — is_forecast_binary","text":"Test whether object forecast object (see as_forecast() information). can test specific forecast_* class using appropriate is_forecast_* function.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Test whether an object is a forecast object — is_forecast_binary","text":"","code":"is_forecast_binary(x) is_forecast_nominal(x) is_forecast_point(x) is_forecast_quantile(x) is_forecast_sample(x) is_forecast(x)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Test whether an object is a forecast object — is_forecast_binary","text":"x R object.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Test whether an object is a forecast object — is_forecast_binary","text":"is_forecast: TRUE object class forecast, FALSE otherwise. is_forecast_*: TRUE object class forecast_* addition class forecast, FALSE otherwise.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/is_forecast.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Test whether an object is a forecast object — is_forecast_binary","text":"","code":"forecast_binary <- as_forecast_binary(example_binary) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. is_forecast(forecast_binary) #> [1] TRUE"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":null,"dir":"Reference","previous_headings":"","what":"Log transformation with an additive shift — log_shift","title":"Log transformation with an additive shift — log_shift","text":"Function shifts value offset applies natural logarithm .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Log transformation with an additive shift — log_shift","text":"","code":"log_shift(x, offset = 0, base = exp(1))"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Log transformation with an additive shift — log_shift","text":"x vector input values transformed offset Number add input value taking natural logarithm. base positive number: base respect logarithms computed. Defaults e = exp(1).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Log transformation with an additive shift — log_shift","text":"numeric vector transformed values","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Log transformation with an additive shift — log_shift","text":"output computed log(x + offset)","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Log transformation with an additive shift — log_shift","text":"Transformation forecasts evaluating predictive performance epidemiological context Nikos . Bosse, Sam Abbott, Anne Cori, Edwin van Leeuwen, Johannes Bracher, Sebastian Funk medRxiv 2023.01.23.23284722 doi:10.1101/2023.01.23.23284722 https://www.medrxiv.org/content/10.1101/2023.01.23.23284722v1 # nolint","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/log_shift.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Log transformation with an additive shift — log_shift","text":"","code":"library(magrittr) # pipe operator log_shift(1:10) #> [1] 0.0000000 0.6931472 1.0986123 1.3862944 1.6094379 1.7917595 1.9459101 #> [8] 2.0794415 2.1972246 2.3025851 log_shift(0:9, offset = 1) #> [1] 0.0000000 0.6931472 1.0986123 1.3862944 1.6094379 1.7917595 1.9459101 #> [8] 2.0794415 2.1972246 2.3025851 example_quantile[observed > 0, ] %>% as_forecast_quantile() %>% transform_forecasts(fun = log_shift, offset = 1) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Forecast type: quantile #> Forecast unit: #> location, target_end_date, target_type, location_name, forecast_date, model, #> horizon, and scale #> #> location target_end_date target_type observed location_name #> #> 1: DE 2021-01-02 Cases 1.273000e+05 Germany #> 2: DE 2021-01-02 Deaths 4.534000e+03 Germany #> 3: DE 2021-01-09 Cases 1.549220e+05 Germany #> 4: DE 2021-01-09 Deaths 6.117000e+03 Germany #> 5: DE 2021-01-16 Cases 1.101830e+05 Germany #> --- #> 40672: IT 2021-07-24 Deaths 4.369448e+00 Italy #> 40673: IT 2021-07-24 Deaths 4.369448e+00 Italy #> 40674: IT 2021-07-24 Deaths 4.369448e+00 Italy #> 40675: IT 2021-07-24 Deaths 4.369448e+00 Italy #> 40676: IT 2021-07-24 Deaths 4.369448e+00 Italy #> forecast_date quantile_level predicted model horizon #> #> 1: NA NA NA #> 2: NA NA NA #> 3: NA NA NA #> 4: NA NA NA #> 5: NA NA NA #> --- #> 40672: 2021-07-12 0.850 5.866468 epiforecasts-EpiNow2 2 #> 40673: 2021-07-12 0.900 5.986452 epiforecasts-EpiNow2 2 #> 40674: 2021-07-12 0.950 6.214608 epiforecasts-EpiNow2 2 #> 40675: 2021-07-12 0.975 6.416732 epiforecasts-EpiNow2 2 #> 40676: 2021-07-12 0.990 6.579251 epiforecasts-EpiNow2 2 #> scale #> #> 1: natural #> 2: natural #> 3: natural #> 4: natural #> 5: natural #> --- #> 40672: log #> 40673: log #> 40674: log #> 40675: log #> 40676: log"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Logarithmic score (sample-based version) — logs_sample","title":"Logarithmic score (sample-based version) — logs_sample","text":"function wrapper around logs_sample() function scoringRules package. log score negative logarithm predictive density evaluated observed value. function used score continuous predictions . Log Score theory also applicable discrete forecasts, problem lies implementation: function uses kernel density estimation, well defined integer-valued Monte Carlo Samples. See scoringRules package details alternatives, e.g. calculating scores specific discrete probability distributions.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Logarithmic score (sample-based version) — logs_sample","text":"","code":"logs_sample(observed, predicted, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Logarithmic score (sample-based version) — logs_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. ... Additional arguments passed logs_sample() scoringRules package.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Logarithmic score (sample-based version) — logs_sample","text":"Vector scores.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Logarithmic score (sample-based version) — logs_sample","text":"Alexander Jordan, Fabian Krüger, Sebastian Lerch, Evaluating Probabilistic Forecasts scoringRules, https://www.jstatsoft.org/article/view/v090i12","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/logs_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Logarithmic score (sample-based version) — logs_sample","text":"","code":"observed <- rpois(30, lambda = 1:30) predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) logs_sample(observed, predicted) #> [1] 2.415978 1.391162 2.876486 1.778651 1.990756 3.187023 2.735783 2.308400 #> [9] 2.217484 2.403781 2.688615 2.813455 2.178733 2.080475 2.668287 2.773460 #> [17] 2.619100 2.600304 2.610290 3.177565 3.065729 2.454865 2.860266 2.676266 #> [25] 3.299807 2.918963 2.538826 2.698623 2.858104 2.869216"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Determine dispersion of a probabilistic forecast — mad_sample","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"Sharpness ability model generate predictions within narrow range dispersion lack thereof. data-independent measure, purely feature forecasts . Dispersion predictive samples corresponding one single observed value measured normalised median absolute deviation median predictive samples. details, see mad() explanations given Funk et al. (2019)","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"","code":"mad_sample(observed = NULL, predicted, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"observed Place holder, argument ignored exists consistency scoring functions. output depend observed values. predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. ... Additional arguments passed mad().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"Vector dispersion values.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"Funk S, Camacho , Kucharski AJ, Lowe R, Eggo RM, Edmunds WJ (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15. PLoS Comput Biol 15(2): e1006785. doi:10.1371/journal.pcbi.1006785","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/mad_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Determine dispersion of a probabilistic forecast — mad_sample","text":"","code":"predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) mad_sample(predicted = predicted) #> [1] 1.4826 1.4826 1.4826 1.4826 2.9652 2.9652 2.9652 2.9652 2.9652 2.9652 #> [11] 2.9652 2.9652 2.9652 4.4478 4.4478 3.7065 4.4478 3.7065 4.4478 4.4478 #> [21] 4.4478 4.4478 4.4478 4.4478 5.9304 5.9304 5.9304 5.9304 5.9304 5.9304"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Class constructor for forecast objects — new_forecast","title":"Class constructor for forecast objects — new_forecast","text":"Construct class based data.frame similar. constructor coerces data data.table assigns class","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Class constructor for forecast objects — new_forecast","text":"","code":"new_forecast(data, classname)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Class constructor for forecast objects — new_forecast","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. classname name class created","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Class constructor for forecast objects — new_forecast","text":"object class indicated classname","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Construct an object of class scores — new_scores","title":"Construct an object of class scores — new_scores","text":"function creates object class scores based data.table similar.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Construct an object of class scores — new_scores","text":"","code":"new_scores(scores, metrics, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Construct an object of class scores — new_scores","text":"scores data.table similar scores produced score(). metrics character vector names scores (.e. names scoring rules used scoring). ... Additional arguments data.table::.data.table()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Construct an object of class scores — new_scores","text":"object class scores","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/new_scores.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Construct an object of class scores — new_scores","text":"","code":"if (FALSE) { # \\dontrun{ df <- data.frame( model = \"A\", wis = \"0.1\" ) new_scores(df, \"wis\") } # }"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pairwise_comparison_one_group.html","id":null,"dir":"Reference","previous_headings":"","what":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","title":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","text":"function pairwise comparison one set forecasts, multiple models involved. gets called get_pairwise_comparisons(). get_pairwise_comparisons() splits data arbitrary subgroups specified user (e.g. pairwise comparison done separately different forecast targets) actual pairwise comparison subgroup managed pairwise_comparison_one_group(). order actually comparison two models subset common forecasts calls compare_forecasts().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pairwise_comparison_one_group.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","text":"","code":"pairwise_comparison_one_group( scores, metric, baseline, compare = \"model\", by, ... )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pairwise_comparison_one_group.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","text":"scores object class scores (data.table scores additional attribute metrics produced score()). metric string name metric relative skill shall computed. default either \"crps\", \"wis\" \"brier_score\" available. baseline string name model. baseline given, scaled relative skill respect baseline returned. default (NULL), relative skill scaled respect baseline model. compare Character vector single colum name defines elements pairwise comparison. example, set \"model\" (default), elements \"model\" column compared. Character vector column names define grouping levels pairwise comparisons. default NULL one relative skill score per distinct entry column selected compare. columns given , example, = \"location\" compare = \"model\", one separate relative skill score calculated every model every location. ... Additional arguments comparison two models. See compare_forecasts() information.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pairwise_comparison_one_group.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Do pairwise comparison for one set of forecasts — pairwise_comparison_one_group","text":"data.table results pairwise comparisons containing mean score ratios (mean_scores_ratio), unadjusted (pval) adjusted (adj_pval) p-values, relative skill values model (..._relative_skill). baseline model given scaled relative skill reported well (..._scaled_relative_skill).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/permutation_test.html","id":null,"dir":"Reference","previous_headings":"","what":"Simple permutation test — permutation_test","title":"Simple permutation test — permutation_test","text":"implementation permutation test follows function permutationTest surveillance package Michael Höhle, Andrea Riebler Michaela Paul. function compares two vectors scores. computes mean vector independently takes either difference ratio two. observed difference ratio compared test statistic based permutations original data. Used get_pairwise_comparisons().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/permutation_test.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Simple permutation test — permutation_test","text":"","code":"permutation_test( scores1, scores2, n_permutation = 999, one_sided = FALSE, comparison_mode = c(\"difference\", \"ratio\") )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/permutation_test.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Simple permutation test — permutation_test","text":"scores1 Vector scores compare another vector scores. scores2 second vector scores compare first n_permutation number replications use permutation test. replications yield exact results, require computation. one_sided Whether compute one-sided test. Default FALSE. comparison_mode compute test statistic comparison two scores. either \"difference\" \"ratio\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/permutation_test.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Simple permutation test — permutation_test","text":"p-value permutation test","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Probability integral transformation for counts — pit_sample","title":"Probability integral transformation for counts — pit_sample","text":"Uses Probability integral transformation (PIT) (randomised PIT integer forecasts) assess calibration predictive Monte Carlo samples.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Probability integral transformation for counts — pit_sample","text":"","code":"pit_sample(observed, predicted, n_replicates = 100)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Probability integral transformation for counts — pit_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n. n_replicates number draws randomised PIT discrete predictions. ignored forecasts continuous.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Probability integral transformation for counts — pit_sample","text":"vector PIT-values. continuous forecasts, vector correspond length observed. integer forecasts, randomised PIT returned length length(observed) * n_replicates.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Probability integral transformation for counts — pit_sample","text":"Calibration reliability forecasts ability model correctly identify uncertainty making predictions. model perfect calibration, observed data time point look came predictive probability distribution time. Equivalently, one can inspect probability integral transform predictive distribution time t, $$ u_t = F_t (x_t) $$ \\(x_t\\) observed data point time \\(t \\textrm{ } t_1, …, t_n\\), n number forecasts, \\(F_t\\) (continuous) predictive cumulative probability distribution time t. true probability distribution outcomes time t \\(G_t\\) forecasts \\(F_t\\) said ideal \\(F_t = G_t\\) times t. case, probabilities \\(u_t\\) distributed uniformly. case discrete nonnegative outcomes incidence counts, PIT longer uniform even forecasts ideal. case randomised PIT can used instead: $$ u_t = P_t(k_t) + v * (P_t(k_t) - P_t(k_t - 1) ) $$ \\(k_t\\) observed count, \\(P_t(x)\\) predictive cumulative probability observing incidence k time t, \\(P_t (-1) = 0\\) definition v standard uniform independent k. \\(P_t\\) true cumulative probability distribution, \\(u_t\\) standard uniform.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Probability integral transformation for counts — pit_sample","text":"Claudia Czado, Tilmann Gneiting Leonhard Held (2009) Predictive model assessment count data. Biometrika, 96(4), 633-648. Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind M. Eggo, W. John Edmunds (2019) Assessing performance real-time epidemic forecasts: case study Ebola Western Area region Sierra Leone, 2014-15, doi:10.1371/journal.pcbi.1006785","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/pit_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Probability integral transformation for counts — pit_sample","text":"","code":"## continuous predictions observed <- rnorm(20, mean = 1:20) predicted <- replicate(100, rnorm(n = 20, mean = 1:20)) pit <- pit_sample(observed, predicted) plot_pit(pit) ## integer predictions observed <- rpois(20, lambda = 1:20) predicted <- replicate(100, rpois(n = 20, lambda = 1:20)) pit <- pit_sample(observed, predicted, n_replicates = 30) plot_pit(pit)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot correlation between metrics — plot_correlations","title":"Plot correlation between metrics — plot_correlations","text":"Plots heatmap correlations different metrics.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot correlation between metrics — plot_correlations","text":"","code":"plot_correlations(correlations, digits = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot correlation between metrics — plot_correlations","text":"correlations data.table correlations scores produced get_correlations(). digits number indicating many decimal places correlations rounded . default (digits = NULL) rounding takes place.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot correlation between metrics — plot_correlations","text":"ggplot object showing coloured matrix correlations metrics. ggplot object visualisation correlations metrics","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_correlations.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot correlation between metrics — plot_correlations","text":"","code":"library(magrittr) # pipe operator scores <- example_quantile %>% as_forecast_quantile %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. correlations <- scores %>% summarise_scores() %>% get_correlations() plot_correlations(correlations, digits = 2)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":null,"dir":"Reference","previous_headings":"","what":"Visualise the number of available forecasts — plot_forecast_counts","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"Visualise Forecasts Available.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"","code":"plot_forecast_counts( forecast_counts, x, y = \"model\", x_as_factor = TRUE, show_counts = TRUE )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"forecast_counts data.table (similar) column count holding forecast counts, produced get_forecast_counts(). x Character vector length one denotes name column appear x-axis plot. y Character vector length one denotes name column appear y-axis plot. Default \"model\". x_as_factor Logical (default TRUE). Whether convert variable x-axis factor. effect e.g. dates shown x-axis. show_counts Logical (default TRUE) indicates whether show actual count numbers plot.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"ggplot object plot forecast counts","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_forecast_counts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Visualise the number of available forecasts — plot_forecast_counts","text":"","code":"library(ggplot2) library(magrittr) # pipe operator forecast_counts <- example_quantile %>% as_forecast_quantile %>% get_forecast_counts(by = c(\"model\", \"target_type\", \"target_end_date\")) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_forecast_counts( forecast_counts, x = \"target_end_date\", show_counts = FALSE ) + facet_wrap(\"target_type\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":null,"dir":"Reference","previous_headings":"","what":"Create a heatmap of a scoring metric — plot_heatmap","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"function can used create heatmap one metric across different groups, e.g. interval score obtained several forecasting models different locations.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"","code":"plot_heatmap(scores, y = \"model\", x, metric)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"scores data.frame scores based quantile forecasts produced score(). y variable scores want show y-Axis. default \"model\" x variable scores want show x-Axis. something like \"horizon\", \"location\" metric String, metric determines value colour shown tiles heatmap.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"ggplot object showing heatmap desired metric","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_heatmap.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Create a heatmap of a scoring metric — plot_heatmap","text":"","code":"library(magrittr) # pipe operator scores <- example_quantile %>% as_forecast_quantile %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. scores <- summarise_scores(scores, by = c(\"model\", \"target_type\")) scores <- summarise_scores( scores, by = c(\"model\", \"target_type\"), fun = signif, digits = 2 ) plot_heatmap(scores, x = \"target_type\", metric = \"bias\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot interval coverage — plot_interval_coverage","title":"Plot interval coverage — plot_interval_coverage","text":"Plot interval coverage values (see get_coverage() information).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot interval coverage — plot_interval_coverage","text":"","code":"plot_interval_coverage(coverage, colour = \"model\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot interval coverage — plot_interval_coverage","text":"coverage data frame coverage values produced get_coverage(). colour According variable shall graphs coloured? Default \"model\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot interval coverage — plot_interval_coverage","text":"ggplot object plot interval coverage","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_interval_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot interval coverage — plot_interval_coverage","text":"","code":"example <- as_forecast_quantile(example_quantile) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. coverage <- get_coverage(example, by = \"model\") plot_interval_coverage(coverage)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"Creates heatmap ratios pvalues pairwise comparison models.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"","code":"plot_pairwise_comparisons( comparison_result, type = c(\"mean_scores_ratio\", \"pval\") )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"comparison_result data.frame produced get_pairwise_comparisons(). type Character vector length one either \"mean_scores_ratio\" \"pval\". denotes whether visualise ratio p-value pairwise comparison. Default \"mean_scores_ratio\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"ggplot object heatmap mean score ratios pairwise comparisons.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pairwise_comparisons.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot heatmap of pairwise comparisons — plot_pairwise_comparisons","text":"","code":"library(ggplot2) library(magrittr) # pipe operator scores <- example_quantile %>% as_forecast_quantile %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. pairwise <- get_pairwise_comparisons(scores, by = \"target_type\") plot_pairwise_comparisons(pairwise, type = \"mean_scores_ratio\") + facet_wrap(~target_type)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pit.html","id":null,"dir":"Reference","previous_headings":"","what":"PIT histogram — plot_pit","title":"PIT histogram — plot_pit","text":"Make simple histogram probability integral transformed values visually check whether uniform distribution seems likely.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pit.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"PIT histogram — plot_pit","text":"","code":"plot_pit(pit, num_bins = \"auto\", breaks = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pit.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"PIT histogram — plot_pit","text":"pit Either vector PIT values, data.table produced get_pit(). num_bins number bins PIT histogram, default \"auto\". num_bins == \"auto\", plot_pit() either display 10 bins, display bin available quantile case passed data quantile-based format. can control number bins supplying number. fine sample-based pit histograms, may fail quantile-based formats. case preferred supply explicit breaks points using breaks argument. breaks Numeric vector break points bins PIT histogram. preferred creating PIT histogram based quantile-based data. Default NULL breaks determined num_bins. breaks used, num_bins ignored.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pit.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"PIT histogram — plot_pit","text":"ggplot object histogram PIT values","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_pit.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"PIT histogram — plot_pit","text":"","code":"library(magrittr) # pipe operator # PIT histogram in vector based format observed <- rnorm(30, mean = 1:30) predicted <- replicate(200, rnorm(n = 30, mean = 1:30)) pit <- pit_sample(observed, predicted) plot_pit(pit) # quantile-based pit pit <- example_quantile %>% as_forecast_quantile() %>% get_pit(by = \"model\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_pit(pit, breaks = seq(0.1, 1, 0.1)) # sample-based pit pit <- example_sample_discrete %>% as_forecast_sample %>% get_pit(by = \"model\") #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. plot_pit(pit)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot quantile coverage — plot_quantile_coverage","title":"Plot quantile coverage — plot_quantile_coverage","text":"Plot quantile coverage values (see get_coverage() information).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot quantile coverage — plot_quantile_coverage","text":"","code":"plot_quantile_coverage(coverage, colour = \"model\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot quantile coverage — plot_quantile_coverage","text":"coverage data frame coverage values produced get_coverage(). colour String, according variable shall graphs coloured? Default \"model\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot quantile coverage — plot_quantile_coverage","text":"ggplot object plot interval coverage","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_quantile_coverage.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot quantile coverage — plot_quantile_coverage","text":"","code":"example <- as_forecast_quantile(example_quantile) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. coverage <- get_coverage(example, by = \"model\") plot_quantile_coverage(coverage)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":null,"dir":"Reference","previous_headings":"","what":"Plot contributions to the weighted interval score — plot_wis","title":"Plot contributions to the weighted interval score — plot_wis","text":"Visualise components weighted interval score: penalties -prediction, -prediction high dispersion (lack sharpness).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Plot contributions to the weighted interval score — plot_wis","text":"","code":"plot_wis(scores, x = \"model\", relative_contributions = FALSE, flip = FALSE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Plot contributions to the weighted interval score — plot_wis","text":"scores data.table scores based quantile forecasts produced score() summarised using summarise_scores(). x variable scores want show x-Axis. Usually \"model\". relative_contributions Logical. Show relative contributions instead absolute contributions? Default FALSE functionality available yet. flip Boolean (default FALSE), whether flip axes.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Plot contributions to the weighted interval score — plot_wis","text":"ggplot object showing contributions three components weighted interval score. ggplot object visualisation WIS decomposition","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Plot contributions to the weighted interval score — plot_wis","text":"Bracher J, Ray E, Gneiting T, Reich, N (2020) Evaluating epidemic forecasts interval format. https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1008618","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/plot_wis.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Plot contributions to the weighted interval score — plot_wis","text":"","code":"library(ggplot2) library(magrittr) # pipe operator scores <- example_quantile %>% as_forecast_quantile %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. scores <- summarise_scores(scores, by = c(\"model\", \"target_type\")) plot_wis(scores, x = \"model\", relative_contributions = TRUE ) + facet_wrap(~target_type) plot_wis(scores, x = \"model\", relative_contributions = FALSE ) + facet_wrap(~target_type, scales = \"free_x\")"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast.html","id":null,"dir":"Reference","previous_headings":"","what":"Print information about a forecast object — print.forecast","title":"Print information about a forecast object — print.forecast","text":"function prints information forecast object, including \"Forecast type\", \"Score columns\", \"Forecast unit\".","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Print information about a forecast object — print.forecast","text":"","code":"# S3 method for class 'forecast' print(x, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Print information about a forecast object — print.forecast","text":"x forecast object (validated data.table predicted observed values, see as_forecast()). ... Additional arguments print().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Print information about a forecast object — print.forecast","text":"Returns x invisibly.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/print.forecast.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Print information about a forecast object — print.forecast","text":"","code":"dat <- as_forecast_quantile(example_quantile) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. print(dat) #> Forecast type: quantile #> Forecast unit: #> location, target_end_date, target_type, location_name, forecast_date, model, #> and horizon #> #> Key: #> location target_end_date target_type observed location_name #> #> 1: DE 2021-01-02 Cases 127300 Germany #> 2: DE 2021-01-02 Deaths 4534 Germany #> 3: DE 2021-01-09 Cases 154922 Germany #> 4: DE 2021-01-09 Deaths 6117 Germany #> 5: DE 2021-01-16 Cases 110183 Germany #> --- #> 20541: IT 2021-07-24 Deaths 78 Italy #> 20542: IT 2021-07-24 Deaths 78 Italy #> 20543: IT 2021-07-24 Deaths 78 Italy #> 20544: IT 2021-07-24 Deaths 78 Italy #> 20545: IT 2021-07-24 Deaths 78 Italy #> forecast_date quantile_level predicted model horizon #> #> 1: NA NA NA #> 2: NA NA NA #> 3: NA NA NA #> 4: NA NA NA #> 5: NA NA NA #> --- #> 20541: 2021-07-12 0.850 352 epiforecasts-EpiNow2 2 #> 20542: 2021-07-12 0.900 397 epiforecasts-EpiNow2 2 #> 20543: 2021-07-12 0.950 499 epiforecasts-EpiNow2 2 #> 20544: 2021-07-12 0.975 611 epiforecasts-EpiNow2 2 #> 20545: 2021-07-12 0.990 719 epiforecasts-EpiNow2 2"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":null,"dir":"Reference","previous_headings":"","what":"Quantile score — quantile_score","title":"Quantile score — quantile_score","text":"Proper Scoring Rule score quantile predictions. Smaller values better. quantile score closely related interval score (see wis()) quantile equivalent works single quantiles instead central prediction intervals. quantile score, also called pinball loss, single quantile level \\(\\tau\\) defined $$ \\text{QS}_\\tau(F, y) = 2 \\cdot \\{ \\mathbf{1}(y \\leq q_\\tau) - \\tau\\} \\cdot (q_\\tau - y) = \\begin{cases} 2 \\cdot (1 - \\tau) * q_\\tau - y, & \\text{} y \\leq q_\\tau\\\\ 2 \\cdot \\tau * |q_\\tau - y|, & \\text{} y > q_\\tau, \\end{cases} $$ \\(q_\\tau\\) \\(\\tau\\)-quantile predictive distribution \\(F\\), \\(\\mathbf{1}(\\cdot)\\) indicator function. weighted interval score single prediction interval can obtained average quantile scores lower upper quantile prediction interval: $$ \\text{WIS}_\\alpha(F, y) = \\frac{\\text{QS}_{\\alpha/2}(F, y) + \\text{QS}_{1 - \\alpha/2}(F, y)}{2}. $$ See SI Bracher et al. (2021) details. quantile_score() returns average quantile score across quantile levels provided. set quantile levels form pairwise central prediction intervals, quantile score equivalent interval score.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Quantile score — quantile_score","text":"","code":"quantile_score(observed, predicted, quantile_level, weigh = TRUE)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Quantile score — quantile_score","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made. weigh Logical. TRUE (default), weigh score \\(\\alpha / 2\\), can averaged interval score , limit (increasing number equally spaced quantiles/prediction intervals), corresponds CRPS. \\(\\alpha\\) value corresponds (\\(\\alpha/2\\)) (\\(1 - \\alpha/2\\)), .e. decimal value represents much outside central prediction interval (E.g. 90 percent central prediction interval, alpha 0.1).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Quantile score — quantile_score","text":"Numeric vector length n quantile score. scores averaged across quantile levels multiple quantile levels provided (result calling rowMeans() matrix quantile scores computed based observed predicted values).","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Quantile score — quantile_score","text":"Strictly Proper Scoring Rules, Prediction,Estimation, Tilmann Gneiting Adrian E. Raftery, 2007, Journal American Statistical Association, Volume 102, 2007 - Issue 477 Evaluating epidemic forecasts interval format, Johannes Bracher, Evan L. Ray, Tilmann Gneiting Nicholas G. Reich, 2021, https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1008618","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_score.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Quantile score — quantile_score","text":"","code":"observed <- rnorm(10, mean = 1:10) alpha <- 0.5 lower <- qnorm(alpha / 2, observed) upper <- qnorm((1 - alpha / 2), observed) qs_lower <- quantile_score(observed, predicted = matrix(lower), quantile_level = alpha / 2 ) qs_upper <- quantile_score(observed, predicted = matrix(upper), quantile_level = 1 - alpha / 2 ) interval_score <- (qs_lower + qs_upper) / 2 interval_score2 <- quantile_score( observed, predicted = cbind(lower, upper), quantile_level = c(alpha / 2, 1 - alpha / 2) ) # this is the same as the following wis( observed, predicted = cbind(lower, upper), quantile_level = c(alpha / 2, 1 - alpha / 2) ) #> [1] 0.3372449 0.3372449 0.3372449 0.3372449 0.3372449 0.3372449 0.3372449 #> [8] 0.3372449 0.3372449 0.3372449"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_to_interval.html","id":null,"dir":"Reference","previous_headings":"","what":"Transform from a quantile format to an interval format — quantile_to_interval","title":"Transform from a quantile format to an interval format — quantile_to_interval","text":"Internal helper function transform quantile format interval format (longer supported forecast format, still used internally. function mimics S3 generic, actually S3 generic, want functions internal exported.) Quantile format quantile format, prediction characterised one multiple predicted values corresponding quantile levels. example, prediction quantile format represented 0.05, 0.25, 0.5, 0.75 0.95 quantiles predictive distribution. Interval format interval format, two quantiles assumed form prediction interval. Prediction intervals need symmetric around median characterised lower upper bound. lower bound defined lower quantile upper bound defined upper quantile. 90% prediction interval, example, covers 90% probability mass defined 5% 95% quantiles. forecast therefore characterised one multiple prediction intervals, e.g. lower upper bounds 50% 90% prediction intervals (corresponding 0.25 0.75 well 0.05 0.095 quantiles).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_to_interval.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Transform from a quantile format to an interval format — quantile_to_interval","text":"","code":"quantile_to_interval(...) quantile_to_interval_dataframe( forecast, format = \"long\", keep_quantile_col = FALSE, ... ) quantile_to_interval_numeric(observed, predicted, quantile_level, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_to_interval.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Transform from a quantile format to an interval format — quantile_to_interval","text":"... Arguments forecast data.table forecasts quantile-based format (see as_forecast()). format format output. Either \"long\" \"wide\". \"long\" (default), column boundary (values either \"upper\" \"lower\" column interval_range contains range interval. \"wide\", column interval_range two columns lower upper contain lower upper bounds prediction interval, respectively. keep_quantile_col keep quantile_level column final output transformation (default FALSE). works format = \"long\". format = \"wide\", quantile_level column always dropped. observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/quantile_to_interval.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Transform from a quantile format to an interval format — quantile_to_interval","text":"data.table forecasts interval format. quantile_to_interval_dataframe: data.table interval format (either \"long\" \"wide\"), without quantile_level column. Rows reordered. quantile_to_interval.numeric: data.table wide interval format columns forecast_id, observed, lower, upper, interval_range. forecast_id column unique identifier forecast. Rows reordered according forecast_id interval_range.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/run_safely.html","id":null,"dir":"Reference","previous_headings":"","what":"Run a function safely — run_safely","title":"Run a function safely — run_safely","text":"wrapper/helper function designed run function safely completely clear arguments passed function. named arguments ... accepted fun removed. unnamed arguments passed function. case fun errors, error converted warning run_safely returns NULL. run_safely can useful constructing functions used metrics score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/run_safely.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Run a function safely — run_safely","text":"","code":"run_safely(..., fun, metric_name)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/run_safely.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Run a function safely — run_safely","text":"... Arguments pass fun. fun function execute. metric_name character string name metric. Used provide informative warning message case fun errors.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/run_safely.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Run a function safely — run_safely","text":"result fun NULL fun errors","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/run_safely.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Run a function safely — run_safely","text":"","code":"f <- function(x) {x} scoringutils:::run_safely(2, fun = f, metric_name = \"f\") #> [1] 2 scoringutils:::run_safely(2, y = 3, fun = f, metric_name = \"f\") #> [1] 2 scoringutils:::run_safely(fun = f, metric_name = \"f\") #> Warning: ! Computation for `f` failed. Error: argument \"x\" is missing, with no default. #> NULL scoringutils:::run_safely(y = 3, fun = f, metric_name = \"f\") #> Warning: ! Computation for `f` failed. Error: argument \"x\" is missing, with no default. #> NULL"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_interval_long.html","id":null,"dir":"Reference","previous_headings":"","what":"Change data from a sample-based format to a long interval range format — sample_to_interval_long","title":"Change data from a sample-based format to a long interval range format — sample_to_interval_long","text":"Transform data format based predictive samples format based interval ranges.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_interval_long.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Change data from a sample-based format to a long interval range format — sample_to_interval_long","text":"","code":"sample_to_interval_long( data, interval_range = c(0, 50, 90), type = 7, keep_quantile_col = TRUE )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_interval_long.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Change data from a sample-based format to a long interval range format — sample_to_interval_long","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. type Type argument passed quantile function. information, see quantile(). keep_quantile_col keep quantile_level column, default TRUE","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/sample_to_interval_long.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Change data from a sample-based format to a long interval range format — sample_to_interval_long","text":"data.table long interval interval range format","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":null,"dir":"Reference","previous_headings":"","what":"Evaluate forecasts — score.forecast_binary","title":"Evaluate forecasts — score.forecast_binary","text":"score() applies selection scoring metrics forecast object (data.table forecasts observations) (see as_forecast()). score() generic dispatches different methods depending class input data. See Forecast types input formats section information forecast types input formats. additional help examples, check Getting Started Vignette well paper Evaluating Forecasts scoringutils R.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Evaluate forecasts — score.forecast_binary","text":"","code":"# S3 method for class 'forecast_binary' score(forecast, metrics = get_metrics(forecast), ...) # S3 method for class 'forecast_nominal' score(forecast, metrics = get_metrics(forecast), ...) # S3 method for class 'forecast_point' score(forecast, metrics = get_metrics(forecast), ...) # S3 method for class 'forecast_quantile' score(forecast, metrics = get_metrics(forecast), ...) # S3 method for class 'forecast_sample' score(forecast, metrics = get_metrics(forecast), ...) score(forecast, metrics, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Evaluate forecasts — score.forecast_binary","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()). metrics named list scoring functions. Names used column names output. See get_metrics() information default metrics used. See Customising metrics section information pass custom arguments scoring functions. ... Currently unused. pass additional arguments scoring functions via .... See Customising metrics section details use purrr::partial() pass arguments individual metrics.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Evaluate forecasts — score.forecast_binary","text":"object class scores. object data.table unsummarised scores (one score per forecast) additional attribute metrics names metrics used scoring. See summarise_scores()) information summarise scores.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Evaluate forecasts — score.forecast_binary","text":"Customising metrics want pass arguments scoring function, need change scoring function via e.g. purrr::partial() pass updated list functions custom metric metrics argument score(). example, use interval_coverage() interval_range = 90, define new function, e.g. interval_coverage_90 <- purrr::partial(interval_coverage, interval_range = 90) pass new function metrics score(). Note want pass variable argument, can unquote !! make sure value evaluated function created. Consider following example:","code":"custom_arg <- \"foo\" print1 <- purrr::partial(print, x = custom_arg) print2 <- purrr::partial(print, x = !!custom_arg) custom_arg <- \"bar\" print1() # prints 'bar' print2() # prints 'foo'"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"forecast-types-and-input-formats","dir":"Reference","previous_headings":"","what":"Forecast types and input formats","title":"Evaluate forecasts — score.forecast_binary","text":"Various different forecast types / forecast formats supported. moment, : point forecasts binary forecasts (\"soft binary classification\") nominal forecasts (\"soft classification multiple unordered classes\") Probabilistic forecasts quantile-based format (forecast represented set predictive quantiles) Probabilistic forecasts sample-based format (forecast represented set predictive samples) Forecast types determined based columns present input data. overview required format forecast type: forecast types require data.frame similar columns observed predicted, model. Point forecasts require column observed type numeric column predicted type numeric. Binary forecasts require column observed type factor exactly two levels column predicted type numeric probabilities, corresponding probability observed equal second factor level. See details information. Nominal forecasts require column observed type factor N levels, (N number possible outcomes), column predicted type numeric probabilities (sum one across possible outcomes), column predicted_label type factor N levels, denoting outcome probability given. Forecasts must complete, .e. must probability assigned every possible outcome. Quantile-based forecasts require column observed type numeric, column predicted type numeric, column quantile_level type numeric quantile-levels (0 1). Sample-based forecasts require column observed type numeric, column predicted type numeric, column sample_id type numeric sample indices. information see vignettes example data (example_quantile, example_sample_continuous, example_sample_discrete, example_point(), example_binary, example_nominal).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"forecast-unit","dir":"Reference","previous_headings":"","what":"Forecast unit","title":"Evaluate forecasts — score.forecast_binary","text":"order score forecasts, scoringutils needs know rows data belong together jointly form single forecasts. easy e.g. point forecast, one row per forecast. quantile sample-based forecasts, however, multiple rows belong single forecast. forecast unit unit single forecast described combination columns uniquely identify single forecast. example, forecasts made different models various locations different time points, several weeks future. forecast unit described forecast_unit = c(\"model\", \"location\", \"forecast_date\", \"forecast_horizon\"). scoringutils automatically tries determine unit single forecast. uses existing columns , means columns must present unrelated forecast unit. simplistic example, additional row, \"even\", one row number even zero otherwise, mess scoring scoringutils thinks column relevant defining forecast unit. order avoid issues, recommend setting forecast unit explicitly, usually forecast_unit argument as_forecast() functions. drop unneeded columns, making sure necessary, 'protected columns' like \"predicted\" \"observed\" retained.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Evaluate forecasts — score.forecast_binary","text":"Bosse NI, Gruson H, Cori , van Leeuwen E, Funk S, Abbott S (2022) Evaluating Forecasts scoringutils R. doi:10.48550/arXiv.2205.07090","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Evaluate forecasts — score.forecast_binary","text":"Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/score.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Evaluate forecasts — score.forecast_binary","text":"","code":"library(magrittr) # pipe operator validated <- as_forecast_quantile(example_quantile) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. score(validated) %>% summarise_scores(by = c(\"model\", \"target_type\")) #> model target_type wis overprediction underprediction #> #> 1: EuroCOVIDhub-ensemble Cases 17943.82383 10043.121943 4237.177310 #> 2: EuroCOVIDhub-baseline Cases 28483.57465 14096.100883 10284.972826 #> 3: epiforecasts-EpiNow2 Cases 20831.55662 11906.823030 3260.355639 #> 4: EuroCOVIDhub-ensemble Deaths 41.42249 7.138247 4.103261 #> 5: EuroCOVIDhub-baseline Deaths 159.40387 65.899117 2.098505 #> 6: UMass-MechBayes Deaths 52.65195 8.978601 16.800951 #> 7: epiforecasts-EpiNow2 Deaths 66.64282 18.892583 15.893314 #> dispersion bias interval_coverage_50 interval_coverage_90 ae_median #> #> 1: 3663.52458 -0.05640625 0.3906250 0.8046875 24101.07031 #> 2: 4102.50094 0.09796875 0.3281250 0.8203125 38473.60156 #> 3: 5664.37795 -0.07890625 0.4687500 0.7890625 27923.81250 #> 4: 30.18099 0.07265625 0.8750000 1.0000000 53.13281 #> 5: 91.40625 0.33906250 0.6640625 1.0000000 233.25781 #> 6: 26.87239 -0.02234375 0.4609375 0.8750000 78.47656 #> 7: 31.85692 -0.00512605 0.4201681 0.9075630 104.74790 # set forecast unit manually (to avoid issues with scoringutils trying to # determine the forecast unit automatically) example_quantile %>% as_forecast_quantile( forecast_unit = c( \"location\", \"target_end_date\", \"target_type\", \"horizon\", \"model\" ) ) %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> location target_end_date target_type horizon model #> #> 1: DE 2021-05-08 Cases 1 EuroCOVIDhub-ensemble #> 2: DE 2021-05-08 Cases 1 EuroCOVIDhub-baseline #> 3: DE 2021-05-08 Cases 1 epiforecasts-EpiNow2 #> 4: DE 2021-05-08 Deaths 1 EuroCOVIDhub-ensemble #> 5: DE 2021-05-08 Deaths 1 EuroCOVIDhub-baseline #> --- #> 883: IT 2021-07-24 Deaths 2 EuroCOVIDhub-baseline #> 884: IT 2021-07-24 Deaths 3 UMass-MechBayes #> 885: IT 2021-07-24 Deaths 2 UMass-MechBayes #> 886: IT 2021-07-24 Deaths 3 epiforecasts-EpiNow2 #> 887: IT 2021-07-24 Deaths 2 epiforecasts-EpiNow2 #> wis overprediction underprediction dispersion bias #> #> 1: 7990.854783 2.549870e+03 0.0000000 5440.985217 0.50 #> 2: 16925.046957 1.527583e+04 0.0000000 1649.220870 0.95 #> 3: 25395.960870 1.722226e+04 0.0000000 8173.700000 0.90 #> 4: 53.880000 0.000000e+00 0.6086957 53.271304 -0.10 #> 5: 46.793043 2.130435e+00 0.0000000 44.662609 0.30 #> --- #> 883: 80.336957 3.608696e+00 0.0000000 76.728261 0.20 #> 884: 4.881739 4.347826e-02 0.0000000 4.838261 0.10 #> 885: 25.581739 1.782609e+01 0.0000000 7.755652 0.80 #> 886: 19.762609 5.478261e+00 0.0000000 14.284348 0.50 #> 887: 66.161739 4.060870e+01 0.0000000 25.553043 0.90 #> interval_coverage_50 interval_coverage_90 ae_median #> #> 1: TRUE TRUE 12271 #> 2: FALSE FALSE 25620 #> 3: FALSE TRUE 44192 #> 4: TRUE TRUE 14 #> 5: TRUE TRUE 15 #> --- #> 883: TRUE TRUE 53 #> 884: TRUE TRUE 1 #> 885: FALSE TRUE 46 #> 886: TRUE TRUE 26 #> 887: FALSE TRUE 108 # forecast formats with different metrics if (FALSE) { # \\dontrun{ score(as_forecast_binary(example_binary)) score(as_forecast_quantile(example_quantile)) score(as_forecast_point(example_point)) score(as_forecast_sample(example_sample_discrete)) score(as_forecast_sample(example_sample_continuous)) } # }"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":null,"dir":"Reference","previous_headings":"","what":"Metrics for binary outcomes — scoring-functions-binary","title":"Metrics for binary outcomes — scoring-functions-binary","text":"Brier score Brier Score mean squared error probabilistic prediction observed outcome. Brier score proper scoring rule. Small values better (best 0, worst 1). $$ \\textrm{Brier\\_Score} = (\\textrm{prediction} - \\textrm{outcome})^2, $$ \\(\\textrm{outcome} \\\\{0, 1\\}\\), \\(\\textrm{prediction} \\[0, 1]\\) represents probability outcome equal 1. Log score binary outcomes Log Score negative logarithm probability assigned observed value. proper scoring rule. Small values better (best zero, worst infinity).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Metrics for binary outcomes — scoring-functions-binary","text":"","code":"brier_score(observed, predicted) logs_binary(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Metrics for binary outcomes — scoring-functions-binary","text":"observed factor length n exactly two levels, holding observed values. highest factor level assumed reference level. means predicted represents probability observed value equal highest factor level. predicted numeric vector length n, holding probabilities. Values represent probability corresponding outcome equal highest level factor observed.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Metrics for binary outcomes — scoring-functions-binary","text":"numeric vector size n Brier scores numeric vector size n log scores","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Metrics for binary outcomes — scoring-functions-binary","text":"functions require users provide observed values factor order distinguish input input format required scoring point forecasts. Internally, however, factors converted numeric values. factor observed = factor(c(0, 1, 1, 0, 1) two levels (0 1) internally coerced numeric vector (case result numeric vector c(1, 2, 2, 1, 1)). subtracting 1, resulting vector (c(0, 1, 1, 0) case) used internal calculations. predictions assumed represent probability outcome equal last/highest factor level (case outcome equal 1). alternatively also provide vector like observed = factor(c(\"\", \"b\", \"b\", \"\")) (two levels, b), result exactly internal representation. Probabilities represent probability outcome equal \"b\". want predictions probabilities outcome \"\", course make observed factor levels swapped, .e. observed = factor(c(\"\", \"b\", \"b\", \"\"), levels = c(\"b\", \"\"))","code":""},{"path":[]},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-binary.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Metrics for binary outcomes — scoring-functions-binary","text":"","code":"observed <- factor(sample(c(0, 1), size = 30, replace = TRUE)) predicted <- runif(n = 30, min = 0, max = 1) brier_score(observed, predicted) #> [1] 0.050968633 0.057656155 0.286927444 0.067535387 0.036218014 0.323334572 #> [7] 0.609439077 0.022618905 0.563870034 0.420850006 0.263388704 0.126243692 #> [13] 0.815464255 0.065961231 0.551322692 0.655667703 0.170750321 0.601418670 #> [19] 0.153762411 0.003327989 0.656657070 0.153676995 0.004340033 0.733833002 #> [25] 0.003680076 0.010143612 0.317605349 0.405392737 0.369087629 0.177242828 logs_binary(observed, predicted) #> [1] 0.25587639 0.27459075 0.76712981 0.30093715 0.21110420 0.84077787 #> [7] 1.51715864 0.16298472 1.38995224 1.04619879 0.71993087 0.43898241 #> [13] 2.33335976 0.29682936 1.35677856 1.65932749 0.53310439 1.49393247 #> [19] 0.49778684 0.05941962 1.66254231 0.49760766 0.06814922 1.94240004 #> [25] 0.06258165 0.10615583 0.82911550 1.01253898 0.93528454 0.54645655"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-nominal.html","id":null,"dir":"Reference","previous_headings":"","what":"Log score for nominal outcomes — logs_nominal","title":"Log score for nominal outcomes — logs_nominal","text":"Log score nominal outcomes Log Score negative logarithm probability assigned observed value. proper scoring rule. Small values better (best zero, worst infinity).","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-nominal.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Log score for nominal outcomes — logs_nominal","text":"","code":"logs_nominal(observed, predicted, predicted_label)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-nominal.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Log score for nominal outcomes — logs_nominal","text":"observed factor length n N levels holding observed values. predicted nxN matrix predictive probabilities, n (number rows) number observations N (number columns) number possible outcomes. predicted_label factor length N, denoting outcome probabilities predicted correspond .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-nominal.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Log score for nominal outcomes — logs_nominal","text":"numeric vector size n log scores","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoring-functions-nominal.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Log score for nominal outcomes — logs_nominal","text":"","code":"factor_levels <- c(\"one\", \"two\", \"three\") predicted_label <- factor(c(\"one\", \"two\", \"three\"), levels = factor_levels) observed <- factor(c(\"one\", \"three\", \"two\"), levels = factor_levels) predicted <- matrix(c(0.8, 0.1, 0.4, 0.1, 0.2, 0.4, 0.1, 0.7, 0.2), nrow = 3) logs_nominal(observed, predicted, predicted_label) #> [1] 0.2231436 0.3566749 0.9162907"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoringutils-package.html","id":null,"dir":"Reference","previous_headings":"","what":"scoringutils: Utilities for Scoring and Assessing Predictions — scoringutils-package","title":"scoringutils: Utilities for Scoring and Assessing Predictions — scoringutils-package","text":"scoringutils facilitates evaluation forecasts convenient framework based data.table. allows user check forecasts diagnose issues, visualise forecasts missing data, transform data scoring, handle missing forecasts, aggregate scores, visualise results evaluation. package mostly focuses evaluation probabilistic forecasts allows evaluating several different forecast types input formats. Find information package Vignettes well accompanying paper, doi:10.48550/arXiv.2205.07090 .","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/scoringutils-package.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"scoringutils: Utilities for Scoring and Assessing Predictions — scoringutils-package","text":"Maintainer: Nikos Bosse nikosbosse@gmail.com (ORCID) Authors: Sam Abbott contact@samabbott.co.uk (ORCID) Hugo Gruson hugo.gruson+R@normalesup.org (ORCID) Sebastian Funk sebastian.funk@lshtm.ac.uk contributors: Johannes Bracher johannes.bracher@kit.edu (ORCID) [contributor] Toshiaki Asakura toshiaki.asa9ra@gmail.com (ORCID) [contributor] James Mba Azam james.azam@lshtm.ac.uk (ORCID) [contributor] Michael Chirico michaelchirico4@gmail.com (ORCID) [contributor]","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/se_mean_sample.html","id":null,"dir":"Reference","previous_headings":"","what":"Squared error of the mean (sample-based version) — se_mean_sample","title":"Squared error of the mean (sample-based version) — se_mean_sample","text":"Squared error mean calculated $$ \\textrm{mean}(\\textrm{observed} - \\textrm{mean prediction})^2 $$ mean prediction calculated mean predictive samples.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/se_mean_sample.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Squared error of the mean (sample-based version) — se_mean_sample","text":"","code":"se_mean_sample(observed, predicted)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/se_mean_sample.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Squared error of the mean (sample-based version) — se_mean_sample","text":"observed vector observed values size n predicted nxN matrix predictive samples, n (number rows) number data points N (number columns) number Monte Carlo samples. Alternatively, predicted can just vector size n.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/se_mean_sample.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Squared error of the mean (sample-based version) — se_mean_sample","text":"","code":"observed <- rnorm(30, mean = 1:30) predicted_values <- matrix(rnorm(30, mean = 1:30)) se_mean_sample(observed, predicted_values) #> [1] 0.014829896 0.316608439 1.142566519 0.086187944 0.058926355 #> [6] 3.073734463 0.002511154 0.008577179 3.999907031 1.048431205 #> [11] 0.281251491 2.816708187 12.593948837 1.990408824 0.123815451 #> [16] 0.569334753 1.956832571 0.052113786 1.198119654 0.285800371 #> [21] 0.113974757 8.105728298 0.019562959 0.737141759 0.375735782 #> [26] 0.062009545 0.058334996 0.492055176 0.007062469 0.978271085"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/select_metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Select metrics from a list of functions — select_metrics","title":"Select metrics from a list of functions — select_metrics","text":"Helper function return scoring rules selected user list possible functions.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/select_metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Select metrics from a list of functions — select_metrics","text":"","code":"select_metrics(metrics, select = NULL, exclude = NULL)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/select_metrics.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Select metrics from a list of functions — select_metrics","text":"metrics list scoring functions. select character vector scoring rules select list. select NULL (default), possible scoring rules returned. exclude character vector scoring rules exclude list. select NULL, argument ignored.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/select_metrics.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Select metrics from a list of functions — select_metrics","text":"list scoring functions.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/select_metrics.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Select metrics from a list of functions — select_metrics","text":"","code":"select_metrics( metrics = get_metrics(example_binary), select = \"brier_score\" ) #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> #> #> select_metrics( metrics = get_metrics(example_binary), exclude = \"log_score\" ) #> $brier_score #> function (observed, predicted) #> { #> assert_input_binary(observed, predicted) #> observed <- as.numeric(observed) - 1 #> brierscore <- (observed - predicted)^2 #> return(brierscore) #> } #> #> #>"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/set_forecast_unit.html","id":null,"dir":"Reference","previous_headings":"","what":"Set unit of a single forecast manually — set_forecast_unit","title":"Set unit of a single forecast manually — set_forecast_unit","text":"Helper function set unit single forecast (.e. combination columns uniquely define single forecast) manually. simple function keeps columns specified forecast_unit (plus additional protected columns, e.g. observed values, predictions quantile levels) removes duplicate rows. set_forecast_unit() mainly called constructing forecast object (see as_forecast()) via forecast_unit argument . done explicitly, scoringutils attempts determine unit single forecast automatically simply assuming column names relevant determine forecast unit. may lead unexpected behaviour, setting forecast unit explicitly can help make code easier debug easier read.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/set_forecast_unit.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Set unit of a single forecast manually — set_forecast_unit","text":"","code":"set_forecast_unit(data, forecast_unit)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/set_forecast_unit.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Set unit of a single forecast manually — set_forecast_unit","text":"data data.frame (similar) predicted observed values. See details section as_forecast() additional information required input formats. forecast_unit Character vector names columns uniquely identify single forecast.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/set_forecast_unit.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Set unit of a single forecast manually — set_forecast_unit","text":"data.table columns kept relevant scoring denote unit single forecast specified user.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/set_forecast_unit.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Set unit of a single forecast manually — set_forecast_unit","text":"","code":"library(magrittr) # pipe operator example_quantile %>% scoringutils:::set_forecast_unit( c(\"location\", \"target_end_date\", \"target_type\", \"horizon\", \"model\") ) #> Forecast type: quantile #> Forecast unit: #> location, target_end_date, target_type, horizon, and model #> #> Key: #> observed quantile_level predicted location target_end_date target_type #> #> 1: 127300 NA NA DE 2021-01-02 Cases #> 2: 4534 NA NA DE 2021-01-02 Deaths #> 3: 154922 NA NA DE 2021-01-09 Cases #> 4: 6117 NA NA DE 2021-01-09 Deaths #> 5: 110183 NA NA DE 2021-01-16 Cases #> --- #> 20541: 78 0.850 352 IT 2021-07-24 Deaths #> 20542: 78 0.900 397 IT 2021-07-24 Deaths #> 20543: 78 0.950 499 IT 2021-07-24 Deaths #> 20544: 78 0.975 611 IT 2021-07-24 Deaths #> 20545: 78 0.990 719 IT 2021-07-24 Deaths #> horizon model #> #> 1: NA #> 2: NA #> 3: NA #> 4: NA #> 5: NA #> --- #> 20541: 2 epiforecasts-EpiNow2 #> 20542: 2 epiforecasts-EpiNow2 #> 20543: 2 epiforecasts-EpiNow2 #> 20544: 2 epiforecasts-EpiNow2 #> 20545: 2 epiforecasts-EpiNow2"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/summarise_scores.html","id":null,"dir":"Reference","previous_headings":"","what":"Summarise scores as produced by score() — summarise_scores","title":"Summarise scores as produced by score() — summarise_scores","text":"Summarise scores produced score(). summarise_scores relies way identify names scores distinguish columns denote unit single forecast. Internally, done via stored attribute, metrics stores names scores. means, however, need careful renaming scores produced score(). , also manually update attribute calling attr(scores, \"metrics\") <- new_names.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/summarise_scores.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Summarise scores as produced by score() — summarise_scores","text":"","code":"summarise_scores(scores, by = \"model\", fun = mean, ...) summarize_scores(scores, by = \"model\", fun = mean, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/summarise_scores.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Summarise scores as produced by score() — summarise_scores","text":"scores object class scores (data.table scores additional attribute metrics produced score()). Character vector column names summarise scores . Default \"model\", .e. scores summarised \"model\" column. fun function used summarising scores. Default mean(). ... Additional parameters can passed summary function provided fun. information see documentation respective function.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/summarise_scores.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Summarise scores as produced by score() — summarise_scores","text":"data.table summarised scores. Scores summarised according names columns original data specified using fun passed summarise_scores().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/summarise_scores.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Summarise scores as produced by score() — summarise_scores","text":"","code":"library(magrittr) # pipe operator scores <- example_sample_continuous %>% as_forecast_sample() %>% score() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. # get scores by model summarise_scores(scores, by = \"model\") #> model bias dss crps overprediction #> #> 1: EuroCOVIDhub-ensemble 0.009765625 16.40496 9876.95886 5281.81845 #> 2: EuroCOVIDhub-baseline 0.177734375 NaN 15309.68627 6701.83844 #> 3: epiforecasts-EpiNow2 -0.024898785 26.10137 11901.43354 6986.16572 #> 4: UMass-MechBayes -0.026953125 10.08582 60.19018 11.20505 #> underprediction dispersion log_score mad ae_median se_mean #> #> 1: 2433.20525 2161.93515 10.747811 8763.6176 12406.03100 2.103026e+09 #> 2: 5864.53649 2743.31134 Inf 9680.3792 18932.50196 2.885063e+09 #> 3: 1740.93388 3174.33393 Inf 12999.5404 14680.12285 3.152268e+09 #> 4: 19.28195 29.70318 5.941622 123.6211 79.66001 1.371418e+04 # get scores by model and target type summarise_scores(scores, by = c(\"model\", \"target_type\")) #> model target_type bias dss crps #> #> 1: EuroCOVIDhub-ensemble Cases -0.04648437 22.89997 19703.05522 #> 2: EuroCOVIDhub-baseline Cases 0.03671875 NaN 30453.58346 #> 3: epiforecasts-EpiNow2 Cases -0.03867188 40.87716 22896.51608 #> 4: EuroCOVIDhub-ensemble Deaths 0.06601562 9.90995 50.86249 #> 5: EuroCOVIDhub-baseline Deaths 0.31875000 12.99360 165.78907 #> 6: UMass-MechBayes Deaths -0.02695313 10.08582 60.19018 #> 7: epiforecasts-EpiNow2 Deaths -0.01008403 10.20807 74.79013 #> overprediction underprediction dispersion log_score mad ae_median #> #> 1: 10552.97603 4861.015121 4289.06407 15.633420 17385.2629 24749.39707 #> 2: 13346.03509 11727.330575 5380.21780 Inf 18982.2128 37648.01693 #> 3: 13462.90822 3346.583024 6087.02483 Inf 24929.3438 28233.04536 #> 4: 10.66088 5.395380 34.80623 5.862203 141.9723 62.66492 #> 5: 57.64179 1.742401 106.40489 6.977391 378.5457 216.98699 #> 6: 11.20505 19.281946 29.70318 5.941622 123.6211 79.66001 #> 7: 19.58555 13.849095 41.35549 6.024092 167.4829 102.18939 #> se_mean #> #> 1: 4.206042e+09 #> 2: 5.769964e+09 #> 3: 6.082863e+09 #> 4: 1.080233e+04 #> 5: 1.622417e+05 #> 6: 1.371418e+04 #> 7: 3.243111e+04 # get standard deviation summarise_scores(scores, by = \"model\", fun = sd) #> model bias dss crps overprediction #> #> 1: EuroCOVIDhub-ensemble 0.5468290 14.869520 39368.24836 37275.23950 #> 2: EuroCOVIDhub-baseline 0.5457971 NA 45020.82814 39070.74445 #> 3: epiforecasts-EpiNow2 0.6083410 108.130107 44957.07746 40776.81690 #> 4: UMass-MechBayes 0.6221914 2.248998 49.62465 21.34675 #> underprediction dispersion log_score mad ae_median se_mean #> #> 1: 8634.87723 5163.42293 21.510119 19799.1620 42801.64123 1.564286e+10 #> 2: 20537.03929 3664.87255 NaN 13610.4174 49458.36446 1.760651e+10 #> 3: 8096.39644 7266.11787 NaN 29616.1714 51129.54601 2.209086e+10 #> 4: 36.98584 29.60927 1.126019 123.3465 76.09471 2.994664e+04 # round digits summarise_scores(scores, by = \"model\") %>% summarise_scores(fun = signif, digits = 2) #> model bias dss crps overprediction underprediction #> #> 1: EuroCOVIDhub-ensemble 0.0098 16 9900 5300 2400 #> 2: EuroCOVIDhub-baseline 0.1800 NaN 15000 6700 5900 #> 3: epiforecasts-EpiNow2 -0.0250 26 12000 7000 1700 #> 4: UMass-MechBayes -0.0270 10 60 11 19 #> dispersion log_score mad ae_median se_mean #> #> 1: 2200 11.0 8800 12000 2.1e+09 #> 2: 2700 Inf 9700 19000 2.9e+09 #> 3: 3200 Inf 13000 15000 3.2e+09 #> 4: 30 5.9 120 80 1.4e+04"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_not_present.html","id":null,"dir":"Reference","previous_headings":"","what":"Test whether column names are NOT present in a data.frame — test_columns_not_present","title":"Test whether column names are NOT present in a data.frame — test_columns_not_present","text":"function checks whether column names present. none columns present, function returns TRUE. one columns present, function returns FALSE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_not_present.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Test whether column names are NOT present in a data.frame — test_columns_not_present","text":"","code":"test_columns_not_present(data, columns)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_not_present.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Test whether column names are NOT present in a data.frame — test_columns_not_present","text":"data data.frame similar checked columns character vector column names check","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_not_present.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Test whether column names are NOT present in a data.frame — test_columns_not_present","text":"Returns TRUE none columns present FALSE otherwise","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_present.html","id":null,"dir":"Reference","previous_headings":"","what":"Test whether all column names are present in a data.frame — test_columns_present","title":"Test whether all column names are present in a data.frame — test_columns_present","text":"function checks whether column names present. one columns missing, function returns FALSE. columns present, function returns TRUE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_present.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Test whether all column names are present in a data.frame — test_columns_present","text":"","code":"test_columns_present(data, columns)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_present.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Test whether all column names are present in a data.frame — test_columns_present","text":"data data.frame similar checked columns character vector column names check","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/test_columns_present.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Test whether all column names are present in a data.frame — test_columns_present","text":"Returns TRUE columns present FALSE otherwise","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/theme_scoringutils.html","id":null,"dir":"Reference","previous_headings":"","what":"Scoringutils ggplot2 theme — theme_scoringutils","title":"Scoringutils ggplot2 theme — theme_scoringutils","text":"theme ggplot2 plots used scoringutils.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/theme_scoringutils.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Scoringutils ggplot2 theme — theme_scoringutils","text":"","code":"theme_scoringutils()"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/theme_scoringutils.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Scoringutils ggplot2 theme — theme_scoringutils","text":"ggplot2 theme","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":null,"dir":"Reference","previous_headings":"","what":"Transform forecasts and observed values — transform_forecasts","title":"Transform forecasts and observed values — transform_forecasts","text":"Function transform forecasts observed values scoring.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Transform forecasts and observed values — transform_forecasts","text":"","code":"transform_forecasts( forecast, fun = log_shift, append = TRUE, label = \"log\", ... )"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Transform forecasts and observed values — transform_forecasts","text":"forecast forecast object (validated data.table predicted observed values, see as_forecast()). fun function used transform observed values predictions. default function log_shift(), custom function essentially log(), additional arguments (offset) allows add offset applying logarithm. often helpful natural log transformation defined zero. common, pragmatic solution, add small offset data applying log transformation. work often used offset 1 precise value depend application. append Logical, defaults TRUE. Whether append transformed version data currently existing data (TRUE). selected, data gets transformed appended existing data, making possible use outcome directly score(). additional column, 'scale', gets created denotes rows untransformed ('scale' value \"natural\") transformed ('scale' value passed argument label). label string newly created 'scale' column denote newly transformed values. relevant append = TRUE. ... Additional parameters pass function supplied. default option log_shift() offset argument.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Transform forecasts and observed values — transform_forecasts","text":"forecast object either transformed version data, one untransformed transformed data. includes original data well transformation original data. one additional column, `scale', present set \"natural\" untransformed forecasts.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Transform forecasts and observed values — transform_forecasts","text":"reasons, depending circumstances, might desirable (check linked reference info). epidemiology, example, may useful log-transform incidence counts evaluating forecasts using scores weighted interval score (WIS) continuous ranked probability score (CRPS). Log-transforming forecasts observations changes interpretation score measure absolute distance forecast observation score evaluates forecast exponential growth rate. Another motivation can apply variance-stabilising transformation standardise incidence counts population. Note want apply transformation, important transform forecasts observations apply score. Applying transformation score risks losing propriety proper scoring rule.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Transform forecasts and observed values — transform_forecasts","text":"Transformation forecasts evaluating predictive performance epidemiological context Nikos . Bosse, Sam Abbott, Anne Cori, Edwin van Leeuwen, Johannes Bracher, Sebastian Funk medRxiv 2023.01.23.23284722 doi:10.1101/2023.01.23.23284722 https://www.medrxiv.org/content/10.1101/2023.01.23.23284722v1","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Transform forecasts and observed values — transform_forecasts","text":"Nikos Bosse nikosbosse@gmail.com","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/transform_forecasts.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Transform forecasts and observed values — transform_forecasts","text":"","code":"library(magrittr) # pipe operator # transform forecasts using the natural logarithm # negative values need to be handled (here by replacing them with 0) example_quantile %>% .[, observed := ifelse(observed < 0, 0, observed)] %>% as_forecast_quantile() %>% # Here we use the default function log_shift() which is essentially the same # as log(), but has an additional arguments (offset) that allows you add an # offset before applying the logarithm. transform_forecasts(append = FALSE) %>% head() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Warning: ! Detected zeros in input values. #> ℹ Try specifying offset = 1 (or any other offset). #> Warning: ! Detected zeros in input values. #> ℹ Try specifying offset = 1 (or any other offset). #> Key: #> location target_end_date target_type observed location_name forecast_date #> #> 1: DE 2021-01-02 Cases 11.754302 Germany #> 2: DE 2021-01-02 Deaths 8.419360 Germany #> 3: DE 2021-01-09 Cases 11.950677 Germany #> 4: DE 2021-01-09 Deaths 8.718827 Germany #> 5: DE 2021-01-16 Cases 11.609898 Germany #> 6: DE 2021-01-16 Deaths 8.677099 Germany #> quantile_level predicted model horizon #> #> 1: NA NA NA #> 2: NA NA NA #> 3: NA NA NA #> 4: NA NA NA #> 5: NA NA NA #> 6: NA NA NA # alternatively, integrating the truncation in the transformation function: example_quantile %>% as_forecast_quantile() %>% transform_forecasts( fun = function(x) {log_shift(pmax(0, x))}, append = FALSE ) %>% head() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Warning: ! Detected zeros in input values. #> ℹ Try specifying offset = 1 (or any other offset). #> Warning: ! Detected zeros in input values. #> ℹ Try specifying offset = 1 (or any other offset). #> Key: #> location target_end_date target_type observed location_name forecast_date #> #> 1: DE 2021-01-02 Cases 11.754302 Germany #> 2: DE 2021-01-02 Deaths 8.419360 Germany #> 3: DE 2021-01-09 Cases 11.950677 Germany #> 4: DE 2021-01-09 Deaths 8.718827 Germany #> 5: DE 2021-01-16 Cases 11.609898 Germany #> 6: DE 2021-01-16 Deaths 8.677099 Germany #> quantile_level predicted model horizon #> #> 1: NA NA NA #> 2: NA NA NA #> 3: NA NA NA #> 4: NA NA NA #> 5: NA NA NA #> 6: NA NA NA # specifying an offset for the log transformation removes the # warning caused by zeros in the data example_quantile %>% as_forecast_quantile() %>% .[, observed := ifelse(observed < 0, 0, observed)] %>% transform_forecasts(offset = 1, append = FALSE) %>% head() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> Key: #> location target_end_date target_type observed location_name forecast_date #> #> 1: DE 2021-01-02 Cases 11.754310 Germany #> 2: DE 2021-01-02 Deaths 8.419580 Germany #> 3: DE 2021-01-09 Cases 11.950683 Germany #> 4: DE 2021-01-09 Deaths 8.718991 Germany #> 5: DE 2021-01-16 Cases 11.609907 Germany #> 6: DE 2021-01-16 Deaths 8.677269 Germany #> quantile_level predicted model horizon #> #> 1: NA NA NA #> 2: NA NA NA #> 3: NA NA NA #> 4: NA NA NA #> 5: NA NA NA #> 6: NA NA NA # adding square root transformed forecasts to the original ones example_quantile %>% .[, observed := ifelse(observed < 0, 0, observed)] %>% as_forecast_quantile() %>% transform_forecasts(fun = sqrt, label = \"sqrt\") %>% score() %>% summarise_scores(by = c(\"model\", \"scale\")) #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> model scale wis overprediction underprediction #> #> 1: EuroCOVIDhub-ensemble natural 5796.064569 1828.5715014 2120.6402853 #> 2: EuroCOVIDhub-baseline natural 11124.930667 3884.4414062 5143.5356658 #> 3: epiforecasts-EpiNow2 natural 7514.375476 2866.4071466 1697.2341137 #> 4: UMass-MechBayes natural 52.651946 8.9786005 16.8009511 #> 5: EuroCOVIDhub-ensemble sqrt 14.974344 5.5037665 5.1827454 #> 6: EuroCOVIDhub-baseline sqrt 27.742316 10.4190016 9.5936380 #> 7: epiforecasts-EpiNow2 sqrt 17.704899 6.5700431 5.7235785 #> 8: UMass-MechBayes sqrt 1.328653 0.3273746 0.4019195 #> dispersion bias interval_coverage_50 interval_coverage_90 #> #> 1: 1846.8527819 0.00812500 0.6328125 0.9023438 #> 2: 2096.9535954 0.21816406 0.4960938 0.9101562 #> 3: 2950.7342158 -0.04336032 0.4453441 0.8461538 #> 4: 26.8723947 -0.02234375 0.4609375 0.8750000 #> 5: 4.2878323 0.00812500 0.6328125 0.9023438 #> 6: 7.7296761 0.21816406 0.4960938 0.9101562 #> 7: 5.4112770 -0.04336032 0.4453441 0.8461538 #> 8: 0.5993586 -0.02234375 0.4609375 0.8750000 #> ae_median #> #> 1: 8880.542969 #> 2: 16156.871094 #> 3: 11208.072874 #> 4: 78.476562 #> 5: 22.458900 #> 6: 39.185406 #> 7: 25.585018 #> 8: 2.069103 # adding multiple transformations example_quantile %>% as_forecast_quantile() %>% .[, observed := ifelse(observed < 0, 0, observed)] %>% transform_forecasts(fun = log_shift, offset = 1) %>% transform_forecasts(fun = sqrt, label = \"sqrt\") %>% head() #> ℹ Some rows containing NA values may be removed. This is fine if not #> unexpected. #> location target_end_date target_type observed location_name forecast_date #> #> 1: DE 2021-01-02 Cases 127300 Germany #> 2: DE 2021-01-02 Deaths 4534 Germany #> 3: DE 2021-01-09 Cases 154922 Germany #> 4: DE 2021-01-09 Deaths 6117 Germany #> 5: DE 2021-01-16 Cases 110183 Germany #> 6: DE 2021-01-16 Deaths 5867 Germany #> quantile_level predicted model horizon scale #> #> 1: NA NA NA natural #> 2: NA NA NA natural #> 3: NA NA NA natural #> 4: NA NA NA natural #> 5: NA NA NA natural #> 6: NA NA NA natural"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"Validate metrics — validate_metrics","title":"Validate metrics — validate_metrics","text":"function validates whether list metrics list valid functions. function used score() make sure metrics valid functions.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Validate metrics — validate_metrics","text":"","code":"validate_metrics(metrics)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_metrics.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Validate metrics — validate_metrics","text":"metrics named list metrics. Every element scoring function applied data.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/validate_metrics.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Validate metrics — validate_metrics","text":"named list metrics, filtered valid functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":null,"dir":"Reference","previous_headings":"","what":"Weighted interval score (WIS) — wis","title":"Weighted interval score (WIS) — wis","text":"WIS proper scoring rule used evaluate forecasts interval- / quantile-based format. See Bracher et al. (2021). Smaller values better. name suggest score assumes forecast comes form one multiple central prediction intervals. prediction interval characterised lower upper bound formed pair predictive quantiles. example, 50% central prediction interval formed 0.25 0.75 quantiles predictive distribution. Interval score interval score () sum three components: overprediction, underprediction dispersion. single prediction interval one components non-zero. single prediction interval observed value lower bound, interval score equal absolute difference lower bound observed value (\"underprediction\"). \"Overprediction\" defined analogously. observed value falls within bounds prediction interval, interval score equal width prediction interval, .e. difference upper lower bound. single interval, therefore : $$ \\textrm{} = (\\textrm{upper} - \\textrm{lower}) + \\frac{2}{\\alpha}(\\textrm{lower} - \\textrm{observed}) * \\mathbf{1}(\\textrm{observed} < \\textrm{lower}) + \\frac{2}{\\alpha}(\\textrm{observed} - \\textrm{upper}) * \\mathbf{1}(\\textrm{observed} > \\textrm{upper}) $$ \\(\\mathbf{1}()\\) indicator function indicates much outside prediction interval. \\(\\alpha\\) decimal value indicates much outside prediction interval. 90% prediction interval, example, \\(\\alpha\\) equal 0.1. specific distribution assumed, interval formed quantiles symmetric around median (.e use 0.1 quantile lower bound 0.7 quantile upper bound). Non-symmetric quantiles can scored using function quantile_score(). set \\(k = 1, \\dots, K\\) prediction intervals median \\(m\\), can compute weighted interval score (WIS) sum interval scores individual intervals: $$ \\text{WIS}_{\\alpha_{\\{0:K\\}}}(F, y) = \\frac{1}{K + 1/2} \\times \\left(w_0 \\times |y - m| + \\sum_{k=1}^{K} \\left\\{ w_k \\times \\text{}_{\\alpha_k}(F, y) \\right\\}\\right) $$ individual scores usually weighted \\(w_k = \\frac{\\alpha_k}{2}\\). weight ensures increasing number equally spaced quantiles, WIS converges continuous ranked probability score (CRPS). Quantile score addition interval score, also exists quantile score (QS) (see quantile_score()), equal -called pinball loss. quantile score can computed single quantile (whereas interval score requires two quantiles form interval). However, intuitive decomposition overprediction, underprediction dispersion exist quantile score. Two versions weighted interval score two ways conceptualise weighted interval score across several quantiles / prediction intervals median. one view, treat WIS average quantile scores (median 0.5-quantile) (default wis()). another view, treat WIS average several interval scores + difference observed value median forecast. effect contrast first view, median twice much weight (weighted like prediction interval, rather like single quantile). valid ways conceptualise WIS can control behaviour count_median_twice-argument. WIS components: WIS components can computed individually using functions overprediction, underprediction, dispersion.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Weighted interval score (WIS) — wis","text":"","code":"wis( observed, predicted, quantile_level, separate_results = FALSE, weigh = TRUE, count_median_twice = FALSE, na.rm = FALSE ) dispersion_quantile(observed, predicted, quantile_level, ...) overprediction_quantile(observed, predicted, quantile_level, ...) underprediction_quantile(observed, predicted, quantile_level, ...)"},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Weighted interval score (WIS) — wis","text":"observed Numeric vector size n observed values. predicted Numeric nxN matrix predictive quantiles, n (number rows) number forecasts (corresponding number observed values) N (number columns) number quantiles per forecast. observed just single number, predicted can just vector size N. quantile_level Vector size N quantile levels predictions made. separate_results Logical. TRUE (default FALSE), separate parts interval score (dispersion penalty, penalties - -prediction get returned separate elements list). want data.frame instead, simply call .data.frame() output. weigh Logical. TRUE (default), weigh score \\(\\alpha / 2\\), can averaged interval score , limit (increasing number equally spaced quantiles/prediction intervals), corresponds CRPS. \\(\\alpha\\) value corresponds (\\(\\alpha/2\\)) (\\(1 - \\alpha/2\\)), .e. decimal value represents much outside central prediction interval (E.g. 90 percent central prediction interval, alpha 0.1). count_median_twice TRUE, count median twice score. na.rm TRUE, ignore NA values computing score. ... Additional arguments passed wis() functions overprediction_quantile(), underprediction_quantile() dispersion_quantile().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Weighted interval score (WIS) — wis","text":"wis(): numeric vector WIS values size n (one per observation), list separate entries separate_results TRUE. dispersion_quantile(): numeric vector dispersion values (one per observation). overprediction_quantile(): numeric vector overprediction values (one per observation). underprediction_quantile(): numeric vector underprediction values (one per observation)","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Weighted interval score (WIS) — wis","text":"Evaluating epidemic forecasts interval format, Johannes Bracher, Evan L. Ray, Tilmann Gneiting Nicholas G. Reich, 2021, https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1008618","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/reference/wis.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Weighted interval score (WIS) — wis","text":"","code":"observed <- c(1, -15, 22) predicted <- rbind( c(-1, 0, 1, 2, 3), c(-2, 1, 2, 2, 4), c(-2, 0, 3, 3, 4) ) quantile_level <- c(0.1, 0.25, 0.5, 0.75, 0.9) wis(observed, predicted, quantile_level) #> [1] 0.36 15.34 19.14"},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-1229000","dir":"Changelog","previous_headings":"","what":"scoringutils 1.2.2.9000","title":"scoringutils 1.2.2.9000","text":"update represents major rewrite package introduces breaking changes. want keep using older version, can download using remotes::install_github(\"epiforecasts/scoringutils@v1.2\"). update aims make package modular customisable overall cleaner easier work . particular, aimed make suggested workflows evaluating forecasts explicit easier follow (see visualisation ). , clarified input formats made consistent across functions. refactord many functions S3-methods introduced forecast objects separate classes different types forecasts. new set as_forecast_() functions introduced validate data convert inputs forecast object (data.table forecast class additional class corresponding forecast type (see )). Another major update possibility users pass scoring functions score(). updated improved function documentation added new vignettes guide users package. Internally, refactored code, improved input checks, updated notifications (now use cli package) increased test coverage. comprehensive documentation new package rewrite revised version original scoringutils paper.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"score-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"score()","title":"scoringutils 1.2.2.9000","text":"previous columns “true_value” “prediction” renamed. score() now requires columns called “observed” “predicted” “model”. column quantile renamed quantile_level sample renamed sample_id score() now generic. S3 methods classes forecast_point, forecast_binary, forecast_quantile, forecast_sample, forecast_nominal, correspond different forecast types can scored scoringutils. score() now calls na.omit() data, instead removing rows missing values columns observed predicted. NA values columns can also mess e.g. grouping forecasts according unit single forecast. score() many functions now require validated forecast object. forecast objects can created using functions as_forecast_point(), as_forecast_binary(), as_forecast_quantile(), as_forecast_sample() (replace previous check_forecast()). forecast object data.table class forecast additional class corresponding forecast type (e.g. forecast_quantile). score() now returns objects class scores stored attribute metrics holds names scoring rules used. Users can call get_metrics() access names scoring rules. score() now returns one score per forecast, instead one score per sample quantile. binary point forecasts, columns “observed” “predicted” now removed consistency forecast types. Users can now also use scoring rules (making use metrics argument, takes named list functions). Default scoring rules can accessed using function get_metrics(), generic S3 methods forecast type. returns named list scoring rules suitable respective forecast object. example, call get_metrics(example_quantile). Column names scores output score() correspond names scoring rules (.e. names functions list metrics). Instead supplying arguments score() manipulate individual scoring rules users now manipulate metric list supplied using purrr::partial() select_metric(). See ?score() information. CRPS now reported decomposition dispersion, overprediction underprediction.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"creating-a-forecast-object-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Creating a forecast object","title":"scoringutils 1.2.2.9000","text":"as_forecast_() functions create forecast object validates . also allow users rename/specify required columns specify forecast unit single step, taking functionality set_forecast_unit() cases. See ?as_forecast() information. as_forecast_() functions like e.g. as_forecast_point() as_forecast_quantile() S3 methods converting another forecast type respective forecast type. example, as_forecast_quantile() method converting forecast_sample object forecast_quantile object estimating quantiles samples.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"updated-workflows-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Updated workflows","title":"scoringutils 1.2.2.9000","text":"example workflow scoring forecast now looks like : Overall, updated suggested workflows users work package. following gives overview (see updated paper details).","code":"forecast_quantile <- as_forecast_quantile( example_quantile, observed = \"observed\", predicted = \"predicted\", model = \"model\", quantile_level = \"quantile_level\", forecast_unit = c(\"model\", \"location\", \"target_end_date\", \"forecast_date\", \"target_type\") ) scores <- score(forecast_quantile)"},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"input-formats-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Input formats","title":"scoringutils 1.2.2.9000","text":"standardised input formats score() well scoring rules exported scoreingutils. following plot gives overview expected input formats different forecast types score(). Support interval format mostly dropped (see PR #525 @nikosbosse reviewed @seabbs). co-existence quantile interval format let confusing user experience many duplicated functions providing functionality. decided simplify interface focusing exclusively quantile format. function bias_range() removed (users now use bias_quantile() instead) function interval_score() made internal function rather exported users. recommend using wis() instead.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"re-validating-forecast-objects-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"(Re-)Validating forecast objects","title":"scoringutils 1.2.2.9000","text":"create validate new forecast object, users can use as_forecast_(). revalidate existing forecast object users can call assert_forecast() (validates input returns invisible(NULL). assert_forecast() generic methods different forecast types. Alternatively, users can call `as_forecast_() re-validate forecast object. Simply printing object also provide additional information. Users can test whether object class forecast_*() using function is_forecast(). Users can also test specific forecast_* class using appropriate is_forecast.forecast_* method. example, check whether object class forecast_quantile, use use scoringutils:::is_forecast.forecast_quantile().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"pairwise-comparisons-and-relative-skill-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Pairwise comparisons and relative skill","title":"scoringutils 1.2.2.9000","text":"functionality computing pairwise comparisons now split summarise_scores(). Instead pairwise comparisons part summarising scores, new function, add_relative_skill(), introduced takes summarised scores input adds columns relative skill scores scaled relative skill scores. function pairwise_comparison() renamed get_pairwise_comparisons(), line get_-functions. Analogously, plot_pairwise_comparison() renamed plot_pairwise_comparisons(). Output columns pairwise comparisons renamed contain name metric used comparing. Replaced warnings errors get_pairwise_comparison avoid returning NULL","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"computing-coverage-values-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Computing coverage values","title":"scoringutils 1.2.2.9000","text":"add_coverage() replaced new function, get_coverage(). function comes updated workflow coverage values computed directly based original data can visualised using plot_interval_coverage() plot_quantile_coverage(). example workflow example_quantile |> as_forecast_quantile() |> get_coverage(= \"model\") |> plot_interval_coverage().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"obtaining-and-plotting-forecast-counts-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Obtaining and plotting forecast counts","title":"scoringutils 1.2.2.9000","text":"clarity, output column get_forecast_counts() renamed “Number forecasts” “count”. get_forecast_counts() now also displays combinations 0 forecasts, instead silently dropping corresponding rows. plot_avail_forecasts() renamed plot_forecast_counts() line change function name. x argument longer default value, value depend data provided user.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"renamed-functions-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Renamed functions","title":"scoringutils 1.2.2.9000","text":"function find_duplicates() renamed get_duplicate_forecasts(). Renamed interval_coverage_quantile() interval_coverage(). “range” consistently renamed “interval_range” code. “range”-format (mostly used internally) renamed “interval”-format Renamed correlation() get_correlations() plot_correlation() plot_correlations() pit() renamed get_pit() converted S3 method.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"deleted-functions-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Deleted functions","title":"scoringutils 1.2.2.9000","text":"Removed abs_error squared_error package favour Metrics::ae Metrics::se.get_duplicate_forecasts() now sorts outputs according forecast unit, making easier spot duplicates. addition, counts option allows user display number duplicates forecast unit, rather raw duplicated rows. Deleted function plot_ranges(). want continue using functionality, can find function code Deprecated-visualisations Vignette. Removed function plot_predictions(), well helper function make_NA(), favour dedicated Vignette shows different ways visualising predictions. future reference, function code can found (Issue #659) Deprecated-visualisations Vignette. Removed function plot_score_table(). can find code Deprecated-visualisations Vignette. Removed function merge_pred_and_obs() used merge two separate data frames forecasts observations. moved contents new “Deprecated functions”-vignette. Removed interval_coverage_sample() users now expected convert quantile format first scoring. Function set_forecast_unit() deleted. Instead now forecast_unit argument as_forecast_() well get_duplicate_forecasts(). Removed interval_coverage_dev_quantile(). Users can still access difference nominal actual interval coverage using get_coverage().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"function-changes-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Function changes","title":"scoringutils 1.2.2.9000","text":"bias_quantile() changed way handles forecasts median missing: median now imputed linear interpolation innermost quantiles. Previously, imputed median simply taking mean innermost quantiles. contrast previous correlation function, get_correlations doesn’t round correlations default. Instead, plot_correlations now digits argument allows users round correlations plotting . Alternatively, using dplyr, call something like mutate(correlations, across((.numeric), \\(x) signif(x, digits = 2))) output get_correlations. wis() now errors default quantile levels form valid prediction intervals returns NA missing values. Previously, na.rm set TRUE default, lead unexpected results, users aware .","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"internal-package-updates-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Internal package updates","title":"scoringutils 1.2.2.9000","text":"deprecated ..density.. replaced after_stat(density) ggplot calls. Files ending “.Rda” renamed “.rds” appropriate used together saveRDS() readRDS(). Added subsetting [ operator scores, score name attribute gets preserved subsetting. Switched using cli condition handling signalling, added tests check_*() test_*() functions. See #583 @jamesmbaazam reviewed @nikosbosse @seabbs. scoringutils now requires R >= 4.0","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"documentation-and-testing-1-2-2-9000","dir":"Changelog","previous_headings":"Package updates","what":"Documentation and testing","title":"scoringutils 1.2.2.9000","text":"Updates documentation functions made sure functions documented return statements Documentation pkgdown pages now created stable dev versions. Added unit tests many functions","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-122","dir":"Changelog","previous_headings":"","what":"scoringutils 1.2.2","title":"scoringutils 1.2.2","text":"CRAN release: 2023-11-29","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-2-2","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.2.2","text":"scoringutils now depends R 3.6. change made since packages testthat lifecycle, used scoringutils now require R 3.6. also updated Github action CI check work R 3.6 now. Added new PR template checklist things included PRs facilitate development review process","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"bug-fixes-1-2-2","dir":"Changelog","previous_headings":"","what":"Bug fixes","title":"scoringutils 1.2.2","text":"Fixes bug set_forecast_unit() function worked data.table, data.frame input. metrics table vignette Details metrics implemented scoringutils duplicated entries. fixed removing duplicated rows.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-2-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.2.1","text":"Gets rid preferably package _pkgdown.yml. theme toggle light dark theme didn’t work properly Updates gh pages deploy action v4 also cleans files triggered Introduces gh action automatically render Readme Readme.Rmd Removes links vignettes renamed","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-120","dir":"Changelog","previous_headings":"","what":"scoringutils 1.2.0","title":"scoringutils 1.2.0","text":"major release contains range new features bug fixes introduced minor releases since 1.1.0. important changes : Documentation updated reflect changes since version 1.1.0, including new transform workflow functions. New set_forecast_unit() function allows manual setting forecast unit. summarise_scores() gains new across argument summarizing across variables. New transform_forecasts() log_shift() functions allow forecast transformations. See documentation transform_forecasts() details example use case. Input checks test coverage improved bias functions. Bug fix get_prediction_type() integer matrix input. Links scoringutils paper citation updates. Warning added interval_score() small interval ranges. Linting updates improvements. Thanks @nikosbosse, @seabbs, @sbfnk code review contributions. Thanks @bisaloo suggestion use linting GitHub Action triggers changes, @adrian-lison suggestion add warning interval_score() interval range 0 1.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-2-0","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.2.0","text":"documentation updated reflect recent changes since scoringutils 1.1.0. particular, usage functions set_forecast_unit(), check_forecasts() transform_forecasts() now documented Vignettes. introduction functions enhances overall workflow help make code readable. functions designed used together pipe operator. example, one can now use something like following: Documentation transform_forecasts() also extended. functions allows user easily add transformations forecasts, suggested paper “Scoring epidemiological forecasts transformed scales”. epidemiological context, example, may make sense apply natural logarithm first scoring forecasts, order obtain scores reflect well models able predict exponential growth rates, rather absolute values. Users can now something like following score transformed version data addition original one: use log_shift() function apply logarithmic transformation forecasts. function introduced scoringutils 1.1.2 helper function acts just like log(), additional argument offset can add number every prediction observed value applying log transformation.","code":"example_quantile |> set_forecast_unit(c(\"model\", \"location\", \"forecast_date\", \"horizon\", \"target_type\")) |> check_forecasts() |> score() data <- example_quantile[true_value > 0, ] data |> transform_forecasts(fun = log_shift, offset = 1) |> score() |> summarise_scores(by = c(\"model\", \"scale\"))"},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-1-2-0","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.2.0","text":"Made check_forecasts() score() pipeable (see issue #290). means users can now directly use output check_forecasts() input score(). score() otherwise runs check_forecasts() internally anyway simply makes step explicit helps writing clearer code.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-117","dir":"Changelog","previous_headings":"","what":"scoringutils 1.1.7","title":"scoringutils 1.1.7","text":"Release @seabbs #305. Reviewed @nikosbosse @sbfnk.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"breaking-changes-1-1-7","dir":"Changelog","previous_headings":"","what":"Breaking changes","title":"scoringutils 1.1.7","text":"prediction_type argument get_forecast_unit() changed dropped. Instead new internal function prediction_is_quantile() used detect quantile variable present. Whilst internal function may impact users accessible via `find_duplicates().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-1-7","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.7","text":"Made imputation median bias_range() bias_quantile() obvious user may cause unexpected behaviour. Simplified bias_range() uses bias_quantile() internally. Added additional input checks bias_range(), bias_quantile(), check_predictions() make sure input valid. Improve coverage unit tests bias_range(), bias_quantile(), bias_sample(). Updated pairwise comparison unit tests use realistic data.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"bug-fixes-1-1-7","dir":"Changelog","previous_headings":"","what":"Bug fixes","title":"scoringutils 1.1.7","text":"Fixed bug get_prediction_type() led unable correctly detect integer (instead categorising continuous) forecasts input matrix. issue impacted bias_sample() also score() used integer forecasts resulting lower bias scores expected.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-1-1-6","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.1.6","text":"Added new argument, across, summarise_scores(). argument allows user summarise scores across different forecast units alternative specifying . See documentation summarise_scores() details example use case.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-1-1-5","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.1.5","text":"Added new function, set_forecast_unit() allows user set forecast unit manually. function removes columns relevant uniquely identifying single forecast. done manually, scoringutils attempts determine unit single automatically simply assuming column names relevant determine forecast unit. can lead unexpected behaviour, setting forecast unit explicitly can help make code easier debug easier read (see issue #268). used part workflow, set_forecast_unit() can directly piped check_forecasts() check everything order.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-1-4","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.4","text":"Added links scoringutils paper Evaluating Forecasts scoringutils R package. Updated citation formatting comply newer standards.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-1-3","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.3","text":"Added warning interval_score() interval range 0 1. Thanks @adrian-lison (see #277) suggestion.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-1-3-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.3","text":"Switched linting GitHub Action triggers changes. Inspired @bisaloo recent contribution epinowcast package. Updated package linters extensive. Inspired @bisaloo recent contribution epinowcast package. Resolved flagged linting issues across package.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-1-1-2","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.1.2","text":"Added new function, transform_forecasts() make easy transform forecasts scoring , suggested Bosse et al. (2023), https://www.medrxiv.org/content/10.1101/2023.01.23.23284722v1. Added function, log_shift() implements default transformation function. function allows add offset applying logarithm.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-111","dir":"Changelog","previous_headings":"","what":"scoringutils 1.1.1","title":"scoringutils 1.1.1","text":"Added small change interval_score() explicitly converts logical vector numeric one. happen implicitly anyway, now done explicitly order avoid issues may come input vector type doesn’t allow implicit conversion.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-110","dir":"Changelog","previous_headings":"","what":"scoringutils 1.1.0","title":"scoringutils 1.1.0","text":"CRAN release: 2023-01-30 minor update package bug fixes minor changes.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-1-1-0","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 1.1.0","text":"Removed attach message warned breaking changes 1.0.0. Renamed metric argument summarise_scores() relative_skill_metric. argument now deprecated removed future version package. Please use new argument instead. Updated documentation score() related functions make soft requirement model column input data explicit. Updated documentation score(), pairwise_comparison() summarise_scores() make clearer unit single forecast required computations Simplified function plot_pairwise_comparison() now supports plotting mean score ratios p-values removed hybrid option print time.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"bug-fixes-1-1-0","dir":"Changelog","previous_headings":"","what":"Bug fixes","title":"scoringutils 1.1.0","text":"Missing baseline forecasts pairwise_comparison() now trigger explicit informative error message. requirements table getting started vignette now correct. Added support optional sample column using quantile forecast format. Previously resulted error.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-100","dir":"Changelog","previous_headings":"","what":"scoringutils 1.0.0","title":"scoringutils 1.0.0","text":"CRAN release: 2022-05-13 Major update package package functions lots breaking changes.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-1-0-0","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 1.0.0","text":"New updated Readme vignette. proposed scoring workflow reworked. Functions changed can easily piped simplified arguments outputs.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"new-functions-and-function-changes-1-0-0","dir":"Changelog","previous_headings":"Feature updates","what":"New functions and function changes","title":"scoringutils 1.0.0","text":"function eval_forecasts() replaced function score() much reduced set function arguments. Functionality summarise scores add relative skill scores moved function summarise_scores() New function check_forecasts() analyse input data scoring New function correlation() compute correlations different metrics New function add_coverage() add coverage specific central prediction intervals. New function avail_forecasts() allows visualise number available forecasts. New function find_duplicates() find duplicate forecasts cause error. plotting functions renamed begin plot_. Arguments simplified. function pit() now works based data.frames. old pit function renamed pit_sample(). PIT p-values removed entirely. function plot_pit() now works directly input produced pit() Many data-handling functions removed input types score() restricted sample-based, quantile-based binary forecasts. function brier_score() now returns brier scores, rather taking mean returning output. crps(), dss() logs() renamed crps_sample(), dss_sample(), logs_sample()","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"bug-fixes-1-0-0","dir":"Changelog","previous_headings":"Feature updates","what":"Bug fixes","title":"scoringutils 1.0.0","text":"Testing expanded Minor bugs fixed, example bug as_forecast_quantile() function (https://github.com/epiforecasts/scoringutils/pull/223)","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-data-updated-1-0-0","dir":"Changelog","previous_headings":"Feature updates","what":"Package data updated","title":"scoringutils 1.0.0","text":"Package data now based forecasts submitted European Forecast Hub (https://covid19forecasthub.eu/). example data files renamed begin example_. new data set, summary_metrics included contains summary metrics implemented scoringutils.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"other-breaking-changes-1-0-0","dir":"Changelog","previous_headings":"","what":"Other breaking changes","title":"scoringutils 1.0.0","text":"‘sharpness’ component weighted interval score renamed dispersion. done make clear component represents maintain consistency used places.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-8","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.8","text":"Added function check_forecasts() runs basic checks input data provides feedback.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-0172","dir":"Changelog","previous_headings":"","what":"scoringutils 0.1.7.2","title":"scoringutils 0.1.7.2","text":"CRAN release: 2021-07-21","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-7-2","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.7.2","text":"Minor bug fixes (previously, ‘interval_score’ needed among selected metrics). data.tables now returned table[] rather table, don’t called twice display contents.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-017","dir":"Changelog","previous_headings":"","what":"scoringutils 0.1.7","title":"scoringutils 0.1.7","text":"CRAN release: 2021-07-14","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-7","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.7","text":"Added function, pairwise_comparison() runs pairwise comparisons models output eval_forecasts() Added functionality compute relative skill within eval_forecasts(). Added function visualise pairwise comparisons.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-7","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.7","text":"WIS definition change introduced version 0.1.5 partly corrected difference weighting introduced summarising scores different interval ranges. “sharpness” renamed ‘mad’ output [score()] sample-based forecasts.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.","text":"eval_forecasts() can now handle separate forecast truth data set input. eval_forecasts() now supports scoring point forecasts along side quantiles quantile-based format. Currently metric used absolute error.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.","text":"Many functions, especially eval_forecasts() got major rewrite. functionality unchanged, code now easier maintain data-handling functions got renamed, old names supported well now.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-5","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.5","text":"Changed default definition weighted interval score. Previously, median prediction counted twice, counted . want go back old behaviour, can call interval_score function argument count_median_twice = FALSE.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"scoringutils-014","dir":"Changelog","previous_headings":"","what":"scoringutils 0.1.4","title":"scoringutils 0.1.4","text":"CRAN release: 2020-11-17","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-4","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.4","text":"Added basic plotting functionality visualise scores. can now easily obtain diagnostic plots based scores produced score. correlation_plot() shows correlation metrics. plot_ranges() shows contribution different prediction intervals chosen metric. plot_heatmap() visualises scores heatmap. plot_score_table() shows coloured summary table scores.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-4","dir":"Changelog","previous_headings":"","what":"package updates","title":"scoringutils 0.1.4","text":"Renamed “calibration” “coverage”. Renamed “true_values” “true_value” data.frames. Renamed “predictions” “prediction” data.frames. Renamed “is_overprediction” “overprediction”. Renamed “is_underprediction” “underprediction”.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"potentially-breaking-changes-0-1-3","dir":"Changelog","previous_headings":"","what":"(Potentially) Breaking changes","title":"scoringutils 0.1.3","text":"argument score now slightly changed meaning. now denotes lowest possible grouping unit, .e. unit one observation needs specified explicitly. default now NULL. reason change metrics need scoring observation level consistent implementation principle. pit function receives grouping now summarise_by. similar spirit, summarise_by specified explicitly e.g. doesn’t assume anymore want ‘range’ included. interval score, weigh = TRUE now default option. Renamed true_values true_value predictions prediction.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-3","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.3","text":"Updated quantile evaluation metrics score. Bias well calibration now take quantiles account. Included option summarise scores according summarise_by argument score() summary can return mean, standard deviation well arbitrary set quantiles. score() can now return pit histograms. Switched ggplot2 plotting.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"potentially-breaking-changes-0-1-2","dir":"Changelog","previous_headings":"","what":"(Potentially) Breaking changes","title":"scoringutils 0.1.2","text":"scores score consistently renamed lower case. Interval_score now interval_score, CRPS now crps etc.","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-2","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.2","text":"Included support grouping scores according vector column names score(). Included support passing arguments lower-level functions score() Included support three new metrics score quantiles score(): bias, sharpness calibration","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-2","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.2","text":"Example data now horizon column illustrate use grouping. Documentation updated explain listed changes.","code":""},{"path":[]},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"feature-updates-0-1-1","dir":"Changelog","previous_headings":"","what":"Feature updates","title":"scoringutils 0.1.1","text":"Included support long well wide input formats quantile forecasts scored score().","code":""},{"path":"https://epiforecasts.io/scoringutils/dev/news/index.html","id":"package-updates-0-1-1","dir":"Changelog","previous_headings":"","what":"Package updates","title":"scoringutils 0.1.1","text":"Updated documentation score(). Added badges README.","code":""}]