From e12dd04d702f2c1b880ca1c0fcd1b27020724ee3 Mon Sep 17 00:00:00 2001 From: Nikos Bosse <37978797+nikosbosse@users.noreply.github.com> Date: Sun, 21 Nov 2021 18:48:06 +0100 Subject: [PATCH 1/5] Update check-full.yaml Update to remove machines that aren't available and therefore failing --- .github/workflows/check-full.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/check-full.yaml b/.github/workflows/check-full.yaml index c4469b908..5dadb6173 100644 --- a/.github/workflows/check-full.yaml +++ b/.github/workflows/check-full.yaml @@ -18,12 +18,10 @@ jobs: fail-fast: false matrix: config: - - {os: macOS-latest, r: 'devel'} - - {os: macOS-latest, r: 'release'} + - {os: macOS-latest, r: 'release'} - {os: windows-latest, r: 'release'} - - {os: ubuntu-16.04, r: 'release', rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/latest"} - - {os: ubuntu-16.04, r: 'oldrel', rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/latest"} - + - {os: ubuntu-20.04, r: 'release'} + - {os: ubuntu-20.04, r: 'oldrel'} env: R_REMOTES_NO_ERRORS_FROM_WARNINGS: true RSPM: ${{ matrix.config.rspm }} From 89dc3771eb042ef911aec2ac1666dce237e12ab6 Mon Sep 17 00:00:00 2001 From: Nikos Bosse <37978797+nikosbosse@users.noreply.github.com> Date: Sun, 21 Nov 2021 21:43:40 +0100 Subject: [PATCH 2/5] Update check-full.yaml --- .github/workflows/check-full.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/check-full.yaml b/.github/workflows/check-full.yaml index 5dadb6173..1cff5c613 100644 --- a/.github/workflows/check-full.yaml +++ b/.github/workflows/check-full.yaml @@ -56,15 +56,18 @@ jobs: env: RHUB_PLATFORM: linux-x86_64-ubuntu-gcc run: | - Rscript -e "remotes::install_github('r-hub/sysreqs')" - sysreqs=$(Rscript -e "cat(sysreqs::sysreq_commands('DESCRIPTION'))") - sudo -s eval "$sysreqs" + while read -r cmd + do + eval sudo $cmd + done < <(Rscript -e 'writeLines(remotes::system_requirements("ubuntu", "20.04"))') + sudo apt-get -y install libudunits2-dev libgdal-dev libqpdf-dev libcurl4-openssl-dev + shell: bash + - name: Install dependencies run: | remotes::install_deps(dependencies = TRUE) remotes::install_cran("rcmdcheck") shell: Rscript {0} - - name: Session info run: | options(width = 100) From 081c45b2ac971c2cc2407fbe6601809325ff1b9a Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Sun, 21 Nov 2021 22:15:22 +0100 Subject: [PATCH 3/5] update news file format --- NEWS.md | 48 +++++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/NEWS.md b/NEWS.md index dbfd7be2e..dc8680033 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,48 +1,50 @@ -## scoringutils 0.1.7.2 +# scoringutils 0.1.7.2 -### Package updates +## Package updates - minor bug fixes (previously, 'interval_score' needed to be among the selected metrics) +- all data.tables are now returned as `table[]` rather than as `table`, +such that they don't have to be called twice to display the contents. -## scoringutils 0.1.7 +# scoringutils 0.1.7 -### Feature updates +## Feature updates - added a function, `pairwise_comparison()` that runs pairwise comparisons between models on the output of `eval_forecasts()` - added functionality to compute relative skill within `eval_forecasts()` - added a function to visualise pairwise comparisons -### Package updates +## Package updates - The WIS definition change introduced in version 0.1.5 was partly corrected such that the difference in weighting is only introduced when summarising over scores from different interval ranges -## scoringutils 0.1. +# scoringutils 0.1. -### Feature updates +## Feature updates - `eval_forecasts()` can now handle a separate forecast and truth data set as as input - `eval_forecasts()` now supports scoring point forecasts along side quantiles in a quantile-based format. Currently the only metric used is the absolute error -### Package updates +## Package updates - Many functions, especially `eval_forecasts()` got a major rewrite. While functionality should be unchanged, the code should now be easier to maintain - Some of the data-handling functions got renamed, but old names are supported as well for now. -## scoringutils 0.1.5 +# scoringutils 0.1.5 -### Package updates +## Package updates - changed the default definition of the weighted interval score. Previously, the median prediction was counted twice, but is no only counted once. If you want to go back to the old behaviour, you can call the interval_score function with the argument `count_median_twice = FALSE`. -## scoringutils 0.1.4 +# scoringutils 0.1.4 -### Feature updates +## Feature updates - we added basic plotting functionality to visualise scores. You can now easily obtain diagnostic plots based on scores as produced by `eval_forecasts`. - `correlation_plot` shows correlation between metrics @@ -51,16 +53,16 @@ chosen metric - `score_heatmap` visualises scores as heatmap - `score_table` shows a coloured summary table of scores -### package updates +## package updates - renamed "calibration" to "coverage" - renamed "true_values" to "true_value" in data.frames - renamed "predictions" to "prediction" in data.frames - renamed "is_overprediction" to "overprediction" - renamed "is_underprediction" to "underprediction" -## scoringutils 0.1.3 +# scoringutils 0.1.3 -### (Potentially) Breaking changes +## (Potentially) Breaking changes - the by argument in `eval_forecasts` now has a slightly changed meaning. It now denotes the lowest possible grouping unit, i.e. the unit of one observation and needs to be specified explicitly. The default is now `NULL`. The reason for @@ -72,7 +74,7 @@ to be included. - for the interval score, `weigh = TRUE` is now the default option. - (potentially planned) rename true_values to true_value and predictions to prediction. -### Feature updates +## Feature updates - updated quantile evaluation metrics in `eval_forecasts`. Bias as well as calibration now take all quantiles into account - Included option to summarise scores according to a `summarise_by` argument in @@ -81,13 +83,13 @@ as an arbitrary set of quantiles. - `eval_forecasts` can now return pit histograms. - switched to ggplot2 for plotting -## scoringutils 0.1.2 +# scoringutils 0.1.2 -### (Potentially) Breaking changes +## (Potentially) Breaking changes - all scores in eval_forecasts were consistently renamed to lower case. Interval_score is now interval_score, CRPS is now crps etc. -### Feature updates +## Feature updates - included support for grouping scores according to a vector of column names in `eval_forecasts` - included support for passing down arguments to lower-level functions in @@ -95,17 +97,17 @@ in `eval_forecasts` - included support for three new metrics to score quantiles with `eval_forecasts`: bias, sharpness and calibration -### Package updates +## Package updates - example data now has a horizon column to illustrate the use of grouping - documentation updated to explain the above listed changes -## scoringutils 0.1.1 +# scoringutils 0.1.1 -### Feature updates +## Feature updates - included support for a long as well as wide input formats for quantile forecasts that are scored with `eval_forecasts` -### Package updates +## Package updates - updated documentation for the `eval_forecasts` - added badges to the Readme From c226829f784d4345fdeda1eb111a259af5f89714 Mon Sep 17 00:00:00 2001 From: Sam Abbott Date: Sun, 21 Nov 2021 21:56:06 +0000 Subject: [PATCH 4/5] Update NAMESPACE --- NAMESPACE | 1 - 1 file changed, 1 deletion(-) diff --git a/NAMESPACE b/NAMESPACE index 3bae902fb..8f8158154 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -38,7 +38,6 @@ export(score_heatmap) export(score_table) export(sharpness) export(show_avail_forecasts) -export(update_list) export(wis_components) importFrom(data.table,"%like%") importFrom(data.table,':=') From 41802cd8985eb3761ec26a5edeffedbd4a8c1bef Mon Sep 17 00:00:00 2001 From: Sam Abbott Date: Sun, 21 Nov 2021 21:58:02 +0000 Subject: [PATCH 5/5] Update utils.R --- R/utils.R | 1 - 1 file changed, 1 deletion(-) diff --git a/R/utils.R b/R/utils.R index ef425e480..7eeaefb46 100644 --- a/R/utils.R +++ b/R/utils.R @@ -177,7 +177,6 @@ extract_from_list <- function(list, what) { #' @param defaults A list of default settings #' @param optional A list of optional settings to override defaults #' @return A list -#' @export #' #' @keywords internal update_list <- function(defaults = list(), optional = list()) {