From 208905b8b10940cd602dd98ab60ef61be123506f Mon Sep 17 00:00:00 2001 From: robinhasse Date: Tue, 26 Sep 2023 09:57:47 +0200 Subject: [PATCH] Make file distribution optional and better error message --- .buildlibrary | 2 +- CITATION.cff | 4 +- DESCRIPTION | 4 +- R/download_distribute.R | 114 ++++++++++++++++++++++++++-------------- R/getfiledestinations.R | 4 ++ README.md | 6 +-- 6 files changed, 87 insertions(+), 47 deletions(-) diff --git a/.buildlibrary b/.buildlibrary index 0ba9de0..01886ff 100644 --- a/.buildlibrary +++ b/.buildlibrary @@ -1,4 +1,4 @@ -ValidationKey: '49190490' +ValidationKey: '5102760' AcceptedWarnings: - 'Warning: package ''.*'' was built under R version' - 'Warning: namespace ''.*'' is not available and has been replaced' diff --git a/CITATION.cff b/CITATION.cff index 791dac6..6ab4823 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -2,8 +2,8 @@ cff-version: 1.2.0 message: If you use this software, please cite it using the metadata from this file. type: software title: 'gms: ''GAMS'' Modularization Support Package' -version: 0.25.11 -date-released: '2023-08-21' +version: 0.26.0 +date-released: '2023-09-26' abstract: A collection of tools to create, use and maintain modularized model code written in the modeling language 'GAMS' (). Out-of-the-box 'GAMS' does not come with support for modularized model code. This package provides diff --git a/DESCRIPTION b/DESCRIPTION index 5c79c97..989e8e3 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,8 +1,8 @@ Package: gms Type: Package Title: 'GAMS' Modularization Support Package -Version: 0.25.11 -Date: 2023-08-21 +Version: 0.26.0 +Date: 2023-09-26 Authors@R: c(person("Jan Philipp", "Dietrich", email = "dietrich@pik-potsdam.de", role = c("aut","cre")), person("David", "Klein", role = "aut"), person("Anastasis", "Giannousakis", role = "aut"), diff --git a/R/download_distribute.R b/R/download_distribute.R index aff53ef..61538b9 100644 --- a/R/download_distribute.R +++ b/R/download_distribute.R @@ -11,7 +11,7 @@ #' no options are required the value has to be NULL. (e.g. #' list("ftp://my_pw_protected_server.de/data"=list(user="me",password=12345), "http://free_server.de/dat"=NULL)) #' @param modelfolder main model folder -#' @param additionalDelete information which additonal data should be deleted before new data are downloaded and distributed +#' @param additionalDelete information which additional data should be deleted before new data are downloaded and distributed #' @param debug switch for debug mode with additional diagnostic information #' @param stopOnMissing Boolean passed along to [download_unpack()] to stop if #' any file in `files` could not be downloaded. Off (`FALSE`) by default. @@ -22,67 +22,103 @@ #' @export download_distribute <- function(files, - repositories=list("/p/projects/rd3mod/inputdata/output"=NULL), - modelfolder=".", - additionalDelete=NULL, - debug=FALSE, + repositories = list("/p/projects/rd3mod/inputdata/output" = NULL), + modelfolder = ".", + additionalDelete = NULL, + debug = FALSE, stopOnMissing = FALSE) { + # set working directory to modelfolder cdir <- getwd() setwd(modelfolder) on.exit(setwd(cdir)) - ########## GENERATE INFORMATION FOR DESTINATION ############################ - file2destination <- getfiledestinations() - ########## GENERAL CLEAN UP ################################################ + + # CLEAN UP ------------------------------------------------------------------- + + # delete old input files to avoid mixed inputs in the case that data # download fails at some point - message("Delete old data in input folders ... ") + + # directories to distribute input files to + file2destination <- getfiledestinations() + # delete files which will be copied/moved later on with copy_input - delete_olddata(file2destination) + if (!is.null(file2destination)) { + message("Delete old data in input folders ... ") + delete_olddata(file2destination) + message("done!\n") + } + # delete additional files not treated by copy_input - if(!is.null(additionalDelete)) delete_olddata(additionalDelete) - message("done!\n") + if (!is.null(additionalDelete)) { + message("Delete additional data ... ") + delete_olddata(additionalDelete) + message("done!\n") + } + + + if (is.null(file2destination)) { + message("No 'files' file found. Input files won't be distributed.") + } + + + + # DATA DOWNLOAD -------------------------------------------------------------- - ########## DATA DOWNLOAD ################################################### # load data from source and unpack it filemap <- download_unpack(input = files, targetdir = "input", repositories = repositories, debug = debug, stopOnMissing = stopOnMissing) - ########## COPY MAGPIE INPUT FILES ######################################### + + + # COPY MAGPIE INPUT FILES ---------------------------------------------------- + # In the following input files in MAgPIE format are converted (if required) to # csX files and copied into the corresponding input folders. In this step also # the resolution information in the file name (if existing) is removed to # allow a resolution-indepedent gams-sourcecode. - low_res <- get_info("input/info.txt","^\\* Output ?resolution:",": ") - - # make an educated guess about what the current low res is - if(is.null(low_res)) { - guessLowRes <- function(folder) { - suf <- sub("^.*_(.*)\\..*$","\\1",dir(folder)) - # remove high res suffix - suf <- suf[!(suf=="0.5")] - if(length(suf)==0) { - warning("Low Res suffix not properly detected, set low res suffix to NULL") - return(NULL) - } else if(length(suf)>1) { - #remove suffixes which only happen to appear once - suf <- suf[duplicated(suf)] - if(length(suf)==0) { - warning("Low Res suffix not properly detected, set low res suffix to NULL") - return(NULL) - } else if(length(suf)>1) { - tmp <- table(suf) - suf <- names(tmp)[order(tmp, decreasing=TRUE)][1] - } + + if (!is.null(file2destination)) { + + low_res <- get_info("input/info.txt","^\\* Output ?resolution:",": ") + + # make an educated guess about what the current low res is + if (is.null(low_res)) { + guessLowRes <- function(folder) { + suf <- sub("^.*_(.*)\\..*$", "\\1", dir(folder)) + # remove high res suffix + suf <- suf[!(suf == "0.5")] + if (length(suf) == 0) { + warning("Low Res suffix not properly detected, set low res suffix to NULL") + return(NULL) + } else if (length(suf) > 1) { + # remove suffixes which only happen to appear once + suf <- suf[duplicated(suf)] + if (length(suf) == 0) { + warning("Low Res suffix not properly detected, set low res suffix to NULL") + return(NULL) + } else if (length(suf) > 1) { + tmp <- table(suf) + suf <- names(tmp)[order(tmp, decreasing = TRUE)][1] + } + } + message("Low resolution suffix automatic determined as \"", suf, "\"") + return(suf) } - message("Low resolution suffix automatic determined as \"",suf,"\"") - return(suf) + low_res <- guessLowRes("input/") } - low_res <- guessLowRes("input/") + + # distribute input files according to the 'files' files + copy_input(x = file2destination, + sourcepath = "input", + suffix = low_res, + move = !debug) } - copy_input(x=file2destination, sourcepath="input", suffix=low_res, move=!debug) + + + return(filemap) } diff --git a/R/getfiledestinations.R b/R/getfiledestinations.R index b5303d5..68e045a 100644 --- a/R/getfiledestinations.R +++ b/R/getfiledestinations.R @@ -2,6 +2,7 @@ #' #' Create file2destination mapping based on information from the model #' +#' #' @author Jan Philipp Dietrich, David Klein @@ -16,6 +17,9 @@ getfiledestinations <- function() { add <- data.frame(file = tmp, destination = dirname(f), stringsAsFactors = FALSE) out <- rbind(out, add) } + if (is.null(out)) { + return(NULL) + } out <- as.data.frame(lapply(out, trimws), stringsAsFactors = FALSE) return(out[out[[1]] != "", ]) } diff --git a/README.md b/README.md index 7551c48..7889b34 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # 'GAMS' Modularization Support Package -R package **gms**, version **0.25.11** +R package **gms**, version **0.26.0** [![CRAN status](https://www.r-pkg.org/badges/version/gms)](https://cran.r-project.org/package=gms) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.4390032.svg)](https://doi.org/10.5281/zenodo.4390032) [![R build status](https://github.com/pik-piam/gms/workflows/check/badge.svg)](https://github.com/pik-piam/gms/actions) [![codecov](https://codecov.io/gh/pik-piam/gms/branch/master/graph/badge.svg)](https://app.codecov.io/gh/pik-piam/gms) [![r-universe](https://pik-piam.r-universe.dev/badges/gms)](https://pik-piam.r-universe.dev/builds) @@ -43,7 +43,7 @@ In case of questions / problems please contact Jan Philipp Dietrich . +Dietrich J, Klein D, Giannousakis A, Beier F, Koch J, Baumstark L, Pflüger M, Richters O (2023). _gms: 'GAMS' Modularization Support Package_. doi:10.5281/zenodo.4390032 , R package version 0.26.0, . A BibTeX entry for LaTeX users is @@ -52,7 +52,7 @@ A BibTeX entry for LaTeX users is title = {gms: 'GAMS' Modularization Support Package}, author = {Jan Philipp Dietrich and David Klein and Anastasis Giannousakis and Felicitas Beier and Johannes Koch and Lavinia Baumstark and Mika Pflüger and Oliver Richters}, year = {2023}, - note = {R package version 0.25.11}, + note = {R package version 0.26.0}, doi = {10.5281/zenodo.4390032}, url = {https://github.com/pik-piam/gms}, }