Merge pull request #294 from ahagen-pik/master

Add PR test routine/ChangeLog

.Rbuildignore

@@ -1,4 +1,7 @@
-^edgetRansport\.Rproj$
+^pull_request_template.md
+^renv$
+^renv\.locks$
+^edgeTransport\.Rproj$
 ^.*\.zenodo.json$
 ^\.buildlibrary$
 ^GDP*\.RDS$
@@ -11,6 +14,7 @@
 ^\.git$
 ^.lintr$
 ^tests/.lintr$
+^tests/PRtest/*
 ^vignettes/.lintr$
 ^Makefile$
 ^.*CITATION.cff$

.buildlibrary

@@ -1,4 +1,4 @@
-ValidationKey: '5648742'
+ValidationKey: '5669056'
 AutocreateReadme: yes
 AcceptedWarnings:
 - 'Warning: package ''.*'' was built under R version'

CITATION.cff

@@ -2,8 +2,8 @@ cff-version: 1.2.0
 message: If you use this software, please cite it using the metadata from this file.
 type: software
 title: 'edgeTransport: Prepare EDGE Transport Data for the REMIND model'
-version: 2.8.2
-date-released: '2024-11-04'
+version: 2.8.3
+date-released: '2024-11-05'
 abstract: EDGE-T is a fork of the GCAM transport module https://jgcri.github.io/gcam-doc/energy.html#transportation
   with a high level of detail in its representation of technological and modal options.
   It is a partial equilibrium model with a nested multinomial logit structure and

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: edgeTransport
 Title: Prepare EDGE Transport Data for the REMIND model
-Version: 2.8.2
+Version: 2.8.3
 Authors@R: c(
     person("Johanna", "Hoppe", , "[email protected]", role = c("aut", "cre"),
            comment = c(ORCID = "0009-0004-6753-5090")),
@@ -19,7 +19,7 @@ Encoding: UTF-8
 LazyData: true
 RoxygenNote: 7.3.2
 VignetteBuilder: knitr
-Date: 2024-11-04
+Date: 2024-11-05
 Config/testthat/edition: 3
 Imports:
     rmndt,

README.md

@@ -1,6 +1,6 @@
 # Prepare EDGE Transport Data for the REMIND model
 
-R package **edgeTransport**, version **2.8.2**
+R package **edgeTransport**, version **2.8.3**
 
 [![CRAN status](https://www.r-pkg.org/badges/version/edgeTransport)](https://cran.r-project.org/package=edgeTransport)  [![R build status](https://github.com/pik-piam/edgeTransport/workflows/check/badge.svg)](https://github.com/pik-piam/edgeTransport/actions) [![codecov](https://codecov.io/gh/pik-piam/edgeTransport/branch/master/graph/badge.svg)](https://app.codecov.io/gh/pik-piam/edgeTransport) [![r-universe](https://pik-piam.r-universe.dev/badges/edgeTransport)](https://pik-piam.r-universe.dev/builds)
 
@@ -46,7 +46,7 @@ In case of questions / problems please contact Johanna Hoppe <johanna.hoppe@pik-
 
 To cite package **edgeTransport** in publications use:
 
-Hoppe J, Dirnaichner A, Rottoli M, Muessel J (2024). _edgeTransport: Prepare EDGE Transport Data for the REMIND model_. R package version 2.8.2, <https://github.com/pik-piam/edgeTransport>.
+Hoppe J, Dirnaichner A, Rottoli M, Muessel J (2024). _edgeTransport: Prepare EDGE Transport Data for the REMIND model_. R package version 2.8.3, <https://github.com/pik-piam/edgeTransport>.
 
 A BibTeX entry for LaTeX users is
 
@@ -55,7 +55,7 @@ A BibTeX entry for LaTeX users is
   title = {edgeTransport: Prepare EDGE Transport Data for the REMIND model},
   author = {Johanna Hoppe and Alois Dirnaichner and Marianna Rottoli and Jarusch Muessel},
   year = {2024},
-  note = {R package version 2.8.2},
+  note = {R package version 2.8.3},
   url = {https://github.com/pik-piam/edgeTransport},
 }
 ```

pull_request_template.md

@@ -0,0 +1,11 @@
+## Purpose of this PR
+
+## Checklist:
+
+- [ ] I used ./test-standard-runs to compare and archive the changes introduced by this PR in /p/projects/edget/PRchangeLog/
+
+## Further information (optional):
+
+* Test runs are here: 
+* Comparison of results (what changes by this PR?): 
+

tests/PRtest/PRtestEDGET.R

@@ -0,0 +1,122 @@
+#!/usr/bin/env Rscript
+
+args <- commandArgs(trailingOnly = TRUE)
+
+folderName <- args[1]
+defScen <- args[2]
+refFolder <- NULL
+if (length(args) == 3) {
+	refFolder <- args[3]
+}
+
+# maybe a check for possible argument errors here? only needed if script is used individually
+
+renv::status()
+renv::restore()
+
+library(madrat)
+library(mrdrivers)
+library(mrremind)
+library(dplyr)
+library(tidyr)
+library(edgeTransport)
+library(reporttransport)
+library(piamPlotComparison)
+
+testStandardScenarios <- function(folderNameD, defScenN){
+
+    allScens <-  tribble(
+    ~SSPscen,         ~transportPolScen,        ~isICEban,    ~demScen,
+    'SSP2',        'Mix1',                 FALSE,       'default',
+    'SSP2',        'Mix2',                 TRUE,        'default',
+    'SSP2',        'Mix3',                 TRUE,        'default',
+    'SSP2',        'Mix4',                 TRUE,        'default'
+    )
+
+    # generate list from data frame rows
+    allScens <- split(allScens, seq(nrow(allScens)))
+    defScen <- allScens[[defScenN]]
+
+    EdgeTransportSAdata <- calcEdgeTransportSA(SSPscen = defScen[["SSPscen"]],
+                                                      transportPolScen = defScen[["transportPolScen"]],
+                                                      isICEban = defScen[["isICEban"]],
+                                                      demScen = defScen[["demScen"]],
+                                                      isTransportReported = TRUE,
+                                                      isTransportExtendedReported = TRUE,
+                                                      isREMINDinputReported = TRUE,
+						      isAnalyticsReported = TRUE,
+                                                      isStored = TRUE,
+                                                      outputFolder = folderNameD)					
+
+  renv::snapshot(type = "all")
+}
+
+plotStandardScenarios <- function(folderNameD, defScenN, refFolderD = NULL){
+
+	n <- as.numeric(defScenN)
+	refFolder <- refFolderD
+	current_path <- getwd()
+	PRchangeLogPath <- dirname(current_path)
+
+	testDirs <- sort(list.dirs(path = PRchangeLogPath, recursive = FALSE))
+	#Only use folders that comply with naming convention
+	testDirs <- testDirs[grepl(".*\\d{8}_.*", testDirs)]
+
+	if (is.character(refFolder)) {
+		testDirs <- testDirs[c(grep(refFolderD, testDirs), grep(current_path, testDirs))]
+	} else {
+		testDirs <- testDirs[c(grep(current_path, testDirs)-1, grep(current_path, testDirs))]
+	}
+
+	mifHist <- "/p/projects/edget/PRchangeLog/historical21.mif"
+
+	allScens <-  tribble(
+    		~transportPolScen,        ~scenName1,    	~scenName2,	~filename,
+    		'Mix1',                   'Mix1before',     'Mix1after',	"ChangeLogMix1",
+    		'Mix2',                   'Mix2before',     'Mix2after',	"ChangeLogMix2",
+    		'Mix3',                   'Mix3before',     'Mix3after',	"ChangeLogMix3",
+    		'Mix4',                   'Mix4before',     'Mix4after',	"ChangeLogMix4"
+    		)
+
+	mifs <- list.files(testDirs, "Transport.mif", full.names = TRUE, recursive = TRUE)
+
+	mifs <- mifs[grep(allScens$transportPolScen[n], mifs)]
+	scenNames <- c(allScens$scenName1[n], allScens$scenName2[n])
+	filename <- allScens$filename[n]
+	outDir <- file.path(folderNameD, filename)
+	dir.create(outDir, showWarnings = FALSE)
+
+	sec <- c("00_info", "01_energy_demand","02_energy_services","03_energy_intensity", "04_stock_and_sales", "05_emissions", "06_input_parameters", "08_transportRemindInputfiles")
+
+	#EU21
+	piamPlotComparison::compareScenarios(
+  	projectLibrary = "reporttransport",
+  	mifs,
+  	mifHist,
+  	outDir,
+  	outputFile = paste0(format(Sys.time(), "%Y-%m-%d_%H.%M.%S"), "_", filename, "EUR.pdf"),
+  	outputFormat = "pdf",
+  	mifScenNames = scenNames,
+  	sections = sec,
+  	reg = c("ENC","EWN","ECS","ESC","ECE","FRA","DEU","UKI","ESW","EUR"),
+  	mainReg = "EUR"
+	)
+
+	#H12
+	piamPlotComparison::compareScenarios(
+  	projectLibrary = "reporttransport",
+  	mifs, 
+  	mifHist,
+  	outDir,
+  	outputFile = paste0(format(Sys.time(), "%Y-%m-%d_%H.%M.%S"), "_", filename, "H12.pdf"),
+  	outputFormat = "pdf",
+  	mifScenNames = scenNames,
+  	sections = sec,
+  	reg = c("OAS","MEA","SSA","LAM","REF","CAZ","CHA","IND","JPN","USA","NEU","EUR","World"),
+  	mainReg = "World"
+	)
+}
+
+
+testStandardScenarios(folderName, defScen)
+plotStandardScenarios(folderName, defScen, refFolder)

tests/PRtest/compScenPR.R

@@ -0,0 +1,95 @@
+#!/usr/bin/env Rscript
+
+args <- commandArgs(trailingOnly = TRUE)
+
+refFolder <- args[1]
+defScen <- args[2]
+
+help <- "
+Usage: Rscript compScenPR.R <refFolder> <ScenN=2> 
+
+Compares a standard scenario run of two different edgeT versions via compScen's. A Transport.mif from the working directory is compared with the corresponding scenario run of <refFolder>. 
+
+<ScenN> : optional, one of 1,2,3,4 corresponding to the standard transport scenarios, Mix1, Mix2ICEban, Mix3ICEban, Mix4ICEban  
+
+  -h, --help   show this text and exit
+"
+
+if (2 != length(args)) {
+    cat("ERROR: Wrong number of Arguments!")
+    cat(help)
+    quit(save = 'no', status = 1)
+}
+
+library(dplyr)
+library(tidyr)
+library(reporttransport)
+library(piamPlotComparison)
+
+plotStandardScenarios <- function(refFolderD, defScenN = 2){
+
+	n <- as.numeric(defScenN)
+	refFolder <- refFolderD
+	current_path <- getwd()
+	PRchangeLogPath <- dirname(current_path)
+
+	testDirs <- sort(list.dirs(path = PRchangeLogPath, recursive = FALSE))
+	#Only use folders that comply with naming convention
+	testDirs <- testDirs[grepl(".*\\d{8}_.*", testDirs)]
+
+	if (is.character(refFolder)) {
+		testDirs <- testDirs[c(grep(refFolderD, testDirs), grep(current_path, testDirs))]
+	} else {
+		testDirs <- testDirs[c(grep(current_path, testDirs)-1, grep(current_path, testDirs))]
+	}
+
+	mifHist <- "/p/projects/edget/PRchangeLog/historical21.mif"
+
+	allScens <-  tribble(
+    		~transportPolScen,        ~scenName1,    	~scenName2,	~filename,
+    		'Mix1',                   'Mix1before',     'Mix1after',	"ChangeLogMix1",
+    		'Mix2',                   'Mix2before',     'Mix2after',	"ChangeLogMix2",
+    		'Mix3',                   'Mix3before',     'Mix3after',	"ChangeLogMix3",
+    		'Mix4',                   'Mix4before',     'Mix4after',	"ChangeLogMix4"
+    		)
+
+	mifs <- list.files(testDirs, "Transport.mif", full.names = TRUE, recursive = TRUE)
+
+	mifs <- mifs[grep(allScens$transportPolScen[n], mifs)]
+	scenNames <- c(allScens$scenName1[n], allScens$scenName2[n])
+	filename <- allScens$filename[n]
+	outDir <- file.path(current_path, filename)
+	dir.create(outDir, showWarnings = FALSE)
+
+	sec <- c("00_info", "01_energy_demand","02_energy_services","03_energy_intensity", "04_stock_and_sales", "05_emissions", "06_input_parameters", "08_transportRemindInputfiles")
+
+	#EU21
+	piamPlotComparison::compareScenarios(
+  	projectLibrary = "reporttransport",
+  	mifs,
+  	mifHist,
+  	outDir,
+  	outputFile = paste0(format(Sys.time(), "%Y-%m-%d_%H.%M.%S"), "_", filename, "EUR.pdf"),
+  	outputFormat = "pdf",
+  	mifScenNames = scenNames,
+  	sections = sec,
+  	reg = c("ENC","EWN","ECS","ESC","ECE","FRA","DEU","UKI","ESW","EUR"),
+  	mainReg = "EUR"
+	)
+
+	#H12
+	piamPlotComparison::compareScenarios(
+  	projectLibrary = "reporttransport",
+  	mifs, 
+  	mifHist,
+  	outDir,
+  	outputFile = paste0(format(Sys.time(), "%Y-%m-%d_%H.%M.%S"), "_", filename, "H12.pdf"),
+  	outputFormat = "pdf",
+  	mifScenNames = scenNames,
+  	sections = sec,
+  	reg = c("OAS","MEA","SSA","LAM","REF","CAZ","CHA","IND","JPN","USA","NEU","EUR","World"),
+  	mainReg = "World"
+	)
+}
+
+plotStandardScenarios(refFolder, defScen)

tests/PRtest/renvfromsource.R

@@ -0,0 +1,9 @@
+#!/usr/bin/env Rscript
+
+sourceRepos <- commandArgs(trailingOnly = TRUE)
+
+renv::load()
+
+renv::install(sourceRepos, type = source)
+
+renv::snapshot(type = "all")

tests/PRtest/submit_compScenPR.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+#SBATCH --qos=short
+#SBATCH --job-name=compScen_edgetPR
+#SBATCH --output=%x-%j.out
+#SBATCH --ntasks=4
+#SBATCH --nodes=4
+#SBATCH --ntasks-per-node=1
+#SBATCH --mem=32000
+#SBATCH --time=02:00:00
+
+srun --ntasks=1 --nodes=1 Rscript compScenPR.R $1 "1" &
+srun --ntasks=1 --nodes=1 Rscript compScenPR.R $1 "2" &
+srun --ntasks=1 --nodes=1 Rscript compScenPR.R $1 "3" &
+srun --ntasks=1 --nodes=1 Rscript compScenPR.R $1 "4" &
+
+wait