Merge branch 'develop' into testfix

CHANGELOG.md

@@ -24,8 +24,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
     [[#1587](https://github.com/remindmodel/remind/pull/1587)]
 - **32_power** extend and reparameterize flexibility tax implementation for electrolysis for hydrogen production
 - **21_tax** add SE tax on electricity going into electrolysis for hydrogen production
-- **scripts** add compareScenarios section for MAGICC7 AR6 output
-    [[#1615](https://github.com/remindmodel/remind/pull/1615)
+- **scripts** add MAGICCv7.5.3 with AR6 settings as output script, add compareScenarios2 option
+    [[#1475](https://github.com/remindmodel/remind/pull/1475), [[#1615](https://github.com/remindmodel/remind/pull/1615)]
 - **scripts** add 'make test-fix' which runs codeCheck in interactive mode, adjusting not_used.txt files
     [[#1625](https://github.com/remindmodel/remind/pull/1625)
 
@@ -40,7 +40,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
     [[#1429](https://github.com/remindmodel/remind/pull/1429), [#1476](https://github.com/remindmodel/remind/pull/1476)]
 - **scripts** '--test' mode for start.R and start_bundle_coupled.R does not write RData files anymore
     [[#1500](https://github.com/remindmodel/remind/pull/1500)]
-- prevent tradtional biomass spillover to other sectors than buildings
+- prevent traditional biomass spillover to other sectors than buildings
     [[#1519](https://github.com/remindmodel/remind/pull/1519)]
 - fully fix landuse and MAGICC6 variables in delayed transition runs to reference run
     [[#1565](https://github.com/remindmodel/remind/pull/1565)]
@@ -50,9 +50,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
     NDC2constant, NPi2018, diffPhaseIn2Constant, diffPhaseIn2Lin, diffPhaseInLin2LinFlex, diffPriceSameCost
     [[#1480](https://github.com/remindmodel/remind/pull/1480)]
 - **36_buildings** remove outdated realizations: services_putty, services_with_capital 
-    [[#1509] (https://github.com/remindmodel/remind/pull/1509)]
+    [[#1509](https://github.com/remindmodel/remind/pull/1509)]
 - **35_transport** remove outdated realization: complex 
-    [[#1543] (https://github.com/remindmodel/remind/pull/1543)]
+    [[#1543](https://github.com/remindmodel/remind/pull/1543)]
 
 ## [3.2.1] - 2023-07-13 (incomplete)
 

CITATION.cff

@@ -251,8 +251,8 @@ authors:
   affiliation: "Potsdam Institute for Climate Impact Research"
 
 title: REMIND - REgional Model of INvestments and Development
-version: "3.2.1.dev1328"
-date-released: 2024-03-18
+version: "3.2.1.dev1351"
+date-released: 2024-03-19
 repository-code: https://github.com/remindmodel/remind
 keywords:
   - energy

config/default.cfg

@@ -69,7 +69,7 @@ cfg$validationmodel_name <- "VALIDATIONREMIND"
 
 #### model version of the overall model (used for run statistics only).
 # automatically generated for development versions, updated by hand for releases
-cfg$model_version <- "3.2.1.dev1328"
+cfg$model_version <- "3.2.1.dev1351"
 
 #### settings ####
 cfg$gms <- list()

main.gms

@@ -216,6 +216,12 @@ $offdigit
 *** turn profiling off (0) or on (1-3, different levels of detail)
 option profile = 0;
 
+file foo_msg;     !! This creates a dummy output file with a well-defined output format:  
+foo_msg.nr = 1;   !! namely F-format (decimal) (and not E-format = scientific notation)
+*** The file can throughout the code be activated with `putclose foo_msg;` and used in the form `put_utility foo_msg "msg" / "xxxx"` to print out xxxx to full.lst 
+*** and be sure that the numeric format is F-format
+
+
 *' @title{extrapage: "00_configuration"} Configuration
 *' @code{extrapage: "00_configuration"}
 *--------------------------------------------------------------------------
@@ -846,7 +852,7 @@ parameter
   cm_LimRock               = 1000;   !! def = 1000
 *'
 parameter
-  cm_expoLinear_yearStart   "time at which carbon price increases lineraly instead of exponentially"
+  cm_expoLinear_yearStart   "time at which carbon price increases linearly instead of exponentially"
 ;
   cm_expoLinear_yearStart  = 2050;   !! def = 2050
 *'
@@ -1004,7 +1010,7 @@ parameter
 *' This switch only has an effect if the flexibility tax is on by cm_flex_tax set to 1
 *' Default value is based on data from German Langfristszenarien (see ./modules/32_power/IntC/datainput.gms).
 parameter
-  cm_FlexTaxFeedback          "switch deciding whether flexibility tax feedback on buildlings and industry electricity prices is on"
+  cm_FlexTaxFeedback          "switch deciding whether flexibility tax feedback on buildings and industry electricity prices is on"
 ;
   cm_FlexTaxFeedback = 0;  !! def = 0  !! regexp = 0|1
 *' cm_FlexTaxFeedback, switches on feedback of flexibility tax on buildings and industry.
@@ -1095,12 +1101,12 @@ parameter
 *' * (any other number) limit of gas demand from 2025 on in Germany in EJ/yr
 *'
 parameter
-  c_SlackMultiplier   "Muliplicative factor to up/downscale the slack size for v_changeProdStartyearSlack"
+  c_SlackMultiplier   "Multiplicative factor to up/downscale the slack size for v_changeProdStartyearSlack"
 ;
   c_SlackMultiplier = 1;  !! def = 1
 *'
 parameter
-  c_changeProdCost   "Muliplicative factor to up/downscale the costs for vm_changeProdStartyearCost"
+  c_changeProdCost   "Multiplicative factor to up/downscale the costs for vm_changeProdStartyearCost"
 ;
   c_changeProdCost = 5;  !! def = 5
 *'
@@ -1400,7 +1406,7 @@ $setGlobal cm_EnSecScen_price  off !! def off
 $setGlobal cm_indstExogScen  off !! def off
 *** cm_exogDem_scen
 *** switch to fix FE or ES demand represented in CES function to trajectories
-*** from exgenous sources (not EDGE models) given in file p47_exogDemScen.
+*** from exogenous sources (not EDGE models) given in file p47_exogDemScen.
 *** This switch fixes demand without recalibration of REMIND CES parameters.
 *** This should be kept in mind when comparing those runs to baseline runs without fixing
 *** as the fixing shifts the CES function away from its optimal point based on the CES parameters used.
@@ -1644,6 +1650,12 @@ $setglobal c_testOneRegi_region  EUR       !! def = EUR  !! regexp = [A-Z]{3}
 *** cm_taxrc_RE     "switch to define whether tax on (CO2 content of) energy imports is recycled to additional direct investments in renewables (wind, solar and storage)"
 $setglobal cm_taxrc_RE  none   !! def = none   !! regexp = none|REdirect
 
+*' cm_repeatNonOpt       "should nonoptimal regions be solved again?"
+*'
+*' *  (off): no, only infeasable regions are repeated, standard setting
+*' *  (yes):  also non-optimal regions are solved again, up to cm_solver_try_max
+$setglobal cm_repeatNonOpt off  
+
 *' @stop
 
 *-------------------------------------------------------------------------------------

modules/80_optimization/nash/declarations.gms

@@ -55,6 +55,7 @@ p80_handle(all_regi)                       "parallel mode handle parameter"
 p80_repy(all_regi,solveinfo80)             "summary report from solver "
 p80_repy_iteration(all_regi,solveinfo80,iteration) "summary report from solver in iteration"
 p80_repyLastOptim(all_regi,solveinfo80)    "p80_repy from last iteration"
+p80_repy_thisSolitr(all_regi,solveinfo80)    "p80_repy from the current solitr - only shows results for regions that were run in this solItr"
 p80_repy_nashitr_solitr(all_regi,solveinfo80,iteration,sol_itr)   "summary report from solver in nash iteration and solver iteration"
 p80_messageFailedMarket(tall,all_enty)     "nash display helper"
 p80_messageShow(convMessage80)             "nash display helper"

modules/80_optimization/nash/postsolve.gms

@@ -274,31 +274,31 @@ loop(trade$(NOT tradeSe(trade)),
  );
 );
 
-*** critertion "infes": and are all solutions optimal?
+*** critertion "infes": is any region neither optimal nor intermediate non-optimal -> then it is infeasible 
 loop(regi,
-     if((p80_repy(regi,'modelstat') ne 2) and (p80_repy(regi,'modelstat') ne 7),
-         s80_bool = 0;
-         p80_messageShow("infes") = YES;
-     );
+  if( (p80_repy(regi,'modelstat') ne 2) and (p80_repy(regi,'modelstat') ne 7),    !! 2 is optimal, 7 nonopt,            
+    s80_bool = 0;
+    p80_messageShow("infes") = YES;
+  );
 *** critertion "nonopt": The next lines are a workaround for the status 7
 *** problem. If the objective value does not differ too much from the last known
 *** optimal solution, accept this solution as if it were optimal. 
-     p80_convNashObjVal_iter(iteration,regi) = p80_repy(regi,'objval') - p80_repyLastOptim(regi,'objval');
-     if (1 le iteration.val,
-        !! no last iteration if this is the first; NA value in p80_repyLastOptim is
-        !! sticky, so test this separately
-        if ( p80_repy(regi,'modelstat') eq 7
-           !! The 1E-4 are quite arbitrary. One should do more research on how
-           !! the solution differs over iteration when status 7 occurs. 
-           AND p80_convNashObjVal_iter(iteration,regi) lt - 1e-4,
-           s80_bool = 0;
-           p80_messageShow("nonopt") = YES;     
-           display "Not all regions were status 2 in the last iteration. The deviation of the objective function from the last optimal solution is too large to be accepted:";
-           s80_dummy = p80_repy(regi,'objval') - p80_repyLastOptim(regi,'objval');
-           display s80_dummy;
-        );
-     );
-); !!regi
+  p80_convNashObjVal_iter(iteration,regi) = p80_repy(regi,'objval') - p80_repyLastOptim(regi,'objval');
+  if (1 le iteration.val,
+    !! no last iteration if this is the first; NA value in p80_repyLastOptim is
+    !! sticky, so test this separately
+    if ( p80_repy(regi,'modelstat') eq 7
+        !! The 1E-4 are quite arbitrary. One should do more research on how
+        !! the solution differs over iteration when status 7 occurs. 
+        AND p80_convNashObjVal_iter(iteration,regi) lt - 1e-4,
+      s80_bool = 0;
+      p80_messageShow("nonopt") = YES;     
+      display "Not all regions were status 2 in the last iteration. The deviation of the objective function from the last optimal solution is too large to be accepted:";
+      s80_dummy = p80_repy(regi,'objval') - p80_repyLastOptim(regi,'objval');
+      display s80_dummy;
+    );
+  );
+); !! loop over regi
 
 *** criterion only for checking, not applied anymore: are the anticipation terms sufficienctly small?
 p80_fadeoutPriceAnticip_iter(iteration) = sm_fadeoutPriceAnticip;
@@ -406,6 +406,7 @@ display p80_repy;
 
 display "trade convergence indicators";
 display p80_surplusMaxTolerance, p80_surplusMax2100;
+display p80_defic_trade, p80_defic_sum,p80_defic_sum_rel;
 
 display "Reasons for non-convergence in this iteration (if not yet converged)";
 
@@ -462,7 +463,8 @@ $ifthen.cm_implicitQttyTarget not "%cm_implicitQttyTarget%" == "off"
         if(sameas(convMessage80, "implicitEnergyTarget"),
 		      display "#### 10) A quantity target has not been reached yet.";
           display "#### Check out the pm_implicitQttyTarget_dev parameter of 47_regipol module.";
-          display "#### The deviation must to be less than cm_implicitQttyTarget_tolerance. By default within 1%, i.e. in between -0.01 and 0.01 of the defined target.";
+          display "#### The relative deviation must to be less than cm_implicitQttyTarget_tolerance, which is 1 percent by default.";
+          display "#### For taxes, this means every value > +0.01, while for subsidies everything < -0.01 is problematic in the following lines.";
           display cm_implicitQttyTarget_tolerance, pm_implicitQttyTarget_dev;
 	      );
 $endif.cm_implicitQttyTarget

modules/80_optimization/nash/solve.gms

@@ -15,13 +15,15 @@ hybrid.solvelink = 0;
 $endif.debug
 
 loop (all_regi,
-  !! only solve for regions that do not have a valid solution for this nash
-  !! iteration
-  if (    sol_itr.val gt 1
-      AND (   p80_repy(all_regi,"modelstat") eq 2
-           OR p80_repy(all_regi,"modelstat") eq 7),
-
-    p80_repy(all_regi,solveinfo80) = 0;
+  !! only solve for regions that do not have a valid solution for this nash iteration
+  if ( sol_itr.val gt 1
+       AND ( p80_repy(all_regi,"modelstat") eq 2
+$ifthen.repeatNonOpt "%cm_repeatNonOpt%" == "off"
+            OR p80_repy(all_regi,"modelstat") eq 7
+$endif.repeatNonOpt
+          ),
+
+    p80_repy_thisSolitr(all_regi,solveinfo80) = 0;
     continue;
   );
 
@@ -49,11 +51,11 @@ loop (all_regi,
   solve hybrid using nlp maximizing vm_welfareGlob;
 
 $ifthene.debug (sameas("%cm_nash_mode%","serial"))OR(sameas("%cm_nash_mode%","debug"))
-  p80_repy(all_regi,"solvestat") = hybrid.solvestat;
-  p80_repy(all_regi,"modelstat") = hybrid.modelstat;
-  p80_repy(all_regi,"resusd")    = hybrid.resusd;
-  p80_repy(all_regi,"objval")    = hybrid.objval;
-  if (p80_repy(all_regi,"modelstat") eq 2,
+  p80_repy_thisSolitr(all_regi,"solvestat") = hybrid.solvestat;
+  p80_repy_thisSolitr(all_regi,"modelstat") = hybrid.modelstat;
+  p80_repy_thisSolitr(all_regi,"resusd")    = hybrid.resusd;
+  p80_repy_thisSolitr(all_regi,"objval")    = hybrid.objval;
+  if (p80_repy_thisSolitr(all_regi,"modelstat") eq 2,
     p80_repyLastOptim(all_regi,"objval") = p80_repy(all_regi,"objval");
   );
 $endif.debug
@@ -65,13 +67,15 @@ $endif.debug
 $ifthen.parallel %cm_nash_mode% == "parallel"
 repeat
   loop (all_regi$handlecollect(p80_handle(all_regi)),
-    p80_repy(all_regi,"solvestat") = hybrid.solvestat;
-    p80_repy(all_regi,"modelstat") = hybrid.modelstat;
-    p80_repy(all_regi,"resusd")    = hybrid.resusd;
-    p80_repy(all_regi,"objval")    = hybrid.objval;
+    p80_repy_thisSolitr(all_regi,"solvestat") = hybrid.solvestat;
+    p80_repy_thisSolitr(all_regi,"modelstat") = hybrid.modelstat;
+    p80_repy_thisSolitr(all_regi,"resusd")    = hybrid.resusd;
+    p80_repy_thisSolitr(all_regi,"objval")    = hybrid.objval;
+
+*    p80_repyLatestSolve(all_regi,solveinfo80) = p80_repy(all_regi,solveinfo80);
 
-    if (p80_repy(all_regi,"modelstat") eq 2,
-      p80_repyLastOptim(all_regi,"objval") = p80_repy(all_regi,"objval");
+    if (p80_repy_thisSolitr(all_regi,"modelstat") eq 2,
+      p80_repyLastOptim(all_regi,"objval") = p80_repy_thisSolitr(all_regi,"objval");
     );
 
     display$handledelete(p80_handle(all_regi)) "trouble deleting handles" ;
@@ -83,19 +87,39 @@ $endif.parallel
 
 regi(all_regi) = YES;
 
+
+display p80_repy_thisSolitr;
+display p80_repy;
+
 *** internal nash helper paramter:
 pm_SolNonInfes(regi) = 0;
 p80_SolNonOpt(regi)  = 0;
+
+putclose foo_msg;  
+*** This putclose serves to make foo_msg the last "active" put file, and thus makes GAMS use the foo_msg formating (namely F-format, not scientific E-format)
+*** Otherwise, the following put messages will try to write modelstat in scientif format, throwing errors because of insufficient space
+
 loop (regi,
+  if( (p80_repy_thisSolitr(regi,"solvestat") > 0) ,
+    put_utility foo_msg "msg" / "Solitr:" sol_itr.tl:2:0 " " regi.tl:4:0 "     updated. Modstat new " p80_repy_thisSolitr(regi,"modelstat"):2:0 ", old " p80_repy(regi,"modelstat"):2:0 "; Resusd new" p80_repy_thisSolitr(regi,"resusd"):5:0 ", old" p80_repy(regi,"resusd"):5:0 "; Obj new" p80_repy_thisSolitr(regi,"objval"):7:3 ", old" p80_repy(regi,"objval"):7:3 ;
+    p80_repy(regi,solveinfo80) = p80_repy_thisSolitr(regi,solveinfo80); !! copy info from this Solitr into p80_repy
+  else
+    put_utility foo_msg "msg" / "Solitr:" sol_itr.tl:2:0 " " regi.tl:4:0 " not updated. Modstat new " p80_repy_thisSolitr(regi,"modelstat"):2:0 ", old " p80_repy(regi,"modelstat"):2:0 "; Resusd new" p80_repy_thisSolitr(regi,"resusd"):5:0 ", old" p80_repy(regi,"resusd"):5:0 "; Obj new" p80_repy_thisSolitr(regi,"objval"):7:3 ", old" p80_repy(regi,"objval"):7:3 ;
+  );
+
   if (p80_repy(regi,"modelstat") eq 2 OR p80_repy(regi,"modelstat") eq 7,
     pm_SolNonInfes(regi) = 1;
   );
   if (p80_repy(regi,"modelstat") eq 7, p80_SolNonOpt(regi) = 1);
 );
 
-*** set o_modelstat to the highest value across all regions, ignoring status 7 
+*** set o_modelstat to the highest value across all regions
 o_modelstat
-  = smax(regi, p80_repy(regi,"modelstat")$(p80_repy(regi,"modelstat") ne 7));
+$ifthen.repeatNonOpt "%cm_repeatNonOpt%" == "off"
+  = smax(regi, p80_repy(regi,"modelstat")$(p80_repy(regi,"modelstat") ne 7));  !! ignoring status 7 
+$else.repeatNonOpt
+  = smax(regi, p80_repy(regi,"modelstat"));                                    !! also taking into account status 7
+$endif.repeatNonOpt
 
 *** in cm_nash_mode=debug mode, enable solprint for next sol_itr when last
 *** iteration was non-optimal:
@@ -105,16 +129,22 @@ if (o_modelstat ne 2,
 );
 $endif.solprint
 
-p80_repy_iteration(all_regi,solveinfo80,iteration)$(
-                                                p80_repy(all_regi,solveinfo80) )
+p80_repy_iteration(all_regi,solveinfo80,iteration)$( p80_repy_thisSolitr(all_regi,solveinfo80) ) !! add information if this region was solved in this iteration
     !! store sum of resusd for all sol_itrs
   = ( p80_repy_iteration(all_regi,solveinfo80,iteration)
-    + p80_repy(all_regi,solveinfo80)
+    + p80_repy_thisSolitr(all_regi,solveinfo80)
     )$( sameas(solveinfo80,"resusd") )
-  + p80_repy(all_regi,solveinfo80)$( NOT sameas(solveinfo80,"resusd") );
+  + p80_repy_thisSolitr(all_regi,solveinfo80)$( NOT sameas(solveinfo80,"resusd") );
+
+p80_repy_nashitr_solitr(all_regi,solveinfo80,iteration,sol_itr)$( p80_repy_thisSolitr(all_regi,solveinfo80) ) !! add information if this region was solved in this iteration
+  = p80_repy_thisSolitr(all_regi,solveinfo80);
+
+put_utility "msg" / "Solve overview: The following are the results for iteration " iteration.tl:3:0  " , sol_itr " sol_itr.tl:3:0 ;
+display o_modelstat;
+display p80_repy;
+display p80_repy_thisSolitr;
+display p80_repy_iteration;
+display p80_repy_nashitr_solitr;
 
-p80_repy_nashitr_solitr(all_regi,solveinfo80,iteration,sol_itr)$(
-                                                p80_repy(all_regi,solveinfo80) )
-  = p80_repy(all_regi,solveinfo80);
 
 *** EOF ./modules/80_optimization/nash/solve.gms

output.R

@@ -93,7 +93,7 @@ choose_slurmConfig_output <- function(output) {
     return("direct")
 
   # Modify slurm options for reporting options that run in parallel (MAGICC) or need more memory
-  if ("ar6Climate" %in% output) {
+  if ("MAGICC7_AR6" %in% output) {
     slurm_options <- paste(slurm_options[1:3], "--tasks-per-node=12 --mem=32000")
   } else if ("nashAnalysis" %in% output) {
     slurm_options <- paste(slurm_options[1:3], "--mem=32000")

scripts/output/single/MAGICC7_AR6.R

@@ -224,8 +224,13 @@ climateAssessmentData <- read.quitte(climateAssessmentOutput) %>%
   filter(period %in% usePeriods) %>%
   interpolate_missing_periods(usePeriods, expand.values = FALSE) %>%
   mutate(variable = gsub("|MAGICCv7.5.3", "", .data$variable, fixed = TRUE)) %>%
-  mutate(variable = gsub("AR6 climate diagnostics|", "MAGICC7 AR6|", .data$variable, fixed = TRUE)) %>%
-  write.mif(remindReportingFile, append = TRUE)
+  mutate(variable = gsub("AR6 climate diagnostics|", "MAGICC7 AR6|", .data$variable, fixed = TRUE))
+
+as.quitte(remindReportingFile) %>%
+  # remove data from old MAGICC7 runs to avoid duplicated
+  filter(! grepl("AR6 climate diagnostics.*MAGICC7", .data$variable), ! grepl("^MAGICC7 AR6", .data$variable)) %>%
+  rbind(climateAssessmentData) %>%
+  write.mif(remindReportingFile)
 
 deletePlus(remindReportingFile, writemif = TRUE)