Merge pull request #1878 from tonnrueter/dev/python_env

Python integration pt. 1

.Rprofile

@@ -42,26 +42,6 @@ if (isTRUE(rownames(installed.packages(priority = "NA")) == "renv")) {
   message("Finished installing R package dependencies.")
 }
 
-# bootstrapping python venv, will only run once after remind is freshly cloned
-if (!dir.exists(".venv/")
-    && (Sys.which("python3") != ""
-        || (Sys.which("python.exe") != ""
-            && suppressWarnings(isTRUE(startsWith(system2("python.exe", "--version", stdout = TRUE), "Python 3")))
-           ))) {
-  message("Python venv is not available, setting up now...")
-  # use system python to set up venv
-  if (.Platform$OS.type == "windows") {
-    system2("python.exe", c("-mvenv", ".venv"))
-    pythonInVenv <- normalizePath(file.path(".venv", "Scripts", "python.exe"), mustWork = TRUE)
-  } else {
-    system2("python3", c("-mvenv", ".venv"))
-    pythonInVenv <- normalizePath(file.path(".venv", "bin", "python"), mustWork = TRUE)
-  }
-  # use venv python to install dependencies in venv
-  system2(pythonInVenv, c("-mpip", "install", "--upgrade", "pip", "wheel"))
-  system2(pythonInVenv, c("-mpip", "install", "-r", "requirements.txt"))
-}
-
 # Configure locations of REMIND input data
 # These can be located in directories on the local machine, remote directories,
 # or default directories on the cluster.

Makefile

@@ -26,16 +26,7 @@ update-renv:     ## Upgrade all pik-piam packages in your renv to the respective
 
 update-renv-all: ## Upgrade all packages (including CRAN packages) in your renv
                  ## to the respective latest release, write renv.lock archive
-                 ## Upgrade all packages in python venv, if python venv exists
 	@Rscript -e 'renv::update(exclude = "renv"); piamenv::archiveRenv()'
-	@if [ -e "./venv/bin/python" ]; then \
-		pv_maj=$$( .venv/bin/python -V | sed 's/^Python \([0-9]\).*/\1/' ); \
-		pv_min=$$( .venv/bin/python -V | sed 's/^Python [0-9]\.\([0-9]\+\).*/\1/' ); \
-		if (( 3 == $$pv_maj )) && (( 7 <= $$pv_min )) && (( $pv_min < 11 )); then \
-			.venv/bin/python -mpip install --upgrade pip wheel; \
-			.venv/bin/python -mpip install --upgrade --upgrade-strategy eager -r requirements.txt; \
-		fi \
-	fi
 
 revert-dev-packages: ## All PIK-PIAM packages that are development versions, i.e.
                      ## that have a non-zero fourth version number component, are
@@ -47,13 +38,6 @@ ensure-reqs:     ## Ensure the REMIND library requirements are fulfilled
                  ## by installing updates and new libraries as necessary. Does not
                  ## install updates unless it is required.
 	@Rscript -e 'source("scripts/start/ensureRequirementsInstalled.R"); ensureRequirementsInstalled(rerunPrompt="make ensure-reqs")'
-	@if [ -e "./venv/bin/python" ]; then \
-		pv_maj=$$( .venv/bin/python -V | sed 's/^Python \([0-9]\).*/\1/' ); \
-		pv_min=$$( .venv/bin/python -V | sed 's/^Python [0-9]\.\([0-9]\+\).*/\1/' ); \
-		if (( 3 == $$pv_maj )) && (( 7 <= $$pv_min )) && (( $pv_min < 11 )); then \
-			.venv/bin/python -mpip -qq install -r requirements.txt; \
-		fi \
-	fi
 
 archive-renv:    ## Write renv.lock into archive.
 	Rscript -e 'piamenv::archiveRenv()'
@@ -62,6 +46,45 @@ restore-renv:    ## Restore renv to the state described in interactively
                  ## selected renv.lock from the archive or a run folder.
 	Rscript -e 'piamenv::restoreRenv()'
 
+clone-conda: ## Clone the specified conda environment or the active environment to a new environment in the user's home directory or specified DEST
+	@if [ -z "$$ENV" ] && [ -z "$$CONDA_DEFAULT_ENV" ]; then \
+		echo "No Conda environment specified and no active Conda environment found."; \
+		exit 1; \
+	elif [ -z "$$ENV" ]; then \
+		ENV=$$CONDA_DEFAULT_ENV; \
+	fi; \
+	TIMESTAMP=$$(date +%Y%m%d); \
+	BASENAME=$$(basename $$ENV); \
+	DEFAULT_CONDA_ENV_DIR=~/.conda/envs; \
+	CLONE_DIR=$${DEST:-$$DEFAULT_CONDA_ENV_DIR/$$BASENAME-clone-$$TIMESTAMP}; \
+	echo "Cloning Conda environment: $$ENV"; \
+	echo "Cloning to: $$CLONE_DIR"; \
+	echo "This might take a few minutes..."; \
+	if conda create --prefix $$CLONE_DIR --clone $$ENV; then \
+		echo "Done!"; \
+		conda env export --prefix $$CLONE_DIR > $$CLONE_DIR.yml; \
+		echo "Cloned environment saved to $$CLONE_DIR.yml"; \
+	else \
+		echo "Cloning failed or was interrupted."; \
+		exit 1; \
+	fi;
+
+create-conda: ## Create a conda environment from the config/py_requirements.txt file
+	@if [ ! -f config/py_requirements.txt ]; then \
+		echo "Requirements file not found: config/py_requirements.txt"; \
+		exit 1; \
+	fi; \
+	ENV=$${ENV:-remind}; \
+	DEST=$${DEST:-$$HOME/.conda/envs}; \
+	echo "Creating Conda environment: $$DEST/$$ENV with Python 3.11"; \
+	mkdir -p $$DEST; \
+	conda create --prefix $$DEST/$$ENV python=3.11 -y; \
+	echo "Activating Conda environment: $$ENV"; \
+	. $$(conda info --base)/etc/profile.d/conda.sh && conda activate $$DEST/$$ENV && \
+	echo "Installing requirements from config/py_requirements.txt"; \
+	pip install -r config/py_requirements.txt; \
+	echo "Conda environment $$ENV created and requirements installed successfully in $$DEST with Python 3.11.";
+
 check:           ## Check if the GAMS code follows the coding etiquette
                  ## using gms::codeCheck
 	Rscript -e 'options(warn = 1); invisible(gms::codeCheck(strict = TRUE));'

config/default.cfg

@@ -109,8 +109,15 @@ cfg$action <- "ce"   #def <- ce
 # --qos, --nodes, --tasks-per-node, -job-name, --output, --mail-type, --comment.
 cfg$slurmConfig <- ""
 
-# Enable Python integration to interface with other Python-based models
-cfg$pythonEnabled <- "off"  # def <- off
+# Enable Python integration to interface with specialized, Python-based simulation models
+# Python/reticulate support is enabled per default in REMIND. If you want to disable it, set the following variable to "off".
+cfg$pythonEnabled <- "on"  
+# Examples:
+# "/p/projects/rd3mod/python/environments/scm_magicc7_hpc/" -> Recommended for users of PIK computing infrastructure
+# "~/.conda/envs/<environment name>" -> Recommended for users outside of PIK: Use a dedicated conda (= virtual Python) environment in your home directory
+# "" -> Empty string: Will use the system Python on typical Linux systems (this is discouraged)
+cfg$pythonPath <- "/p/projects/rd3mod/python/environments/scm_magicc7_hpc/"
+cfg$ignorePythonRequirements <- "false"
 
 # Should REMIND output be shown in console? (0:no, 2:logfile, 3:yes)
 cfg$logoption <- 2

config/py_requirements.txt

@@ -0,0 +1,15 @@
+# |  (C) 2006-2024 Potsdam Institute for Climate Impact Research (PIK)
+# |  authors, and contributors see CITATION.cff file. This file is part
+# |  of REMIND and licensed under AGPL-3.0-or-later. Under Section 7 of
+# |  AGPL-3.0, you are granted additional permissions described in the
+# |  REMIND License Exception, version 1.0 (see LICENSE file).
+# |  Contact: [email protected]
+# Use Gabriels version of climate_assessment
+climate-assessment @ git+https://github.com/gabriel-abrahao/climate-assessment.git@5d4d81ffed1d2af9cb463608b9a41d91ed02ea63
+
+# netcdf for HDF5 support
+netCDF4>=1.6.0
+
+# No new numpy, no old pandas
+numpy<3
+pandas>2

scripts/start/run.R

@@ -9,11 +9,6 @@ run <- function() {
 
   load("config.Rdata")
 
-  if (cfg$pythonEnabled == "on"){
-    # Set environment variables so that reticulate finds the configured Python virtual env
-    Sys.setenv(RETICULATE_PYTHON = piamenv::pythonBinPath(".venv"))
-  }
-
   # Save start time
   timeGAMSStart <- Sys.time()
 

scripts/start/submit.R

@@ -96,13 +96,6 @@ submit <- function(cfg, restart = FALSE, stopOnFolderCreateError = TRUE) {
                stdout = renvLogPath, stderr = "2>&1")
     }
 
-    if (cfg$pythonEnabled == "on") {
-      piamenv::createResultsfolderPythonVirtualEnv(normalizePath(cfg$results_folder))
-    } else {
-      # create empty .venv folder so that new venv won't be initialized automatically by .Rprofile
-      dir.create(file.path(cfg$results_folder, ".venv"))
-    }
-
     # Save the cfg (with the updated name of the result folder) into the results folder.
     # Do not save the new name of the results folder to the .RData file in REMINDs main folder, because it
     # might be needed to restart subsequent runs manually and should not contain the time stamp in this case.

start.R

@@ -301,10 +301,6 @@ if (any(c("--reprepare", "--restart") %in% flags)) {
         cfg$slurmConfig      <- slurmConfig
         if (testOneRegi_region != "") cfg$gms$c_testOneRegi_region <- testOneRegi_region
       }
-      # Make sure all python requirements are installed
-      if (cfg$pythonEnabled == "on") {
-        piamenv::updatePythonVirtualEnv()
-      }
       # Directly start runs that have a gdx file location given as path_gdx... or where this field is empty
       gdx_specified <- grepl(".gdx", cfg$files2export$start[path_gdx_list], fixed = TRUE)
       gdx_na <- is.na(cfg$files2export$start[path_gdx_list])