From b82e4029f4a6158ce83fa56af774855a6078fb93 Mon Sep 17 00:00:00 2001 From: yerbol-akhmetov Date: Wed, 9 Oct 2024 21:56:41 +0500 Subject: [PATCH 01/14] add absolute path to config.default.yaml --- scripts/_helpers.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index e3ceb2adf..0a8b625ff 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -36,8 +36,13 @@ # filename of the regions definition config file REGIONS_CONFIG = "regions_definition_config.yaml" +# absolute path to base directory (pypsa-earth) +BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) -def check_config_version(config, fp_config="config.default.yaml"): +# absolute path to config.default.yaml +CONFIG_DEFAULT_PATH = os.path.join(BASE_DIR, 'config.default.yaml') + +def check_config_version(config, fp_config=CONFIG_DEFAULT_PATH): """ Check that a version of the local config.yaml matches to the actual config version as defined in config.default.yaml. From 271c78ae5a2f53a0d962cf45b26df1fe72792672 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 10 Oct 2024 09:38:51 +0000 Subject: [PATCH 02/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/_helpers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 0a8b625ff..229903601 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -40,7 +40,8 @@ BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) # absolute path to config.default.yaml -CONFIG_DEFAULT_PATH = os.path.join(BASE_DIR, 'config.default.yaml') +CONFIG_DEFAULT_PATH = os.path.join(BASE_DIR, "config.default.yaml") + def check_config_version(config, fp_config=CONFIG_DEFAULT_PATH): """ From c1ffc88e12a4d954fc718e413149cfee75130a9e Mon Sep 17 00:00:00 2001 From: yerbol-akhmetov Date: Thu, 10 Oct 2024 14:42:22 +0500 Subject: [PATCH 03/14] add release notes --- doc/release_notes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 1ea2f5393..a4df9209e 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -26,6 +26,8 @@ E.g. if a new rule becomes available describe how to use it `make test` and in o * Include a dedicated cutout for Europe in bundle_config.yaml `PR #1125 `_ +* Use absolute path for `config.default.yaml` in `_helpers.py` for facilitate smooth module import `PR #1137 `_ + PyPSA-Earth 0.4.1 ================= From 7513a04135e707feb01802482d7e42daa15db084 Mon Sep 17 00:00:00 2001 From: = Date: Fri, 11 Oct 2024 15:24:34 +0000 Subject: [PATCH 04/14] download files using absolute path, not based on os.getcwd() --- scripts/build_shapes.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index 22e6b68cf..39da3a72a 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -24,6 +24,7 @@ three_2_two_digits_country, two_2_three_digits_country, two_digits_2_name_country, + BASE_DIR ) from numba import njit from numba.core import types @@ -85,7 +86,7 @@ def download_GADM(country_code, update=False, out_logging=False): GADM_url = f"https://geodata.ucdavis.edu/gadm/gadm4.1/gpkg/{GADM_filename}.gpkg" GADM_inputfile_gpkg = os.path.join( - os.getcwd(), + BASE_DIR, "data", "gadm", GADM_filename, @@ -489,7 +490,7 @@ def download_WorldPop_standard( ] WorldPop_inputfile = os.path.join( - os.getcwd(), "data", "WorldPop", WorldPop_filename + BASE_DIR, "data", "WorldPop", WorldPop_filename ) # Input filepath tif if not os.path.exists(WorldPop_inputfile) or update is True: @@ -543,7 +544,7 @@ def download_WorldPop_API( WorldPop_filename = f"{two_2_three_digits_country(country_code).lower()}_ppp_{year}_UNadj_constrained.tif" # Request to get the file WorldPop_inputfile = os.path.join( - os.getcwd(), "data", "WorldPop", WorldPop_filename + BASE_DIR, "data", "WorldPop", WorldPop_filename ) # Input filepath tif os.makedirs(os.path.dirname(WorldPop_inputfile), exist_ok=True) year_api = int(str(year)[2:]) @@ -580,11 +581,11 @@ def convert_GDP(name_file_nc, year=2015, out_logging=False): name_file_tif = name_file_nc[:-2] + "tif" # path of the nc file - GDP_nc = os.path.join(os.getcwd(), "data", "GDP", name_file_nc) # Input filepath nc + GDP_nc = os.path.join(BASE_DIR, "data", "GDP", name_file_nc) # Input filepath nc # path of the tif file GDP_tif = os.path.join( - os.getcwd(), "data", "GDP", name_file_tif + BASE_DIR, "data", "GDP", name_file_tif ) # Input filepath nc # Check if file exists, otherwise throw exception @@ -629,7 +630,7 @@ def load_GDP( # path of the nc file name_file_tif = name_file_nc[:-2] + "tif" GDP_tif = os.path.join( - os.getcwd(), "data", "GDP", name_file_tif + BASE_DIR, "data", "GDP", name_file_tif ) # Input filepath tif if update | (not os.path.exists(GDP_tif)): From 6fabdccf35e7daf69b67c84754fbc345bc573fd5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 11 Oct 2024 15:27:14 +0000 Subject: [PATCH 05/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/build_shapes.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index 39da3a72a..8d3722de6 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -19,12 +19,12 @@ import requests import xarray as xr from _helpers import ( + BASE_DIR, configure_logging, create_logger, three_2_two_digits_country, two_2_three_digits_country, two_digits_2_name_country, - BASE_DIR ) from numba import njit from numba.core import types @@ -584,9 +584,7 @@ def convert_GDP(name_file_nc, year=2015, out_logging=False): GDP_nc = os.path.join(BASE_DIR, "data", "GDP", name_file_nc) # Input filepath nc # path of the tif file - GDP_tif = os.path.join( - BASE_DIR, "data", "GDP", name_file_tif - ) # Input filepath nc + GDP_tif = os.path.join(BASE_DIR, "data", "GDP", name_file_tif) # Input filepath nc # Check if file exists, otherwise throw exception if not os.path.exists(GDP_nc): @@ -629,9 +627,7 @@ def load_GDP( # path of the nc file name_file_tif = name_file_nc[:-2] + "tif" - GDP_tif = os.path.join( - BASE_DIR, "data", "GDP", name_file_tif - ) # Input filepath tif + GDP_tif = os.path.join(BASE_DIR, "data", "GDP", name_file_tif) # Input filepath tif if update | (not os.path.exists(GDP_tif)): if out_logging: From 61a1821fa4d094275374c2824e753ecf574197e8 Mon Sep 17 00:00:00 2001 From: = Date: Sun, 13 Oct 2024 13:07:05 +0000 Subject: [PATCH 06/14] add prefix to outputs of retrieve_databundle_light if exists --- scripts/retrieve_databundle_light.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/scripts/retrieve_databundle_light.py b/scripts/retrieve_databundle_light.py index cf6e4c3b9..e37841ce8 100644 --- a/scripts/retrieve_databundle_light.py +++ b/scripts/retrieve_databundle_light.py @@ -143,7 +143,7 @@ def download_and_unzip_zenodo(config, rootpath, hot_run=True, disable_progress=F """ resource = config["category"] file_path = os.path.join(rootpath, "tempfile.zip") - destination = os.path.relpath(config["destination"]) + destination = os.path.join(PREFIX, config["destination"]) url = config["urls"]["zenodo"] if hot_run: @@ -188,7 +188,7 @@ def download_and_unzip_gdrive(config, rootpath, hot_run=True, disable_progress=F """ resource = config["category"] file_path = os.path.join(rootpath, "tempfile.zip") - destination = os.path.relpath(config["destination"]) + destination = os.path.join(PREFIX, config["destination"]) url = config["urls"]["gdrive"] # retrieve file_id from path @@ -266,7 +266,7 @@ def download_and_unzip_protectedplanet( """ resource = config["category"] file_path = os.path.join(rootpath, "tempfile_wpda.zip") - destination = os.path.relpath(config["destination"]) + destination = os.path.join(PREFIX, config["destination"]) url = config["urls"]["protectedplanet"] def get_first_day_of_month(date): @@ -438,7 +438,7 @@ def download_and_unzip_direct(config, rootpath, hot_run=True, disable_progress=F True when download is successful, False otherwise """ resource = config["category"] - destination = os.path.relpath(config["destination"]) + destination = os.path.join(PREFIX, config["destination"]) url = config["urls"]["direct"] file_path = os.path.join(destination, os.path.basename(url)) @@ -492,7 +492,7 @@ def download_and_unzip_hydrobasins( True when download is successful, False otherwise """ resource = config["category"] - destination = os.path.relpath(config["destination"]) + destination = os.path.join(PREFIX, config["destination"]) url_templ = config["urls"]["hydrobasins"]["base_url"] suffix_list = config["urls"]["hydrobasins"]["suffixes"] @@ -543,7 +543,7 @@ def download_and_unzip_post(config, rootpath, hot_run=True, disable_progress=Fal True when download is successful, False otherwise """ resource = config["category"] - destination = os.path.relpath(config["destination"]) + destination = os.path.join(PREFIX, config["destination"]) # load data for post method postdata = config["urls"]["post"] @@ -792,8 +792,8 @@ def datafiles_retrivedatabundle(config): def merge_hydrobasins_shape(config_hydrobasin, hydrobasins_level): - basins_path = config_hydrobasin["destination"] - output_fl = config_hydrobasin["output"][0] + basins_path = os.path.join(PREFIX, config_hydrobasin["destination"]) + output_fl = os.path.join(PREFIX, config_hydrobasin["output"][0]) files_to_merge = [ "hybas_{0:s}_lev{1:02d}_v1c.shp".format(suffix, hydrobasins_level) @@ -824,6 +824,9 @@ def merge_hydrobasins_shape(config_hydrobasin, hydrobasins_level): countries = snakemake.params.countries logger.info(f"Retrieving data for {len(countries)} countries.") + # get prefix (exists only if importing rule as import, else empty string) + PREFIX = snakemake.log[0].split("log")[0] + # load enable configuration config_enable = snakemake.config["enable"] # load databundle configuration From 722e9cfefc1d04f19423de07e1d79a86b03673de Mon Sep 17 00:00:00 2001 From: = Date: Tue, 15 Oct 2024 13:52:05 +0000 Subject: [PATCH 07/14] use absolute path for osm data download --- scripts/download_osm_data.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/download_osm_data.py b/scripts/download_osm_data.py index c92fdc2b4..f0b77a3bf 100644 --- a/scripts/download_osm_data.py +++ b/scripts/download_osm_data.py @@ -30,7 +30,7 @@ import shutil from pathlib import Path -from _helpers import configure_logging, create_logger, read_osm_config +from _helpers import configure_logging, create_logger, read_osm_config, BASE_DIR from earth_osm import eo logger = create_logger(__name__) @@ -99,8 +99,8 @@ def convert_iso_to_geofk( run = snakemake.config.get("run", {}) RDIR = run["name"] + "/" if run.get("name") else "" - store_path_resources = Path.joinpath(Path().cwd(), "resources", RDIR, "osm", "raw") - store_path_data = Path.joinpath(Path().cwd(), "data", "osm") + store_path_resources = Path.joinpath(Path(BASE_DIR), "resources", RDIR, "osm", "raw") + store_path_data = Path.joinpath(Path(BASE_DIR), "data", "osm") country_list = country_list_to_geofk(snakemake.params.countries) eo.save_osm_data( From 02f63b198999840759b779eb7ae7095c727835e1 Mon Sep 17 00:00:00 2001 From: yerbol-akhmetov Date: Sat, 19 Oct 2024 21:30:36 +0500 Subject: [PATCH 08/14] define prefix in _helpers --- scripts/_helpers.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 5b1ca8457..d8395fd32 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -36,11 +36,13 @@ # filename of the regions definition config file REGIONS_CONFIG = "regions_definition_config.yaml" -# absolute path to base directory (pypsa-earth) +# prefix when running pypsa-earth rules in different directories (if running in pypsa-earth directory PREFIX='.') BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) +RUN_DIR = os.path.abspath(os.getcwd()) +PREFIX = os.path.relpath(BASE_DIR, RUN_DIR) # absolute path to config.default.yaml -CONFIG_DEFAULT_PATH = os.path.join(BASE_DIR, "config.default.yaml") +CONFIG_DEFAULT_PATH = os.path.join(PREFIX, "config.default.yaml") def check_config_version(config, fp_config=CONFIG_DEFAULT_PATH): From 7cee0cd80f644ae03b1707bd39dd1214dbf961ed Mon Sep 17 00:00:00 2001 From: yerbol-akhmetov Date: Sat, 19 Oct 2024 21:31:26 +0500 Subject: [PATCH 09/14] import PREFIX in retrieve_databundle_light --- scripts/retrieve_databundle_light.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/retrieve_databundle_light.py b/scripts/retrieve_databundle_light.py index e37841ce8..307fb5883 100644 --- a/scripts/retrieve_databundle_light.py +++ b/scripts/retrieve_databundle_light.py @@ -93,6 +93,7 @@ create_country_list, create_logger, progress_retrieve, + PREFIX ) from google_drive_downloader import GoogleDriveDownloader as gdd from tqdm import tqdm @@ -824,9 +825,6 @@ def merge_hydrobasins_shape(config_hydrobasin, hydrobasins_level): countries = snakemake.params.countries logger.info(f"Retrieving data for {len(countries)} countries.") - # get prefix (exists only if importing rule as import, else empty string) - PREFIX = snakemake.log[0].split("log")[0] - # load enable configuration config_enable = snakemake.config["enable"] # load databundle configuration From 4058cc66ee9ff709b53a739add6a329727157f03 Mon Sep 17 00:00:00 2001 From: yerbol-akhmetov Date: Sat, 19 Oct 2024 21:37:52 +0500 Subject: [PATCH 10/14] add PREFIX to search for .nc or .csv file in PREFIX/data --- scripts/build_demand_profiles.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index 51f1193c0..ed030f808 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -50,7 +50,7 @@ import pypsa import scipy.sparse as sparse import xarray as xr -from _helpers import configure_logging, create_logger, read_csv_nafix, read_osm_config +from _helpers import configure_logging, create_logger, read_csv_nafix, read_osm_config, PREFIX from shapely.prepared import prep from shapely.validation import make_valid @@ -121,7 +121,7 @@ def get_load_paths_gegis(ssp_parentfolder, config): for continent in region_load: sel_ext = ".nc" for ext in [".nc", ".csv"]: - load_path = os.path.join(str(load_dir), str(continent) + str(ext)) + load_path = os.path.join(PREFIX, str(load_dir), str(continent) + str(ext)) if os.path.exists(load_path): sel_ext = ext break From 0f784cbc8eeade356490b412fa0d6171b163229c Mon Sep 17 00:00:00 2001 From: = Date: Thu, 24 Oct 2024 14:00:11 +0000 Subject: [PATCH 11/14] revert to BASE_DIR usage --- scripts/_helpers.py | 6 ++---- scripts/build_demand_profiles.py | 4 ++-- scripts/build_renewable_profiles.py | 4 ++-- scripts/retrieve_databundle_light.py | 18 +++++++++--------- 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index d8395fd32..253f7b9d1 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -36,13 +36,11 @@ # filename of the regions definition config file REGIONS_CONFIG = "regions_definition_config.yaml" -# prefix when running pypsa-earth rules in different directories (if running in pypsa-earth directory PREFIX='.') +# prefix when running pypsa-earth rules in different directories (if running in pypsa-earth as subworkflow) BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) -RUN_DIR = os.path.abspath(os.getcwd()) -PREFIX = os.path.relpath(BASE_DIR, RUN_DIR) # absolute path to config.default.yaml -CONFIG_DEFAULT_PATH = os.path.join(PREFIX, "config.default.yaml") +CONFIG_DEFAULT_PATH = os.path.join(BASE_DIR, "config.default.yaml") def check_config_version(config, fp_config=CONFIG_DEFAULT_PATH): diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index ed030f808..6e843e104 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -50,7 +50,7 @@ import pypsa import scipy.sparse as sparse import xarray as xr -from _helpers import configure_logging, create_logger, read_csv_nafix, read_osm_config, PREFIX +from _helpers import configure_logging, create_logger, read_csv_nafix, read_osm_config, BASE_DIR from shapely.prepared import prep from shapely.validation import make_valid @@ -121,7 +121,7 @@ def get_load_paths_gegis(ssp_parentfolder, config): for continent in region_load: sel_ext = ".nc" for ext in [".nc", ".csv"]: - load_path = os.path.join(PREFIX, str(load_dir), str(continent) + str(ext)) + load_path = os.path.join(BASE_DIR, str(load_dir), str(continent) + str(ext)) if os.path.exists(load_path): sel_ext = ext break diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 1ebf220b4..4fcc80d9a 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -202,7 +202,7 @@ import pandas as pd import progressbar as pgb import xarray as xr -from _helpers import configure_logging, create_logger +from _helpers import configure_logging, create_logger, BASE_DIR from add_electricity import load_powerplants from dask.distributed import Client from pypsa.geo import haversine @@ -559,7 +559,7 @@ def create_scaling_factor( # filter plants for hydro if snakemake.wildcards.technology.startswith("hydro"): country_shapes = gpd.read_file(paths.country_shapes) - hydrobasins = gpd.read_file(resource["hydrobasins"]) + hydrobasins = gpd.read_file(os.path.join(BASE_DIR, resource["hydrobasins"])) ppls = load_powerplants(snakemake.input.powerplants) hydro_ppls = ppls[ppls.carrier == "hydro"] diff --git a/scripts/retrieve_databundle_light.py b/scripts/retrieve_databundle_light.py index 307fb5883..ed5ea9055 100644 --- a/scripts/retrieve_databundle_light.py +++ b/scripts/retrieve_databundle_light.py @@ -93,7 +93,7 @@ create_country_list, create_logger, progress_retrieve, - PREFIX + BASE_DIR ) from google_drive_downloader import GoogleDriveDownloader as gdd from tqdm import tqdm @@ -144,7 +144,7 @@ def download_and_unzip_zenodo(config, rootpath, hot_run=True, disable_progress=F """ resource = config["category"] file_path = os.path.join(rootpath, "tempfile.zip") - destination = os.path.join(PREFIX, config["destination"]) + destination = os.path.join(BASE_DIR, config["destination"]) url = config["urls"]["zenodo"] if hot_run: @@ -189,7 +189,7 @@ def download_and_unzip_gdrive(config, rootpath, hot_run=True, disable_progress=F """ resource = config["category"] file_path = os.path.join(rootpath, "tempfile.zip") - destination = os.path.join(PREFIX, config["destination"]) + destination = os.path.join(BASE_DIR, config["destination"]) url = config["urls"]["gdrive"] # retrieve file_id from path @@ -267,7 +267,7 @@ def download_and_unzip_protectedplanet( """ resource = config["category"] file_path = os.path.join(rootpath, "tempfile_wpda.zip") - destination = os.path.join(PREFIX, config["destination"]) + destination = os.path.join(BASE_DIR, config["destination"]) url = config["urls"]["protectedplanet"] def get_first_day_of_month(date): @@ -439,7 +439,7 @@ def download_and_unzip_direct(config, rootpath, hot_run=True, disable_progress=F True when download is successful, False otherwise """ resource = config["category"] - destination = os.path.join(PREFIX, config["destination"]) + destination = os.path.join(BASE_DIR, config["destination"]) url = config["urls"]["direct"] file_path = os.path.join(destination, os.path.basename(url)) @@ -493,7 +493,7 @@ def download_and_unzip_hydrobasins( True when download is successful, False otherwise """ resource = config["category"] - destination = os.path.join(PREFIX, config["destination"]) + destination = os.path.join(BASE_DIR, config["destination"]) url_templ = config["urls"]["hydrobasins"]["base_url"] suffix_list = config["urls"]["hydrobasins"]["suffixes"] @@ -544,7 +544,7 @@ def download_and_unzip_post(config, rootpath, hot_run=True, disable_progress=Fal True when download is successful, False otherwise """ resource = config["category"] - destination = os.path.join(PREFIX, config["destination"]) + destination = os.path.join(BASE_DIR, config["destination"]) # load data for post method postdata = config["urls"]["post"] @@ -793,8 +793,8 @@ def datafiles_retrivedatabundle(config): def merge_hydrobasins_shape(config_hydrobasin, hydrobasins_level): - basins_path = os.path.join(PREFIX, config_hydrobasin["destination"]) - output_fl = os.path.join(PREFIX, config_hydrobasin["output"][0]) + basins_path = os.path.join(BASE_DIR, config_hydrobasin["destination"]) + output_fl = os.path.join(BASE_DIR, config_hydrobasin["output"][0]) files_to_merge = [ "hybas_{0:s}_lev{1:02d}_v1c.shp".format(suffix, hydrobasins_level) From e1c6ddf856e3d8c09813c713a20c5eaa7258a7c2 Mon Sep 17 00:00:00 2001 From: yerbol-akhmetov Date: Sat, 26 Oct 2024 18:23:16 +0500 Subject: [PATCH 12/14] BASE_DIR to prepare_network for emission data --- scripts/prepare_network.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 9106fc90d..a008611cc 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -65,7 +65,7 @@ import pandas as pd import pypsa import requests -from _helpers import configure_logging, create_logger +from _helpers import configure_logging, create_logger, BASE_DIR from add_electricity import load_costs, update_transmission_costs idx = pd.IndexSlice @@ -85,11 +85,11 @@ def download_emission_data(): try: url = "https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/EDGAR/datasets/v60_GHG/CO2_excl_short-cycle_org_C/v60_GHG_CO2_excl_short-cycle_org_C_1970_2018.zip" with requests.get(url) as rq: - with open("data/co2.zip", "wb") as file: + with open(os.path.join(BASE_DIR, "data/co2.zip"), "wb") as file: file.write(rq.content) - file_path = "data/co2.zip" + file_path = os.path.join(BASE_DIR, "data/co2.zip") with ZipFile(file_path, "r") as zipObj: - zipObj.extract("v60_CO2_excl_short-cycle_org_C_1970_2018.xls", "data") + zipObj.extract("v60_CO2_excl_short-cycle_org_C_1970_2018.xls", os.path.join(BASE_DIR, "data")) os.remove(file_path) return "v60_CO2_excl_short-cycle_org_C_1970_2018.xls" except: @@ -117,7 +117,7 @@ def emission_extractor(filename, emission_year, country_names): """ # data reading process - datapath = os.path.join(os.getcwd(), "data", filename) + datapath = os.path.join(BASE_DIR, "data", filename) df = pd.read_excel(datapath, sheet_name="v6.0_EM_CO2_fossil_IPCC1996", skiprows=8) df.columns = df.iloc[0] df = df.set_index("Country_code_A3") From 46f60e38bc0c5995f00182c5a4f52abbc2dc3d15 Mon Sep 17 00:00:00 2001 From: yerbol-akhmetov Date: Sun, 27 Oct 2024 10:36:03 +0500 Subject: [PATCH 13/14] store config.yaml using absolute path --- scripts/_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 253f7b9d1..ce97f6171 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -93,7 +93,7 @@ def handle_exception(exc_type, exc_value, exc_traceback): def copy_default_files(): - fn = Path("config.yaml") + fn = Path(os.path.join(BASE_DIR, "config.yaml")) if not fn.exists(): fn.write_text( "# Write down config entries differing from config.default.yaml\n\nrun: {}" From c161e5968de9efc1345fb56c8fb693a060452a3f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 29 Oct 2024 08:50:08 +0000 Subject: [PATCH 14/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/build_demand_profiles.py | 8 +++++++- scripts/build_renewable_profiles.py | 2 +- scripts/download_osm_data.py | 6 ++++-- scripts/prepare_network.py | 7 +++++-- scripts/retrieve_databundle_light.py | 2 +- 5 files changed, 18 insertions(+), 7 deletions(-) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index 6e843e104..3686364b2 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -50,7 +50,13 @@ import pypsa import scipy.sparse as sparse import xarray as xr -from _helpers import configure_logging, create_logger, read_csv_nafix, read_osm_config, BASE_DIR +from _helpers import ( + BASE_DIR, + configure_logging, + create_logger, + read_csv_nafix, + read_osm_config, +) from shapely.prepared import prep from shapely.validation import make_valid diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 4fcc80d9a..2dce6032e 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -202,7 +202,7 @@ import pandas as pd import progressbar as pgb import xarray as xr -from _helpers import configure_logging, create_logger, BASE_DIR +from _helpers import BASE_DIR, configure_logging, create_logger from add_electricity import load_powerplants from dask.distributed import Client from pypsa.geo import haversine diff --git a/scripts/download_osm_data.py b/scripts/download_osm_data.py index f0b77a3bf..c327a7ae4 100644 --- a/scripts/download_osm_data.py +++ b/scripts/download_osm_data.py @@ -30,7 +30,7 @@ import shutil from pathlib import Path -from _helpers import configure_logging, create_logger, read_osm_config, BASE_DIR +from _helpers import BASE_DIR, configure_logging, create_logger, read_osm_config from earth_osm import eo logger = create_logger(__name__) @@ -99,7 +99,9 @@ def convert_iso_to_geofk( run = snakemake.config.get("run", {}) RDIR = run["name"] + "/" if run.get("name") else "" - store_path_resources = Path.joinpath(Path(BASE_DIR), "resources", RDIR, "osm", "raw") + store_path_resources = Path.joinpath( + Path(BASE_DIR), "resources", RDIR, "osm", "raw" + ) store_path_data = Path.joinpath(Path(BASE_DIR), "data", "osm") country_list = country_list_to_geofk(snakemake.params.countries) diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index a008611cc..47d847b78 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -65,7 +65,7 @@ import pandas as pd import pypsa import requests -from _helpers import configure_logging, create_logger, BASE_DIR +from _helpers import BASE_DIR, configure_logging, create_logger from add_electricity import load_costs, update_transmission_costs idx = pd.IndexSlice @@ -89,7 +89,10 @@ def download_emission_data(): file.write(rq.content) file_path = os.path.join(BASE_DIR, "data/co2.zip") with ZipFile(file_path, "r") as zipObj: - zipObj.extract("v60_CO2_excl_short-cycle_org_C_1970_2018.xls", os.path.join(BASE_DIR, "data")) + zipObj.extract( + "v60_CO2_excl_short-cycle_org_C_1970_2018.xls", + os.path.join(BASE_DIR, "data"), + ) os.remove(file_path) return "v60_CO2_excl_short-cycle_org_C_1970_2018.xls" except: diff --git a/scripts/retrieve_databundle_light.py b/scripts/retrieve_databundle_light.py index ed5ea9055..297599d4a 100644 --- a/scripts/retrieve_databundle_light.py +++ b/scripts/retrieve_databundle_light.py @@ -89,11 +89,11 @@ import pandas as pd import yaml from _helpers import ( + BASE_DIR, configure_logging, create_country_list, create_logger, progress_retrieve, - BASE_DIR ) from google_drive_downloader import GoogleDriveDownloader as gdd from tqdm import tqdm