From 3988a49d78132df356a8a3ea9d2e8aac261ce448 Mon Sep 17 00:00:00 2001 From: Wolfgang Preimesberger Date: Fri, 25 Oct 2024 23:27:20 +0200 Subject: [PATCH] Code formatting --- README.rst | 5 +- src/ecmwf_models/cli.py | 200 +++++++++++++++-------------- src/ecmwf_models/era5/download.py | 29 ++--- src/ecmwf_models/era5/reader.py | 1 - src/ecmwf_models/era5/reshuffle.py | 18 +-- src/ecmwf_models/extract.py | 56 ++++---- src/ecmwf_models/globals.py | 12 +- src/ecmwf_models/grid.py | 20 +-- src/ecmwf_models/interface.py | 98 +++++++------- src/ecmwf_models/utils.py | 55 ++++---- 10 files changed, 252 insertions(+), 242 deletions(-) diff --git a/README.rst b/README.rst index fafc955..54bc8ae 100644 --- a/README.rst +++ b/README.rst @@ -2,11 +2,10 @@ ecmwf_models ============ - |ci| |cov| |pip| |doc| .. |ci| image:: https://github.com/TUW-GEO/ecmwf_models/actions/workflows/ci.yml/badge.svg?branch=master - :target: https://github.com/ecmwf_models/c3s_sm/actions + :target: https://github.com/TUW-GEO/ecmwf_models/actions .. |cov| image:: https://coveralls.io/repos/TUW-GEO/ecmwf_models/badge.png?branch=master :target: https://coveralls.io/r/TUW-GEO/ecmwf_models?branch=master @@ -18,7 +17,7 @@ ecmwf_models :target: https://ecmwf-models.readthedocs.io/en/latest/ -Readers and converters for data from the `ECMWF reanalysis models +Readers and converters for `ECMWF reanalysis (ERA) data `_. Written in Python. Works great in combination with `pytesmo `_. diff --git a/src/ecmwf_models/cli.py b/src/ecmwf_models/cli.py index bd2a33e..38c0cbe 100644 --- a/src/ecmwf_models/cli.py +++ b/src/ecmwf_models/cli.py @@ -2,31 +2,33 @@ from datetime import datetime from ecmwf_models.era5.download import download_and_move, download_record_extension -from ecmwf_models.utils import ( - default_variables -) +from ecmwf_models.utils import (default_variables) from ecmwf_models.era5.reshuffle import img2ts, extend_ts @click.command( "download", - context_settings={'show_default': True, - 'help_option_names': ['-h', '--help']}, + context_settings={ + 'show_default': True, + 'help_option_names': ['-h', '--help'] + }, short_help="Download ERA5 reanalysis image data between two " - "dates from the Copernicus Climate Data Store (CDS). " - "Before this program can be used, you have to " - "register an account at CDS, and should setup a `.cdsapirc` " - "file as described here: " - "https://cds.climate.copernicus.eu/how-to-api") + "dates from the Copernicus Climate Data Store (CDS). " + "Before this program can be used, you have to " + "register an account at CDS, and should setup a `.cdsapirc` " + "file as described here: " + "https://cds.climate.copernicus.eu/how-to-api") @click.argument("PATH", type=click.Path(writable=True)) @click.option( - '--start', '-s', + '--start', + '-s', type=click.STRING, default=str(datetime(1979, 1, 1)), help="First date of the period to download data for. Format: YYYY-MM-DD") @click.option( - '--end', '-e', + '--end', + '-e', type=click.STRING, default=str(datetime.now().date()), help="Last date of the period to download data for. Format: YYYY-MM-DD") @@ -36,51 +38,51 @@ type=click.STRING, default=','.join(default_variables('era5', 'short_name')), help="Name of variables to download. To download multiple variables pass " - "comma-separated names (e.g. `-v swvl1,stl1`). " - "For all available variables see the docs. ") + "comma-separated names (e.g. `-v swvl1,stl1`). " + "For all available variables see the docs. ") @click.option( "-k", "--keep_original", type=click.BOOL, default=False, help="Also keep the original image stacks as downloaded from CDS, " - "instead of deleting them after extracting individual images. " - "Default is False.") + "instead of deleting them after extracting individual images. " + "Default is False.") @click.option( "-grb", "--as_grib", type=click.BOOL, default=False, help="Download data in grib format instead of netcdf. " - "Default is False.") + "Default is False.") @click.option( "--h_steps", type=str, default="0,6,12,18", help="Temporal sampling of downloaded images. To download multiple time " - "stamps or each day, pass comma-separated values. " - "Pass a set of full hours here, like '--h_steps 0,12' to download " - "two images for each day, at 0:00 and 12:00 respectively. " - "By default, we download 6-hourly images starting at 0:00 UTC, " - "(i.e. `--h_steps 0,6,12,18`") + "stamps or each day, pass comma-separated values. " + "Pass a set of full hours here, like '--h_steps 0,12' to download " + "two images for each day, at 0:00 and 12:00 respectively. " + "By default, we download 6-hourly images starting at 0:00 UTC, " + "(i.e. `--h_steps 0,6,12,18`") @click.option( "--max_request_size", type=int, default=1000, help="Maximum number of requests to pass to the CDS API. " - "The default is 1000, but what is allowed, depends on server " - "settings. Server settings may change at some point. Change " - "accordingly here in case that 'the request is too large'. " - "A smaller number will results in smaller download chunks (slower).") + "The default is 1000, but what is allowed, depends on server " + "settings. Server settings may change at some point. Change " + "accordingly here in case that 'the request is too large'. " + "A smaller number will results in smaller download chunks (slower).") @click.option( "--cds_token", type=click.STRING, default=None, help="To identify with the CDS. Required only if no `.cdsapirc` file " - "exists in the home directory (see documentation). " - "You can find your token/key on your CDS user profile page. " - "Alternatively, you can also set an environment variable " - "`CDSAPI_KEY` with your token.") + "exists in the home directory (see documentation). " + "You can find your token/key on your CDS user profile page. " + "Alternatively, you can also set an environment variable " + "`CDSAPI_KEY` with your token.") def cli_download_era5(path, start, end, variables, keep_original, as_grib, h_steps, max_request_size, cds_token): """ @@ -121,22 +123,26 @@ def cli_download_era5(path, start, end, variables, keep_original, as_grib, @click.command( "download", - context_settings={'show_default': True, - 'help_option_names': ['-h', '--help']}, + context_settings={ + 'show_default': True, + 'help_option_names': ['-h', '--help'] + }, short_help="Download ERA5 reanalysis image data between two " - "dates from the Copernicus Climate Data Store (CDS). " - "Before this program can be used, you have to " - "register an account at CDS, and should setup a `.cdsapirc` " - "file as described here: " - "https://cds.climate.copernicus.eu/how-to-api") + "dates from the Copernicus Climate Data Store (CDS). " + "Before this program can be used, you have to " + "register an account at CDS, and should setup a `.cdsapirc` " + "file as described here: " + "https://cds.climate.copernicus.eu/how-to-api") @click.argument("PATH", type=click.Path(writable=True)) @click.option( - '--start', '-s', + '--start', + '-s', type=click.STRING, default=str(datetime(1979, 1, 1)), help="First date of the period to download data for. Format: YYYY-MM-DD") @click.option( - '--end', '-e', + '--end', + '-e', type=click.STRING, default=str(datetime.now().date()), help="Last date of the period to download data for. Format: YYYY-MM-DD") @@ -146,51 +152,51 @@ def cli_download_era5(path, start, end, variables, keep_original, as_grib, type=click.STRING, default=','.join(default_variables('era5-land', 'short_name')), help="Name of variables to download. To download multiple variables pass " - "comma-separated names (e.g. `-v swvl1,stl1`). " - "For all available variables see the docs. ") + "comma-separated names (e.g. `-v swvl1,stl1`). " + "For all available variables see the docs. ") @click.option( "-k", "--keep_original", type=click.BOOL, default=False, help="Also keep the original image stacks as downloaded from CDS, " - "instead of deleting them after extracting individual images. " - "Default is False.") + "instead of deleting them after extracting individual images. " + "Default is False.") @click.option( "-grb", "--as_grib", type=click.BOOL, default=False, help="Download data in grib format instead of netcdf. " - "Default is False.") + "Default is False.") @click.option( "--h_steps", type=click.STRING, default='0,6,12,18', help="Temporal sampling of downloaded images. To download multiple time " - "stamps or each day, pass comma-separated values. " - "Pass a set of full hours here, like '--h_steps 0,12' to download " - "two images for each day, at 0:00 and 12:00 respectively. " - "By default, we download 6-hourly images starting at 0:00 UTC, " - "(i.e. `--h_steps 0,6,12,18`") + "stamps or each day, pass comma-separated values. " + "Pass a set of full hours here, like '--h_steps 0,12' to download " + "two images for each day, at 0:00 and 12:00 respectively. " + "By default, we download 6-hourly images starting at 0:00 UTC, " + "(i.e. `--h_steps 0,6,12,18`") @click.option( "--max_request_size", type=int, default=1000, help="Maximum number of requests to pass to the CDS API. " - "The default is 1000, but what is allowed, depends on server " - "settings. Server settings may change at some point. Change " - "accordingly here in case that 'the request is too large'. " - "A smaller number will results in smaller download chunks (slower).") + "The default is 1000, but what is allowed, depends on server " + "settings. Server settings may change at some point. Change " + "accordingly here in case that 'the request is too large'. " + "A smaller number will results in smaller download chunks (slower).") @click.option( "--cds_token", type=click.STRING, default=None, help="To identify with the CDS. Required only if no `.cdsapirc` file " - "exists in the home directory (see documentation). " - "You can find your token/key on your CDS user profile page. " - "Alternatively, you can also set an environment variable " - "`CDSAPI_KEY` with your token.") + "exists in the home directory (see documentation). " + "You can find your token/key on your CDS user profile page. " + "Alternatively, you can also set an environment variable " + "`CDSAPI_KEY` with your token.") def cli_download_era5land(path, start, end, variables, keep_original, as_grib, h_steps, max_request_size, cds_token): """ @@ -224,30 +230,31 @@ def cli_download_era5land(path, start, end, variables, keep_original, as_grib, keep_original=keep_original, stepsize='month', n_max_request=max_request_size, - cds_token=cds_token - ) + cds_token=cds_token) return status_code @click.command( "update_img", - context_settings={'show_default': True, - 'help_option_names': ['-h', '--help']}, + context_settings={ + 'show_default': True, + 'help_option_names': ['-h', '--help'] + }, short_help="Extend an existing set of images by downloading new data " - "with the same settings as before." - "NOTE: First use the `era5 download` or `era5land download` " - "programs.") + "with the same settings as before." + "NOTE: First use the `era5 download` or `era5land download` " + "programs.") @click.argument("path", type=click.Path(writable=True)) @click.option( "--cds_token", type=click.STRING, default=None, help="To identify with the CDS. Required only if no `.cdsapirc` file " - "exists in the home directory (see documentation). " - "You can find your token/key on your CDS user profile page. " - "Alternatively, you can also set an environment variable " - "`CDSAPI_KEY` with your token.") + "exists in the home directory (see documentation). " + "You can find your token/key on your CDS user profile page. " + "Alternatively, you can also set an environment variable " + "`CDSAPI_KEY` with your token.") def cli_update_img(path, cds_token=None): """ Download new images from CDS to your existing local archive. Use the same @@ -272,10 +279,12 @@ def cli_update_img(path, cds_token=None): @click.command( "reshuffle", - context_settings={'show_default': True, - 'help_option_names': ['-h', '--help']}, + context_settings={ + 'show_default': True, + 'help_option_names': ['-h', '--help'] + }, short_help="Convert previously downloaded ERA5/ERA5-Land image " - "data into a time series format.") + "data into a time series format.") @click.argument("IMG_PATH", type=click.Path(readable=True)) @click.argument("TS_PATH", type=click.Path(writable=True)) @click.argument("START", type=click.STRING) @@ -286,39 +295,42 @@ def cli_update_img(path, cds_token=None): type=click.STRING, default=None, help="Subset of variables to convert. Pass comma-separated names" - "to select multiple variables (short names, as in the input images, " - "e.g. `-v swvl1,stl1`). If not specified, all variables from the " - "first image file are used.") + "to select multiple variables (short names, as in the input images, " + "e.g. `-v swvl1,stl1`). If not specified, all variables from the " + "first image file are used.") @click.option( - '--land_points', '-l', is_flag=True, show_default=True, + '--land_points', + '-l', + is_flag=True, + show_default=True, default=False, help="Store only time series for points that are over land. " - "Default is False.") + "Default is False.") @click.option( '--bbox', nargs=4, type=click.FLOAT, default=None, help="4 NUMBERS | min_lon min_lat max_lon max_lat. " - "Set Bounding Box (lower left and upper right corner) " - "of area to reshuffle (WGS84). Default is global.") + "Set Bounding Box (lower left and upper right corner) " + "of area to reshuffle (WGS84). Default is global.") @click.option( "--h_steps", type=click.STRING, default="0,6,12,18", help="Full hour time stamps of images to include in time series. " - "To select multiple, pass comma-separated values here, " - "e.g. '--h_steps 0,12' will only include data from images at " - "0:00 and 12:00 UTC and ignore all other available time stamps." - "By default, we include data for every 6th hour each day.") + "To select multiple, pass comma-separated values here, " + "e.g. '--h_steps 0,12' will only include data from images at " + "0:00 and 12:00 UTC and ignore all other available time stamps." + "By default, we include data for every 6th hour each day.") @click.option( '--imgbuffer', '-b', type=click.INT, default=50, help="Number of images to read into memory at once before " - "conversion to time series. A larger buffer means faster " - "processing but requires more memory.") + "conversion to time series. A larger buffer means faster " + "processing but requires more memory.") def cli_reshuffle(img_path, ts_path, start, end, variables, land_points, bbox, h_steps, imgbuffer): """ @@ -365,8 +377,10 @@ def cli_reshuffle(img_path, ts_path, start, end, variables, land_points, bbox, @click.command( "update_ts", - context_settings={'show_default': True, - 'help_option_names': ['-h', '--help']}, + context_settings={ + 'show_default': True, + 'help_option_names': ['-h', '--help'] + }, short_help="Append new image data to an existing time series archive.") @click.argument("TS_PATH", type=click.Path(writable=True)) @click.option( @@ -375,17 +389,17 @@ def cli_reshuffle(img_path, ts_path, start, end, variables, land_points, bbox, type=click.STRING, default=None, help="Manually specify where the image data to extend the time " - "series are located. If this is not specified, we use the previously " - "used path from the `overview.yml` file stored with the time series " - "data.") + "series are located. If this is not specified, we use the previously " + "used path from the `overview.yml` file stored with the time series " + "data.") @click.option( '--imgbuffer', '-b', type=click.INT, default=50, help="Number of images to read into memory at once before " - "conversion to time series. A larger buffer means faster " - "processing but requires more memory.") + "conversion to time series. A larger buffer means faster " + "processing but requires more memory.") def cli_extend_ts(ts_path, imgpath, imgbuffer): """ Append new image data to an existing time series archive. The archive must @@ -410,7 +424,7 @@ def cli_extend_ts(ts_path, imgpath, imgbuffer): @click.group(short_help="ERA5 Command Line Programs imported from the " - "`ecmwf_models` pip package.") + "`ecmwf_models` pip package.") def era5(): pass @@ -421,7 +435,7 @@ def era5(): @click.group(short_help="ERA5-Land Command Line Programs imported from the " - "`ecmwf_models` pip package.") + "`ecmwf_models` pip package.") def era5land(): pass diff --git a/src/ecmwf_models/era5/download.py b/src/ecmwf_models/era5/download.py index a554410..13be6d8 100644 --- a/src/ecmwf_models/era5/download.py +++ b/src/ecmwf_models/era5/download.py @@ -18,7 +18,9 @@ from ecmwf_models.utils import ( lookup, update_image_summary_file, - default_variables, split_array, check_api_ready, + default_variables, + split_array, + check_api_ready, ) from ecmwf_models.extract import save_ncs_from_nc, save_gribs_from_grib @@ -329,8 +331,7 @@ def download_and_move( n_hsteps=len(h_steps), max_req_size=n_max_request, reduce=True, - daily_request=True if stepsize == "day" else False - ) + daily_request=True if stepsize == "day" else False) logger.info(f"Request is split into {len(req_periods)} chunks") logger.info(f"Target directory {target_path}") @@ -348,8 +349,7 @@ def _download(curr_start, curr_end): fname = "{start}_{end}.{ext}".format( start=curr_start.strftime("%Y%m%d"), end=curr_end.strftime("%Y%m%d"), - ext="grb" if grb else "nc" - ) + ext="grb" if grb else "nc") dl_file = os.path.join(downloaded_data_path, fname) @@ -394,8 +394,7 @@ def _download(curr_start, curr_end): target_path, product_name=product.upper(), keep_original=keep_original, - keep_prelim=keep_prelim - ) + keep_prelim=keep_prelim) else: save_ncs_from_nc( dl_file, @@ -404,13 +403,10 @@ def _download(curr_start, curr_end): grid=grid, remap_method=remap_method, keep_original=keep_original, - keep_prelim=keep_prelim - ) + keep_prelim=keep_prelim) return status_code - - # Since we download month/month or day/day we need to # collect all the status codes to return a valid # status code for the whole time period @@ -420,8 +416,8 @@ def _download(curr_start, curr_end): 'curr_start': [p[0] for p in req_periods], 'curr_end': [p[1] for p in req_periods] }, - logger_name='cdsapi', loglevel='DEBUG' - ) + logger_name='cdsapi', + loglevel='DEBUG') # remove temporary files if not keep_original: @@ -501,6 +497,9 @@ def download_record_extension(path, dry_run=False, cds_token=None): - timedelta(days=1) # yesterday return download_and_move( - path, startdate=startdate, enddate=enddate, - cds_token=cds_token, dry_run=dry_run, + path, + startdate=startdate, + enddate=enddate, + cds_token=cds_token, + dry_run=dry_run, **props['download_settings']) diff --git a/src/ecmwf_models/era5/reader.py b/src/ecmwf_models/era5/reader.py index b364863..7992b6d 100644 --- a/src/ecmwf_models/era5/reader.py +++ b/src/ecmwf_models/era5/reader.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - """ This module contains ERA5/ERA5-Land specific child classes of the netcdf and grib base classes, that are used for reading all ecmwf products. diff --git a/src/ecmwf_models/era5/reshuffle.py b/src/ecmwf_models/era5/reshuffle.py index 020dcec..2e21e39 100644 --- a/src/ecmwf_models/era5/reshuffle.py +++ b/src/ecmwf_models/era5/reshuffle.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - """ Image to time series conversion tools. """ @@ -17,6 +16,7 @@ from ecmwf_models.grid import ERA_RegularImgGrid, ERA5_RegularImgLandGrid from ecmwf_models.utils import parse_filetype, parse_product, get_first_last_image_date + def extend_ts(ts_path, **img2ts_kwargs): """ Append any new data from the image path to the time series data. @@ -41,8 +41,7 @@ def extend_ts(ts_path, **img2ts_kwargs): "No overview.yml file found in the passed directory. " "This file is required to use the same settings to extend an " "existing record. NOTE: Use the `era5 download` or " - "`era5land download` programs first." - ) + "`era5land download` programs first.") ts_props = read_summary_yml(ts_path) @@ -59,7 +58,7 @@ def extend_ts(ts_path, **img2ts_kwargs): img2ts_kwargs[k] = v if 'enddate' not in img2ts_kwargs: - img2ts_kwargs['enddate'] = None # All available images + img2ts_kwargs['enddate'] = None # All available images img2ts(outputpath=ts_path, **img2ts_kwargs) @@ -136,8 +135,7 @@ def img2ts( if land_points: if product == "era5": - grid = ERA5_RegularImgLandGrid( - resolution=0.25, bbox=bbox) + grid = ERA5_RegularImgLandGrid(resolution=0.25, bbox=bbox) elif product == "era5-land": grid = ERA5_RegularImgLandGrid(resolution=0.1, bbox=bbox) else: @@ -187,8 +185,11 @@ def img2ts( data = input_dataset.read(first_date_time) ts_attributes = data.metadata - props = {'product': product, 'filetype': filetype, - 'last_update': datetime.now()} + props = { + 'product': product, + 'filetype': filetype, + 'last_update': datetime.now() + } kwargs = { 'input_root': input_root, @@ -225,4 +226,3 @@ def img2ts( reshuffler.calc() update_ts_summary_file(outputpath, props) - diff --git a/src/ecmwf_models/extract.py b/src/ecmwf_models/extract.py index d01e574..e5648f6 100644 --- a/src/ecmwf_models/extract.py +++ b/src/ecmwf_models/extract.py @@ -5,25 +5,26 @@ import xarray as xr from datedown.fname_creator import create_dt_fpath +from ecmwf_models.globals import (IMG_FNAME_TEMPLATE, + IMG_FNAME_DATETIME_FORMAT, EXPVER, SUBDIRS) from ecmwf_models.globals import ( - IMG_FNAME_TEMPLATE, - IMG_FNAME_DATETIME_FORMAT, - EXPVER, SUBDIRS -) -from ecmwf_models.globals import ( - Cdo, cdo_available, CdoNotFoundError, - pygrib, pygrib_available, PygribNotFoundError, + Cdo, + cdo_available, + CdoNotFoundError, + pygrib, + pygrib_available, + PygribNotFoundError, ) def save_ncs_from_nc( - input_nc, - output_path, - product_name, - grid=None, - keep_original=True, - remap_method="bil", - keep_prelim=True, + input_nc, + output_path, + product_name, + grid=None, + keep_original=True, + remap_method="bil", + keep_prelim=True, ): """ Split the downloaded netcdf file into daily files and add to folder @@ -48,8 +49,7 @@ def save_ncs_from_nc( product="{product}", type='AN', datetime=IMG_FNAME_DATETIME_FORMAT, - ext='nc' - ) + ext='nc') nc_in = xr.open_dataset(input_nc, mask_and_scale=True) if 'valid_time' in nc_in.variables: @@ -86,8 +86,8 @@ def save_ncs_from_nc( continue if len(ext) > 0: - filename_templ = _filename_templ.format( - product=product_name + '-' + ext) + filename_templ = _filename_templ.format(product=product_name + + '-' + ext) else: filename_templ = _filename_templ.format(product=product_name) @@ -130,11 +130,11 @@ def save_ncs_from_nc( def save_gribs_from_grib( - input_grib, - output_path, - product_name, - keep_original=True, - keep_prelim=True, + input_grib, + output_path, + product_name, + keep_original=True, + keep_prelim=True, ): """ Split the downloaded grib file into daily files and add to folder structure @@ -163,8 +163,7 @@ def save_gribs_from_grib( product="{product}", type='AN', datetime=IMG_FNAME_DATETIME_FORMAT, - ext='grb' - ) + ext='grb') grib_in.seek(0) prev_date = None @@ -184,14 +183,13 @@ def save_gribs_from_grib( continue if len(ext) > 0: - filename_templ = _filename_templ.format( - product=product_name + '-' + ext) + filename_templ = _filename_templ.format(product=product_name + + '-' + ext) else: filename_templ = _filename_templ.format(product=product_name) filepath = create_dt_fpath( - filedate, root=output_path, fname=filename_templ, - subdirs=SUBDIRS) + filedate, root=output_path, fname=filename_templ, subdirs=SUBDIRS) if not os.path.exists(os.path.dirname(filepath)): os.makedirs(os.path.dirname(filepath)) diff --git a/src/ecmwf_models/globals.py b/src/ecmwf_models/globals.py index eb9c535..c573416 100644 --- a/src/ecmwf_models/globals.py +++ b/src/ecmwf_models/globals.py @@ -1,7 +1,6 @@ import os from pathlib import Path - IMG_FNAME_TEMPLATE = "{product}_{type}_{datetime}.{ext}" IMG_FNAME_DATETIME_FORMAT = "%Y%m%d_%H%M" @@ -33,6 +32,7 @@ class PygribNotFoundError(ModuleNotFoundError): + def __init__(self, msg=None): _default_msg = ("pygrib could not be imported. " "Pleas run `conda install -c conda-forge pygrib` to " @@ -41,8 +41,10 @@ def __init__(self, msg=None): class CdoNotFoundError(ModuleNotFoundError): + def __init__(self, msg=None): - _default_msg = ("cdo and/or python-cdo not installed. " - "Pleas run `conda install -c conda-forge cdo` and also " - "`pip install cdo`.") - self.msg = _default_msg if msg is None else msg \ No newline at end of file + _default_msg = ( + "cdo and/or python-cdo not installed. " + "Pleas run `conda install -c conda-forge cdo` and also " + "`pip install cdo`.") + self.msg = _default_msg if msg is None else msg diff --git a/src/ecmwf_models/grid.py b/src/ecmwf_models/grid.py index 39f0087..a5fc178 100644 --- a/src/ecmwf_models/grid.py +++ b/src/ecmwf_models/grid.py @@ -9,6 +9,7 @@ from typing import Tuple import xarray as xr + def trafo_lon(lon): """ 0...360 -> 0...180...-180 @@ -27,6 +28,7 @@ def trafo_lon(lon): lon[lons_gt_180] = lon[lons_gt_180] - 360.0 return lon + def safe_arange(start, stop, step): """ Like numpy.arange, but floating point precision is kept. @@ -47,10 +49,13 @@ def safe_arange(start, stop, step): Range of values in interval at the given step size / sampling """ f_step = (1. / float(step)) - vals = np.arange(float(start) * f_step, float(stop) * f_step, - float(step) * f_step) + vals = np.arange( + float(start) * f_step, + float(stop) * f_step, + float(step) * f_step) return vals / f_step + def get_grid_resolution(lats: np.ndarray, lons: np.ndarray) -> (float, float): """ try to derive the grid resolution from given coords. @@ -113,10 +118,9 @@ def ERA5_RegularImgLandGrid( ds = ds.assign_coords({'longitude': trafo_lon(ds['longitude'].values)}) if bbox is not None: ds = ds.sel(latitude=slice(bbox[3], bbox[1])) - ds = ds.isel(longitude=np.where(( - (ds['longitude'].values >= bbox[0]) & - (ds['longitude'].values <= bbox[2]) - ))[0]) + ds = ds.isel( + longitude=np.where(((ds['longitude'].values >= bbox[0]) + & (ds['longitude'].values <= bbox[2])))[0]) land_mask = np.array(ds.values == 1.0) @@ -167,8 +171,8 @@ def ERA_RegularImgGrid( grid = grid.to_cell_grid(cellsize=5.0) if bbox is not None: - subgpis = grid.get_bbox_grid_points(latmin=bbox[1], latmax=bbox[3], - lonmin=bbox[0], lonmax=bbox[2]) + subgpis = grid.get_bbox_grid_points( + latmin=bbox[1], latmax=bbox[3], lonmin=bbox[0], lonmax=bbox[2]) grid = grid.subgrid_from_gpis(subgpis) return grid diff --git a/src/ecmwf_models/interface.py b/src/ecmwf_models/interface.py index eed3da6..3af58df 100644 --- a/src/ecmwf_models/interface.py +++ b/src/ecmwf_models/interface.py @@ -19,7 +19,6 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. - """ Base classes for reading downloaded ERA netcdf and grib images and stacks """ @@ -39,12 +38,13 @@ from ecmwf_models.grid import trafo_lon from ecmwf_models.utils import lookup from ecmwf_models.globals import ( - IMG_FNAME_TEMPLATE, IMG_FNAME_DATETIME_FORMAT, - SUPPORTED_PRODUCTS, SUBDIRS, -) -from ecmwf_models.globals import ( - pygrib, pygrib_available, PygribNotFoundError + IMG_FNAME_TEMPLATE, + IMG_FNAME_DATETIME_FORMAT, + SUPPORTED_PRODUCTS, + SUBDIRS, ) +from ecmwf_models.globals import (pygrib, pygrib_available, + PygribNotFoundError) class ERANcImg(ImageBase): @@ -75,14 +75,14 @@ class ERANcImg(ImageBase): """ def __init__( - self, - filename, - product, - parameter=None, - subgrid=None, - mask_seapoints=False, - array_1D=False, - mode='r', + self, + filename, + product, + parameter=None, + subgrid=None, + mask_seapoints=False, + array_1D=False, + mode='r', ): super(ERANcImg, self).__init__(filename, mode=mode) @@ -90,7 +90,7 @@ def __init__( if parameter is not None: # look up short names self.parameter = lookup( - product, np.atleast_1d(parameter))["short_name"].values + product, np.atleast_1d(parameter))["short_name"].values else: self.parameter = None @@ -135,8 +135,10 @@ def read(self, timestamp=None): return_img = {} return_metadata = {} - grid = gridfromdims(trafo_lon(dataset['longitude'].values), - dataset['latitude'].values, origin='top') + grid = gridfromdims( + trafo_lon(dataset['longitude'].values), + dataset['latitude'].values, + origin='top') if self.subgrid is not None: gpis = grid.find_nearest_gpi(self.subgrid.activearrlon, @@ -149,11 +151,10 @@ def read(self, timestamp=None): variable = dataset[name] except KeyError: path, f = os.path.split(self.filename) - warnings.warn( - f"Cannot load variable {name} from file {f}. " - f"Filling image with NaNs.") - dat = np.full(grid.shape if gpis is None else len(gpis), - np.nan) + warnings.warn(f"Cannot load variable {name} from file {f}. " + f"Filling image with NaNs.") + dat = np.full( + grid.shape if gpis is None else len(gpis), np.nan) return_img[name] = dat continue @@ -268,7 +269,8 @@ def __init__( if parameter is not None: # look up short names - self.parameter = lookup(product, np.atleast_1d(parameter))["short_name"].values + self.parameter = lookup( + product, np.atleast_1d(parameter))["short_name"].values else: self.parameter = None @@ -282,20 +284,18 @@ def __init__( # the goal is to use ERA5-T*.nc if necessary, but prefer ERA5*.nc self.fn_templ_priority = [ - IMG_FNAME_TEMPLATE.format(product=(p + ext).upper(), - type='*', - datetime="{datetime}", - ext='nc') - for ext in ['', '-T'] for p in SUPPORTED_PRODUCTS + IMG_FNAME_TEMPLATE.format( + product=(p + ext).upper(), + type='*', + datetime="{datetime}", + ext='nc') for ext in ['', '-T'] for p in SUPPORTED_PRODUCTS ] super(ERANcDs, self).__init__( root_path, ERANcImg, - fname_templ=IMG_FNAME_TEMPLATE.format(product='*', - type='*', - datetime='{datetime}', - ext='nc'), + fname_templ=IMG_FNAME_TEMPLATE.format( + product='*', type='*', datetime='{datetime}', ext='nc'), datetime_format=IMG_FNAME_DATETIME_FORMAT, subpath_templ=SUBDIRS, exact_templ=False, @@ -384,16 +384,14 @@ def tstamps_for_daterange(self, start_date, end_date): class ERAGrbImg(ImageBase): - def __init__( - self, - filename, - product, - parameter=None, - subgrid=None, - mask_seapoints=False, - array_1D=True, - mode='r' - ): + def __init__(self, + filename, + product, + parameter=None, + subgrid=None, + mask_seapoints=False, + array_1D=True, + mode='r'): """ Reader for a single ERA grib file. The main purpose of this class is to use it in the time series conversion routine. To read downloaded image @@ -476,8 +474,10 @@ def read(self, timestamp=None): param_data = message.values if grid is None: - grid = BasicGrid(trafo_lon(lons).flatten(), lats.flatten(), - shape=param_data.shape) + grid = BasicGrid( + trafo_lon(lons).flatten(), + lats.flatten(), + shape=param_data.shape) param_data = param_data.flatten() @@ -532,8 +532,7 @@ def read(self, timestamp=None): param_data = np.full(np.prod(self.subgrid.shape), np.nan) warnings.warn( f"Cannot load variable {param_name} from file " - f"{self.filename}. Filling image with NaNs." - ) + f"{self.filename}. Filling image with NaNs.") return_img[param_name] = param_data return_metadata[param_name] = {} return_metadata[param_name]["long_name"] = lookup( @@ -574,6 +573,7 @@ def close(self): class ERAGrbDs(MultiTemporalImageBase): + def __init__( self, root_path, @@ -627,10 +627,8 @@ def __init__( "array_1D": array_1D, } - fname_templ = IMG_FNAME_TEMPLATE.format(product="*", - type="*", - datetime="{datetime}", - ext="grb") + fname_templ = IMG_FNAME_TEMPLATE.format( + product="*", type="*", datetime="{datetime}", ext="grb") super(ERAGrbDs, self).__init__( root_path, diff --git a/src/ecmwf_models/utils.py b/src/ecmwf_models/utils.py index f132cee..7839b95 100644 --- a/src/ecmwf_models/utils.py +++ b/src/ecmwf_models/utils.py @@ -20,7 +20,6 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. - """ Utility functions for all data products in this package. """ @@ -36,10 +35,10 @@ from ecmwf_models.extract import save_gribs_from_grib from repurpose.misc import find_first_at_depth -from ecmwf_models.globals import ( - DOTRC, CDS_API_URL, IMG_FNAME_TEMPLATE, - IMG_FNAME_DATETIME_FORMAT, SUPPORTED_PRODUCTS -) +from ecmwf_models.globals import (DOTRC, CDS_API_URL, IMG_FNAME_TEMPLATE, + IMG_FNAME_DATETIME_FORMAT, + SUPPORTED_PRODUCTS) + def parse_product(inpath: str) -> str: """ @@ -63,7 +62,8 @@ def parse_product(inpath: str) -> str: elif "era5" in props['product'].lower(): return "era5" # also era5-t else: - raise ValueError(f"Could not derive product name from data in {inpath}") + raise ValueError( + f"Could not derive product name from data in {inpath}") def parse_filetype(inpath: str) -> str: @@ -176,12 +176,12 @@ def default_variables(product="era5", format='dl_name'): def make_era5_land_definition_file( - data_file, - out_file, - data_file_y_res=0.25, - ref_var="lsm", - threshold=0.5, - exclude_antarctica=True, + data_file, + out_file, + data_file_y_res=0.25, + ref_var="lsm", + threshold=0.5, + exclude_antarctica=True, ): """ Create a land grid definition file from a variable within a downloaded, @@ -215,8 +215,7 @@ def make_era5_land_definition_file( for dim_name in ds_in.dimensions.keys(): ds_out.createDimension(dim_name, size=ds_in.dimensions[dim_name].size) - ds_out.createVariable(dim_name, "float32", (dim_name,), - zlib=True) + ds_out.createVariable(dim_name, "float32", (dim_name,), zlib=True) ds_out.variables[dim_name][:] = ds_in.variables[dim_name][:] ref = ds_in.variables[ref_var] @@ -232,13 +231,12 @@ def make_era5_land_definition_file( if exclude_antarctica: cut_off_lat = -60.0 index_thres_lat = ((180.0 / data_file_y_res) + 1) - ( - (90.0 + cut_off_lat) / data_file_y_res) + (90.0 + cut_off_lat) / data_file_y_res) land_mask[int(index_thres_lat):, :] = np.nan else: cut_off_lat = None - ds_out.createVariable("land", "float32", - (lat_name, lon_name), zlib=True) + ds_out.createVariable("land", "float32", (lat_name, lon_name), zlib=True) ds_out.variables["land"][:] = land_mask land_attrs = OrderedDict([ @@ -298,8 +296,7 @@ def check_api_ready() -> bool: 'Neither CDSAPI_KEY variable nor .cdsapirc file found, ' 'download will not work! ' 'Please create a .cdsapirc file with your API key. ' - 'See: https://cds.climate.copernicus.eu/how-to-api' - ) + 'See: https://cds.climate.copernicus.eu/how-to-api') else: return True else: @@ -353,9 +350,8 @@ def get_first_last_image_date(path, start_from_last=True): Parse date from the last found image file that matches `fntempl`. """ try: - props = img_infer_file_props(path, - fntempl=IMG_FNAME_TEMPLATE, - start_from_last=start_from_last) + props = img_infer_file_props( + path, fntempl=IMG_FNAME_TEMPLATE, start_from_last=start_from_last) dt = datetime.strptime(props['datetime'], IMG_FNAME_DATETIME_FORMAT) except ValueError: raise ValueError('Could not infer date from filenames. ' @@ -382,10 +378,10 @@ def update_image_summary_file(data_path: str, If not specified, then `data_path` is used. If a file already exists, it will be overwritten. """ - first_image_date = get_first_last_image_date(data_path, - start_from_last=False) - last_image_date = get_first_last_image_date(data_path, - start_from_last=True) + first_image_date = get_first_last_image_date( + data_path, start_from_last=False) + last_image_date = get_first_last_image_date( + data_path, start_from_last=True) props = img_infer_file_props(data_path, start_from_last=False) _ = props.pop("datetime") @@ -414,7 +410,8 @@ def assert_product(product: str) -> str: return product - if __name__ == '__main__': - save_gribs_from_grib("/tmp/era5/grb/temp_downloaded/20240730_20240731.grb", - output_path='/tmp/era5/grb', product_name='ERA5') \ No newline at end of file + save_gribs_from_grib( + "/tmp/era5/grb/temp_downloaded/20240730_20240731.grb", + output_path='/tmp/era5/grb', + product_name='ERA5')