diff --git a/.cruft.json b/.cruft.json index 9ef0d39..3f3c44e 100644 --- a/.cruft.json +++ b/.cruft.json @@ -1,6 +1,6 @@ { "template": "https://github.com/ecmwf-projects/cookiecutter-conda-package", - "commit": "c7cd369f71b4cfc647fc7c1972cc3c45860256d5", + "commit": "c4101cc46ee7312553f246d8806aebc7160edcc4", "checkout": null, "context": { "cookiecutter": { diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3cbce5d..f63d7ca 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,10 +41,10 @@ repos: args: [--autofix] additional_dependencies: [toml-sort<0.22.0] - repo: https://github.com/PyCQA/pydocstyle.git - rev: 6.1.1 + rev: 6.2.2 hooks: - id: pydocstyle - additional_dependencies: [toml] + additional_dependencies: [tomli] exclude: tests|docs - repo: https://github.com/nbQA-dev/nbQA rev: 1.5.3 diff --git a/notebooks/climate_projections/01-Application_Template_Global_Timeseries_Monthly.ipynb b/notebooks/climate_projections/01-Application_Template_Global_Timeseries_Monthly.ipynb new file mode 100644 index 0000000..ca70777 --- /dev/null +++ b/notebooks/climate_projections/01-Application_Template_Global_Timeseries_Monthly.ipynb @@ -0,0 +1,189 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Climate Projections Monthly: Global timeseries" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Import libraries\n", + "\n", + "Switch warnings off for better readability." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import warnings\n", + "\n", + "from c3s_eqc_automatic_quality_control import diagnostics, download, plot\n", + "\n", + "warnings.filterwarnings(\"ignore\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define request" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "collection_id = \"projections-cmip6\"\n", + "\n", + "year_start = 2008\n", + "year_stop = 2009\n", + "\n", + "request = {\n", + " \"format\": \"zip\",\n", + " \"temporal_resolution\": \"monthly\",\n", + " \"experiment\": \"historical\",\n", + " \"variable\": \"near_surface_air_temperature\",\n", + " \"model\": \"cmcc_cm2_sr5\",\n", + " \"year\": [str(year) for year in range(year_start, year_stop + 1)],\n", + " \"month\": [f\"{month:02d}\" for month in range(1, 12 + 1)],\n", + "}" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define transform function\n", + "\n", + "Drop bounds as they are not used and create issues with dask" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def drop_bounds_and_spatial_weighted_mean(ds):\n", + " return diagnostics.spatial_weighted_mean(ds.drop_dims(\"bnds\"))\n", + "\n", + "\n", + "def drop_bounds_and_spatial_weighted_std(ds):\n", + " return diagnostics.spatial_weighted_std(ds.drop_dims(\"bnds\"))" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compute spatially-weighted global mean timeseries" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ds_mean = download.download_and_transform(\n", + " collection_id,\n", + " request,\n", + " transform_func=drop_bounds_and_spatial_weighted_mean,\n", + " split_all=True,\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compute spatially-weighted global std timeseries" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ds_std = download.download_and_transform(\n", + " collection_id,\n", + " request,\n", + " transform_func=drop_bounds_and_spatial_weighted_std,\n", + " split_all=True,\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Plot and save figure" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "filename = \"_\".join(\n", + " [\"global-timeseries\", collection_id, request[\"temporal_resolution\"]]\n", + ")\n", + "title = filename.replace(\"-\", \" \").replace(\"_\", \" \").title()\n", + "fig = plot.shaded_std(\"tas\", ds_mean, ds_std, title=title)\n", + "fig.show()\n", + "fig.write_image(filename + \".png\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "eqc", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.8" + }, + "vscode": { + "interpreter": { + "hash": "39a16a1176456aec0710d6d8dd097fdfd8eece03838aebbaaddfca0f16ac2477" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/climate_projections/02-Application_Template_Global_Timeseries_Daily.ipynb b/notebooks/climate_projections/02-Application_Template_Global_Timeseries_Daily.ipynb new file mode 100644 index 0000000..a38fee5 --- /dev/null +++ b/notebooks/climate_projections/02-Application_Template_Global_Timeseries_Daily.ipynb @@ -0,0 +1,204 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Climate Projections Daily: Global timeseries" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Import libraries\n", + "\n", + "Switch warnings off for better readability." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import warnings\n", + "\n", + "from c3s_eqc_automatic_quality_control import diagnostics, download, plot\n", + "\n", + "warnings.filterwarnings(\"ignore\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define request" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "collection_id = \"projections-cmip6\"\n", + "request = {\n", + " \"format\": \"zip\",\n", + " \"temporal_resolution\": \"daily\",\n", + " \"experiment\": \"historical\",\n", + " \"variable\": \"near_surface_air_temperature\",\n", + " \"model\": \"cmcc_cm2_sr5\",\n", + "}\n", + "start = \"2008-01\"\n", + "stop = \"2008-03\"" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Generate requests" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "requests = download.update_request_date(\n", + " request, start=start, stop=stop, stringify_dates=True\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define transform function\n", + "\n", + "Drop bounds as they are not used and create issues with dask" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def drop_bounds_and_spatial_weighted_mean(ds):\n", + " return diagnostics.spatial_weighted_mean(ds.drop_dims(\"bnds\"))\n", + "\n", + "\n", + "def drop_bounds_and_spatial_weighted_std(ds):\n", + " return diagnostics.spatial_weighted_std(ds.drop_dims(\"bnds\"))" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compute spatially-weighted global mean timeseries" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ds_mean = download.download_and_transform(\n", + " collection_id,\n", + " requests,\n", + " transform_func=drop_bounds_and_spatial_weighted_mean,\n", + " chunks={\"year\": 1, \"month\": 1},\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compute spatially-weighted global std timeseries" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ds_std = download.download_and_transform(\n", + " collection_id,\n", + " requests,\n", + " transform_func=drop_bounds_and_spatial_weighted_std,\n", + " chunks={\"year\": 1, \"month\": 1},\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Plot and save figure" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "filename = \"_\".join(\n", + " [\"global-timeseries\", collection_id, request[\"temporal_resolution\"]]\n", + ")\n", + "title = filename.replace(\"-\", \" \").replace(\"_\", \" \").title()\n", + "fig = plot.shaded_std(\"tas\", ds_mean, ds_std, title=title)\n", + "fig.show()\n", + "fig.write_image(filename + \".png\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "eqc", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.8" + }, + "vscode": { + "interpreter": { + "hash": "39a16a1176456aec0710d6d8dd097fdfd8eece03838aebbaaddfca0f16ac2477" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/renalysis/01-Application_Template_Global_Timeseries_Single_Level.ipynb b/notebooks/renalysis/01-Application_Template_Global_Timeseries_Single_Level.ipynb index 6040d98..27aff78 100644 --- a/notebooks/renalysis/01-Application_Template_Global_Timeseries_Single_Level.ipynb +++ b/notebooks/renalysis/01-Application_Template_Global_Timeseries_Single_Level.ipynb @@ -113,7 +113,7 @@ " collection_id,\n", " requests,\n", " transform_func=diagnostics.spatial_weighted_std,\n", - " chunks={\"month\": 1},\n", + " chunks={\"year\": 1, \"month\": 1},\n", ")" ] }, diff --git a/notebooks/satellite/01-Application_Template_Global_Timeseries_Monthly.ipynb b/notebooks/satellite/01-Application_Template_Global_Timeseries_Monthly.ipynb new file mode 100644 index 0000000..4b31fdc --- /dev/null +++ b/notebooks/satellite/01-Application_Template_Global_Timeseries_Monthly.ipynb @@ -0,0 +1,201 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Satellite Monthly: Global timeseries" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Import libraries\n", + "\n", + "Switch warnings off for better readability." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import warnings\n", + "\n", + "import pandas as pd\n", + "from c3s_eqc_automatic_quality_control import diagnostics, download, plot\n", + "\n", + "warnings.filterwarnings(\"ignore\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define request" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "collection_id = \"satellite-aerosol-properties\"\n", + "\n", + "year_start = 2018\n", + "year_stop = 2019\n", + "\n", + "request = {\n", + " \"format\": \"zip\",\n", + " \"time_aggregation\": \"monthly_average\",\n", + " \"variable\": \"aerosol_optical_depth\",\n", + " \"sensor_on_satellite\": \"slstr_on_sentinel_3a\",\n", + " \"algorithm\": \"ens\",\n", + " \"year\": [str(year) for year in range(year_start, year_stop + 1)],\n", + " \"month\": [f\"{month:02d}\" for month in range(1, 12 + 1)],\n", + " \"version\": \"v2.1\",\n", + "}" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define transform function:\n", + "\n", + "`source_to_time` is a temporary workaround.\n", + "The harmonisation of satellite data has not been implemented in the toolbox yet.\n", + "Therefore, the toolbox is not able to concatenate satellite data.\n", + "\n", + "In EQC we concatenate satellite data along the dimension `source`, which corresponds to the name of the source file.\n", + "Then, we use the name of the source file to infer the `time` dimension.\n", + "Unfortunately, there are several naming conventions and this step cannot be automatised." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def source_to_time_monthly(ds):\n", + " # Naming convention: YYYYMM-*.nc\n", + " ds[\"source\"] = pd.to_datetime(ds[\"source\"].str.slice(None, 6), format=\"%Y%m\")\n", + " return ds.rename(source=\"time\")\n", + "\n", + "\n", + "def source_to_time_monthly_and_spatial_weighted_mean(ds):\n", + " return diagnostics.spatial_weighted_mean(source_to_time_monthly(ds))\n", + "\n", + "\n", + "def source_to_time_monthly_and_spatial_weighted_std(ds):\n", + " return diagnostics.spatial_weighted_std(source_to_time_monthly(ds))" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compute spatially-weighted global mean timeseries" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ds_mean = download.download_and_transform(\n", + " collection_id,\n", + " request,\n", + " transform_func=source_to_time_monthly_and_spatial_weighted_mean,\n", + " split_all=True,\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compute spatially-weighted global std timeseries" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ds_std = download.download_and_transform(\n", + " collection_id,\n", + " request,\n", + " transform_func=source_to_time_monthly_and_spatial_weighted_std,\n", + " split_all=True,\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Plot and save figure" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "filename = \"_\".join([\"global-timeseries\", collection_id, request[\"time_aggregation\"]])\n", + "title = filename.replace(\"-\", \" \").replace(\"_\", \" \").title()\n", + "fig = plot.shaded_std(\"AOD550\", ds_mean, ds_std, title=title)\n", + "fig.show()\n", + "fig.write_image(filename + \".png\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "eqc", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.8 | packaged by conda-forge | (main, Nov 22 2022, 08:27:35) [Clang 14.0.6 ]" + }, + "vscode": { + "interpreter": { + "hash": "39a16a1176456aec0710d6d8dd097fdfd8eece03838aebbaaddfca0f16ac2477" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/satellite/02-Application_Template_Global_Timeseries_Daily.ipynb b/notebooks/satellite/02-Application_Template_Global_Timeseries_Daily.ipynb new file mode 100644 index 0000000..7eb1167 --- /dev/null +++ b/notebooks/satellite/02-Application_Template_Global_Timeseries_Daily.ipynb @@ -0,0 +1,216 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Satellite Daily: Global timeseries" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Import libraries\n", + "\n", + "Switch warnings off for better readability." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import warnings\n", + "\n", + "import pandas as pd\n", + "from c3s_eqc_automatic_quality_control import diagnostics, download, plot\n", + "\n", + "warnings.filterwarnings(\"ignore\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define request" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "collection_id = \"satellite-aerosol-properties\"\n", + "request = {\n", + " \"format\": \"zip\",\n", + " \"time_aggregation\": \"daily_average\",\n", + " \"variable\": \"aerosol_optical_depth\",\n", + " \"sensor_on_satellite\": \"slstr_on_sentinel_3a\",\n", + " \"algorithm\": \"ens\",\n", + " \"version\": \"v2.1\",\n", + "}\n", + "start = \"2018-01\"\n", + "stop = \"2018-03\"" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Generate requests" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "requests = download.update_request_date(\n", + " request, start=start, stop=stop, stringify_dates=True\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define transform function:\n", + "\n", + "`source_to_time` is a temporary workaround.\n", + "The harmonisation of satellite data has not been implemented in the toolbox yet.\n", + "Therefore, the toolbox is not able to concatenate satellite data.\n", + "\n", + "In EQC we concatenate satellite data along the dimension `source`, which corresponds to the name of the source file.\n", + "Then, we use the name of the source file to infer the `time` dimension.\n", + "Unfortunately, there are several naming conventions and this step cannot be automatised." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def source_to_time_daily(ds):\n", + " # Naming convention: YYYYMMDD-*.nc\n", + " ds[\"source\"] = pd.to_datetime(ds[\"source\"].str.slice(None, 8), format=\"%Y%m%d\")\n", + " return ds.rename(source=\"time\")\n", + "\n", + "\n", + "def source_to_time_daily_and_spatial_weighted_mean(ds):\n", + " return diagnostics.spatial_weighted_mean(source_to_time_daily(ds))\n", + "\n", + "\n", + "def source_to_time_daily_and_spatial_weighted_std(ds):\n", + " return diagnostics.spatial_weighted_std(source_to_time_daily(ds))" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compute spatially-weighted global mean timeseries" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ds_mean = download.download_and_transform(\n", + " collection_id,\n", + " requests,\n", + " transform_func=source_to_time_daily_and_spatial_weighted_std,\n", + " chunks={\"year\": 1, \"month\": 1},\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compute spatially-weighted global std timeseries" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ds_std = download.download_and_transform(\n", + " collection_id,\n", + " requests,\n", + " transform_func=source_to_time_daily_and_spatial_weighted_std,\n", + " chunks={\"year\": 1, \"month\": 1},\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Plot and save figure" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "filename = \"_\".join([\"global-timeseries\", collection_id, request[\"time_aggregation\"]])\n", + "title = filename.replace(\"-\", \" \").replace(\"_\", \" \").title()\n", + "fig = plot.shaded_std(\"AOD550\", ds_mean, ds_std, title=title)\n", + "fig.show()\n", + "fig.write_image(filename + \".png\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "eqc", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.8" + }, + "vscode": { + "interpreter": { + "hash": "39a16a1176456aec0710d6d8dd097fdfd8eece03838aebbaaddfca0f16ac2477" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}