From 2e657fedb2a203acc406acded79b3f9028a6506f Mon Sep 17 00:00:00 2001 From: Lisa Bock Date: Fri, 9 Dec 2022 11:18:14 +0100 Subject: [PATCH 1/8] Fix siconc in KIOST-ESM (#1829) Co-authored-by: Manuel Schlund --- esmvalcore/cmor/_fixes/cmip6/kiost_esm.py | 29 +++++- .../cmor/_fixes/cmip6/test_kiost_esm.py | 88 +++++++++++-------- 2 files changed, 76 insertions(+), 41 deletions(-) diff --git a/esmvalcore/cmor/_fixes/cmip6/kiost_esm.py b/esmvalcore/cmor/_fixes/cmip6/kiost_esm.py index 518a82e541..77ab4805e5 100644 --- a/esmvalcore/cmor/_fixes/cmip6/kiost_esm.py +++ b/esmvalcore/cmor/_fixes/cmip6/kiost_esm.py @@ -1,4 +1,6 @@ """Fixes for KIOST-ESM model.""" +from dask import array as da + from ..common import SiconcFixScalarCoord from ..fix import Fix from ..shared import add_scalar_height_coord @@ -17,7 +19,8 @@ def fix_metadata(self, cubes): Returns ------- - iris.cube.Cube + iris.cube.Cubes + Fixed cubes. """ cube = self.get_cube_from_list(cubes) add_scalar_height_coord(cube, 2.0) @@ -45,7 +48,8 @@ def fix_metadata(self, cubes): Returns ------- - iris.cube.Cube + iris.cube.Cubes + Fixed cubes. """ cube = self.get_cube_from_list(cubes) add_scalar_height_coord(cube, 10.0) @@ -60,4 +64,23 @@ class Vas(SfcWind): """Fixes for vas.""" -Siconc = SiconcFixScalarCoord +class Siconc(SiconcFixScalarCoord): + """Fixes for siconc.""" + + def fix_data(self, cube): + """Fix data. + + Fix missing values. + + Parameters + ---------- + cube: iris.cube.Cube + Input cube. + + Returns + ------- + iris.cube.Cube + Fixed cube. + """ + cube.data = da.ma.masked_invalid(cube.core_data()) + return cube diff --git a/tests/integration/cmor/_fixes/cmip6/test_kiost_esm.py b/tests/integration/cmor/_fixes/cmip6/test_kiost_esm.py index 41de6089c5..9b460db15a 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_kiost_esm.py +++ b/tests/integration/cmor/_fixes/cmip6/test_kiost_esm.py @@ -1,13 +1,12 @@ """Test fixes for KIOST-ESM.""" import iris +import numpy as np import pytest from cf_units import Unit +from iris.coords import DimCoord +from iris.cube import Cube, CubeList -from esmvalcore.cmor._fixes.cmip6.kiost_esm import ( - SfcWind, - Siconc, - Tas, -) +from esmvalcore.cmor._fixes.cmip6.kiost_esm import SfcWind, Siconc, Tas from esmvalcore.cmor._fixes.common import SiconcFixScalarCoord from esmvalcore.cmor._fixes.fix import Fix from esmvalcore.cmor.table import get_var_info @@ -15,60 +14,60 @@ @pytest.fixture def sfcwind_cubes(): - correct_lat_coord = iris.coords.DimCoord([0.0], - var_name='lat', - standard_name='latitude') - wrong_lat_coord = iris.coords.DimCoord([0.0], - var_name='latitudeCoord', - standard_name='latitude') - correct_lon_coord = iris.coords.DimCoord([0.0], - var_name='lon', - standard_name='longitude') - wrong_lon_coord = iris.coords.DimCoord([0.0], - var_name='longitudeCoord', - standard_name='longitude') - correct_cube = iris.cube.Cube( + correct_lat_coord = DimCoord([0.0], + var_name='lat', + standard_name='latitude') + wrong_lat_coord = DimCoord([0.0], + var_name='latitudeCoord', + standard_name='latitude') + correct_lon_coord = DimCoord([0.0], + var_name='lon', + standard_name='longitude') + wrong_lon_coord = DimCoord([0.0], + var_name='longitudeCoord', + standard_name='longitude') + correct_cube = Cube( [[10.0]], var_name='sfcWind', dim_coords_and_dims=[(correct_lat_coord, 0), (correct_lon_coord, 1)], ) - wrong_cube = iris.cube.Cube( + wrong_cube = Cube( [[10.0]], var_name='ta', dim_coords_and_dims=[(wrong_lat_coord, 0), (wrong_lon_coord, 1)], attributes={'parent_time_units': 'days since 0000-00-00 (noleap)'}, ) - scalar_cube = iris.cube.Cube(0.0, var_name='ps') - return iris.cube.CubeList([correct_cube, wrong_cube, scalar_cube]) + scalar_cube = Cube(0.0, var_name='ps') + return CubeList([correct_cube, wrong_cube, scalar_cube]) @pytest.fixture def tas_cubes(): - correct_lat_coord = iris.coords.DimCoord([0.0], - var_name='lat', - standard_name='latitude') - wrong_lat_coord = iris.coords.DimCoord([0.0], - var_name='latitudeCoord', - standard_name='latitude') - correct_lon_coord = iris.coords.DimCoord([0.0], - var_name='lon', - standard_name='longitude') - wrong_lon_coord = iris.coords.DimCoord([0.0], - var_name='longitudeCoord', - standard_name='longitude') - correct_cube = iris.cube.Cube( + correct_lat_coord = DimCoord([0.0], + var_name='lat', + standard_name='latitude') + wrong_lat_coord = DimCoord([0.0], + var_name='latitudeCoord', + standard_name='latitude') + correct_lon_coord = DimCoord([0.0], + var_name='lon', + standard_name='longitude') + wrong_lon_coord = DimCoord([0.0], + var_name='longitudeCoord', + standard_name='longitude') + correct_cube = Cube( [[10.0]], var_name='tas', dim_coords_and_dims=[(correct_lat_coord, 0), (correct_lon_coord, 1)], ) - wrong_cube = iris.cube.Cube( + wrong_cube = Cube( [[10.0]], var_name='ta', dim_coords_and_dims=[(wrong_lat_coord, 0), (wrong_lon_coord, 1)], attributes={'parent_time_units': 'days since 0000-00-00 (noleap)'}, ) - scalar_cube = iris.cube.Cube(0.0, var_name='ps') - return iris.cube.CubeList([correct_cube, wrong_cube, scalar_cube]) + scalar_cube = Cube(0.0, var_name='ps') + return CubeList([correct_cube, wrong_cube, scalar_cube]) def test_get_sfcwind_fix(): @@ -110,7 +109,20 @@ def test_get_siconc_fix(): def test_siconc_fix(): """Test fix for ``siconc``.""" - assert Siconc is SiconcFixScalarCoord + assert issubclass(Siconc, SiconcFixScalarCoord) + + +def test_siconc_fix_data(): + """Test fix for ``siconc``.""" + vardef = get_var_info('CMIP6', 'SImon', 'siconc') + fix = Siconc(vardef) + + cube = Cube([0.0, np.nan, 1.0], var_name='siconc') + assert not np.ma.is_masked(cube.data) + + out_cube = fix.fix_data(cube) + np.testing.assert_array_almost_equal(out_cube.data, [0.0, np.nan, 1.0]) + np.testing.assert_array_equal(out_cube.data.mask, [False, True, False]) def test_get_tas_fix(): From 6b9a95d406d287b3afb25c3fac829bc6bc8b9c66 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 11 Dec 2022 11:36:38 +0000 Subject: [PATCH 2/8] [Condalock] Update Linux condalock file (#1865) Co-authored-by: valeriupredoi --- conda-linux-64.lock | 60 ++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/conda-linux-64.lock b/conda-linux-64.lock index b49f7e34d5..1fe61c661c 100644 --- a/conda-linux-64.lock +++ b/conda-linux-64.lock @@ -3,7 +3,7 @@ # input_hash: 153b7f6d9833c25e981b17319838235a50f9fd0c93fdaa09952a3919c2a09afc @EXPLICIT https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2#d7c89558ba9fa0495403155b64376d81 -https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2022.9.24-ha878542_0.tar.bz2#41e4e87062433e283696cf384f952ef6 +https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2022.12.7-ha878542_0.conda#ff9f73d45c4a07d6f424495288a26080 https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2#0c96522c6bdaed4b1566d11387caaf45 https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2#34893075a5c9e55cdafac56607368fc6 https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2#4d59c254e01d9cde7957100457e2d5fb @@ -58,7 +58,7 @@ https://conda.anaconda.org/conda-forge/linux-64/lz4-c-1.9.3-h9c3ff4c_1.tar.bz2#f https://conda.anaconda.org/conda-forge/linux-64/mpich-4.0.3-h846660c_100.tar.bz2#50d66bb751cfa71ee2a48b2d3eb90ac1 https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.3-h27087fc_1.tar.bz2#4acfc691e64342b9dae57cf2adc63238 https://conda.anaconda.org/conda-forge/linux-64/nspr-4.35-h27087fc_0.conda#da0ec11a6454ae19bff5b02ed881a2b1 -https://conda.anaconda.org/conda-forge/linux-64/openssl-3.0.7-h166bdaf_0.tar.bz2#d1ad1824c71e67dea42f07e06cd177dc +https://conda.anaconda.org/conda-forge/linux-64/openssl-3.0.7-h0b41bf4_1.conda#7adaac6ff98219bcb99b45e408b80f4e https://conda.anaconda.org/conda-forge/linux-64/pixman-0.40.0-h36c2ea0_0.tar.bz2#660e72c82f2e75a6b3fe6a6e75c79f19 https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-h36c2ea0_1001.tar.bz2#22dad4df6e8630e8dff2428f6f6a7036 https://conda.anaconda.org/conda-forge/linux-64/snappy-1.1.9-hbd366e4_2.tar.bz2#48018e187dacc6002d3ede9c824238ac @@ -93,7 +93,7 @@ https://conda.anaconda.org/conda-forge/linux-64/udunits2-2.2.28-hc3e0081_0.tar.b https://conda.anaconda.org/conda-forge/linux-64/xorg-libsm-1.2.3-hd9c2040_1000.tar.bz2#9e856f78d5c80d5a78f61e72d1d473a3 https://conda.anaconda.org/conda-forge/linux-64/zlib-1.2.13-h166bdaf_4.tar.bz2#4b11e365c0275b808be78b30f904e295 https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.2-h6239696_4.tar.bz2#adcf0be7897e73e312bd24353b613f74 -https://conda.anaconda.org/conda-forge/linux-64/blosc-1.21.1-h83bc5f7_3.tar.bz2#37baca23e60af4130cfc03e8ab9f8e22 +https://conda.anaconda.org/conda-forge/linux-64/blosc-1.21.2-hafa529b_0.conda#9fc20ab886b80d1029b828ef7ee79a35 https://conda.anaconda.org/conda-forge/linux-64/boost-cpp-1.78.0-h75c5d50_1.tar.bz2#accc1f1ca33809bbf9ad067a0a69e236 https://conda.anaconda.org/conda-forge/linux-64/brotli-bin-1.0.9-h166bdaf_8.tar.bz2#e5613f2bc717e9945840ff474419b8e4 https://conda.anaconda.org/conda-forge/linux-64/freetype-2.12.1-hca18f0e_1.conda#e1232042de76d24539a436d37597eb06 @@ -120,7 +120,7 @@ https://conda.anaconda.org/conda-forge/noarch/attrs-22.1.0-pyh71513ae_1.tar.bz2# https://conda.anaconda.org/conda-forge/linux-64/backports.zoneinfo-0.2.1-py310hff52083_7.tar.bz2#02d7c823f5e6fd4bbe5562c612465aed https://conda.anaconda.org/conda-forge/linux-64/brotli-1.0.9-h166bdaf_8.tar.bz2#2ff08978892a3e8b954397c461f18418 https://conda.anaconda.org/conda-forge/linux-64/c-compiler-1.5.1-h166bdaf_0.tar.bz2#0667d7da14e682c9d07968601f6233ef -https://conda.anaconda.org/conda-forge/noarch/certifi-2022.9.24-pyhd8ed1ab_0.tar.bz2#f66309b099374af91369e67e84af397d +https://conda.anaconda.org/conda-forge/noarch/certifi-2022.12.7-pyhd8ed1ab_0.conda#fb9addc3db06e56abe03e0e9f21a63e6 https://conda.anaconda.org/conda-forge/noarch/cfgv-3.3.1-pyhd8ed1ab_0.tar.bz2#ebb5f5f7dc4f1a3780ef7ea7738db08c https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-2.1.1-pyhd8ed1ab_0.tar.bz2#c1d5b294fbf9a795dec349a6f4d8be8e https://conda.anaconda.org/conda-forge/noarch/click-8.1.3-unix_pyhd8ed1ab_2.tar.bz2#20e4087407c7cb04a40817114b333dbf @@ -135,7 +135,7 @@ https://conda.anaconda.org/conda-forge/linux-64/docutils-0.17.1-py310hff52083_3. https://conda.anaconda.org/conda-forge/noarch/dodgy-0.2.1-py_0.tar.bz2#62a69d073f7446c90f417b0787122f5b https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.0.4-pyhd8ed1ab_0.tar.bz2#e0734d1f12de77f9daca98bda3428733 https://conda.anaconda.org/conda-forge/noarch/execnet-1.9.0-pyhd8ed1ab_0.tar.bz2#0e521f7a5e60d508b121d38b04874fb2 -https://conda.anaconda.org/conda-forge/noarch/filelock-3.8.0-pyhd8ed1ab_0.tar.bz2#10f0218dbd493ab2e5dc6759ddea4526 +https://conda.anaconda.org/conda-forge/noarch/filelock-3.8.2-pyhd8ed1ab_0.conda#0f09c2bc17ddd8732be8e5b99297c7ce https://conda.anaconda.org/conda-forge/linux-64/fontconfig-2.14.1-hc2a2eb6_0.tar.bz2#78415f0180a8d9c5bcc47889e00d5fb1 https://conda.anaconda.org/conda-forge/noarch/fsspec-2022.11.0-pyhd8ed1ab_0.tar.bz2#eb919f2119a6db5d0192f9e9c3711572 https://conda.anaconda.org/conda-forge/linux-64/gdk-pixbuf-2.42.8-hff1cb4f_1.tar.bz2#a61c6312192e7c9de71548a6706a21e6 @@ -156,7 +156,7 @@ https://conda.anaconda.org/conda-forge/linux-64/lazy-object-proxy-1.8.0-py310h57 https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.14-h6ed2654_0.tar.bz2#dcc588839de1445d90995a0a2c4f3a39 https://conda.anaconda.org/conda-forge/linux-64/libcurl-7.86.0-h2283fc2_1.tar.bz2#fdca8cd67ec2676f90a70ac73a32538b https://conda.anaconda.org/conda-forge/linux-64/libkml-1.3.0-h37653c0_1015.tar.bz2#37d3747dd24d604f63d2610910576e63 -https://conda.anaconda.org/conda-forge/linux-64/libpq-15.1-he2d8382_0.conda#d04d0e442f0fe23c281a24ffdb3f66f8 +https://conda.anaconda.org/conda-forge/linux-64/libpq-15.1-h67c24c5_1.conda#e1389a8d9a907133b3e6483c2807d243 https://conda.anaconda.org/conda-forge/linux-64/libwebp-1.2.4-h522a892_0.tar.bz2#802e43f480122a85ae6a34c1909f8f98 https://conda.anaconda.org/conda-forge/noarch/locket-1.0.0-pyhd8ed1ab_0.tar.bz2#91e27ef3d05cc772ce627e51cff111c4 https://conda.anaconda.org/conda-forge/linux-64/lxml-4.9.1-py310ha00c094_1.tar.bz2#4b7ed16f7db1eea5b53442aeab2d3b9e @@ -170,8 +170,9 @@ https://conda.anaconda.org/conda-forge/linux-64/mypy_extensions-0.4.3-py310hff52 https://conda.anaconda.org/conda-forge/noarch/networkx-2.8.8-pyhd8ed1ab_0.tar.bz2#bb45ff9deddb045331fd039949f39650 https://conda.anaconda.org/conda-forge/linux-64/numpy-1.23.5-py310h53a5b5f_0.conda#3b114b1559def8bad228fec544ac1812 https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.0-h7d73246_1.tar.bz2#a11b4df9271a8d7917686725aa04c8f2 -https://conda.anaconda.org/conda-forge/noarch/pathspec-0.10.2-pyhd8ed1ab_0.tar.bz2#4e88b9285fca81694005eda5d85b35b8 -https://conda.anaconda.org/conda-forge/noarch/platformdirs-2.5.2-pyhd8ed1ab_1.tar.bz2#2fb3f88922e7aec26ba652fcdfe13950 +https://conda.anaconda.org/conda-forge/noarch/packaging-22.0-pyhd8ed1ab_0.conda#0e8e1bd93998978fc3125522266d12db +https://conda.anaconda.org/conda-forge/noarch/pathspec-0.10.3-pyhd8ed1ab_0.conda#0f7d2186dd12ef3277e70fd85f9ff6a7 +https://conda.anaconda.org/conda-forge/noarch/platformdirs-2.6.0-pyhd8ed1ab_0.conda#b1b2ab02d1ece1719f7fa002ad4bc70d https://conda.anaconda.org/conda-forge/noarch/pluggy-1.0.0-pyhd8ed1ab_5.tar.bz2#7d301a0d25f424d96175f810935f0da9 https://conda.anaconda.org/conda-forge/linux-64/psutil-5.9.4-py310h5764c6d_0.tar.bz2#c3c55664e9becc48e6a652e2b641961f https://conda.anaconda.org/conda-forge/noarch/py-1.11.0-pyh6c4a22f_0.tar.bz2#b4613d7e7a493916d867842a6a148054 @@ -201,7 +202,7 @@ https://conda.anaconda.org/conda-forge/noarch/termcolor-2.1.1-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/noarch/toml-0.10.2-pyhd8ed1ab_0.tar.bz2#f832c45a477c78bebd107098db465095 https://conda.anaconda.org/conda-forge/noarch/tomli-2.0.1-pyhd8ed1ab_0.tar.bz2#5844808ffab9ebdb694585b50ba02a96 https://conda.anaconda.org/conda-forge/noarch/toolz-0.12.0-pyhd8ed1ab_0.tar.bz2#92facfec94bc02d6ccf42e7173831a36 -https://conda.anaconda.org/conda-forge/linux-64/tornado-6.1-py310h5764c6d_3.tar.bz2#8a5770e6392d29d99c9bc9c3635bba60 +https://conda.anaconda.org/conda-forge/linux-64/tornado-6.2-py310h5764c6d_1.tar.bz2#be4a201ac582c11d89ed7d15b3157cc3 https://conda.anaconda.org/conda-forge/noarch/types-pkg_resources-0.1.3-pyhd8ed1ab_0.tar.bz2#82e2a50752d5a512ab88e66778f9a7a8 https://conda.anaconda.org/conda-forge/noarch/types-pyyaml-6.0.12.2-pyhd8ed1ab_0.tar.bz2#e7f8004e3dcfb6a0e1751573cb79fb15 https://conda.anaconda.org/conda-forge/noarch/types-urllib3-1.26.25.4-pyhd8ed1ab_0.tar.bz2#ca86be78a819654cf93d81649600437f @@ -240,7 +241,7 @@ https://conda.anaconda.org/conda-forge/noarch/geopy-2.3.0-pyhd8ed1ab_0.tar.bz2#5 https://conda.anaconda.org/conda-forge/linux-64/hdf5-1.12.2-mpi_mpich_h5d83325_0.tar.bz2#6b5c2d276f306df759cfbdb0f41c4db9 https://conda.anaconda.org/conda-forge/noarch/html5lib-1.1-pyh9f0ad1d_0.tar.bz2#b2355343d6315c892543200231d7154a https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-5.1.0-pyha770c72_0.conda#46a62e35b9ae515cf0e49afc7fe0e7ef -https://conda.anaconda.org/conda-forge/noarch/importlib_resources-5.10.0-pyhd8ed1ab_0.tar.bz2#eb521efe7f5550de7573bc1629b74a05 +https://conda.anaconda.org/conda-forge/noarch/importlib_resources-5.10.1-pyhd8ed1ab_0.conda#db5d88c84c769798bf4b2f6776446b32 https://conda.anaconda.org/conda-forge/noarch/isodate-0.6.1-pyhd8ed1ab_0.tar.bz2#4a62c93c1b5c0b920508ae3fd285eaf5 https://conda.anaconda.org/conda-forge/noarch/isort-5.10.1-pyhd8ed1ab_0.tar.bz2#83df9ffefe1cc01d9e60405bf871bfdb https://conda.anaconda.org/conda-forge/noarch/jinja2-3.1.2-pyhd8ed1ab_1.tar.bz2#c8490ed5c70966d232fdd389d0dbed37 @@ -251,14 +252,14 @@ https://conda.anaconda.org/conda-forge/noarch/munch-2.5.0-py_0.tar.bz2#31d9e9be5 https://conda.anaconda.org/conda-forge/linux-64/mypy-0.991-py310h5764c6d_0.tar.bz2#aaa26cf19bd0903cfed601a4da61ec25 https://conda.anaconda.org/conda-forge/noarch/nested-lookup-0.2.25-pyhd8ed1ab_1.tar.bz2#2f59daeb14581d41b1e2dda0895933b2 https://conda.anaconda.org/conda-forge/noarch/nodeenv-1.7.0-pyhd8ed1ab_0.tar.bz2#fbe1182f650c04513046d6894046cd6c -https://conda.anaconda.org/conda-forge/noarch/packaging-21.3-pyhd8ed1ab_0.tar.bz2#71f1ab2de48613876becddd496371c85 https://conda.anaconda.org/conda-forge/noarch/partd-1.3.0-pyhd8ed1ab_0.tar.bz2#af8c82d121e63082926062d61d9abb54 https://conda.anaconda.org/conda-forge/linux-64/pillow-9.2.0-py310h454ad03_3.tar.bz2#eb354ff791f505b1d6f13f776359d88e https://conda.anaconda.org/conda-forge/noarch/pip-22.3.1-pyhd8ed1ab_0.tar.bz2#da66f2851b9836d3a7c5190082a45f7d -https://conda.anaconda.org/conda-forge/linux-64/postgresql-15.1-ha105346_0.conda#e93c67ae270664fe0c71519bfbf5abd8 +https://conda.anaconda.org/conda-forge/linux-64/postgresql-15.1-ha105346_1.conda#81cfa38baa2a8741f0566f8815fef4e3 https://conda.anaconda.org/conda-forge/linux-64/proj-9.1.0-h93bde94_0.tar.bz2#255c7204dda39747c3ba380d28b026d7 https://conda.anaconda.org/conda-forge/noarch/pydocstyle-6.1.1-pyhd8ed1ab_0.tar.bz2#e417954a987d382b3142886726ab3aad https://conda.anaconda.org/conda-forge/noarch/pygments-2.13.0-pyhd8ed1ab_0.tar.bz2#9f478e8eedd301008b5f395bad0caaed +https://conda.anaconda.org/conda-forge/noarch/pytest-7.2.0-pyhd8ed1ab_2.tar.bz2#ac82c7aebc282e6ac0450fca012ca78c https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.8.2-pyhd8ed1ab_0.tar.bz2#dd999d1cc9f79e67dbb855c8924c7984 https://conda.anaconda.org/conda-forge/linux-64/python-stratify-0.2.post0-py310hde88566_3.tar.bz2#0b686f306a76fba9a61e7019f854321f https://conda.anaconda.org/conda-forge/linux-64/scipy-1.9.3-py310hdfbd76f_2.tar.bz2#0582a434d03f6b06d5defbb142c96f4f @@ -266,7 +267,7 @@ https://conda.anaconda.org/conda-forge/linux-64/shapely-1.8.5-py310h5b266fc_2.ta https://conda.anaconda.org/conda-forge/noarch/tomlkit-0.11.6-pyha770c72_0.tar.bz2#471bf9e605820b59988e830382b8d654 https://conda.anaconda.org/conda-forge/noarch/types-requests-2.28.11.5-pyhd8ed1ab_0.tar.bz2#bcd389a385a8b6bd866238f1a16d13cf https://conda.anaconda.org/conda-forge/noarch/url-normalize-1.4.3-pyhd8ed1ab_0.tar.bz2#7c4076e494f0efe76705154ac9302ba6 -https://conda.anaconda.org/conda-forge/linux-64/virtualenv-20.17.0-py310hff52083_0.conda#c6fc5e3f0a463ddb59cfda9a1582cfa0 +https://conda.anaconda.org/conda-forge/linux-64/virtualenv-20.17.1-py310hff52083_0.conda#d26ee3f6561669ec1f118d6d3404e42a https://conda.anaconda.org/conda-forge/linux-64/xerces-c-3.2.4-h55805fa_1.tar.bz2#d127dc8efe24033b306180939e51e6af https://conda.anaconda.org/conda-forge/noarch/yamale-4.0.4-pyh6c4a22f_0.tar.bz2#cc9f59f147740d88679bf1bd94dbe588 https://conda.anaconda.org/conda-forge/noarch/yamllint-1.28.0-pyhd8ed1ab_0.tar.bz2#f28e487a994b504cc5fdb6cb5e6a0bf9 @@ -276,8 +277,8 @@ https://conda.anaconda.org/conda-forge/linux-64/brotlipy-0.7.0-py310h5764c6d_100 https://conda.anaconda.org/conda-forge/linux-64/cf-units-3.1.1-py310hde88566_2.tar.bz2#7433944046deda7775c5b1f7e0b6fe18 https://conda.anaconda.org/conda-forge/linux-64/compilers-1.5.1-ha770c72_0.tar.bz2#8a0ff3c519396696bbe9ca786606372f https://conda.anaconda.org/conda-forge/linux-64/cryptography-38.0.4-py310h600f1e7_0.conda#f999dcc21fe27ad97a8afcfa590daa14 -https://conda.anaconda.org/conda-forge/noarch/dask-core-2022.11.1-pyhd8ed1ab_0.conda#383ee12e7c9c27adab310a884bc359ab -https://conda.anaconda.org/conda-forge/noarch/django-4.1.3-pyhd8ed1ab_0.tar.bz2#9b2bb1af9f01f3730b916f4ac134570b +https://conda.anaconda.org/conda-forge/noarch/dask-core-2022.12.0-pyhd8ed1ab_0.conda#3a0f020d07998e1ae711df071f97fc19 +https://conda.anaconda.org/conda-forge/noarch/django-4.1.4-pyhd8ed1ab_0.conda#14f8b26ca366987ec53655f6a7a6cba0 https://conda.anaconda.org/conda-forge/noarch/flake8-4.0.1-pyhd8ed1ab_2.tar.bz2#a824bd55ce47e9c637427f730c651231 https://conda.anaconda.org/conda-forge/linux-64/geotiff-1.7.1-ha76d385_4.tar.bz2#6a613710a0f19aba3a5dfe83bf1c1c0f https://conda.anaconda.org/conda-forge/linux-64/harfbuzz-5.3.0-h418a68e_0.tar.bz2#888056bd4b12e110b10d4d1f29161c5e @@ -289,9 +290,14 @@ https://conda.anaconda.org/conda-forge/linux-64/matplotlib-base-3.6.2-py310h8d5e https://conda.anaconda.org/conda-forge/linux-64/pandas-1.5.2-py310h769672d_0.conda#bc363997d22f3b058fb17f1e89d4c96f https://conda.anaconda.org/conda-forge/linux-64/poppler-22.11.0-h92391eb_0.tar.bz2#833285a4a9d03a93f3e3e5f31b319de8 https://conda.anaconda.org/conda-forge/noarch/pybtex-0.24.0-pyhd8ed1ab_2.tar.bz2#2099b86a7399c44c0c61cdb6de6915ba -https://conda.anaconda.org/conda-forge/noarch/pylint-2.15.7-pyhd8ed1ab_0.conda#9ad4613cb7a16fcfdc92170d1e394cdd +https://conda.anaconda.org/conda-forge/noarch/pylint-2.15.8-pyhd8ed1ab_0.conda#26a62404bbfc93452c4d22aa2cda6f08 https://conda.anaconda.org/conda-forge/linux-64/pyproj-3.4.0-py310hb1338dc_2.tar.bz2#e1648c222911ad7559d62831e4bc447c -https://conda.anaconda.org/conda-forge/noarch/pytest-7.2.0-pyhd8ed1ab_2.tar.bz2#ac82c7aebc282e6ac0450fca012ca78c +https://conda.anaconda.org/conda-forge/noarch/pytest-cov-4.0.0-pyhd8ed1ab_0.tar.bz2#c9e3f8bfdb9bfc34aa1836a6ed4b25d7 +https://conda.anaconda.org/conda-forge/noarch/pytest-env-0.6.2-py_0.tar.bz2#2fe6beb5d2d87cf252a2575043919696 +https://conda.anaconda.org/conda-forge/noarch/pytest-metadata-2.0.4-pyhd8ed1ab_0.tar.bz2#7ac02a65917993d38ca1bfd7b87208e4 +https://conda.anaconda.org/conda-forge/noarch/pytest-mock-3.10.0-pyhd8ed1ab_0.tar.bz2#db93caa9fe182f0cd20291aeb22f57ac +https://conda.anaconda.org/conda-forge/noarch/pytest-mypy-0.8.0-pyhd8ed1ab_0.tar.bz2#4e81c96e5f875c09e5b9f999035b9d8e +https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.1.0-pyhd8ed1ab_0.conda#e82f8fb903d7c4a59c77954759c341f9 https://conda.anaconda.org/conda-forge/noarch/rdflib-6.2.0-pyhd8ed1ab_0.tar.bz2#b9acd5fbaf467f7447746b1ecac50e83 https://conda.anaconda.org/conda-forge/linux-64/requirements-detector-0.7-py310hff52083_3.tar.bz2#0cd9f8972da2c8ad624676d47643805f https://conda.anaconda.org/conda-forge/linux-64/tiledb-2.11.3-h3f4058f_1.tar.bz2#4ae02ca7d1da6e2b4e3005108df36812 @@ -299,23 +305,18 @@ https://conda.anaconda.org/conda-forge/linux-64/ukkonen-1.0.1-py310hbf28c38_3.ta https://conda.anaconda.org/conda-forge/linux-64/cartopy-0.21.0-py310h83f2385_3.tar.bz2#4ec35f7eebe4221c1c00fdd6540db4dc https://conda.anaconda.org/conda-forge/noarch/flake8-polyfill-1.0.2-py_0.tar.bz2#a53db35e3d07f0af2eccd59c2a00bffe https://conda.anaconda.org/conda-forge/noarch/identify-2.5.9-pyhd8ed1ab_0.conda#e7ecbbb61a37daed2a13de43d35d5282 -https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.5.3-hd850ce1_7.conda#eca3717db78338df707af836346f65c1 +https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.5.3-h867e046_8.conda#4d3739bedfe7a58cb9095ec6b24f7544 https://conda.anaconda.org/conda-forge/noarch/nc-time-axis-1.4.1-pyhd8ed1ab_0.tar.bz2#281b58948bf60a2582de9e548bcc5369 https://conda.anaconda.org/conda-forge/linux-64/netcdf-fortran-4.6.0-mpi_mpich_hd09bd1e_1.tar.bz2#0b69750bb937cab0db14f6bcef6fd787 -https://conda.anaconda.org/conda-forge/linux-64/netcdf4-1.6.0-nompi_py310h55e1e36_102.tar.bz2#588d5bd8f16287b766c509ef173b892d +https://conda.anaconda.org/conda-forge/linux-64/netcdf4-1.6.0-nompi_py310h0a86a1f_103.conda#7f69695b684f2595d9ba1ce26d693b7d https://conda.anaconda.org/conda-forge/linux-64/pango-1.50.12-h382ae3d_0.conda#627bea5af786dbd8013ef26127d8115a https://conda.anaconda.org/conda-forge/noarch/pylint-plugin-utils-0.7-pyhd8ed1ab_0.tar.bz2#1657976383aee04dbb3ae3bdf654bb58 https://conda.anaconda.org/conda-forge/noarch/pyopenssl-22.1.0-pyhd8ed1ab_0.tar.bz2#fbfa0a180d48c800f922a10a114a8632 -https://conda.anaconda.org/conda-forge/noarch/pytest-cov-4.0.0-pyhd8ed1ab_0.tar.bz2#c9e3f8bfdb9bfc34aa1836a6ed4b25d7 -https://conda.anaconda.org/conda-forge/noarch/pytest-env-0.6.2-py_0.tar.bz2#2fe6beb5d2d87cf252a2575043919696 -https://conda.anaconda.org/conda-forge/noarch/pytest-metadata-2.0.4-pyhd8ed1ab_0.tar.bz2#7ac02a65917993d38ca1bfd7b87208e4 -https://conda.anaconda.org/conda-forge/noarch/pytest-mock-3.10.0-pyhd8ed1ab_0.tar.bz2#db93caa9fe182f0cd20291aeb22f57ac -https://conda.anaconda.org/conda-forge/noarch/pytest-mypy-0.8.0-pyhd8ed1ab_0.tar.bz2#4e81c96e5f875c09e5b9f999035b9d8e -https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.0.2-pyhd8ed1ab_0.tar.bz2#18bdfe034d1187a34d860ed8e6fec788 +https://conda.anaconda.org/conda-forge/noarch/pytest-html-3.2.0-pyhd8ed1ab_1.tar.bz2#d5c7a941dfbceaab4b172a56d7918eb0 https://conda.anaconda.org/conda-forge/linux-64/esmf-8.2.0-mpi_mpich_h5a1934d_102.tar.bz2#bb8bdfa5e3e9e3f6ec861f05cd2ad441 -https://conda.anaconda.org/conda-forge/linux-64/gdal-3.5.3-py310he53f9b6_7.conda#0100b9758152cdb58945fd842821d0fc +https://conda.anaconda.org/conda-forge/linux-64/gdal-3.5.3-py310hc1b7723_8.conda#bcdf3bbfc55928afc900a92a70f745d5 https://conda.anaconda.org/conda-forge/linux-64/gtk2-2.24.33-h90689f9_2.tar.bz2#957a0255ab58aaf394a91725d73ab422 -https://conda.anaconda.org/conda-forge/noarch/iris-3.3.1-pyhd8ed1ab_0.tar.bz2#176ab70a4fdd63397fdbfdbcd082a36e +https://conda.anaconda.org/conda-forge/noarch/iris-3.4.0-pyhd8ed1ab_0.conda#6a6c17cfff8564de41d2a952c97bc2e8 https://conda.anaconda.org/conda-forge/linux-64/librsvg-2.54.4-h7abd40a_0.tar.bz2#921e53675ed5ea352f022b79abab076a https://conda.anaconda.org/conda-forge/noarch/myproxyclient-2.1.0-pyhd8ed1ab_2.tar.bz2#363b0816e411feb0df925d4f224f026a https://conda.anaconda.org/conda-forge/noarch/pep8-naming-0.10.0-pyh9f0ad1d_0.tar.bz2#b3c5536e4f9f58a4b16adb6f1e11732d @@ -323,15 +324,14 @@ https://conda.anaconda.org/conda-forge/linux-64/pre-commit-2.20.0-py310hff52083_ https://conda.anaconda.org/conda-forge/noarch/pylint-celery-0.3-py_1.tar.bz2#e29456a611a62d3f26105a2f9c68f759 https://conda.anaconda.org/conda-forge/noarch/pylint-django-2.5.3-pyhd8ed1ab_0.tar.bz2#00d8853fb1f87195722ea6a582cc9b56 https://conda.anaconda.org/conda-forge/noarch/pylint-flask-0.6-py_0.tar.bz2#5a9afd3d0a61b08d59eed70fab859c1b -https://conda.anaconda.org/conda-forge/noarch/pytest-html-3.2.0-pyhd8ed1ab_1.tar.bz2#d5c7a941dfbceaab4b172a56d7918eb0 https://conda.anaconda.org/conda-forge/noarch/urllib3-1.26.13-pyhd8ed1ab_0.conda#3078ef2359efd6ecadbc7e085c5e0592 -https://conda.anaconda.org/conda-forge/noarch/distributed-2022.11.1-pyhd8ed1ab_0.conda#b416f82d39b8e0cd14c697aafc0536ef +https://conda.anaconda.org/conda-forge/noarch/distributed-2022.12.0-pyhd8ed1ab_0.conda#2c934253dc579d314ac2a1f43b4be15c https://conda.anaconda.org/conda-forge/linux-64/esmpy-8.2.0-mpi_mpich_py310hd9c82d4_101.tar.bz2#0333d51ee594be40f50b157ac6f27b5a https://conda.anaconda.org/conda-forge/linux-64/fiona-1.8.22-py310h60a68a4_2.tar.bz2#fc3bb0d986e4ede4ddedb62ae2d99c55 https://conda.anaconda.org/conda-forge/linux-64/graphviz-6.0.2-h99bc08f_0.conda#8f247587d1520a2bbc6f79a821b74c07 https://conda.anaconda.org/conda-forge/noarch/prospector-1.7.7-pyhd8ed1ab_0.tar.bz2#01010f8ea38d650158703a581e51b979 https://conda.anaconda.org/conda-forge/noarch/requests-2.28.1-pyhd8ed1ab_1.tar.bz2#089382ee0e2dc2eae33a04cc3c2bddb0 -https://conda.anaconda.org/conda-forge/noarch/dask-2022.11.1-pyhd8ed1ab_0.conda#cbaf14c6796cadd8d43aba812a6abef4 +https://conda.anaconda.org/conda-forge/noarch/dask-2022.12.0-pyhd8ed1ab_0.conda#db0f9c381d65054afdb6ae1e547d8b2e https://conda.anaconda.org/conda-forge/linux-64/pydot-1.4.2-py310hff52083_3.tar.bz2#45231e3f8fa29b6cea52e2cfe9b47a22 https://conda.anaconda.org/conda-forge/noarch/requests-cache-0.9.6-pyhd8ed1ab_0.tar.bz2#276b447a966e8f3d3daecb3c5a156330 https://conda.anaconda.org/conda-forge/noarch/sphinx-5.3.0-pyhd8ed1ab_0.tar.bz2#f9e1fcfe235d655900bfeb6aee426472 From bccc8e00a5ca8708f52ff3d303d1a6d401a3d542 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Tue, 13 Dec 2022 17:20:28 +0100 Subject: [PATCH 3/8] Add `esmvalcore.local`, a module to search data on the local filesystem (#1835) Co-authored-by: Manuel Schlund --- doc/api/esmvalcore.local.rst | 5 + doc/api/esmvalcore.rst | 2 + doc/api/esmvalcore.typing.rst | 6 + doc/develop/fixing_data.rst | 2 +- doc/quickstart/configure.rst | 12 +- doc/quickstart/find_data.rst | 38 +- esmvalcore/_provenance.py | 2 +- esmvalcore/_recipe.py | 62 +-- esmvalcore/_recipe_checks.py | 25 +- esmvalcore/cmor/_fixes/ipslcm/ipsl_cm6.py | 8 +- esmvalcore/config-developer.yml | 28 +- esmvalcore/config/_config.py | 8 + esmvalcore/esgf/_download.py | 9 +- esmvalcore/esgf/_search.py | 27 +- esmvalcore/{_data_finder.py => local.py} | 374 +++++++++++------- esmvalcore/preprocessor/__init__.py | 5 +- esmvalcore/preprocessor/_ancillary_vars.py | 3 +- esmvalcore/preprocessor/_io.py | 1 + esmvalcore/preprocessor/_regrid.py | 3 +- esmvalcore/typing.py | 11 + tests/integration/conftest.py | 51 +-- tests/integration/data_finder.yml | 130 ++++-- tests/integration/test_data_finder.py | 125 ------ tests/integration/test_local.py | 114 ++++++ tests/integration/test_provenance.py | 2 +- tests/integration/test_recipe.py | 268 ++++++------- tests/integration/test_recipe_checks.py | 31 +- .../experimental/test_run_recipe.py | 11 +- tests/unit/{data_finder => local}/__init__.py | 0 tests/unit/local/test_facets.py | 25 ++ .../test_replace_tags.py | 61 +-- .../test_select_files.py | 22 +- .../test_time.py} | 95 ++++- tests/unit/test_data_finder.py | 65 --- tests/unit/test_recipe.py | 31 +- 35 files changed, 951 insertions(+), 711 deletions(-) create mode 100644 doc/api/esmvalcore.local.rst create mode 100644 doc/api/esmvalcore.typing.rst rename esmvalcore/{_data_finder.py => local.py} (63%) create mode 100644 esmvalcore/typing.py delete mode 100644 tests/integration/test_data_finder.py create mode 100644 tests/integration/test_local.py rename tests/unit/{data_finder => local}/__init__.py (100%) create mode 100644 tests/unit/local/test_facets.py rename tests/unit/{data_finder => local}/test_replace_tags.py (54%) rename tests/unit/{data_finder => local}/test_select_files.py (87%) rename tests/unit/{data_finder/test_get_start_end_year.py => local/test_time.py} (67%) delete mode 100644 tests/unit/test_data_finder.py diff --git a/doc/api/esmvalcore.local.rst b/doc/api/esmvalcore.local.rst new file mode 100644 index 0000000000..12326929fa --- /dev/null +++ b/doc/api/esmvalcore.local.rst @@ -0,0 +1,5 @@ +Find files on the local filesystem +================================== + +.. automodule:: esmvalcore.local + :no-inherited-members: diff --git a/doc/api/esmvalcore.rst b/doc/api/esmvalcore.rst index 23787500da..fe61d3bfbc 100644 --- a/doc/api/esmvalcore.rst +++ b/doc/api/esmvalcore.rst @@ -14,5 +14,7 @@ library. This section documents the public API of ESMValCore. esmvalcore.esgf esmvalcore.exceptions esmvalcore.iris_helpers + esmvalcore.local esmvalcore.preprocessor + esmvalcore.typing esmvalcore.experimental diff --git a/doc/api/esmvalcore.typing.rst b/doc/api/esmvalcore.typing.rst new file mode 100644 index 0000000000..f0f45e7469 --- /dev/null +++ b/doc/api/esmvalcore.typing.rst @@ -0,0 +1,6 @@ +Type hints +========== + +.. automodule:: esmvalcore.typing + :no-inherited-members: + :no-special-members: diff --git a/doc/develop/fixing_data.rst b/doc/develop/fixing_data.rst index d097327e65..8902fb5dbf 100644 --- a/doc/develop/fixing_data.rst +++ b/doc/develop/fixing_data.rst @@ -377,7 +377,7 @@ To allow ESMValCore to locate the data files, use the following steps: native6: ... input_dir: - default: 'Tier{tier}/{dataset}/{latestversion}/{frequency}/{short_name}' + default: 'Tier{tier}/{dataset}/{version}/{frequency}/{short_name}' MY_DATA_ORG: '{dataset}/{exp}/{simulation}/{version}/{type}' input_file: default: '*.nc' diff --git a/doc/quickstart/configure.rst b/doc/quickstart/configure.rst index 810e7484c8..5eaac1b8d2 100644 --- a/doc/quickstart/configure.rst +++ b/doc/quickstart/configure.rst @@ -438,8 +438,8 @@ Example of the CMIP6 project configuration: CMIP6: input_dir: default: '/' - BADC: '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{latestversion}' - DKRZ: '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{latestversion}' + BADC: '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{version}' + DKRZ: '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{version}' ETHZ: '{exp}/{mip}/{short_name}/{dataset}/{ensemble}/{grid}/' input_file: '{short_name}_{mip}_{dataset}_{exp}_{ensemble}_{grid}*.nc' output_file: '{project}_{dataset}_{mip}_{exp}_{ensemble}_{short_name}' @@ -462,7 +462,7 @@ at each site. As an example, the CMIP6 directory path on BADC would be: .. code-block:: yaml - '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{latestversion}' + '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{version}' The resulting directory path would look something like this: @@ -475,8 +475,8 @@ which may be needed: .. code-block:: yaml - - '{exp}/{ensemble}/original/{mip}/{short_name}/{grid}/{latestversion}' - - '{exp}/{ensemble}/computed/{mip}/{short_name}/{grid}/{latestversion}' + - '{exp}/{ensemble}/original/{mip}/{short_name}/{grid}/{version}' + - '{exp}/{ensemble}/computed/{mip}/{short_name}/{grid}/{version}' In that case, the resultant directories will be: @@ -629,7 +629,7 @@ Example: native6: cmor_strict: false input_dir: - default: 'Tier{tier}/{dataset}/{latestversion}/{frequency}/{short_name}' + default: 'Tier{tier}/{dataset}/{version}/{frequency}/{short_name}' input_file: default: '*.nc' output_file: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}' diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst index c43b9b63bc..3eba26400c 100644 --- a/doc/quickstart/find_data.rst +++ b/doc/quickstart/find_data.rst @@ -33,16 +33,16 @@ ensures that files and paths to them are named according to a standardized convention. Examples of this convention, also used by ESMValTool for file discovery and data retrieval, include: -* CMIP6 file: ``[variable_short_name]_[mip]_[dataset_name]_[experiment]_[ensemble]_[grid]_[start-date]-[end-date].nc`` -* CMIP5 file: ``[variable_short_name]_[mip]_[dataset_name]_[experiment]_[ensemble]_[start-date]-[end-date].nc`` -* OBS file: ``[project]_[dataset_name]_[type]_[version]_[mip]_[short_name]_[start-date]-[end-date].nc`` +* CMIP6 file: ``{variable_short_name}_{mip}_{dataset_name}_{experiment}_{ensemble}_{grid}_{start-date}-{end-date}.nc`` +* CMIP5 file: ``{variable_short_name}_{mip}_{dataset_name}_{experiment}_{ensemble}_{start-date}-{end-date}.nc`` +* OBS file: ``{project}_{dataset_name}_{type}_{version}_{mip}_{short_name}_{start-date}-{end-date}.nc`` Similar standards exist for the standard paths (input directories); for the ESGF data nodes, these paths differ slightly, for example: -* CMIP6 path for BADC: ``ROOT-BADC/[institute]/[dataset_name]/[experiment]/[ensemble]/[mip]/ - [variable_short_name]/[grid]``; -* CMIP6 path for ETHZ: ``ROOT-ETHZ/[experiment]/[mip]/[variable_short_name]/[dataset_name]/[ensemble]/[grid]`` +* CMIP6 path for BADC: ``ROOT-BADC/{institute}/{dataset_name}/{experiment}/{ensemble}/{mip}/ + {variable_short_name}/{grid}``; +* CMIP6 path for ETHZ: ``ROOT-ETHZ/{experiment}/{mip}/{variable_short_name}/{dataset_name}/{ensemble}/{grid}`` From the ESMValTool user perspective the number of data input parameters is optimized to allow for ease of use. We detail this procedure in the next @@ -130,7 +130,7 @@ MSWEP - Supported frequencies: ``mon``, ``day``, ``3hr``. - Tier: 3 -For example for monthly data, place the files in the ``/Tier3/MSWEP/latestversion/mon/pr`` subdirectory of your ``native6`` project location. +For example for monthly data, place the files in the ``/Tier3/MSWEP/version/mon/pr`` subdirectory of your ``native6`` project location. .. note:: For monthly data (``V220``), the data must be postfixed with the date, i.e. rename ``global_monthly_050deg.nc`` to ``global_monthly_050deg_197901-201710.nc`` @@ -168,9 +168,9 @@ The default naming conventions for input directories and files for CESM are * input directories: 3 different types supported: * ``/`` (run directory) - * ``[case]/[gcomp]/hist`` (short-term archiving) - * ``[case]/[gcomp]/proc/[tdir]/[tperiod]`` (post-processed data) -* input files: ``[case].[scomp].[type].[string]*nc`` + * ``{case}/{gcomp}/hist`` (short-term archiving) + * ``{case}/{gcomp}/proc/{tdir}/{tperiod}`` (post-processed data) +* input files: ``{case}.{scomp}.{type}.{string}*nc`` as configured in the :ref:`config-developer file ` (using the default DRS ``drs: default`` in the :ref:`user configuration file`). @@ -179,12 +179,12 @@ More information about CESM naming conventions are given `here .. note:: - The ``[string]`` entry in the input file names above does not only + The ``{string}`` entry in the input file names above does not only correspond to the (optional) ``$string`` entry for `CESM model output files `__, but can also be used to read `post-processed files `__. - In the latter case, ``[string]`` corresponds to the combination + In the latter case, ``{string}`` corresponds to the combination ``$SSTRING.$TSTRING``. Thus, example dataset entries could look like this: @@ -244,8 +244,8 @@ model output. The default naming conventions for input directories and files for EMAC are -* input directories: ``[exp]/[channel]`` -* input files: ``[exp]*[channel][postproc_flag].nc`` +* input directories: ``{exp}/{channel}`` +* input files: ``{exp}*{channel}{postproc_flag}.nc`` as configured in the :ref:`config-developer file ` (using the default DRS ``drs: default`` in the :ref:`user configuration file`). @@ -313,8 +313,8 @@ ESMValTool is able to read native `ICON The default naming conventions for input directories and files for ICON are -* input directories: ``[exp]`` or ``{exp}/outdata`` -* input files: ``[exp]_[var_type]*.nc`` +* input directories: ``{exp}`` or ``{exp}/outdata`` +* input files: ``{exp}_{var_type}*.nc`` as configured in the :ref:`config-developer file ` (using the default DRS ``drs: default`` in the :ref:`user configuration file`). @@ -478,11 +478,11 @@ type of root paths they need the data from, e.g.: will tell the tool that the user needs data from a repository structured according to the BADC DRS structure, i.e.: -``ROOT/[institute]/[dataset_name]/[experiment]/[ensemble]/[mip]/[variable_short_name]/[grid]``; +``ROOT/{institute}/{dataset_name}/{experiment}/{ensemble}/{mip}/{variable_short_name}/{grid}``; setting the ``ROOT`` parameter is explained below. This is a strictly-structured repository tree and if there are any sort of irregularities -(e.g. there is no ``[mip]`` directory) the data will not be found! ``BADC`` can +(e.g. there is no ``{mip}`` directory) the data will not be found! ``BADC`` can be replaced with ``DKRZ`` or ``ETHZ`` depending on the existing ``ROOT`` directory structure. The snippet @@ -561,7 +561,7 @@ datasets are listed in any recipe, under either the ``datasets`` and/or - {dataset: HadGEM2-CC, project: CMIP5, exp: historical, ensemble: r1i1p1, start_year: 2001, end_year: 2004} - {dataset: UKESM1-0-LL, project: CMIP6, exp: historical, ensemble: r1i1p1f2, grid: gn, start_year: 2004, end_year: 2014} -``_data_finder`` will use this information to find data for **all** the variables specified in ``diagnostics/variables``. +The data finding feature will use this information to find data for **all** the variables specified in ``diagnostics/variables``. Recap and example ================= diff --git a/esmvalcore/_provenance.py b/esmvalcore/_provenance.py index 2aa1ff1860..193a823c56 100644 --- a/esmvalcore/_provenance.py +++ b/esmvalcore/_provenance.py @@ -194,7 +194,7 @@ def _initialize_entity(self): for k, v in self.attributes.items() if k not in ('authors', 'projects') } - self.entity = self.provenance.entity('file:' + self.filename, + self.entity = self.provenance.entity(f'file:{self.filename}', attributes) attribute_to_authors(self.entity, self.attributes.get('authors', [])) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 66c1285c99..86411a3e8a 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -16,17 +16,6 @@ from . import __version__ from . import _recipe_checks as check from . import esgf -from ._data_finder import ( - _find_input_files, - _get_timerange_from_years, - _parse_period, - _truncate_dates, - dates_to_timerange, - get_input_filelist, - get_multiproduct_filename, - get_output_file, - get_start_end_date, -) from ._provenance import TrackedFile, get_recipe_provenance from ._task import DiagnosticTask, ResumeTask, TaskSet from .cmor.check import CheckLevels @@ -39,6 +28,16 @@ ) from .config._diagnostics import TAGS from .exceptions import InputFilesNotFound, RecipeError +from .local import _dates_to_timerange as dates_to_timerange +from .local import _get_multiproduct_filename as get_multiproduct_filename +from .local import _get_output_file as get_output_file +from .local import _get_start_end_date as get_start_end_date +from .local import ( + _get_timerange_from_years, + _parse_period, + _truncate_dates, + find_files, +) from .preprocessor import ( DEFAULT_ORDER, FINAL_STEPS, @@ -225,7 +224,7 @@ def _augment(base, update): def _dataset_to_file(variable, config_user): """Find the first file belonging to dataset from variable info.""" - (files, dirnames, filenames) = _get_input_files(variable, config_user) + (files, globs) = _get_input_files(variable, config_user) if not files and variable.get('derive'): required_vars = get_required(variable['short_name'], variable['project']) @@ -233,12 +232,11 @@ def _dataset_to_file(variable, config_user): _augment(required_var, variable) _add_cmor_info(required_var, override=True) _add_extra_facets(required_var, config_user['extra_facets_dir']) - (files, dirnames, - filenames) = _get_input_files(required_var, config_user) + (files, globs) = _get_input_files(required_var, config_user) if files: variable = required_var break - check.data_availability(files, variable, dirnames, filenames) + check.data_availability(files, variable, globs) return files[0] @@ -584,10 +582,13 @@ def _get_input_files(variable, config_user): variable['start_year'] = start_year variable['end_year'] = end_year - (input_files, dirnames, - filenames) = get_input_filelist(variable=variable, - rootpath=config_user['rootpath'], - drs=config_user['drs']) + + variable = dict(variable) + if variable['project'] == 'CMIP5' and variable['frequency'] == 'fx': + variable['ensemble'] = 'r0i0p0' + if variable['frequency'] == 'fx': + variable.pop('timerange', None) + input_files, globs = find_files(debug=True, **variable) # Set up downloading from ESGF if requested. if (not config_user['offline'] @@ -596,8 +597,7 @@ def _get_input_files(variable, config_user): check.data_availability( input_files, variable, - dirnames, - filenames, + globs, log=False, ) except RecipeError: @@ -611,15 +611,14 @@ def _get_input_files(variable, config_user): DOWNLOAD_FILES.add(file) input_files.append(str(local_copy)) - dirnames.append('ESGF:') + globs.append('ESGF') - return (input_files, dirnames, filenames) + return (input_files, globs) def _get_ancestors(variable, config_user): """Get the input files for a single dataset and setup provenance.""" - (input_files, dirnames, - filenames) = _get_input_files(variable, config_user) + (input_files, globs) = _get_input_files(variable, config_user) logger.debug( "Using input files for variable %s of dataset %s:\n%s", @@ -629,7 +628,7 @@ def _get_ancestors(variable, config_user): f'{f} (will be downloaded)' if not os.path.exists(f) else str(f) for f in input_files), ) - check.data_availability(input_files, variable, dirnames, filenames) + check.data_availability(input_files, variable, globs) logger.info("Found input files for %s", variable['alias'].replace('_', ' ')) @@ -836,11 +835,10 @@ def _update_timerange(variable, config_user): check.valid_time_selection(timerange) if '*' in timerange: - (files, _, _) = _find_input_files( - variable, config_user['rootpath'], config_user['drs']) + facets = deepcopy(variable) + facets.pop('timerange', None) + files = find_files(**facets) if not files and not config_user.get('offline', True): - facets = deepcopy(variable) - facets.pop('timerange', None) files = [file.name for file in esgf.find_files(**facets)] if not files: @@ -928,6 +926,8 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, preproc_dir = config_user['preproc_dir'] for variable in variables: + if variable['frequency'] == 'fx': + variable.pop('timerange', None) _update_timerange(variable, config_user) variable['filename'] = get_output_file(variable, config_user['preproc_dir']) @@ -1094,7 +1094,7 @@ def _get_single_preprocessor_task(variables, logger.info("PreprocessingTask %s created.", task.name) logger.debug("PreprocessingTask %s will create the files:\n%s", task.name, - '\n'.join(p.filename for p in task.products)) + '\n'.join(str(p.filename) for p in task.products)) return task diff --git a/esmvalcore/_recipe_checks.py b/esmvalcore/_recipe_checks.py index a58dfd7e75..948319544b 100644 --- a/esmvalcore/_recipe_checks.py +++ b/esmvalcore/_recipe_checks.py @@ -1,5 +1,4 @@ """Module with functions to check a recipe.""" -import itertools import logging import os import re @@ -10,8 +9,8 @@ import isodate import yamale -from ._data_finder import get_start_end_year from .exceptions import InputFilesNotFound, RecipeError +from .local import _get_start_end_year from .preprocessor import TIME_PREPROCESSORS, PreprocessingTask from .preprocessor._multimodel import STATISTIC_MAPPING @@ -94,28 +93,18 @@ def variable(var, required_keys): missing, var.get('short_name'), var.get('diagnostic'))) -def _log_data_availability_errors(input_files, var, dirnames, filenames): +def _log_data_availability_errors(input_files, var, patterns): """Check if the required input data is available.""" var = dict(var) if not input_files: var.pop('filename', None) logger.error("No input files found for variable %s", var) - if dirnames and filenames: - patterns = itertools.product(dirnames, filenames) - patterns = [os.path.join(d, f) for (d, f) in patterns] + if patterns: if len(patterns) == 1: msg = f': {patterns[0]}' else: - msg = '\n{}'.format('\n'.join(patterns)) + msg = '\n{}'.format('\n'.join(str(p) for p in patterns)) logger.error("Looked for files matching%s", msg) - elif dirnames and not filenames: - logger.error( - "Looked for files in %s, but did not find any file pattern " - "to match against", dirnames) - elif filenames and not dirnames: - logger.error( - "Looked for files matching %s, but did not find any existing " - "input directory", filenames) logger.error("Set 'log_level' to 'debug' to get more information") @@ -145,10 +134,10 @@ def _group_years(years): return ", ".join(ranges) -def data_availability(input_files, var, dirnames, filenames, log=True): +def data_availability(input_files, var, patterns, log=True): """Check if input_files cover the required years.""" if log: - _log_data_availability_errors(input_files, var, dirnames, filenames) + _log_data_availability_errors(input_files, var, patterns) if not input_files: raise InputFilesNotFound( @@ -163,7 +152,7 @@ def data_availability(input_files, var, dirnames, filenames, log=True): available_years = set() for filename in input_files: - start, end = get_start_end_year(filename) + start, end = _get_start_end_year(filename) available_years.update(range(start, end + 1)) missing_years = required_years - available_years diff --git a/esmvalcore/cmor/_fixes/ipslcm/ipsl_cm6.py b/esmvalcore/cmor/_fixes/ipslcm/ipsl_cm6.py index dd978b33af..d590daf7af 100644 --- a/esmvalcore/cmor/_fixes/ipslcm/ipsl_cm6.py +++ b/esmvalcore/cmor/_fixes/ipslcm/ipsl_cm6.py @@ -31,8 +31,8 @@ def fix_file(self, filepath, output_dir): However, we take care of ESMValTool policy re. dependencies licence """ - if "_" + self.extra_facets.get("group", - "non-sense") + ".nc" not in filepath: + if "_" + self.extra_facets.get( + "group", "non-sense") + ".nc" not in str(filepath): # No need to filter the file logger.debug("Not filtering for %s", filepath) return filepath @@ -47,11 +47,11 @@ def fix_file(self, filepath, output_dir): # Proceed with CDO selvar varname = self.extra_facets.get(VARNAME_KEY, self.vardef.short_name) - alt_filepath = filepath.replace(".nc", "_cdo_selected.nc") + alt_filepath = str(filepath).replace(".nc", "_cdo_selected.nc") outfile = self.get_fixed_filepath(output_dir, alt_filepath) tim1 = time.time() logger.debug("Using CDO for selecting %s in %s", varname, filepath) - command = ["cdo", "-selvar,%s" % varname, filepath, outfile] + command = ["cdo", "-selvar,%s" % varname, str(filepath), outfile] subprocess.run(command, check=True) logger.debug("CDO selection done in %.2f seconds", time.time() - tim1) return outfile diff --git a/esmvalcore/config-developer.yml b/esmvalcore/config-developer.yml index aec3c3df52..eaf00d67de 100644 --- a/esmvalcore/config-developer.yml +++ b/esmvalcore/config-developer.yml @@ -31,11 +31,11 @@ CMIP6: cmor_strict: true input_dir: default: '/' - BADC: '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{latestversion}' - DKRZ: '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{latestversion}' - ESGF: '{project}/{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{latestversion}' + BADC: '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{version}' + DKRZ: '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{version}' + ESGF: '{project}/{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{version}' ETHZ: '{exp}/{mip}/{short_name}/{dataset}/{ensemble}/{grid}/' - SYNDA: '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{latestversion}' + SYNDA: '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{version}' input_file: '{short_name}_{mip}_{dataset}_{exp}_{ensemble}_{grid}*.nc' output_file: '{project}_{dataset}_{mip}_{exp}_{ensemble}_{short_name}_{grid}' cmor_type: 'CMIP6' @@ -44,15 +44,15 @@ CMIP5: cmor_strict: true input_dir: default: '/' - BADC: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{latestversion}/{short_name}' + BADC: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{version}/{short_name}' BSC: '{type}/{project}/{exp}/{dataset.lower}' CP4CDS: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{short_name}/latest/' - DKRZ: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{latestversion}/{short_name}' + DKRZ: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{version}/{short_name}' ETHZ: '{exp}/{mip}/{short_name}/{dataset}/{ensemble}/' - ESGF: '{project.lower}/{product}/{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{latestversion}' + ESGF: '{project.lower}/{product}/{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{version}' RCAST: '{exp}/{mip}/{short_name}/{dataset}/{ensemble}/' SMHI: '{dataset}/{ensemble}/{exp}/{frequency}' - SYNDA: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{latestversion}' + SYNDA: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{version}' input_file: '{short_name}_{mip}_{dataset}_{exp}_{ensemble}*.nc' output_file: '{project}_{dataset}_{mip}_{exp}_{ensemble}_{short_name}' @@ -60,9 +60,9 @@ CMIP3: cmor_strict: true input_dir: default: '/' - BADC: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{short_name}/{ensemble}/{latestversion}' + BADC: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{short_name}/{ensemble}/{version}' DKRZ: '{exp}/{modeling_realm}/{frequency}/{short_name}/{dataset}/{ensemble}' - ESGF: '{project.lower}/{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{ensemble}/{short_name}/{latestversion}' + ESGF: '{project.lower}/{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{ensemble}/{short_name}/{version}' IPSL: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{ensemble}/{short_name}/{version}/{short_name}' input_file: '{short_name}_*.nc' output_file: '{project}_{institute}_{dataset}_{mip}_{exp}_{ensemble}_{short_name}' @@ -95,7 +95,7 @@ OBS6: native6: cmor_strict: false input_dir: - default: 'Tier{tier}/{dataset}/{latestversion}/{frequency}/{short_name}' + default: 'Tier{tier}/{dataset}/{version}/{frequency}/{short_name}' input_file: default: '*.nc' output_file: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}' @@ -106,7 +106,7 @@ obs4MIPs: cmor_strict: false input_dir: default: 'Tier{tier}/{dataset}' - ESGF: '{project}/{dataset}/{latestversion}' + ESGF: '{project}/{dataset}/{version}' RCAST: '/' IPSL: '{realm}/{short_name}/{freq}/{grid}/{institute}/{dataset}/{latest_version}' input_file: @@ -146,8 +146,8 @@ CORDEX: input_dir: default: '/' spec: '{domain}/{institute}/{driver}/{exp}/{ensemble}/{dataset}/{rcm_version}/{mip}/{short_name}' - BADC: '{domain}/{institute}/{driver}/{exp}/{ensemble}/{dataset}/{rcm_version}/{mip}/{short_name}/{latestversion}' - ESGF: '{project.lower}/output/{domain}/{institute}/{driver}/{exp}/{ensemble}/{dataset}/{rcm_version}/{frequency}/{short_name}/{latestversion}' + BADC: '{domain}/{institute}/{driver}/{exp}/{ensemble}/{dataset}/{rcm_version}/{mip}/{short_name}/{version}' + ESGF: '{project.lower}/output/{domain}/{institute}/{driver}/{exp}/{ensemble}/{dataset}/{rcm_version}/{frequency}/{short_name}/{version}' input_file: '{short_name}_{domain}_{driver}_{exp}_{ensemble}_{dataset}_{rcm_version}_{mip}*.nc' output_file: '{project}_{dataset}_{rcm_version}_{driver}_{domain}_{mip}_{exp}_{ensemble}_{short_name}' cmor_type: 'CMIP5' diff --git a/esmvalcore/config/_config.py b/esmvalcore/config/_config.py index ae7441197d..595a4ded00 100644 --- a/esmvalcore/config/_config.py +++ b/esmvalcore/config/_config.py @@ -97,7 +97,15 @@ def load_config_developer(cfg_file): cfg['obs4MIPs'] = cfg.pop('obs4mips') for project, settings in cfg.items(): + for site, drs in settings['input_dir'].items(): + # Since v2.8, 'version' can be used instead of 'latestversion' + if isinstance(drs, list): + drs = [d.replace('{latestversion}', '{version}') for d in drs] + else: + drs = drs.replace('{latestversion}', '{version}') + settings['input_dir'][site] = drs CFG[project] = settings + read_cmor_tables(cfg_file) diff --git a/esmvalcore/esgf/_download.py b/esmvalcore/esgf/_download.py index 6749c59aaa..ce4a0e8fc3 100644 --- a/esmvalcore/esgf/_download.py +++ b/esmvalcore/esgf/_download.py @@ -19,6 +19,7 @@ import yaml from humanfriendly import format_size, format_timespan +from ..local import LocalFile from ._logon import get_credentials from .facets import DATASET_MAP, FACETS @@ -337,14 +338,16 @@ def local_file(self, dest_folder): Returns ------- - Path + LocalFile The path where the file will be located after download. """ - return Path( + file = LocalFile( dest_folder, *self.dataset.split('.'), self.name, ).absolute() + file.facets = self.facets + return file def download(self, dest_folder): """Download the file. @@ -361,7 +364,7 @@ def download(self, dest_folder): Returns ------- - Path + LocalFile The path where the file will be located after download. """ local_file = self.local_file(dest_folder) diff --git a/esmvalcore/esgf/_search.py b/esmvalcore/esgf/_search.py index 7e3949596f..decbbc90fb 100644 --- a/esmvalcore/esgf/_search.py +++ b/esmvalcore/esgf/_search.py @@ -6,13 +6,13 @@ import pyesgf.search import requests.exceptions -from .._data_finder import ( +from ..config._esgf_pyclient import get_esgf_config +from ..local import ( + _get_start_end_date, _get_timerange_from_years, _parse_period, _truncate_dates, - get_start_end_date, ) -from ..config._esgf_pyclient import get_esgf_config from ._download import ESGFFile from .facets import DATASET_MAP, FACETS @@ -169,7 +169,7 @@ def select_by_time(files, timerange): for file in files: start_date, end_date = _parse_period(timerange) try: - start, end = get_start_end_date(file.name) + start, end = _get_start_end_date(file.name) except ValueError: # If start and end year cannot be read from the filename # just select everything. @@ -195,10 +195,21 @@ def find_files(*, project, short_name, dataset, **facets): dataset : str The name of the dataset. **facets : typing.Union[str, list[str]] - Any other search facets. The special value ``'*'`` will match anything. - If no ``version`` facet is specified, the function returns only the - latest version of each file, while other omitted facets will default - to ``'*'``. + Any other search facets. An ``'*'`` can be used to match + any value. By default, only the latest version of a file will + be returned. To select all versions use ``version='*'`` while other + omitted facets will default to ``'*'``. It is also + possible to specify multiple values for a facet, e.g. + ``exp=['historical', 'ssp585']`` will match any file that belongs + to either the historical or ssp585 experiment. + The ``timerange`` facet can be specified in `ISO 8601 format + `__. + + Note + ---- + A value of ``timerange='*'`` is supported, but combining a ``'*'`` with + a time or period :ref:`as supported in the recipe ` is currently + not supported and will return all found files. Examples -------- diff --git a/esmvalcore/_data_finder.py b/esmvalcore/local.py similarity index 63% rename from esmvalcore/_data_finder.py rename to esmvalcore/local.py index a2c4b90ff2..c9c3f1e5c4 100644 --- a/esmvalcore/_data_finder.py +++ b/esmvalcore/local.py @@ -1,34 +1,25 @@ -"""Data finder module for the ESMValTool.""" -import glob +"""Find files on the local filesystem.""" +from __future__ import annotations + +import itertools import logging import os import re +from glob import glob from pathlib import Path +from typing import Any, Union import iris import isodate +from .config import CFG from .config._config import get_project_config from .exceptions import RecipeError +from .typing import Facets, FacetValue logger = logging.getLogger(__name__) -def find_files(dirnames, filenames): - """Find files matching filenames in dirnames.""" - logger.debug("Looking for files matching %s in %s", filenames, dirnames) - - result = [] - for dirname in dirnames: - for filename_pattern in filenames: - pat = os.path.join(dirname, filename_pattern) - files = glob.glob(pat) - files.sort() # sorting makes it easier to see what was found - result.extend(files) - - return result - - def _get_from_pattern(pattern, date_range_pattern, stem, group): """Get time, date or datetime from date range patterns in file names.""" # @@ -68,7 +59,7 @@ def _get_from_pattern(pattern, date_range_pattern, stem, group): return start_point, end_point -def get_start_end_date(filename): +def _get_start_end_date(filename): """Get the start and end dates as a string from a file name. Examples of allowed dates : 1980, 198001, 19801231, @@ -118,13 +109,13 @@ def get_start_end_date(filename): break if start_date is None or end_date is None: - raise ValueError(f'File {filename} dates do not match a recognized' + raise ValueError(f'File {filename} dates do not match a recognized ' 'pattern and time can not be read from the file') return start_date, end_date -def dates_to_timerange(start_date, end_date): +def _dates_to_timerange(start_date, end_date): """Convert ``start_date`` and ``end_date`` to ``timerange``. Note @@ -162,16 +153,16 @@ def _get_timerange_from_years(variable): start_year = variable.get('start_year') end_year = variable.get('end_year') if start_year and end_year: - variable['timerange'] = dates_to_timerange(start_year, end_year) + variable['timerange'] = _dates_to_timerange(start_year, end_year) elif start_year: - variable['timerange'] = dates_to_timerange(start_year, start_year) + variable['timerange'] = _dates_to_timerange(start_year, start_year) elif end_year: - variable['timerange'] = dates_to_timerange(end_year, end_year) + variable['timerange'] = _dates_to_timerange(end_year, end_year) variable.pop('start_year', None) variable.pop('end_year', None) -def get_start_end_year(filename): +def _get_start_end_year(filename): """Get the start and end year from a file name. Examples of allowed dates : 1980, 198001, 19801231, @@ -215,7 +206,7 @@ def get_start_end_year(filename): break if start_year is None or end_year is None: - raise ValueError(f'File {filename} dates do not match a recognized' + raise ValueError(f'File {filename} dates do not match a recognized ' 'pattern and time can not be read from the file') return int(start_year), int(end_year) @@ -293,7 +284,7 @@ def _truncate_dates(date, file_date): return int(date), int(file_date) -def select_files(filenames, timerange): +def _select_files(filenames, timerange): """Select files containing data between a given timerange. If the timerange is given as a period, the file selection @@ -302,11 +293,15 @@ def select_files(filenames, timerange): Otherwise, the file selection occurs taking into account the time resolution of the file. """ + if '*' in timerange: + # TODO: support * combined with a period + return filenames + selection = [] for filename in filenames: start_date, end_date = _parse_period(timerange) - start, end = get_start_end_date(filename) + start, end = _get_start_end_date(filename) start_date, start = _truncate_dates(start_date, start) end_date, end = _truncate_dates(end_date, end) @@ -317,37 +312,40 @@ def select_files(filenames, timerange): return selection -def _replace_tags(paths, variable): +def _replace_tags( + paths: Union[str, list[str]], + variable: Facets, +) -> list[Path]: """Replace tags in the config-developer's file with actual values.""" if isinstance(paths, str): - paths = set((paths.strip('/'), )) + pathset = set((paths.strip('/'), )) else: - paths = set(path.strip('/') for path in paths) - tlist = set() - for path in paths: + pathset = set(path.strip('/') for path in paths) + tlist: set[str] = set() + for path in pathset: tlist = tlist.union(re.findall(r'{([^}]*)}', path)) if 'sub_experiment' in variable: - new_paths = [] - for path in paths: - new_paths.extend( + new_paths: set[str] = set() + for path in pathset: + new_paths.update( (re.sub(r'(\b{ensemble}\b)', r'{sub_experiment}-\1', path), re.sub(r'({ensemble})', r'{sub_experiment}-\1', path))) tlist.add('sub_experiment') - paths = new_paths + pathset = new_paths for tag in tlist: original_tag = tag tag, _, _ = _get_caps_options(tag) - if tag == 'latestversion': # handled separately later - continue if tag in variable: replacewith = variable[tag] + elif tag == 'version': + replacewith = '*' else: raise RecipeError(f"Dataset key '{tag}' must be specified for " f"{variable}, check your recipe entry") - paths = _replace_tag(paths, original_tag, replacewith) - return paths + pathset = _replace_tag(pathset, original_tag, replacewith) + return [Path(p) for p in pathset] def _replace_tag(paths, tag, replacewith): @@ -383,37 +381,14 @@ def _apply_caps(original, lower, upper): return original -def _resolve_latestversion(dirname_template): - """Resolve the 'latestversion' tag. - - This implementation avoid globbing on centralized clusters with very - large data root dirs (i.e. ESGF nodes like Jasmin/DKRZ). - """ - if '{latestversion}' not in dirname_template: - return dirname_template - - # Find latest version - part1, part2 = dirname_template.split('{latestversion}') - part2 = part2.lstrip(os.sep) - if os.path.exists(part1): - versions = os.listdir(part1) - versions.sort(reverse=True) - for version in ['latest'] + versions: - dirname = os.path.join(part1, version, part2) - if os.path.isdir(dirname): - return dirname - - return None - - -def _select_drs(input_type, drs, project): +def _select_drs(input_type, project): """Select the directory structure of input path.""" cfg = get_project_config(project) input_path = cfg[input_type] if isinstance(input_path, str): return input_path - structure = drs.get(project, 'default') + structure = CFG['drs'].get(project, 'default') if structure in input_path: return input_path[structure] @@ -422,87 +397,61 @@ def _select_drs(input_type, drs, project): structure, project)) -ROOTPATH_WARNED = set() +_ROOTPATH_WARNED = set() -def get_rootpath(rootpath, project): +def _get_rootpath(project): """Select the rootpath.""" + rootpath = CFG['rootpath'] for key in (project, 'default'): if key in rootpath: nonexistent = tuple(p for p in rootpath[key] if not os.path.exists(p)) - if nonexistent and (key, nonexistent) not in ROOTPATH_WARNED: + if nonexistent and (key, nonexistent) not in _ROOTPATH_WARNED: logger.warning( "'%s' rootpaths '%s' set in config-user.yml do not exist", key, ', '.join(str(p) for p in nonexistent)) - ROOTPATH_WARNED.add((key, nonexistent)) + _ROOTPATH_WARNED.add((key, nonexistent)) return rootpath[key] raise KeyError('default rootpath must be specified in config-user file') -def _find_input_dirs(variable, rootpath, drs): - """Return a the full paths to input directories.""" +def _get_globs(variable): + """Compose the globs that will be used to look for files.""" project = variable['project'] - root = get_rootpath(rootpath, project) - path_template = _select_drs('input_dir', drs, project) + rootpaths = _get_rootpath(project) - dirnames = [] - for dirname_template in _replace_tags(path_template, variable): - for base_path in root: - dirname = os.path.join(base_path, dirname_template) - dirname = _resolve_latestversion(dirname) - if dirname is None: - continue - matches = glob.glob(dirname) - matches = [match for match in matches if os.path.isdir(match)] - if matches: - for match in matches: - dirnames.append(match) - else: - logger.debug("Skipping non-existent %s", dirname) + dirname_template = _select_drs('input_dir', project) + dirname_globs = _replace_tags(dirname_template, variable) - return dirnames + filename_template = _select_drs('input_file', project) + filename_globs = _replace_tags(filename_template, variable) + globs = sorted(r / d / f for r in rootpaths for d in dirname_globs + for f in filename_globs) + return globs -def _get_filenames_glob(variable, drs): - """Return patterns that can be used to look for input files.""" - path_template = _select_drs('input_file', drs, variable['project']) - filenames_glob = _replace_tags(path_template, variable) - return filenames_glob +def _get_input_filelist(variable): + """Return the full path to input files.""" + variable = dict(variable) + if 'original_short_name' in variable: + variable['short_name'] = variable['original_short_name'] -def _find_input_files(variable, rootpath, drs): - """Find available input files. + globs = _get_globs(variable) + logger.debug("Looking for files matching %s", globs) - Return the files, the directory in which they are located in, and - the file name. - """ - short_name = variable['short_name'] - variable['short_name'] = variable['original_short_name'] - input_dirs = _find_input_dirs(variable, rootpath, drs) - filenames_glob = _get_filenames_glob(variable, drs) - files = find_files(input_dirs, filenames_glob) - variable['short_name'] = short_name - return (files, input_dirs, filenames_glob) + files = list(Path(file) for glob_ in globs for file in glob(str(glob_))) + files.sort() # sorting makes it easier to see what was found + if 'timerange' in variable: + files = _select_files(files, variable['timerange']) -def get_input_filelist(variable, rootpath, drs): - """Return the full path to input files.""" - # change ensemble to fixed r0i0p0 for fx variables - # this is needed and is not a duplicate effort - if variable['project'] == 'CMIP5' and variable['frequency'] == 'fx': - variable['ensemble'] = 'r0i0p0' - (files, dirnames, filenames) = _find_input_files(variable, rootpath, drs) - # do time gating only for non-fx variables - if variable['frequency'] != 'fx': - files = select_files( - files, - variable['timerange']) - return (files, dirnames, filenames) - - -def get_output_file(variable, preproc_dir): + return files, globs + + +def _get_output_file(variable: dict[str, Any], preproc_dir: Path) -> Path: """Return the full path to the output (preprocessed) file.""" cfg = get_project_config(variable['project']) @@ -510,22 +459,20 @@ def get_output_file(variable, preproc_dir): if isinstance(variable.get('exp'), (list, tuple)): variable = dict(variable) variable['exp'] = '-'.join(variable['exp']) - - outfile = os.path.join( + outfile = _replace_tags(cfg['output_file'], variable)[0] + if 'timerange' in variable: + timerange = variable['timerange'].replace('/', '-') + outfile = Path(f'{outfile}_{timerange}') + outfile = Path(f"{outfile}.nc") + return Path( preproc_dir, - variable['diagnostic'], - variable['variable_group'], - _replace_tags(cfg['output_file'], variable)[0], + variable.get('diagnostic', ''), + variable.get('variable_group', ''), + outfile, ) - if variable['frequency'] != 'fx': - timerange = variable['timerange'].replace('/', '-') - outfile += f'_{timerange}' - - outfile += '.nc' - return outfile -def get_multiproduct_filename(attributes, preproc_dir): +def _get_multiproduct_filename(attributes: dict, preproc_dir: Path) -> Path: """Get ensemble/multi-model filename depending on settings.""" relevant_keys = [ 'project', 'dataset', 'exp', 'ensemble_statistics', @@ -547,7 +494,7 @@ def get_multiproduct_filename(attributes, preproc_dir): filename_segments.append( f"{attributes['timerange'].replace('/', '-')}.nc") - outfile = os.path.join( + outfile = Path( preproc_dir, attributes['diagnostic'], attributes['variable_group'], @@ -555,3 +502,162 @@ def get_multiproduct_filename(attributes, preproc_dir): ) return outfile + + +def _path2facets(path: Path, drs: str) -> dict[str, str]: + """Extract facets from a path using a DRS like '{facet1}/{facet2}'.""" + keys = [] + for key in re.findall(r"{(.*?)}", drs): + key = key.split('.')[0] # Remove trailing .lower and .upper + keys.append(key) + start, end = -len(keys) - 1, -1 + values = path.parts[start:end] + facets = {key: values[idx] for idx, key in enumerate(keys)} + return facets + + +def _filter_versions_called_latest( + files: list['LocalFile'], +) -> list['LocalFile']: + """Filter out versions called 'latest' if they are duplicates. + + On compute clusters it is usual to have a symbolic link to the + latest version called 'latest'. Those need to be skipped in order to + find valid version names and avoid duplicate results. + """ + resolved_valid_versions = { + f.resolve(strict=False) + for f in files if f.facets.get('version') != 'latest' + } + return [ + f for f in files if f.facets.get('version') != 'latest' or f.resolve( + strict=False) not in resolved_valid_versions + ] + + +def _select_latest_version(files: list['LocalFile']) -> list['LocalFile']: + """Select only the latest version of files.""" + + def filename(file): + return file.name + + def version(file): + return file.facets.get('version', '') + + result = [] + for _, group in itertools.groupby(sorted(files, key=filename), + key=filename): + duplicates = sorted(group, key=version) + latest = duplicates[-1] + result.append(latest) + return result + + +def find_files( + *, + debug: bool = False, + **facets: FacetValue, +) -> Union[list[LocalFile], tuple[list[LocalFile], list[Path]]]: + """Find files on the local filesystem. + + The directories that are searched for files are defined in + :data:`esmvalcore.config.CFG` under the ``'rootpath'`` key using the + directory structure defined under the ``'drs'`` key. + If ``esmvalcore.config.CFG['rootpath']`` contains a key that matches the + value of the ``project`` facet, those paths will be used. If there is no + project specific key, the directories in + ``esmvalcore.config.CFG['rootpath']['default']`` will be searched. + + See :ref:`findingdata` for extensive instructions on configuring ESMValCore + so it can find files locally. + + Parameters + ---------- + debug + When debug is set to :obj:`True`, the function will return a tuple + with the first element containing the files that were found + and the second element containing the :func:`glob.glob` patterns that + were used to search for files. + **facets + Facets used to search for files. An ``'*'`` can be used to match + any value. By default, only the latest version of a file will + be returned. To select all versions use ``version='*'``. It is also + possible to specify multiple values for a facet, e.g. + ``exp=['historical', 'ssp585']`` will match any file that belongs + to either the historical or ssp585 experiment. + The ``timerange`` facet can be specified in `ISO 8601 format + `__. + + Note + ---- + A value of ``timerange='*'`` is supported, but combining a ``'*'`` with + a time or period :ref:`as supported in the recipe ` is currently + not supported and will return all found files. + + Examples + -------- + Search for files containing surface air temperature from any CMIP6 model + for the historical experiment: + + >>> esmvalcore.local.find_files( + ... project='CMIP6', + ... activity='CMIP', + ... mip='Amon', + ... short_name='tas', + ... exp='historical', + ... dataset='*', + ... ensemble='*', + ... grid='*', + ... institute='*', + ... ) # doctest: +SKIP + [LocalFile('/home/bandela/climate_data/CMIP6/CMIP/BCC/BCC-ESM1/historical/r1i1p1f1/Amon/tas/gn/v20181214/tas_Amon_BCC-ESM1_historical_r1i1p1f1_gn_185001-201412.nc')] + + Returns + ------- + list[LocalFile] + The files that were found. + """ # pylint: disable=line-too-long + filenames, globs = _get_input_filelist(facets) + drs = _select_drs('input_dir', facets['project']) + if isinstance(drs, list): + # Not sure how to handle a list of DRSs + drs = '' + files = [] + filter_latest = False + for filename in filenames: + file = LocalFile(filename) + file.facets.update(_path2facets(file, drs)) + if file.facets.get('version') == 'latest': + filter_latest = True + files.append(file) + + if filter_latest: + files = _filter_versions_called_latest(files) + + if 'version' not in facets: + files = _select_latest_version(files) + + if debug: + return files, globs + return files + + +class LocalFile(type(Path())): # type: ignore + """File on the local filesystem.""" + + @property + def facets(self) -> Facets: + """Facets describing the file. + + Note + ---- + When using :func:`find_files`, facets are read from the directory + structure. Facets stored in filenames are not yet supported. + """ + if not hasattr(self, '_facets'): + self._facets: Facets = {} + return self._facets + + @facets.setter + def facets(self, value: Facets): + self._facets = value diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index 8f806ecebe..c7ca5a8818 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -2,6 +2,7 @@ import copy import inspect import logging +from pathlib import Path from pprint import pformat from iris.cube import Cube @@ -325,7 +326,7 @@ def _run_preproc_function(function, items, kwargs, input_files=None): f"here; refer to the debug log for a full list)") # Make sure that the arguments are indexable - if isinstance(items, (PreprocessorFile, Cube, str)): + if isinstance(items, (PreprocessorFile, Cube, str, Path)): items = [items] if isinstance(items, set): items = list(items) @@ -361,7 +362,7 @@ def preprocess(items, step, input_files=None, **settings): items = [] for item in result: - if isinstance(item, (PreprocessorFile, Cube, str)): + if isinstance(item, (PreprocessorFile, Cube, str, Path)): items.append(item) else: items.extend(item) diff --git a/esmvalcore/preprocessor/_ancillary_vars.py b/esmvalcore/preprocessor/_ancillary_vars.py index 482f2c8803..dd77e6b946 100644 --- a/esmvalcore/preprocessor/_ancillary_vars.py +++ b/esmvalcore/preprocessor/_ancillary_vars.py @@ -1,6 +1,7 @@ """Preprocessor functions for ancillary variables and cell measures.""" import logging +from pathlib import Path import dask.array as da import iris @@ -158,7 +159,7 @@ def add_fx_variables(cube, fx_variables, check_level): for fx_info in fx_variables.values(): if not fx_info: continue - if isinstance(fx_info['filename'], str): + if isinstance(fx_info['filename'], (str, Path)): fx_info['filename'] = [fx_info['filename']] fx_cube = _load_fx(cube, fx_info, check_level) diff --git a/esmvalcore/preprocessor/_io.py b/esmvalcore/preprocessor/_io.py index fff3548539..3690863c48 100644 --- a/esmvalcore/preprocessor/_io.py +++ b/esmvalcore/preprocessor/_io.py @@ -135,6 +135,7 @@ def load(file, callback=None, ignore_warnings=None): ValueError Cubes are empty. """ + file = str(file) logger.debug("Loading:\n%s", file) if ignore_warnings is None: ignore_warnings = [] diff --git a/esmvalcore/preprocessor/_regrid.py b/esmvalcore/preprocessor/_regrid.py index eabbfd853e..e4edbce9fe 100644 --- a/esmvalcore/preprocessor/_regrid.py +++ b/esmvalcore/preprocessor/_regrid.py @@ -7,6 +7,7 @@ import re from copy import deepcopy from decimal import Decimal +from pathlib import Path from typing import Dict import iris @@ -548,7 +549,7 @@ def regrid(cube, target_grid, scheme, lat_offset=True, lon_offset=True): reference: esmf_regrid.schemes:ESMFAreaWeighted """ - if isinstance(target_grid, str): + if isinstance(target_grid, (str, Path)): if os.path.isfile(target_grid): target_grid = iris.load_cube(target_grid) else: diff --git a/esmvalcore/typing.py b/esmvalcore/typing.py new file mode 100644 index 0000000000..a217bf46ee --- /dev/null +++ b/esmvalcore/typing.py @@ -0,0 +1,11 @@ +"""Type aliases for providing type hints.""" +from __future__ import annotations + +from numbers import Number +from typing import Dict, Sequence, Union + +FacetValue = Union[str, Sequence[str], Number] +"""Type describing a single facet.""" + +Facets = Dict[str, FacetValue] +"""Type describing a collection of facets.""" diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 3bdd9a9c42..d40e0dbea0 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,9 +1,10 @@ import os +from pathlib import Path import iris import pytest -from esmvalcore import _data_finder +import esmvalcore.local from esmvalcore.config import CFG, _config from esmvalcore.config._config_object import CFG_DEFAULT @@ -15,9 +16,9 @@ def session(tmp_path, monkeypatch): session.update(CFG_DEFAULT) session['output_dir'] = tmp_path / 'esmvaltool_output' - # The patched_data_finder fixture does not return the correct input + # The patched_datafinder fixture does not return the correct input # directory structure, so make sure it is set to flat for every project - session['drs'] = {} + monkeypatch.setitem(CFG, 'drs', {}) for project in _config.CFG: monkeypatch.setitem(_config.CFG[project]['input_dir'], 'default', '/') return session @@ -36,8 +37,8 @@ def create_test_file(filename, tracking_id=None): iris.save(cube, filename) -def _get_filenames(root_path, filenames, tracking_id): - filename = filenames[0] +def _get_filenames(root_path, filename, tracking_id): + filename = Path(filename).name filename = str(root_path / 'input' / filename) filenames = [] if filename.endswith('[_.]*nc'): @@ -71,14 +72,10 @@ def tracking_ids(i=0): tracking_id = tracking_ids() - def find_files(_, filenames): - # Any occurrence of [something] in filename should have - # been replaced before this function is called. - for filename in filenames: - assert '{' not in filename - return _get_filenames(tmp_path, filenames, tracking_id) + def glob(file_glob): + return _get_filenames(tmp_path, file_glob, tracking_id) - monkeypatch.setattr(_data_finder, 'find_files', find_files) + monkeypatch.setattr(esmvalcore.local, 'glob', glob) @pytest.fixture @@ -91,22 +88,16 @@ def tracking_ids(i=0): tracking_id = tracking_ids() - def find_files(_, filenames): - # Any occurrence of [something] in filename should have - # been replaced before this function is called. - for filename in filenames: - assert '{' not in filename - + def glob(filename): # Fail for specified fx variables - for filename in filenames: - if 'fx_' in filename: - return [] - if 'sftlf' in filename: - return [] - if 'IyrAnt_' in filename: - return [] - if 'IyrGre_' in filename: - return [] - return _get_filenames(tmp_path, filenames, tracking_id) - - monkeypatch.setattr(_data_finder, 'find_files', find_files) + if 'fx_' in filename: + return [] + if 'sftlf' in filename: + return [] + if 'IyrAnt_' in filename: + return [] + if 'IyrGre_' in filename: + return [] + return _get_filenames(tmp_path, filename, tracking_id) + + monkeypatch.setattr(esmvalcore.local, 'glob', glob) diff --git a/tests/integration/data_finder.yml b/tests/integration/data_finder.yml index 74b50980d5..7179014173 100644 --- a/tests/integration/data_finder.yml +++ b/tests/integration/data_finder.yml @@ -36,6 +36,7 @@ get_output_file: frequency: mon mip: Amon exp: amip + channel: Amon timerange: '1960/1980' diagnostic: test_diag preprocessor: test_preproc @@ -70,6 +71,7 @@ get_output_file: frequency: mon mip: Amon exp: amip + var_type: atm_2d_ml timerange: '1960/1980' diagnostic: test_diag preprocessor: test_preproc @@ -104,6 +106,8 @@ get_output_file: mip: Amon exp: amip case: f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1 + gcomp: atm + scomp: cam type: h0 timerange: '2000/2002' diagnostic: test_diag @@ -275,7 +279,8 @@ get_input_filelist: - drs: default variable: *variable - dirs: null + dirs: + - '' file_patterns: - ta_Amon_HadGEM2-ES_historical_r1i1p1*.nc found_files: [] @@ -293,13 +298,48 @@ get_input_filelist: - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_195912-198411.nc - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc dirs: - - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta + - INPE/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/*/ta + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/*/ta file_patterns: - ta_Amon_HadGEM2-ES_historical_r1i1p1*.nc found_files: - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_195912-198411.nc - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc + - drs: BADC + variable: + <<: *variable + timerange: '2000/2005' + version: v20110329 + available_files: + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20110329/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc + dirs: + - INPE/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20110329/ta + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20110329/ta + file_patterns: + - ta_Amon_HadGEM2-ES_historical_r1i1p1*.nc + found_files: + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20110329/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc + + - drs: BADC + variable: + <<: *variable + ensemble: '*' + timerange: '2000/2005' + available_files: + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20110329/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r2i1p1/v20110329/ta/ta_Amon_HadGEM2-ES_historical_r2i1p1_198412-200511.nc + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r2i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r2i1p1_198412-200511.nc + dirs: + - INPE/HadGEM2-ES/historical/mon/atmos/Amon/*/*/ta + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/*/*/ta + file_patterns: + - ta_Amon_HadGEM2-ES_historical_**.nc + found_files: + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r2i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r2i1p1_198412-200511.nc - drs: BADC variable: @@ -316,12 +356,13 @@ get_input_filelist: - link_name: MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/latest target: v20120928 dirs: - - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/latest/ta + - INPE/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/*/ta + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/*/ta file_patterns: - ta_Amon_HadGEM2-ES_historical_r1i1p1*.nc found_files: - - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/latest/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_195912-198411.nc - - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/latest/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_195912-198411.nc + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc - drs: DKRZ variable: @@ -335,7 +376,8 @@ get_input_filelist: - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20110330/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_195912-198411.nc - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20110330/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc dirs: - - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20110330/ta + - INPE/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/*/ta + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/*/ta file_patterns: - ta_Amon_HadGEM2-ES_historical_r1i1p1*.nc found_files: @@ -357,9 +399,12 @@ get_input_filelist: - MOHC/HadGEM2-ES/rcp45/mon/atmos/Amon/r1i1p1/v20110330/ta/ta_Amon_HadGEM2-ES_rcp45_r1i1p1_200601-210012.nc - MOHC/HadGEM2-ES/rcp85/mon/atmos/Amon/r1i1p1/v20110330/ta/ta_Amon_HadGEM2-ES_rcp85_r1i1p1_200601-210012.nc dirs: - - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20110330/ta - - MOHC/HadGEM2-ES/rcp45/mon/atmos/Amon/r1i1p1/v20110330/ta - - MOHC/HadGEM2-ES/rcp85/mon/atmos/Amon/r1i1p1/v20110330/ta + - INPE/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/*/ta + - INPE/HadGEM2-ES/rcp45/mon/atmos/Amon/r1i1p1/*/ta + - INPE/HadGEM2-ES/rcp85/mon/atmos/Amon/r1i1p1/*/ta + - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/*/ta + - MOHC/HadGEM2-ES/rcp45/mon/atmos/Amon/r1i1p1/*/ta + - MOHC/HadGEM2-ES/rcp85/mon/atmos/Amon/r1i1p1/*/ta file_patterns: - ta_Amon_HadGEM2-ES_historical_r1i1p1*.nc - ta_Amon_HadGEM2-ES_rcp45_r1i1p1*.nc @@ -434,7 +479,8 @@ get_input_filelist: - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/Amon/ta/gn/v20200101/ta_Amon_HadGEM3-GC31-LL_historical_r1i1p1f1_gn_195001-199912.nc - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/Amon/ta/gn/v20200101/ta_Amon_HadGEM3-GC31-LL_historical_r1i1p1f1_gn_200001-201412.nc dirs: - - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/Amon/ta/gn/v20200101/ + - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/Amon/ta/gn/*/ + - CMIP/NERC/HadGEM3-GC31-LL/historical/r1i1p1f1/Amon/ta/gn/*/ file_patterns: - ta_Amon_HadGEM3-GC31-LL_historical_r1i1p1f1_gn*.nc found_files: @@ -591,7 +637,7 @@ get_input_filelist: modeling_realm: [atmos] mip: fx exp: historical - ensemble: r1i1p1 + ensemble: r0i0p0 diagnostic: test_diag preprocessor: test_preproc available_files: @@ -616,14 +662,15 @@ get_input_filelist: modeling_realm: [atmos] mip: fx exp: historical - ensemble: r1i1p1 + ensemble: r0i0p0 diagnostic: test_diag preprocessor: test_preproc available_files: - MOHC/HadGEM2-ES/historical/fx/atmos/fx/r1i1p1/v20110330/sftlf/sftlf_fx_HadGEM2-ES_historical_r1i1p1.nc - MOHC/HadGEM2-ES/historical/fx/atmos/fx/r0i0p0/v20110330/sftlf/sftlf_fx_HadGEM2-ES_historical_r0i0p0.nc dirs: - - MOHC/HadGEM2-ES/historical/fx/atmos/fx/r0i0p0/v20110330/sftlf + - INPE/HadGEM2-ES/historical/fx/atmos/fx/r0i0p0/*/sftlf + - MOHC/HadGEM2-ES/historical/fx/atmos/fx/r0i0p0/*/sftlf file_patterns: - sftlf_fx_HadGEM2-ES_historical_r0i0p0*.nc found_files: @@ -641,13 +688,15 @@ get_input_filelist: modeling_realm: [atmos] mip: fx exp: historical - ensemble: r1i1p1 + ensemble: r0i0p0 diagnostic: test_diag preprocessor: test_preproc available_files: - - MOHC/HadGEM2-ES/historical/fx/atmos/fx/r1i1p1/v20110330/sftlf/sftlf_fx_HadGEM2-ES_historical_r0i0p0.nc - - MOHC/HadGEM2-ES/historical/fx/atmos/fx/r1i1p1/v20110330/areacella/areacella_fx_HadGEM2-ES_historical_r0i0p0.nc - dirs: [] + - MOHC/HadGEM2-ES/historical/fx/atmos/fx/r0i0p0/v20110330/sftlf/sftlf_fx_HadGEM2-ES_historical_r0i0p0.nc + - MOHC/HadGEM2-ES/historical/fx/atmos/fx/r0i0p0/v20110330/areacella/areacella_fx_HadGEM2-ES_historical_r0i0p0.nc + dirs: + - INPE/HadGEM2-ES/historical/fx/atmos/fx/r0i0p0/*/orog/ + - MOHC/HadGEM2-ES/historical/fx/atmos/fx/r0i0p0/*/orog/ file_patterns: - orog_fx_HadGEM2-ES_historical_r0i0p0*.nc found_files: [] @@ -674,7 +723,8 @@ get_input_filelist: - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/Omon/areacello/gn/v20200101/areacello_Omon-GC31-LL_historical_r1i1p1f1_gn_199901-200012.nc - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/Ofx/areacello/gn/v20200101/areacello_Ofx_HadGEM3-GC31-LL_historical_r1i1p1f1_gn.nc dirs: - - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/Ofx/areacello/gn/v20200101/ + - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/Ofx/areacello/gn/*/ + - CMIP/NERC/HadGEM3-GC31-LL/historical/r1i1p1f1/Ofx/areacello/gn/*/ file_patterns: - areacello_Ofx_HadGEM3-GC31-LL_historical_r1i1p1f1_gn*.nc found_files: @@ -703,7 +753,8 @@ get_input_filelist: - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/Omon/areacello/gn/v20200101/areacello_Omon_HadGEM3-GC31-LL_historical_r1i1p1f1_gn_199901-200012.nc - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/Ofx/areacello/gn/v20200101/areacello_Ofx_HadGEM3-GC31-LL_historical_r1i1p1f1_gn.nc dirs: - - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/Omon/areacello/gn/v20200101/ + - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/Omon/areacello/gn/*/ + - CMIP/NERC/HadGEM3-GC31-LL/historical/r1i1p1f1/Omon/areacello/gn/*/ file_patterns: - areacello_Omon_HadGEM3-GC31-LL_historical_r1i1p1f1_gn*.nc found_files: @@ -732,7 +783,8 @@ get_input_filelist: - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/Omon/volcello/gn/v20200101/this_is_a_wrong_file.nc - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/Ofx/volcello/gn/v20200101/volcello_Ofx_HadGEM3-GC31-LL_historical_r1i1p1f1_gn.nc dirs: - - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/Omon/volcello/gn/v20200101/ + - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/Omon/volcello/gn/*/ + - CMIP/NERC/HadGEM3-GC31-LL/historical/r1i1p1f1/Omon/volcello/gn/*/ file_patterns: - volcello_Omon_HadGEM3-GC31-LL_historical_r1i1p1f1_gn*.nc found_files: [] @@ -757,7 +809,9 @@ get_input_filelist: preprocessor: test_preproc available_files: - CMIP/MOHC/HadGEM3-GC31-LL/historical/r0i0p0/Ofx/volcello/gn/v20200101/volcello_Ofx_HadGEM3-GC31-LL_historical_r0i0p0_gn.nc - dirs: [] + dirs: + - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/Ofx/volcello/gn/*/ + - CMIP/NERC/HadGEM3-GC31-LL/historical/r1i1p1f1/Ofx/volcello/gn/*/ file_patterns: - volcello_Ofx_HadGEM3-GC31-LL_historical_r1i1p1f1_gn*.nc found_files: [] @@ -775,7 +829,6 @@ get_input_filelist: mip: fx exp: historical ensemble: r1i1p1 - timerange: '1999/2000' diagnostic: test_diag preprocessor: test_preproc available_files: @@ -868,6 +921,30 @@ get_input_filelist: - OBS6_ERA-Interim_reanaly_42_Omon_deptho[_.]*nc found_files: [] + - drs: default + variable: + short_name: tas + dataset: ERA5 + project: native6 + frequency: mon + mip: Amon + tier: 3 + type: reanaly + timerange: '2000/2010' + available_files: + - Tier3/ERA5/1/mon/tas/era5_2m_temperature_2000_monthly.nc + - Tier3/ERA5/1/mon/tas/era5_2m_temperature_2001_monthly.nc + dirs: + - Tier3/ERA5/*/mon/tas + file_patterns: + - '*.nc' + found_files: + - Tier3/ERA5/1/mon/tas/era5_2m_temperature_2000_monthly.nc + - Tier3/ERA5/1/mon/tas/era5_2m_temperature_2001_monthly.nc + available_symlinks: + - link_name: Tier3/ERA5/latest + target: '1' + # EMAC - drs: default @@ -880,6 +957,8 @@ get_input_filelist: frequency: mon mip: Amon exp: amip + channel: Amon + postproc_flag: '' timerange: '200002/200003' diagnostic: test_diag preprocessor: test_preproc @@ -952,6 +1031,7 @@ get_input_filelist: frequency: mon mip: Amon exp: amip + var_type: atm_2d_ml timerange: '200002/200003' diagnostic: test_diag preprocessor: test_preproc @@ -1014,7 +1094,12 @@ get_input_filelist: frequency: mon mip: Amon case: f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1 + gcomp: atm + scomp: cam type: h0 + tdir: '' + tperiod: '' + string: '' timerange: '2000/2002' diagnostic: test_diag preprocessor: test_preproc @@ -1024,6 +1109,7 @@ get_input_filelist: - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1/atm/hist/f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.cam.h0.2002.nc dirs: - '' + - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1/atm/proc - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1/atm/hist file_patterns: - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.cam.h0.*nc diff --git a/tests/integration/test_data_finder.py b/tests/integration/test_data_finder.py deleted file mode 100644 index 7e74266f45..0000000000 --- a/tests/integration/test_data_finder.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Tests for _data_finder.py.""" -import os -import shutil -import tempfile - -import pytest -import yaml - -import esmvalcore.config -from esmvalcore._data_finder import ( - _find_input_files, - get_input_filelist, - get_output_file, -) - -# Load test configuration -with open(os.path.join(os.path.dirname(__file__), 'data_finder.yml')) as file: - CONFIG = yaml.safe_load(file) - - -def _augment_with_extra_facets(variable): - """Augment variable dict with extra facets.""" - extra_facets = esmvalcore.config._config.get_extra_facets( - variable['project'], - variable['dataset'], - variable['mip'], - variable['short_name'], - (), - ) - for (key, val) in extra_facets.items(): - if key not in variable: - variable[key] = val - - -def print_path(path): - """Print path.""" - txt = path - if os.path.isdir(path): - txt += '/' - if os.path.islink(path): - txt += ' -> ' + os.readlink(path) - print(txt) - - -def tree(path): - """Print path, similar to the the `tree` command.""" - print_path(path) - for dirpath, dirnames, filenames in os.walk(path): - for dirname in dirnames: - print_path(os.path.join(dirpath, dirname)) - for filename in filenames: - print_path(os.path.join(dirpath, filename)) - - -def create_file(filename): - """Create an empty file.""" - dirname = os.path.dirname(filename) - if not os.path.exists(dirname): - os.makedirs(dirname) - - with open(filename, 'a'): - pass - - -def create_tree(path, filenames=None, symlinks=None): - """Create directory structure and files.""" - for filename in filenames or []: - create_file(os.path.join(path, filename)) - - for symlink in symlinks or []: - link_name = os.path.join(path, symlink['link_name']) - os.symlink(symlink['target'], link_name) - - -@pytest.mark.parametrize('cfg', CONFIG['get_output_file']) -def test_get_output_file(cfg): - """Test getting output name for preprocessed files.""" - _augment_with_extra_facets(cfg['variable']) - output_file = get_output_file(cfg['variable'], cfg['preproc_dir']) - assert output_file == cfg['output_file'] - - -@pytest.fixture -def root(): - """Root function for tests.""" - dirname = tempfile.mkdtemp() - yield os.path.join(dirname, 'output1') - print("Directory structure was:") - tree(dirname) - shutil.rmtree(dirname) - - -@pytest.mark.parametrize('cfg', CONFIG['get_input_filelist']) -def test_get_input_filelist(root, cfg): - """Test retrieving input filelist.""" - create_tree(root, cfg.get('available_files'), - cfg.get('available_symlinks')) - - # Augment variable dict with extra facets - _augment_with_extra_facets(cfg['variable']) - - # Find files - rootpath = {cfg['variable']['project']: [root]} - drs = {cfg['variable']['project']: cfg['drs']} - timerange = cfg['variable'].get('timerange') - if timerange and '*' in timerange: - (files, _, _) = _find_input_files(cfg['variable'], rootpath, drs) - ref_files = [ - os.path.join(root, file) for file in cfg['found_files']] - # Test result - assert sorted(files) == sorted(ref_files) - else: - (input_filelist, dirnames, - filenames) = get_input_filelist(cfg['variable'], rootpath, drs) - # Test result - ref_files = [os.path.join(root, file) for file in cfg['found_files']] - if cfg['dirs'] is None: - ref_dirs = [] - else: - ref_dirs = [os.path.join(root, dir) for dir in cfg['dirs']] - ref_patterns = cfg['file_patterns'] - - assert sorted(input_filelist) == sorted(ref_files) - assert sorted(dirnames) == sorted(ref_dirs) - assert sorted(filenames) == sorted(ref_patterns) diff --git a/tests/integration/test_local.py b/tests/integration/test_local.py new file mode 100644 index 0000000000..02982177a9 --- /dev/null +++ b/tests/integration/test_local.py @@ -0,0 +1,114 @@ +"""Tests for `esmvalcore.local`.""" +import os +import pprint +from pathlib import Path + +import pytest +import yaml + +from esmvalcore.config import CFG +from esmvalcore.local import LocalFile, _get_output_file, find_files + +# Load test configuration +with open(os.path.join(os.path.dirname(__file__), 'data_finder.yml')) as file: + CONFIG = yaml.safe_load(file) + + +def print_path(path): + """Print path.""" + txt = path + if os.path.isdir(path): + txt += '/' + if os.path.islink(path): + txt += ' -> ' + os.readlink(path) + print(txt) + + +def tree(path): + """Print path, similar to the the `tree` command.""" + print_path(path) + for dirpath, dirnames, filenames in os.walk(path): + for dirname in dirnames: + print_path(os.path.join(dirpath, dirname)) + for filename in filenames: + print_path(os.path.join(dirpath, filename)) + + +def create_file(filename): + """Create an empty file.""" + dirname = os.path.dirname(filename) + if not os.path.exists(dirname): + os.makedirs(dirname) + + with open(filename, 'a'): + pass + + +def create_tree(path, filenames=None, symlinks=None): + """Create directory structure and files.""" + for filename in filenames or []: + create_file(os.path.join(path, filename)) + + for symlink in symlinks or []: + link_name = os.path.join(path, symlink['link_name']) + os.symlink(symlink['target'], link_name) + + +@pytest.mark.parametrize('cfg', CONFIG['get_output_file']) +def test_get_output_file(cfg): + """Test getting output name for preprocessed files.""" + output_file = _get_output_file(cfg['variable'], cfg['preproc_dir']) + expected = Path(cfg['output_file']) + assert output_file == expected + + +@pytest.fixture +def root(tmp_path): + """Root function for tests.""" + dirname = str(tmp_path) + yield dirname + print("Directory structure was:") + tree(dirname) + + +@pytest.mark.parametrize('cfg', CONFIG['get_input_filelist']) +def test_find_files(monkeypatch, root, cfg): + """Test retrieving input filelist.""" + print(f"Testing DRS {cfg['drs']} with variable:\n", + pprint.pformat(cfg['variable'])) + project = cfg['variable']['project'] + monkeypatch.setitem(CFG, 'drs', {project: cfg['drs']}) + monkeypatch.setitem(CFG, 'rootpath', {project: root}) + create_tree(root, cfg.get('available_files'), + cfg.get('available_symlinks')) + + # Find files + input_filelist, globs = find_files(debug=True, **cfg['variable']) + # Test result + ref_files = [Path(root, file) for file in cfg['found_files']] + ref_globs = [ + Path(root, d, f) for d in cfg['dirs'] for f in cfg['file_patterns'] + ] + assert sorted([Path(f) for f in input_filelist]) == sorted(ref_files) + assert sorted([Path(g) for g in globs]) == sorted(ref_globs) + + +def test_find_files_with_facets(monkeypatch, root): + """Test that a LocalFile with populated `facets` is returned.""" + for cfg in CONFIG['get_input_filelist']: + if cfg['drs'] != 'default': + break + + project = cfg['variable']['project'] + monkeypatch.setitem(CFG, 'drs', {project: cfg['drs']}) + monkeypatch.setitem(CFG, 'rootpath', {project: root}) + + create_tree(root, cfg.get('available_files'), + cfg.get('available_symlinks')) + + # Find files + input_filelist = find_files(**cfg['variable']) + ref_files = [Path(root, file) for file in cfg['found_files']] + assert sorted([Path(f) for f in input_filelist]) == sorted(ref_files) + assert isinstance(input_filelist[0], LocalFile) + assert input_filelist[0].facets diff --git a/tests/integration/test_provenance.py b/tests/integration/test_provenance.py index 630a7c0517..091069f3b6 100644 --- a/tests/integration/test_provenance.py +++ b/tests/integration/test_provenance.py @@ -3,7 +3,7 @@ def get_file_record(prov, filename): - records = prov.get_record('file:' + filename) + records = prov.get_record(f'file:{filename}') assert records return records[0] diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py index 06a7dcdbb0..d4a348871d 100644 --- a/tests/integration/test_recipe.py +++ b/tests/integration/test_recipe.py @@ -136,7 +136,7 @@ def _get_default_settings_for_chl(fix_dir, save_filename, preprocessor): 'diagnostic': 'diagnostic_name', 'ensemble': 'r1i1p1', 'exp': 'historical', - 'filename': fix_dir.replace('_fixed', '.nc'), + 'filename': Path(fix_dir.replace('_fixed', '.nc')), 'frequency': 'yr', 'institute': ['CCCma'], 'long_name': 'Total Chlorophyll Mass Concentration', @@ -161,7 +161,7 @@ def _get_default_settings_for_chl(fix_dir, save_filename, preprocessor): 'diagnostic': 'diagnostic_name', 'ensemble': 'r1i1p1', 'exp': 'historical', - 'filename': fix_dir.replace('_fixed', '.nc'), + 'filename': Path(fix_dir.replace('_fixed', '.nc')), 'frequency': 'yr', 'institute': ['CCCma'], 'long_name': 'Total Chlorophyll Mass Concentration', @@ -185,7 +185,7 @@ def _get_default_settings_for_chl(fix_dir, save_filename, preprocessor): 'diagnostic': 'diagnostic_name', 'ensemble': 'r1i1p1', 'exp': 'historical', - 'filename': fix_dir.replace('_fixed', '.nc'), + 'filename': Path(fix_dir.replace('_fixed', '.nc')), 'frequency': 'yr', 'institute': ['CCCma'], 'long_name': 'Total Chlorophyll Mass Concentration', @@ -595,7 +595,7 @@ def test_default_fx_preprocessor(tmp_path, patched_datafinder, config_user): 'diagnostic': 'diagnostic_name', 'ensemble': 'r0i0p0', 'exp': 'historical', - 'filename': fix_dir.replace('_fixed', '.nc'), + 'filename': Path(fix_dir.replace('_fixed', '.nc')), 'frequency': 'fx', 'institute': ['CCCma'], 'long_name': 'Land Area Fraction', @@ -619,7 +619,7 @@ def test_default_fx_preprocessor(tmp_path, patched_datafinder, config_user): 'diagnostic': 'diagnostic_name', 'ensemble': 'r0i0p0', 'exp': 'historical', - 'filename': fix_dir.replace('_fixed', '.nc'), + 'filename': Path(fix_dir.replace('_fixed', '.nc')), 'frequency': 'fx', 'institute': ['CCCma'], 'long_name': 'Land Area Fraction', @@ -642,7 +642,7 @@ def test_default_fx_preprocessor(tmp_path, patched_datafinder, config_user): 'diagnostic': 'diagnostic_name', 'ensemble': 'r0i0p0', 'exp': 'historical', - 'filename': fix_dir.replace('_fixed', '.nc'), + 'filename': Path(fix_dir.replace('_fixed', '.nc')), 'frequency': 'fx', 'institute': ['CCCma'], 'long_name': 'Land Area Fraction', @@ -874,7 +874,7 @@ def test_simple_cordex_recipe(tmp_path, patched_datafinder, config_user): recipe = get_recipe(tmp_path, content, config_user) variable = recipe.diagnostics['test']['preprocessor_output']['tas'][0] - filename = variable.pop('filename').split('/')[-1] + filename = variable.pop('filename').name assert (filename == 'CORDEX_MOHC-HadGEM3-RA_v1_ECMWF-ERAINT_AFR-44_mon_evaluation_' 'r1i1p1_tas_1991-1993.nc') @@ -939,7 +939,7 @@ def test_simple_cordex_recipe(tmp_path, patched_datafinder, config_user): def test_recipe_iso_timerange(tmp_path, patched_datafinder, config_user, input_time, output_time): """Test recipe with timerange tag.""" - content = dedent(""" + content = dedent(f""" diagnostics: test: additional_datasets: @@ -948,44 +948,46 @@ def test_recipe_iso_timerange(tmp_path, patched_datafinder, config_user, exp: historical ensemble: r2i1p1f1 grid: gn - timerange: variables: pr: mip: 3hr + timerange: '{input_time}' areacella: mip: fx scripts: null """) - recipe = yaml.safe_load(content) - (recipe['diagnostics']['test']['additional_datasets'][0]['timerange'] - ) = input_time - content = yaml.safe_dump(recipe) - recipe = get_recipe(tmp_path, content, config_user) - variable = recipe.diagnostics['test']['preprocessor_output']['pr'][0] - filename = variable.pop('filename').split('/')[-1] - assert (filename == 'CMIP6_HadGEM3-GC31-LL_3hr_historical_r2i1p1f1_' - f'pr_gn_{output_time}.nc') - fx_variable = ( - recipe.diagnostics['test']['preprocessor_output']['areacella'][0]) - fx_filename = fx_variable.pop('filename').split('/')[-1] - assert (fx_filename == - 'CMIP6_HadGEM3-GC31-LL_fx_historical_r2i1p1f1_areacella_gn.nc') + assert len(recipe.tasks) == 2 + pr_task = [t for t in recipe.tasks if t.name.endswith('pr')][0] + assert len(pr_task.products) == 1 + pr_product = pr_task.products.pop() + + filename = ('CMIP6_HadGEM3-GC31-LL_3hr_historical_r2i1p1f1_' + f'pr_gn_{output_time}.nc') + assert pr_product.filename.name == filename + + areacella_task = [t for t in recipe.tasks + if t.name.endswith('areacella')][0] + assert len(areacella_task.products) == 1 + areacella_product = areacella_task.products.pop() + + filename = 'CMIP6_HadGEM3-GC31-LL_fx_historical_r2i1p1f1_areacella_gn.nc' + assert areacella_product.filename.name == filename @pytest.mark.parametrize('input_time,output_time', TEST_ISO_TIMERANGE) def test_recipe_iso_timerange_as_dataset(tmp_path, patched_datafinder, config_user, input_time, output_time): """Test recipe with timerange tag in the datasets section.""" - content = dedent(""" + content = dedent(f""" datasets: - dataset: HadGEM3-GC31-LL project: CMIP6 exp: historical ensemble: r2i1p1f1 grid: gn - timerange: + timerange: '{input_time}' diagnostics: test: variables: @@ -996,18 +998,14 @@ def test_recipe_iso_timerange_as_dataset(tmp_path, patched_datafinder, scripts: null """) - recipe = yaml.safe_load(content) - (recipe['datasets'][0]['timerange']) = input_time - content = yaml.safe_dump(recipe) - recipe = get_recipe(tmp_path, content, config_user) variable = recipe.diagnostics['test']['preprocessor_output']['pr'][0] - filename = variable.pop('filename').split('/')[-1] + filename = variable.pop('filename').name assert (filename == 'CMIP6_HadGEM3-GC31-LL_3hr_historical_r2i1p1f1_' f'pr_gn_{output_time}.nc') fx_variable = ( recipe.diagnostics['test']['preprocessor_output']['areacella'][0]) - fx_filename = fx_variable.pop('filename').split('/')[-1] + fx_filename = fx_variable.pop('filename').name assert (fx_filename == 'CMIP6_HadGEM3-GC31-LL_fx_historical_r2i1p1f1_areacella_gn.nc') @@ -2584,8 +2582,8 @@ def test_empty_fxvar_dict(tmp_path, patched_datafinder, config_user): assert product.settings['add_fx_variables']['fx_variables'] == {} -def test_user_defined_fxvar(tmp_path, patched_datafinder, config_user): - content = dedent(""" +@pytest.mark.parametrize('content', [ + pytest.param(dedent(""" preprocessors: landmask: mask_landsea: @@ -2620,45 +2618,46 @@ def test_user_defined_fxvar(tmp_path, patched_datafinder, config_user): additional_datasets: - {dataset: CanESM2} scripts: null - """) - recipe = get_recipe(tmp_path, content, config_user) - - # Check custom fx variables - task = recipe.tasks.pop() - product = task.products.pop() - - # landsea - settings = product.settings['mask_landsea'] - assert len(settings) == 1 - assert settings['mask_out'] == 'sea' - fx_variables = product.settings['add_fx_variables']['fx_variables'] - assert isinstance(fx_variables, dict) - assert len(fx_variables) == 4 - assert '_fx_' in fx_variables['sftlf']['filename'] - assert '_piControl_' in fx_variables['sftlf']['filename'] - - # landseaice - settings = product.settings['mask_landseaice'] - assert len(settings) == 1 - assert settings['mask_out'] == 'sea' - assert '_fx_' in fx_variables['sftgif']['filename'] - assert '_piControl_' in fx_variables['sftgif']['filename'] - - # volume statistics - settings = product.settings['volume_statistics'] - assert len(settings) == 1 - assert settings['operator'] == 'mean' - assert 'volcello' in fx_variables - - # area statistics - settings = product.settings['area_statistics'] - assert len(settings) == 1 - assert settings['operator'] == 'mean' - assert '_fx_' in fx_variables['areacello']['filename'] - assert '_piControl_' in fx_variables['areacello']['filename'] - - -def test_user_defined_fxlist(tmp_path, patched_datafinder, config_user): + """), + id='fx_variables_as_dict_of_dicts'), + pytest.param(dedent(""" + preprocessors: + landmask: + mask_landsea: + mask_out: sea + fx_variables: [{'short_name': 'sftlf', 'exp': 'piControl'}] + mask_landseaice: + mask_out: sea + fx_variables: [{'short_name': 'sftgif', 'exp': 'piControl'}] + volume_statistics: + operator: mean + area_statistics: + operator: mean + fx_variables: [{'short_name': 'areacello', 'mip': 'fx', + 'exp': 'piControl'}] + diagnostics: + diagnostic_name: + variables: + gpp: + preprocessor: landmask + project: CMIP5 + mip: Lmon + exp: historical + start_year: 2000 + end_year: 2005 + ensemble: r1i1p1 + additional_datasets: + - {dataset: CanESM2} + scripts: null + """), + id='fx_variables_as_list_of_dicts'), +]) +def test_user_defined_fxvar( + tmp_path, + patched_datafinder, + config_user, + content, +): content = dedent(""" preprocessors: landmask: @@ -2702,15 +2701,15 @@ def test_user_defined_fxlist(tmp_path, patched_datafinder, config_user): fx_variables = product.settings['add_fx_variables']['fx_variables'] assert isinstance(fx_variables, dict) assert len(fx_variables) == 4 - assert '_fx_' in fx_variables['sftlf']['filename'] - assert '_piControl_' in fx_variables['sftlf']['filename'] + assert '_fx_' in fx_variables['sftlf']['filename'].name + assert '_piControl_' in fx_variables['sftlf']['filename'].name # landseaice settings = product.settings['mask_landseaice'] assert len(settings) == 1 assert settings['mask_out'] == 'sea' - assert '_fx_' in fx_variables['sftlf']['filename'] - assert '_piControl_' in fx_variables['sftlf']['filename'] + assert '_fx_' in fx_variables['sftlf']['filename'].name + assert '_piControl_' in fx_variables['sftlf']['filename'].name # volume statistics settings = product.settings['volume_statistics'] @@ -2722,8 +2721,8 @@ def test_user_defined_fxlist(tmp_path, patched_datafinder, config_user): settings = product.settings['area_statistics'] assert len(settings) == 1 assert settings['operator'] == 'mean' - assert '_fx_' in fx_variables['areacello']['filename'] - assert '_piControl_' in fx_variables['areacello']['filename'] + assert '_fx_' in fx_variables['areacello']['filename'].name + assert '_piControl_' in fx_variables['areacello']['filename'].name def test_landmask_no_fx(tmp_path, patched_failing_datafinder, config_user): @@ -2779,14 +2778,17 @@ def test_fx_vars_fixed_mip_cmip6(tmp_path, patched_datafinder, config_user): content = dedent(""" preprocessors: preproc: - area_statistics: - operator: mean - fx_variables: - sftgif: - mip: fx - volcello: - ensemble: r2i1p1f1 - mip: Ofx + volume_statistics: + operator: mean + fx_variables: + volcello: + ensemble: r2i1p1f1 + mip: Ofx + mask_landseaice: + mask_out: ice + fx_variables: + sftgif: + mip: fx diagnostics: diagnostic_name: @@ -2813,9 +2815,9 @@ def test_fx_vars_fixed_mip_cmip6(tmp_path, patched_datafinder, config_user): assert len(task.products) == 1 product = task.products.pop() - # Check area_statistics - assert 'area_statistics' in product.settings - settings = product.settings['area_statistics'] + # Check volume_statistics + assert 'volume_statistics' in product.settings + settings = product.settings['volume_statistics'] assert len(settings) == 1 assert settings['operator'] == 'mean' @@ -2823,9 +2825,9 @@ def test_fx_vars_fixed_mip_cmip6(tmp_path, patched_datafinder, config_user): fx_variables = product.settings['add_fx_variables']['fx_variables'] assert isinstance(fx_variables, dict) assert len(fx_variables) == 2 - assert '_fx_' in fx_variables['sftgif']['filename'] - assert '_r2i1p1f1_' in fx_variables['volcello']['filename'] - assert '_Ofx_' in fx_variables['volcello']['filename'] + assert '_fx_' in fx_variables['sftgif']['filename'].name + assert '_r2i1p1f1_' in fx_variables['volcello']['filename'].name + assert '_Ofx_' in fx_variables['volcello']['filename'].name def test_fx_vars_invalid_mip_cmip6(tmp_path, patched_datafinder, config_user): @@ -2960,11 +2962,11 @@ def test_fx_vars_mip_search_cmip6(tmp_path, patched_datafinder, config_user): fx_variables = product.settings['add_fx_variables']['fx_variables'] assert isinstance(fx_variables, dict) assert len(fx_variables) == 5 - assert '_fx_' in fx_variables['areacella']['filename'] - assert '_Ofx_' in fx_variables['areacello']['filename'] - assert '_Efx_' in fx_variables['clayfrac']['filename'] - assert '_fx_' in fx_variables['sftlf']['filename'] - assert '_Ofx_' in fx_variables['sftof']['filename'] + assert '_fx_' in fx_variables['areacella']['filename'].name + assert '_Ofx_' in fx_variables['areacello']['filename'].name + assert '_Efx_' in fx_variables['clayfrac']['filename'].name + assert '_fx_' in fx_variables['sftlf']['filename'].name + assert '_Ofx_' in fx_variables['sftof']['filename'].name def test_fx_list_mip_search_cmip6(tmp_path, patched_datafinder, config_user): @@ -2972,15 +2974,18 @@ def test_fx_list_mip_search_cmip6(tmp_path, patched_datafinder, config_user): content = dedent(""" preprocessors: preproc: - area_statistics: - operator: mean - fx_variables: [ - 'areacella', - 'areacello', - 'clayfrac', - 'sftlf', - 'sftof', - ] + area_statistics: + operator: mean + fx_variables: [ + 'areacella', + 'areacello', + ] + mask_landsea: + mask_out: sea + fx_variables: [ + 'sftlf', + 'sftof', + ] diagnostics: diagnostic_name: @@ -3016,12 +3021,11 @@ def test_fx_list_mip_search_cmip6(tmp_path, patched_datafinder, config_user): # Check add_fx_variables fx_variables = product.settings['add_fx_variables']['fx_variables'] assert isinstance(fx_variables, dict) - assert len(fx_variables) == 5 - assert '_fx_' in fx_variables['areacella']['filename'] - assert '_Ofx_' in fx_variables['areacello']['filename'] - assert '_Efx_' in fx_variables['clayfrac']['filename'] - assert '_fx_' in fx_variables['sftlf']['filename'] - assert '_Ofx_' in fx_variables['sftof']['filename'] + assert len(fx_variables) == 4 + assert '_fx_' in fx_variables['areacella']['filename'].name + assert '_Ofx_' in fx_variables['areacello']['filename'].name + assert '_fx_' in fx_variables['sftlf']['filename'].name + assert '_Ofx_' in fx_variables['sftof']['filename'].name def test_fx_vars_volcello_in_ofx_cmip6(tmp_path, patched_datafinder, @@ -3070,8 +3074,8 @@ def test_fx_vars_volcello_in_ofx_cmip6(tmp_path, patched_datafinder, fx_variables = product.settings['add_fx_variables']['fx_variables'] assert isinstance(fx_variables, dict) assert len(fx_variables) == 1 - assert '_Omon_' not in fx_variables['volcello']['filename'] - assert '_Ofx_' in fx_variables['volcello']['filename'] + assert '_Omon_' not in fx_variables['volcello']['filename'].name + assert '_Ofx_' in fx_variables['volcello']['filename'].name def test_fx_dicts_volcello_in_ofx_cmip6(tmp_path, patched_datafinder, @@ -3119,9 +3123,9 @@ def test_fx_dicts_volcello_in_ofx_cmip6(tmp_path, patched_datafinder, fx_variables = product.settings['add_fx_variables']['fx_variables'] assert isinstance(fx_variables, dict) assert len(fx_variables) == 1 - assert '_Oyr_' in fx_variables['volcello']['filename'][0] - assert '_piControl_' in fx_variables['volcello']['filename'][0] - assert '_Omon_' not in fx_variables['volcello']['filename'][0] + assert '_Oyr_' in fx_variables['volcello']['filename'][0].name + assert '_piControl_' in fx_variables['volcello']['filename'][0].name + assert '_Omon_' not in fx_variables['volcello']['filename'][0].name def test_fx_vars_list_no_preproc_cmip6(tmp_path, patched_datafinder, @@ -3221,8 +3225,8 @@ def test_fx_vars_volcello_in_omon_cmip6(tmp_path, patched_failing_datafinder, fx_variables = product.settings['add_fx_variables']['fx_variables'] assert isinstance(fx_variables, dict) assert len(fx_variables) == 1 - assert '_Ofx_' not in fx_variables['volcello']['filename'][0] - assert '_Omon_' in fx_variables['volcello']['filename'][0] + assert '_Ofx_' not in fx_variables['volcello']['filename'][0].name + assert '_Omon_' in fx_variables['volcello']['filename'][0].name def test_fx_vars_volcello_in_oyr_cmip6(tmp_path, patched_failing_datafinder, @@ -3269,8 +3273,8 @@ def test_fx_vars_volcello_in_oyr_cmip6(tmp_path, patched_failing_datafinder, fx_variables = product.settings['add_fx_variables']['fx_variables'] assert isinstance(fx_variables, dict) assert len(fx_variables) == 1 - assert '_Ofx_' not in fx_variables['volcello']['filename'][0] - assert '_Oyr_' in fx_variables['volcello']['filename'][0] + assert '_Ofx_' not in fx_variables['volcello']['filename'][0].name + assert '_Oyr_' in fx_variables['volcello']['filename'][0].name def test_fx_vars_volcello_in_fx_cmip5(tmp_path, patched_datafinder, @@ -3315,8 +3319,8 @@ def test_fx_vars_volcello_in_fx_cmip5(tmp_path, patched_datafinder, fx_variables = product.settings['add_fx_variables']['fx_variables'] assert isinstance(fx_variables, dict) assert len(fx_variables) == 1 - assert '_fx_' in fx_variables['volcello']['filename'] - assert '_Omon_' not in fx_variables['volcello']['filename'] + assert '_fx_' in fx_variables['volcello']['filename'].name + assert '_Omon_' not in fx_variables['volcello']['filename'].name def test_wrong_project(tmp_path, patched_datafinder, config_user): @@ -3434,8 +3438,8 @@ def test_unique_fx_var_in_multiple_mips_cmip6(tmp_path, content = dedent(""" preprocessors: preproc: - area_statistics: - operator: mean + mask_landseaice: + mask_out: ice fx_variables: sftgif: @@ -3464,11 +3468,11 @@ def test_unique_fx_var_in_multiple_mips_cmip6(tmp_path, assert len(task.products) == 1 product = task.products.pop() - # Check area_statistics - assert 'area_statistics' in product.settings - settings = product.settings['area_statistics'] + # Check mask_landseaice + assert 'mask_landseaice' in product.settings + settings = product.settings['mask_landseaice'] assert len(settings) == 1 - assert settings['operator'] == 'mean' + assert settings['mask_out'] == 'ice' # Check add_fx_variables # Due to failing datafinder, only files in LImon are found even though @@ -3479,7 +3483,7 @@ def test_unique_fx_var_in_multiple_mips_cmip6(tmp_path, sftgif_files = fx_variables['sftgif']['filename'] assert isinstance(sftgif_files, list) assert len(sftgif_files) == 1 - assert '_LImon_' in sftgif_files[0] + assert '_LImon_' in sftgif_files[0].name def test_multimodel_mask(tmp_path, patched_datafinder, config_user): @@ -3639,8 +3643,8 @@ def test_dataset_to_file_derived_var(mock_get_input_files, mock_data_availability, config_user): """Test ``_dataset_to_file`` with derived variable.""" mock_get_input_files.side_effect = [ - ([], [], []), - ([sentinel.out_file], [sentinel.dirname], [sentinel.filename]), + ([], []), + ([sentinel.out_file], [sentinel.globs]), ] variable = { 'dataset': 'ICON', diff --git a/tests/integration/test_recipe_checks.py b/tests/integration/test_recipe_checks.py index 065a6e1301..08e9d39624 100644 --- a/tests/integration/test_recipe_checks.py +++ b/tests/integration/test_recipe_checks.py @@ -1,4 +1,5 @@ """Integration tests for :mod:`esmvalcore._recipe_checks`.""" +import os.path from typing import Any, List from unittest import mock @@ -11,13 +12,8 @@ from esmvalcore.preprocessor import PreprocessorFile ERR_ALL = 'Looked for files matching%s' -ERR_D = ('Looked for files in %s, but did not find any file pattern to match ' - 'against') -ERR_F = ('Looked for files matching %s, but did not find any existing input ' - 'directory') ERR_RANGE = 'No input data available for years {} in files:\n{}' VAR = { - 'filename': 'a/c.nc', 'frequency': 'mon', 'short_name': 'tas', 'timerange': '2020/2025', @@ -26,7 +22,6 @@ 'end_year': 2025 } FX_VAR = { - 'filename': 'a/b.nc', 'frequency': 'fx', 'short_name': 'areacella', } @@ -69,17 +64,8 @@ def test_data_availability_data(mock_logger, input_files, var, error): DATA_AVAILABILITY_NO_DATA: List[Any] = [ ([], [], None), - ([], None, None), - (None, [], None), - (None, None, None), - (['dir1'], [], (ERR_D, ['dir1'])), - (['dir1', 'dir2'], [], (ERR_D, ['dir1', 'dir2'])), - (['dir1'], None, (ERR_D, ['dir1'])), - (['dir1', 'dir2'], None, (ERR_D, ['dir1', 'dir2'])), - ([], ['a*.nc'], (ERR_F, ['a*.nc'])), - ([], ['a*.nc', 'b*.nc'], (ERR_F, ['a*.nc', 'b*.nc'])), - (None, ['a*.nc'], (ERR_F, ['a*.nc'])), - (None, ['a*.nc', 'b*.nc'], (ERR_F, ['a*.nc', 'b*.nc'])), + ([''], ['a*.nc'], (ERR_ALL, ': a*.nc')), + ([''], ['a*.nc', 'b*.nc'], (ERR_ALL, '\na*.nc\nb*.nc')), (['1'], ['a'], (ERR_ALL, ': 1/a')), (['1'], ['a', 'b'], (ERR_ALL, '\n1/a\n1/b')), (['1', '2'], ['a'], (ERR_ALL, '\n1/a\n2/a')), @@ -100,10 +86,13 @@ def test_data_availability_no_data(mock_logger, dirnames, filenames, error): 'start_year': 2020, 'end_year': 2025 } + patterns = [ + os.path.join(d, f) for d in dirnames for f in filenames + ] error_first = ('No input files found for variable %s', var_no_filename) error_last = ("Set 'log_level' to 'debug' to get more information", ) with pytest.raises(RecipeError) as rec_err: - check.data_availability([], var, dirnames, filenames) + check.data_availability([], var, patterns) assert str(rec_err.value) == 'Missing data for alias: tas' if error is None: assert mock_logger.error.call_count == 2 @@ -137,7 +126,7 @@ def test_data_availability_no_data(mock_logger, dirnames, filenames, error): '*/P2Y21DT12H00M00S', '1/301', '1/*', - '*/301' + '*/301', ] @@ -161,7 +150,7 @@ def test_valid_time_selection(timerange): @pytest.mark.parametrize('timerange,message', BAD_TIMERANGES) -def test_valid_time_selection_rehections(timerange, message): +def test_valid_time_selection_rejections(timerange, message): """Check that bad definitions raise RecipeError.""" with pytest.raises(check.RecipeError) as rec_err: check.valid_time_selection(timerange) @@ -189,7 +178,7 @@ def test_data_availability_nonexistent(tmp_path): ) dest_folder = tmp_path input_files = [esmvalcore.esgf.ESGFFile([result]).local_file(dest_folder)] - check.data_availability(input_files, var, dirnames=[], filenames=[]) + check.data_availability(input_files, var, patterns=[]) def test_reference_for_bias_preproc_empty(): diff --git a/tests/sample_data/experimental/test_run_recipe.py b/tests/sample_data/experimental/test_run_recipe.py index 053385d821..86f31a438a 100644 --- a/tests/sample_data/experimental/test_run_recipe.py +++ b/tests/sample_data/experimental/test_run_recipe.py @@ -40,7 +40,7 @@ def recipe(): @pytest.mark.use_sample_data @pytest.mark.parametrize('task', (None, 'example/ta')) -def test_run_recipe(task, recipe, tmp_path): +def test_run_recipe(monkeypatch, task, recipe, tmp_path): """Test running a basic recipe using sample data. Recipe contains no provenance and no diagnostics. @@ -50,12 +50,13 @@ def test_run_recipe(task, recipe, tmp_path): assert isinstance(recipe, Recipe) assert isinstance(recipe._repr_html_(), str) + sample_data_config = esmvaltool_sample_data.get_rootpaths() + monkeypatch.setitem(CFG, 'rootpath', sample_data_config['rootpath']) + monkeypatch.setitem(CFG, 'drs', {'CMIP6': 'SYNDA'}) session = CFG.start_session(recipe.path.stem) session.clear() session.update(CFG_DEFAULT) session['output_dir'] = tmp_path / 'esmvaltool_output' - session.update(esmvaltool_sample_data.get_rootpaths()) - session['drs'] = {'CMIP6': 'SYNDA'} session['max_parallel_tasks'] = 1 session['remove_preproc_dir'] = False @@ -82,14 +83,14 @@ def test_run_recipe(task, recipe, tmp_path): @pytest.mark.use_sample_data -def test_run_recipe_diagnostic_failing(recipe, tmp_path): +def test_run_recipe_diagnostic_failing(monkeypatch, recipe, tmp_path): """Test running a single diagnostic using sample data. Recipe contains no provenance and no diagnostics. """ TAGS.set_tag_values(AUTHOR_TAGS) - CFG['output_dir'] = tmp_path + monkeypatch.setitem(CFG, 'output_dir', tmp_path) session = CFG.start_session(recipe.path.stem) diff --git a/tests/unit/data_finder/__init__.py b/tests/unit/local/__init__.py similarity index 100% rename from tests/unit/data_finder/__init__.py rename to tests/unit/local/__init__.py diff --git a/tests/unit/local/test_facets.py b/tests/unit/local/test_facets.py new file mode 100644 index 0000000000..1f7f3c35af --- /dev/null +++ b/tests/unit/local/test_facets.py @@ -0,0 +1,25 @@ +from pathlib import Path + +from esmvalcore.local import LocalFile, _path2facets + + +def test_path2facets(): + """Test `_path2facets1.""" + filepath = Path("/climate_data/value1/value2/filename.nc") + drs = "{facet1}/{facet2.lower}" + + expected = { + 'facet1': 'value1', + 'facet2': 'value2', + } + + result = _path2facets(filepath, drs) + + assert result == expected + + +def test_localfile(): + file = LocalFile('/a/b.nc') + file.facets = {'a': 'A'} + assert Path(file) == Path('/a/b.nc') + assert file.facets == {'a': 'A'} diff --git a/tests/unit/data_finder/test_replace_tags.py b/tests/unit/local/test_replace_tags.py similarity index 54% rename from tests/unit/data_finder/test_replace_tags.py rename to tests/unit/local/test_replace_tags.py index 9215e8ebd5..38d0f63f75 100644 --- a/tests/unit/data_finder/test_replace_tags.py +++ b/tests/unit/local/test_replace_tags.py @@ -1,8 +1,10 @@ -"""Tests for _replace_tags in _data_finder.py.""" +"""Tests for `_replace_tags` in `esmvalcore.local`.""" +from pathlib import Path + import pytest -from esmvalcore._data_finder import _replace_tags from esmvalcore.exceptions import RecipeError +from esmvalcore.local import _replace_tags VARIABLE = { 'project': 'CMIP6', @@ -18,20 +20,23 @@ def test_replace_tags(): - """Tests for get_start_end_year function.""" + """Tests for `_replace_tags` function.""" path = _replace_tags( '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/' - '{grid}/{latestversion}', VARIABLE) + '{grid}/{version}', VARIABLE) input_file = _replace_tags( '{short_name}_{mip}_{dataset}_{exp}_{ensemble}_{grid}*.nc', VARIABLE) output_file = _replace_tags( '{project}_{dataset}_{mip}_{exp}_{ensemble}_{short_name}', VARIABLE) assert path == [ - 'act/HMA/ACCURATE-MODEL/experiment/r1i1p1f1/Amon/tas/gr/' - '{latestversion}' + Path('act/HMA/ACCURATE-MODEL/experiment/r1i1p1f1/Amon/tas/gr/*') + ] + assert input_file == [ + Path('tas_Amon_ACCURATE-MODEL_experiment_r1i1p1f1_gr*.nc') + ] + assert output_file == [ + Path('CMIP6_ACCURATE-MODEL_Amon_experiment_r1i1p1f1_tas') ] - assert input_file == ['tas_Amon_ACCURATE-MODEL_experiment_r1i1p1f1_gr*.nc'] - assert output_file == ['CMIP6_ACCURATE-MODEL_Amon_experiment_r1i1p1f1_tas'] def test_replace_tags_missing_facet(): @@ -45,34 +50,38 @@ def test_replace_tags_missing_facet(): def test_replace_tags_list_of_str(): - assert sorted( - _replace_tags(('folder/subfolder/{short_name}', 'folder2/{short_name}', - 'subfolder/{short_name}'), VARIABLE)) == sorted([ - 'folder2/tas', - 'folder/subfolder/tas', - 'subfolder/tas', - ]) + paths = [ + 'folder/subfolder/{short_name}', + 'folder2/{short_name}', + 'subfolder/{short_name}', + ] + reference = [ + Path('folder/subfolder/tas'), + Path('folder2/tas'), + Path('subfolder/tas'), + ] + assert sorted(_replace_tags(paths, VARIABLE)) == reference def test_replace_tags_with_subexperiment(): - """Tests for get_start_end_year function.""" + """Tests for `_replace_tags` function.""" variable = {'sub_experiment': '199411', **VARIABLE} - path = _replace_tags( + paths = _replace_tags( '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/' - '{grid}/{latestversion}', variable) + '{grid}/{version}', variable) input_file = _replace_tags( '{short_name}_{mip}_{dataset}_{exp}_{ensemble}_{grid}*.nc', variable) output_file = _replace_tags( '{project}_{dataset}_{mip}_{exp}_{ensemble}_{short_name}', variable) - assert sorted(path) == sorted([ - 'act/HMA/ACCURATE-MODEL/experiment/r1i1p1f1/Amon/tas/gr/' - '{latestversion}', - 'act/HMA/ACCURATE-MODEL/experiment/199411-r1i1p1f1/Amon/tas/gr/' - '{latestversion}' - ]) + expected_paths = [ + Path( + 'act/HMA/ACCURATE-MODEL/experiment/199411-r1i1p1f1/Amon/tas/gr/*'), + Path('act/HMA/ACCURATE-MODEL/experiment/r1i1p1f1/Amon/tas/gr/*'), + ] + assert sorted(paths) == expected_paths assert input_file == [ - 'tas_Amon_ACCURATE-MODEL_experiment_199411-r1i1p1f1_gr*.nc' + Path('tas_Amon_ACCURATE-MODEL_experiment_199411-r1i1p1f1_gr*.nc') ] assert output_file == [ - 'CMIP6_ACCURATE-MODEL_Amon_experiment_199411-r1i1p1f1_tas' + Path('CMIP6_ACCURATE-MODEL_Amon_experiment_199411-r1i1p1f1_tas') ] diff --git a/tests/unit/data_finder/test_select_files.py b/tests/unit/local/test_select_files.py similarity index 87% rename from tests/unit/data_finder/test_select_files.py rename to tests/unit/local/test_select_files.py index a3dcf7618b..162cfc89c5 100644 --- a/tests/unit/data_finder/test_select_files.py +++ b/tests/unit/local/test_select_files.py @@ -1,4 +1,4 @@ -from esmvalcore._data_finder import select_files +from esmvalcore.local import _select_files def test_select_files(): @@ -10,7 +10,7 @@ def test_select_files(): "pr_Amon_MPI-ESM1-2-HR_historical_r1i1p1f1_gn_197001-197412.nc", ] - result = select_files(files, '1962/1967') + result = _select_files(files, '1962/1967') expected = [ "pr_Amon_MPI-ESM1-2-HR_historical_r1i1p1f1_gn_196001-196412.nc", @@ -29,7 +29,7 @@ def test_select_files_monthly_resolution(): "pr_Amon_EC-Earth3_dcppA-hindcast_s1960-r1i1p1f1_gr_196311-196410.nc", ] - result = select_files(files, '196201/196205') + result = _select_files(files, '196201/196205') expected = [ "pr_Amon_EC-Earth3_dcppA-hindcast_s1960-r1i1p1f1_gr_196111-196210.nc" @@ -48,7 +48,7 @@ def test_select_files_daily_resolution(): filename + "19621101-19631031.nc" ] - result = select_files(files, '19600101/19611215') + result = _select_files(files, '19600101/19611215') expected = [ filename + "19601101-19611031.nc", @@ -73,10 +73,10 @@ def test_select_files_sub_daily_resolution(): filename + "19621101T0300-19631031T2100.nc", ] - result_no_separator = select_files( + result_no_separator = _select_files( files_no_separator, '19600101T0900/19610101T09HH00MM') - result_separator = select_files( + result_separator = _select_files( files_separator, '19600101T0900/19610101T0900') @@ -113,8 +113,8 @@ def test_select_files_time_period(): filename_datetime + "196211010300-196310312100.nc", ] - result_date = select_files(files_date, '196211/P2Y5M') - result_datetime = select_files(files_datetime, '19601101T1300/P1Y0M0DT6H') + result_date = _select_files(files_date, '196211/P2Y5M') + result_datetime = _select_files(files_datetime, '19601101T1300/P1Y0M0DT6H') expected_date = [ filename_date + "196211-196310.nc", @@ -142,9 +142,9 @@ def test_select_files_varying_format(): filename + "196211010300-196310312100.nc", ] - result_yearly = select_files(files, '1960/1962') - result_monthly = select_files(files, '196011/196210') - result_daily = select_files(files, '19601101/19601105') + result_yearly = _select_files(files, '1960/1962') + result_monthly = _select_files(files, '196011/196210') + result_daily = _select_files(files, '19601101/19601105') assert result_yearly == files assert result_monthly == files[0:2] diff --git a/tests/unit/data_finder/test_get_start_end_year.py b/tests/unit/local/test_time.py similarity index 67% rename from tests/unit/data_finder/test_get_start_end_year.py rename to tests/unit/local/test_time.py index 1fb02c2e9b..257cf0ed4e 100644 --- a/tests/unit/data_finder/test_get_start_end_year.py +++ b/tests/unit/local/test_time.py @@ -1,12 +1,13 @@ -"""Unit tests for :func:`esmvalcore._data_finder.regrid._stock_cube`""" - +"""Unit tests for time related functions in `esmvalcore.local`.""" import iris import pytest -from esmvalcore._data_finder import ( +from esmvalcore.local import ( + _dates_to_timerange, + _get_start_end_date, + _get_start_end_year, _get_timerange_from_years, - get_start_end_date, - get_start_end_year, + _truncate_dates, ) FILENAME_CASES = [ @@ -61,32 +62,32 @@ @pytest.mark.parametrize('case', FILENAME_CASES) def test_get_start_end_year(case): - """Tests for get_start_end_year function.""" + """Tests for _get_start_end_year function.""" filename, case_start, case_end = case if case_start is None and case_end is None: # If the filename is inconclusive or too difficult # we resort to reading the file, which fails here # because the file is not there. with pytest.raises(ValueError): - get_start_end_year(filename) + _get_start_end_year(filename) else: - start, end = get_start_end_year(filename) + start, end = _get_start_end_year(filename) assert case_start == start assert case_end == end @pytest.mark.parametrize('case', FILENAME_DATE_CASES) def test_get_start_end_date(case): - """Tests for get_start_end_date function.""" + """Tests for _get_start_end_date function.""" filename, case_start, case_end = case if case_start is None and case_end is None: # If the filename is inconclusive or too difficult # we resort to reading the file, which fails here # because the file is not there. with pytest.raises(ValueError): - get_start_end_date(filename) + _get_start_end_date(filename) else: - start, end = get_start_end_date(filename) + start, end = _get_start_end_date(filename) assert case_start == start assert case_end == end @@ -101,7 +102,7 @@ def test_read_time_from_cube(monkeypatch, tmp_path): units='days since 1990-01-01') cube.add_dim_coord(time, 0) iris.save(cube, temp_file) - start, end = get_start_end_year(temp_file) + start, end = _get_start_end_year(temp_file) assert start == 1990 assert end == 1991 @@ -116,7 +117,7 @@ def test_read_datetime_from_cube(monkeypatch, tmp_path): units='days since 1990-01-01') cube.add_dim_coord(time, 0) iris.save(cube, temp_file) - start, end = get_start_end_date(temp_file) + start, end = _get_start_end_date(temp_file) assert start == '19900101' assert end == '19910102' @@ -128,14 +129,14 @@ def test_raises_if_unable_to_deduce(monkeypatch, tmp_path): cube = iris.cube.Cube([0, 0], var_name='var') iris.save(cube, temp_file) with pytest.raises(ValueError): - get_start_end_date(temp_file) + _get_start_end_date(temp_file) def test_fails_if_no_date_present(): """Test raises if no date is present.""" with pytest.raises((ValueError, OSError)): - get_start_end_date('var_whatever') - get_start_end_year('var_whatever') + _get_start_end_date('var_whatever') + _get_start_end_year('var_whatever') def test_get_timerange_from_years(): @@ -176,3 +177,65 @@ def test_get_timerange_from_end_year(): assert 'end_year' not in variable assert variable['timerange'] == '2002/2002' + + +TEST_DATES_TO_TIMERANGE = [ + (2000, 2000, '2000/2000'), + (1, 2000, '0001/2000'), + (2000, 1, '2000/0001'), + (1, 2, '0001/0002'), + ('2000', '2000', '2000/2000'), + ('1', '2000', '0001/2000'), + (2000, '1', '2000/0001'), + ('1', 2, '0001/0002'), + ('*', '*', '*/*'), + (2000, '*', '2000/*'), + ('2000', '*', '2000/*'), + (1, '*', '0001/*'), + ('1', '*', '0001/*'), + ('*', 2000, '*/2000'), + ('*', '2000', '*/2000'), + ('*', 1, '*/0001'), + ('*', '1', '*/0001'), + ('P5Y', 'P5Y', 'P5Y/P5Y'), + (2000, 'P5Y', '2000/P5Y'), + ('2000', 'P5Y', '2000/P5Y'), + (1, 'P5Y', '0001/P5Y'), + ('1', 'P5Y', '0001/P5Y'), + ('P5Y', 2000, 'P5Y/2000'), + ('P5Y', '2000', 'P5Y/2000'), + ('P5Y', 1, 'P5Y/0001'), + ('P5Y', '1', 'P5Y/0001'), + ('*', 'P5Y', '*/P5Y'), + ('P5Y', '*', 'P5Y/*'), +] + + +@pytest.mark.parametrize('start_date,end_date,expected_timerange', + TEST_DATES_TO_TIMERANGE) +def test_dates_to_timerange(start_date, end_date, expected_timerange): + """Test ``_dates_to_timerange``.""" + timerange = _dates_to_timerange(start_date, end_date) + assert timerange == expected_timerange + + +TEST_TRUNCATE_DATES = [ + ('2000', '2000', (2000, 2000)), + ('200001', '2000', (2000, 2000)), + ('2000', '200001', (2000, 2000)), + ('200001', '2000', (2000, 2000)), + ('200001', '200001', (200001, 200001)), + ('20000102', '200001', (200001, 200001)), + ('200001', '20000102', (200001, 200001)), + ('20000102', '20000102', (20000102, 20000102)), + ('20000102T23:59:59', '20000102', (20000102, 20000102)), + ('20000102', '20000102T23:59:59', (20000102, 20000102)), + ('20000102T235959', '20000102T01:02:03', (20000102235959, 20000102010203)), +] + + +@pytest.mark.parametrize('date,date_file,expected_output', TEST_TRUNCATE_DATES) +def test_truncate_dates(date, date_file, expected_output): + """Test ``_truncate_dates``.""" + output = _truncate_dates(date, date_file) + assert output == expected_output diff --git a/tests/unit/test_data_finder.py b/tests/unit/test_data_finder.py deleted file mode 100644 index 5210b9b2cb..0000000000 --- a/tests/unit/test_data_finder.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Unit tests for ``_data_finder.py``.""" -import pytest - -from esmvalcore._data_finder import _truncate_dates, dates_to_timerange - -TEST_DATES_TO_TIMERANGE = [ - (2000, 2000, '2000/2000'), - (1, 2000, '0001/2000'), - (2000, 1, '2000/0001'), - (1, 2, '0001/0002'), - ('2000', '2000', '2000/2000'), - ('1', '2000', '0001/2000'), - (2000, '1', '2000/0001'), - ('1', 2, '0001/0002'), - ('*', '*', '*/*'), - (2000, '*', '2000/*'), - ('2000', '*', '2000/*'), - (1, '*', '0001/*'), - ('1', '*', '0001/*'), - ('*', 2000, '*/2000'), - ('*', '2000', '*/2000'), - ('*', 1, '*/0001'), - ('*', '1', '*/0001'), - ('P5Y', 'P5Y', 'P5Y/P5Y'), - (2000, 'P5Y', '2000/P5Y'), - ('2000', 'P5Y', '2000/P5Y'), - (1, 'P5Y', '0001/P5Y'), - ('1', 'P5Y', '0001/P5Y'), - ('P5Y', 2000, 'P5Y/2000'), - ('P5Y', '2000', 'P5Y/2000'), - ('P5Y', 1, 'P5Y/0001'), - ('P5Y', '1', 'P5Y/0001'), - ('*', 'P5Y', '*/P5Y'), - ('P5Y', '*', 'P5Y/*'), -] - - -@pytest.mark.parametrize('start_date,end_date,expected_timerange', - TEST_DATES_TO_TIMERANGE) -def test_dates_to_timerange(start_date, end_date, expected_timerange): - """Test ``dates_to_timerange``.""" - timerange = dates_to_timerange(start_date, end_date) - assert timerange == expected_timerange - - -TEST_TRUNCATE_DATES = [ - ('2000', '2000', (2000, 2000)), - ('200001', '2000', (2000, 2000)), - ('2000', '200001', (2000, 2000)), - ('200001', '2000', (2000, 2000)), - ('200001', '200001', (200001, 200001)), - ('20000102', '200001', (200001, 200001)), - ('200001', '20000102', (200001, 200001)), - ('20000102', '20000102', (20000102, 20000102)), - ('20000102T23:59:59', '20000102', (20000102, 20000102)), - ('20000102', '20000102T23:59:59', (20000102, 20000102)), - ('20000102T235959', '20000102T01:02:03', (20000102235959, 20000102010203)), -] - - -@pytest.mark.parametrize('date,date_file,expected_output', TEST_TRUNCATE_DATES) -def test_truncate_dates(date, date_file, expected_output): - """Test ``_truncate_dates``.""" - output = _truncate_dates(date, date_file) - assert output == expected_output diff --git a/tests/unit/test_recipe.py b/tests/unit/test_recipe.py index d74f570370..30c53aaf89 100644 --- a/tests/unit/test_recipe.py +++ b/tests/unit/test_recipe.py @@ -1,4 +1,5 @@ from collections import defaultdict +from pathlib import Path from unittest import mock import iris @@ -260,9 +261,9 @@ def test_search_esgf(mocker, tmp_path, local_availability, already_downloaded): local_files = local_file_options[local_availability] mocker.patch.object(_recipe, - 'get_input_filelist', + 'find_files', autospec=True, - return_value=(list(local_files), [], [])) + return_value=(list(local_files), [])) mocker.patch.object( _recipe.esgf, 'find_files', @@ -310,9 +311,9 @@ def test_search_esgf_timerange(mocker, tmp_path, timerange): esgf_files = create_esgf_search_results() mocker.patch.object(_recipe, - '_find_input_files', + 'find_files', autospec=True, - return_value=([], [], [])) + return_value=[]) mocker.patch.object( _recipe.esgf, 'find_files', @@ -464,8 +465,10 @@ def test_update_multiproduct_multi_model_statistics(): assert len(output) == 2 filenames = [p.filename for p in output] - assert '/preproc/d/var/CMIP6_MultiModelMean_2002-2004.nc' in filenames - assert '/preproc/d/var/CMIP6_MultiModelStd_Dev_2002-2004.nc' in filenames + assert Path( + '/preproc/d/var/CMIP6_MultiModelMean_2002-2004.nc') in filenames + assert Path( + '/preproc/d/var/CMIP6_MultiModelStd_Dev_2002-2004.nc') in filenames for product in output: for attr in common_attributes: @@ -480,12 +483,12 @@ def test_update_multiproduct_multi_model_statistics(): assert product.attributes['start_year'] == 2002 assert 'end_year' in product.attributes assert product.attributes['end_year'] == 2004 - if 'MultiModelStd_Dev' in product.filename: + if 'MultiModelStd_Dev' in str(product.filename): assert product.attributes['alias'] == 'MultiModelStd_Dev' assert product.attributes['dataset'] == 'MultiModelStd_Dev' assert (product.attributes['multi_model_statistics'] == 'MultiModelStd_Dev') - elif 'MultiModelMean' in product.filename: + elif 'MultiModelMean' in str(product.filename): assert product.attributes['alias'] == 'MultiModelMean' assert product.attributes['dataset'] == 'MultiModelMean' assert (product.attributes['multi_model_statistics'] == @@ -498,8 +501,8 @@ def test_update_multiproduct_multi_model_statistics(): assert len(stats) == 2 assert 'mean' in stats assert 'std_dev' in stats - assert 'MultiModelMean' in stats['mean'].filename - assert 'MultiModelStd_Dev' in stats['std_dev'].filename + assert 'MultiModelMean' in str(stats['mean'].filename) + assert 'MultiModelStd_Dev' in str(stats['std_dev'].filename) def test_update_multiproduct_ensemble_statistics(): @@ -536,8 +539,8 @@ def test_update_multiproduct_ensemble_statistics(): assert len(output) == 1 product = list(output)[0] - assert (product.filename == - '/preproc/d/var/CMIP6_CanESM2_EnsembleMedian_2000-2000.nc') + assert product.filename == Path( + '/preproc/d/var/CMIP6_CanESM2_EnsembleMedian_2000-2000.nc') for attr in common_attributes: assert attr in product.attributes @@ -559,8 +562,8 @@ def test_update_multiproduct_ensemble_statistics(): stats = output_products['CMIP6_CanESM2'] assert len(stats) == 1 assert 'median' in stats - assert (stats['median'].filename == - '/preproc/d/var/CMIP6_CanESM2_EnsembleMedian_2000-2000.nc') + assert stats['median'].filename == Path( + '/preproc/d/var/CMIP6_CanESM2_EnsembleMedian_2000-2000.nc') def test_update_multiproduct_no_product(): From b0c5321cb66ba283698a980473356e407393cc8b Mon Sep 17 00:00:00 2001 From: FranziskaWinterstein <119339136+FranziskaWinterstein@users.noreply.github.com> Date: Wed, 14 Dec 2022 13:02:09 +0100 Subject: [PATCH 4/8] Add standard variable names for EMAC (#1853) --- esmvalcore/cmor/_fixes/emac/emac.py | 81 +++++++++++++++++++---------- 1 file changed, 54 insertions(+), 27 deletions(-) diff --git a/esmvalcore/cmor/_fixes/emac/emac.py b/esmvalcore/cmor/_fixes/emac/emac.py index 6a3dd3528c..a65a0a21fa 100644 --- a/esmvalcore/cmor/_fixes/emac/emac.py +++ b/esmvalcore/cmor/_fixes/emac/emac.py @@ -208,8 +208,10 @@ class Clwvi(EmacFix): def fix_metadata(self, cubes): """Fix metadata.""" cube = ( - self.get_cube(cubes, var_name=['xlvi_cav', 'xlvi_ave']) + - self.get_cube(cubes, var_name=['xivi_cav', 'xivi_ave']) + self.get_cube(cubes, var_name=['xlvi_cav', 'xlvi_ave', + 'xlvi']) + + self.get_cube(cubes, var_name=['xivi_cav', 'xivi_ave', + 'xivi']) ) cube.var_name = self.vardef.short_name return CubeList([cube]) @@ -245,8 +247,10 @@ class Pr(EmacFix): def fix_metadata(self, cubes): """Fix metadata.""" cube = ( - self.get_cube(cubes, var_name=['aprl_cav', 'aprl_ave']) + - self.get_cube(cubes, var_name=['aprc_cav', 'aprc_ave']) + self.get_cube(cubes, var_name=['aprl_cav', 'aprl_ave', + 'aprl']) + + self.get_cube(cubes, var_name=['aprc_cav', 'aprc_ave', + 'aprc']) ) cube.var_name = self.vardef.short_name return CubeList([cube]) @@ -258,8 +262,10 @@ class Rlds(EmacFix): def fix_metadata(self, cubes): """Fix metadata.""" cube = ( - self.get_cube(cubes, var_name=['flxtbot_cav', 'flxtbot_ave']) - - self.get_cube(cubes, var_name=['tradsu_cav', 'tradsu_ave']) + self.get_cube(cubes, var_name=['flxtbot_cav', 'flxtbot_ave', + 'flxsbot']) - + self.get_cube(cubes, var_name=['tradsu_cav', 'tradsu_ave', + 'tradsu']) ) cube.var_name = self.vardef.short_name return CubeList([cube]) @@ -280,8 +286,10 @@ class Rsds(EmacFix): def fix_metadata(self, cubes): """Fix metadata.""" cube = ( - self.get_cube(cubes, var_name=['flxsbot_cav', 'flxsbot_ave']) - - self.get_cube(cubes, var_name=['sradsu_cav', 'sradsu_ave']) + self.get_cube(cubes, var_name=['flxsbot_cav', 'flxsbot_ave', + 'flxsbot']) - + self.get_cube(cubes, var_name=['sradsu_cav', 'sradsu_ave', + 'sradsu']) ) cube.var_name = self.vardef.short_name return CubeList([cube]) @@ -293,8 +301,10 @@ class Rsdt(EmacFix): def fix_metadata(self, cubes): """Fix metadata.""" cube = ( - self.get_cube(cubes, var_name=['flxstop_cav', 'flxstop_ave']) - - self.get_cube(cubes, var_name=['srad0u_cav', 'srad0u_ave']) + self.get_cube(cubes, var_name=['flxstop_cav', 'flxstop_ave', + 'flxstop']) - + self.get_cube(cubes, var_name=['srad0u_cav', 'srad0u_ave', + 'srad0u']) ) cube.var_name = self.vardef.short_name return CubeList([cube]) @@ -315,8 +325,10 @@ class Rtmt(EmacFix): def fix_metadata(self, cubes): """Fix metadata.""" cube = ( - self.get_cube(cubes, var_name=['flxttop_cav', 'flxttop_ave']) + - self.get_cube(cubes, var_name=['flxstop_cav', 'flxstop_ave']) + self.get_cube(cubes, var_name=['flxttop_cav', 'flxttop_ave', + 'flxttop']) + + self.get_cube(cubes, var_name=['flxstop_cav', 'flxstop_ave', + 'flxstop']) ) cube.var_name = self.vardef.short_name return CubeList([cube]) @@ -379,10 +391,14 @@ class MP_BC_tot(EmacFix): # noqa: N801 def fix_metadata(self, cubes): """Fix metadata.""" cube = ( - self.get_cube(cubes, var_name=['MP_BC_ki_cav', 'MP_BC_ki_ave']) + - self.get_cube(cubes, var_name=['MP_BC_ks_cav', 'MP_BC_ks_ave']) + - self.get_cube(cubes, var_name=['MP_BC_as_cav', 'MP_BC_as_ave']) + - self.get_cube(cubes, var_name=['MP_BC_cs_cav', 'MP_BC_cs_ave']) + self.get_cube(cubes, var_name=['MP_BC_ki_cav', 'MP_BC_ki_ave', + 'MP_BC_ki']) + + self.get_cube(cubes, var_name=['MP_BC_ks_cav', 'MP_BC_ks_ave', + 'MP_BC_ks']) + + self.get_cube(cubes, var_name=['MP_BC_as_cav', 'MP_BC_as_ave', + 'MP_BC_as']) + + self.get_cube(cubes, var_name=['MP_BC_cs_cav', 'MP_BC_cs_ave', + 'MP_BC_cs']) ) cube.var_name = self.vardef.short_name return CubeList([cube]) @@ -394,10 +410,14 @@ class MP_DU_tot(EmacFix): # noqa: N801 def fix_metadata(self, cubes): """Fix metadata.""" cube = ( - self.get_cube(cubes, var_name=['MP_DU_ai_cav', 'MP_DU_ai_ave']) + - self.get_cube(cubes, var_name=['MP_DU_as_cav', 'MP_DU_as_ave']) + - self.get_cube(cubes, var_name=['MP_DU_ci_cav', 'MP_DU_ci_ave']) + - self.get_cube(cubes, var_name=['MP_DU_cs_cav', 'MP_DU_cs_ave']) + self.get_cube(cubes, var_name=['MP_DU_ai_cav', 'MP_DU_ai_ave', + 'MP_DU_ai']) + + self.get_cube(cubes, var_name=['MP_DU_as_cav', 'MP_DU_as_ave', + 'MP_DU_as']) + + self.get_cube(cubes, var_name=['MP_DU_ci_cav', 'MP_DU_ci_ave', + 'MP_DU_ci']) + + self.get_cube(cubes, var_name=['MP_DU_cs_cav', 'MP_DU_cs_ave', + 'MP_DU_cs']) ) cube.var_name = self.vardef.short_name return CubeList([cube]) @@ -410,13 +430,17 @@ def fix_metadata(self, cubes): """Fix metadata.""" cube = ( self.get_cube( - cubes, var_name=['MP_SO4mm_ns_cav', 'MP_SO4mm_ns_ave']) + + cubes, var_name=['MP_SO4mm_ns_cav', 'MP_SO4mm_ns_ave', + 'MP_SO4mm_ns']) + self.get_cube( - cubes, var_name=['MP_SO4mm_ks_cav', 'MP_SO4mm_ks_ave']) + + cubes, var_name=['MP_SO4mm_ks_cav', 'MP_SO4mm_ks_ave', + 'MP_SO4mm_ks']) + self.get_cube( - cubes, var_name=['MP_SO4mm_as_cav', 'MP_SO4mm_as_ave']) + + cubes, var_name=['MP_SO4mm_as_cav', 'MP_SO4mm_as_ave', + 'MP_SO4mm_as']) + self.get_cube( - cubes, var_name=['MP_SO4mm_cs_cav', 'MP_SO4mm_cs_ave']) + cubes, var_name=['MP_SO4mm_cs_cav', 'MP_SO4mm_cs_ave', + 'MP_SO4mm_cs']) ) cube.var_name = self.vardef.short_name return CubeList([cube]) @@ -428,9 +452,12 @@ class MP_SS_tot(EmacFix): # noqa: N801 def fix_metadata(self, cubes): """Fix metadata.""" cube = ( - self.get_cube(cubes, var_name=['MP_SS_ks_cav', 'MP_SS_ks_ave']) + - self.get_cube(cubes, var_name=['MP_SS_as_cav', 'MP_SS_as_ave']) + - self.get_cube(cubes, var_name=['MP_SS_cs_cav', 'MP_SS_cs_ave']) + self.get_cube(cubes, var_name=['MP_SS_ks_cav', 'MP_SS_ks_ave', + 'MP_SS_ks']) + + self.get_cube(cubes, var_name=['MP_SS_as_cav', 'MP_SS_as_ave', + 'MP_SS_as']) + + self.get_cube(cubes, var_name=['MP_SS_cs_cav', 'MP_SS_cs_ave', + 'MP_SS_cs']) ) cube.var_name = self.vardef.short_name return CubeList([cube]) From aba81fdff02bea78d75f87084b35c067cfeb2dba Mon Sep 17 00:00:00 2001 From: Klaus Zimmermann Date: Fri, 16 Dec 2022 17:45:47 +0100 Subject: [PATCH 5/8] Pin esmpy<8.4 (#1871) --- environment.yml | 2 +- setup.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index 43e16206ea..b7f30b1167 100644 --- a/environment.yml +++ b/environment.yml @@ -11,7 +11,7 @@ dependencies: - dask - compilers - esgf-pyclient>=0.3.1 - - esmpy!=8.1.0 # see github.com/ESMValGroup/ESMValCore/issues/1208 + - esmpy!=8.1.0,<8.4 # see github.com/ESMValGroup/ESMValCore/issues/1208 - fiona - fire - geopy diff --git a/setup.py b/setup.py index 3a732d09c3..49daac53ca 100755 --- a/setup.py +++ b/setup.py @@ -33,7 +33,8 @@ 'dask[array]', 'esgf-pyclient>=0.3.1', 'esmf-regrid', - 'esmpy!=8.1.0', # see github.com/ESMValGroup/ESMValCore/issues/1208 + # see github.com/ESMValGroup/ESMValCore/issues/1208 + 'esmpy!=8.1.0,<8.4', 'fiona', 'fire', 'geopy', From 7b5e40f5f497162d402ebbeb85c0861889824884 Mon Sep 17 00:00:00 2001 From: sloosvel <45196700+sloosvel@users.noreply.github.com> Date: Mon, 19 Dec 2022 18:01:11 +0100 Subject: [PATCH 6/8] Fix rotated coordinate grids and `tas` and `pr` for CORDEX datasets (#1765) Co-authored-by: Pep Cos <66776374+pepcos@users.noreply.github.com> Co-authored-by: Klaus Zimmermann --- environment.yml | 1 + esmvalcore/cmor/_fixes/cordex/__init__ .py | 1 + .../cordex/cnrm_cerfacs_cnrm_cm5/__init__.py | 1 + .../cnrm_cerfacs_cnrm_cm5/cnrm_aladin63.py | 33 +++ .../mohc_hadrem3_ga7_05.py | 7 + esmvalcore/cmor/_fixes/cordex/cordex_fixes.py | 206 ++++++++++++++++ .../_fixes/cordex/ichec_ec_earth/__init__.py | 1 + .../ichec_ec_earth/clmcom_cclm4_8_17.py | 5 + .../cordex/ichec_ec_earth/gerics_remo2015.py | 5 + .../cordex/ichec_ec_earth/knmi_racmo22e.py | 5 + .../ichec_ec_earth/mohc_hadrem3_ga7_05.py | 7 + .../_fixes/cordex/ichec_ec_earth/smhi_rca4.py | 7 + .../_fixes/cordex/miroc_miroc5/__init__.py | 1 + .../cordex/miroc_miroc5/clmcom_cclm4_8_17.py | 5 + .../cordex/miroc_miroc5/gerics_remo2015.py | 5 + .../cordex/miroc_miroc5/uhoh_wrf361h.py | 34 +++ .../_fixes/cordex/mohc_hadgem2_es/__init__.py | 1 + .../cordex/mohc_hadgem2_es/dmi_hirham5.py | 25 ++ .../cordex/mohc_hadgem2_es/gerics_remo2015.py | 7 + .../mohc_hadgem2_es/mohc_hadrem3_ga7_05.py | 7 + .../cordex/mohc_hadgem2_es/smhi_rca4.py | 7 + .../cordex/mpi_m_mpi_esm_lr/__init__.py | 1 + .../cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py | 7 + .../cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py | 5 + .../mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py | 7 + .../_fixes/cordex/ncc_noresm1_m/__init__.py | 1 + .../cordex/ncc_noresm1_m/gerics_remo2015.py | 5 + .../cordex/ncc_noresm1_m/knmi_racmo22e.py | 5 + .../ncc_noresm1_m/mohc_hadrem3_ga7_05.py | 7 + .../_fixes/cordex/ncc_noresm1_m/smhi_rca4.py | 7 + esmvalcore/cmor/_fixes/fix.py | 26 +- esmvalcore/cmor/check.py | 4 + setup.py | 1 + .../cmor/_fixes/cordex/__init__.py | 1 + .../cordex/test_cnrm_cerfacs_cnrm_cm5.py | 61 +++++ .../cmor/_fixes/cordex/test_cordex_fixes.py | 229 ++++++++++++++++++ .../cmor/_fixes/cordex/test_ichec_ec_earth.py | 46 ++++ .../cmor/_fixes/cordex/test_miroc_miroc5.py | 62 +++++ .../_fixes/cordex/test_mohc_hadgem2_es.py | 105 ++++++++ .../_fixes/cordex/test_mpi_m_mpi_esm_lr.py | 36 +++ .../cmor/_fixes/cordex/test_ncc_noresm1_m.py | 46 ++++ tests/integration/cmor/_fixes/test_fix.py | 23 ++ tests/unit/cmor/test_cmor_check.py | 10 +- 43 files changed, 1060 insertions(+), 6 deletions(-) create mode 100644 esmvalcore/cmor/_fixes/cordex/__init__ .py create mode 100644 esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/__init__.py create mode 100644 esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/cnrm_aladin63.py create mode 100644 esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/mohc_hadrem3_ga7_05.py create mode 100644 esmvalcore/cmor/_fixes/cordex/cordex_fixes.py create mode 100644 esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/__init__.py create mode 100644 esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/clmcom_cclm4_8_17.py create mode 100644 esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/gerics_remo2015.py create mode 100644 esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/knmi_racmo22e.py create mode 100644 esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/mohc_hadrem3_ga7_05.py create mode 100644 esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/smhi_rca4.py create mode 100644 esmvalcore/cmor/_fixes/cordex/miroc_miroc5/__init__.py create mode 100644 esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py create mode 100644 esmvalcore/cmor/_fixes/cordex/miroc_miroc5/gerics_remo2015.py create mode 100644 esmvalcore/cmor/_fixes/cordex/miroc_miroc5/uhoh_wrf361h.py create mode 100644 esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/__init__.py create mode 100644 esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py create mode 100644 esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/gerics_remo2015.py create mode 100644 esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py create mode 100644 esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py create mode 100644 esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/__init__.py create mode 100644 esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py create mode 100644 esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py create mode 100644 esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py create mode 100644 esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/__init__.py create mode 100644 esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py create mode 100644 esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/knmi_racmo22e.py create mode 100644 esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/mohc_hadrem3_ga7_05.py create mode 100644 esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/smhi_rca4.py create mode 100644 tests/integration/cmor/_fixes/cordex/__init__.py create mode 100644 tests/integration/cmor/_fixes/cordex/test_cnrm_cerfacs_cnrm_cm5.py create mode 100644 tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py create mode 100644 tests/integration/cmor/_fixes/cordex/test_ichec_ec_earth.py create mode 100644 tests/integration/cmor/_fixes/cordex/test_miroc_miroc5.py create mode 100644 tests/integration/cmor/_fixes/cordex/test_mohc_hadgem2_es.py create mode 100644 tests/integration/cmor/_fixes/cordex/test_mpi_m_mpi_esm_lr.py create mode 100644 tests/integration/cmor/_fixes/cordex/test_ncc_noresm1_m.py diff --git a/environment.yml b/environment.yml index b7f30b1167..c17f9de011 100644 --- a/environment.yml +++ b/environment.yml @@ -30,6 +30,7 @@ dependencies: - pip!=21.3 - prov - psutil + - py-cordex - pybtex - python>=3.8 - python-stratify diff --git a/esmvalcore/cmor/_fixes/cordex/__init__ .py b/esmvalcore/cmor/_fixes/cordex/__init__ .py new file mode 100644 index 0000000000..093969370f --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/__init__ .py @@ -0,0 +1 @@ +"""Fixes for CORDEX data.""" diff --git a/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/__init__.py b/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/__init__.py new file mode 100644 index 0000000000..093969370f --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/__init__.py @@ -0,0 +1 @@ +"""Fixes for CORDEX data.""" diff --git a/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/cnrm_aladin63.py b/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/cnrm_aladin63.py new file mode 100644 index 0000000000..af0348767a --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/cnrm_aladin63.py @@ -0,0 +1,33 @@ +"""Fixes for rcm CNRM-ALADIN63 driven by CNRM-CERFACS-CNRM-CM5.""" +import numpy as np + +from esmvalcore.cmor._fixes.cordex.cordex_fixes import TimeLongName as BaseFix +from esmvalcore.cmor._fixes.shared import add_scalar_height_coord +from esmvalcore.cmor.fix import Fix + + +class Tas(Fix): + """Fixes for tas.""" + + def fix_metadata(self, cubes): + """Add height (2m) coordinate and correct long_name for time. + + Parameters + ---------- + cubes : iris.cube.CubeList + Input cubes. + + Returns + ------- + iris.cube.CubeList + """ + for cube in cubes: + add_scalar_height_coord(cube) + if cube.coord('height').points != 2.: + cube.coord('height').points = np.ma.array([2.0]) + cube.coord('time').long_name = 'time' + + return cubes + + +Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/mohc_hadrem3_ga7_05.py new file mode 100644 index 0000000000..fd31e9b8a6 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/mohc_hadrem3_ga7_05.py @@ -0,0 +1,7 @@ +"""Fixes for rcm MOHC-HadREM3-GA7-05 driven by CNRM-CERFACS-CNRM-CM5.""" +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + MOHCHadREM3GA705 as BaseFix) + +Tas = BaseFix + +Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py new file mode 100644 index 0000000000..9ccbfd4ed3 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py @@ -0,0 +1,206 @@ +"""Fixes that are shared between datasets and drivers.""" +import logging +from functools import lru_cache + +import cordex as cx +import iris +import numpy as np +from cf_units import Unit +from iris.coord_systems import LambertConformal, RotatedGeogCS + +from esmvalcore.cmor.fix import Fix +from esmvalcore.exceptions import RecipeError + +logger = logging.getLogger(__name__) + + +@lru_cache +def _get_domain(data_domain): + domain = cx.cordex_domain(data_domain, add_vertices=True) + return domain + + +@lru_cache +def _get_domain_info(data_domain): + domain_info = cx.domain_info(data_domain) + return domain_info + + +class MOHCHadREM3GA705(Fix): + """General fix for MOHC-HadREM3-GA7-05.""" + + def fix_metadata(self, cubes): + """Fix time long_name, and latitude and longitude var_name. + + Parameters + ---------- + cubes : iris.cube.CubeList + Input cubes. + + Returns + ------- + iris.cube.CubeList + """ + for cube in cubes: + cube.coord('latitude').var_name = 'lat' + cube.coord('longitude').var_name = 'lon' + cube.coord('time').long_name = 'time' + + return cubes + + +class TimeLongName(Fix): + """Fixes for time coordinate.""" + + def fix_metadata(self, cubes): + """Fix time long_name. + + Parameters + ---------- + cubes : iris.cube.CubeList + Input cubes. + + Returns + ------- + iris.cube.CubeList + """ + for cube in cubes: + cube.coord('time').long_name = 'time' + + return cubes + + +class CLMcomCCLM4817(Fix): + """Fixes for CLMcom-CCLM4-8-17.""" + + def fix_metadata(self, cubes): + """Fix calendars. + + Set calendar to 'proleptic_gregorian' to avoid + concatenation issues between historical and + scenario runs. + + Fix dtype value of coordinates and coordinate bounds. + + Parameters + ---------- + cubes : iris.cube.CubeList + Input cubes. + + Returns + ------- + iris.cube.CubeList + """ + for cube in cubes: + time_unit = cube.coord('time').units + if time_unit.calendar == 'standard': + new_unit = time_unit.change_calendar('proleptic_gregorian') + cube.coord('time').units = new_unit + for coord in cube.coords(): + if coord.dtype in ['>f8', '>f4']: + coord.points = coord.core_points().astype( + np.float64, casting='same_kind') + if coord.bounds is not None: + coord.bounds = coord.core_bounds().astype( + np.float64, casting='same_kind') + return cubes + + +class AllVars(Fix): + """General CORDEX grid fix.""" + + def _check_grid_differences(self, old_coord, new_coord): + """Check differences between coords.""" + diff = np.max(np.abs(old_coord.points - new_coord.points)) + logger.debug( + "Maximum difference between original %s" + "points and standard %s domain points " + "for dataset %s and driver %s is: %s.", new_coord.var_name, + self.extra_facets['domain'], self.extra_facets['dataset'], + self.extra_facets['driver'], str(diff)) + + if diff > 10e-4: + raise RecipeError( + "Differences between the original grid and the " + f"standardised grid are above 10e-4 {new_coord.units}.") + + def _fix_rotated_coords(self, cube, domain, domain_info): + """Fix rotated coordinates.""" + for dim_coord in ['rlat', 'rlon']: + old_coord = cube.coord(domain[dim_coord].standard_name) + old_coord_dims = old_coord.cube_dims(cube) + points = domain[dim_coord].data + coord_system = iris.coord_systems.RotatedGeogCS( + grid_north_pole_latitude=domain_info['pollat'], + grid_north_pole_longitude=domain_info['pollon']) + new_coord = iris.coords.DimCoord( + points, + var_name=dim_coord, + standard_name=domain[dim_coord].standard_name, + long_name=domain[dim_coord].long_name, + units=Unit('degrees'), + coord_system=coord_system, + ) + self._check_grid_differences(old_coord, new_coord) + new_coord.guess_bounds() + cube.remove_coord(old_coord) + cube.add_dim_coord(new_coord, old_coord_dims) + + def _fix_geographical_coords(self, cube, domain): + """Fix geographical coordinates.""" + for aux_coord in ['lat', 'lon']: + old_coord = cube.coord(domain[aux_coord].standard_name) + cube.remove_coord(old_coord) + points = domain[aux_coord].data + bounds = domain[f'{aux_coord}_vertices'].data + new_coord = iris.coords.AuxCoord( + points, + var_name=aux_coord, + standard_name=domain[aux_coord].standard_name, + long_name=domain[aux_coord].long_name, + units=Unit(domain[aux_coord].units), + bounds=bounds, + ) + self._check_grid_differences(old_coord, new_coord) + aux_coord_dims = (cube.coord(var_name='rlat').cube_dims(cube) + + cube.coord(var_name='rlon').cube_dims(cube)) + cube.add_aux_coord(new_coord, aux_coord_dims) + + def fix_metadata(self, cubes): + """Fix CORDEX rotated grids. + + Set rotated and geographical coordinates to the + values given by each domain specification. + + The domain specifications are retrieved from the + py-cordex package. + + Parameters + ---------- + cubes : iris.cube.CubeList + Input cubes. + + Returns + ------- + iris.cube.CubeList + """ + data_domain = self.extra_facets['domain'] + domain = _get_domain(data_domain) + domain_info = _get_domain_info(data_domain) + for cube in cubes: + coord_system = cube.coord_system() + if isinstance(coord_system, RotatedGeogCS): + self._fix_rotated_coords(cube, domain, domain_info) + self._fix_geographical_coords(cube, domain) + elif isinstance(coord_system, LambertConformal): + logger.warning( + "Support for CORDEX datasets in a Lambert Conformal " + "coordinate system is ongoing. Certain preprocessor " + "functions may fail.") + else: + raise RecipeError( + f"Coordinate system {coord_system.grid_mapping_name} " + "not supported in CORDEX datasets. Must be " + "rotated_latitude_longitude or lambert_conformal_conic.") + + return cubes diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/__init__.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/__init__.py new file mode 100644 index 0000000000..093969370f --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/__init__.py @@ -0,0 +1 @@ +"""Fixes for CORDEX data.""" diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/clmcom_cclm4_8_17.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/clmcom_cclm4_8_17.py new file mode 100644 index 0000000000..883e38cecd --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/clmcom_cclm4_8_17.py @@ -0,0 +1,5 @@ +"""Fixes for rcm CLMcom-CCLM4-8-17 driven by ICHEC-EC-EARTH.""" +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + CLMcomCCLM4817 as BaseFix) + +AllVars = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/gerics_remo2015.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/gerics_remo2015.py new file mode 100644 index 0000000000..2f6b9a4d62 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/gerics_remo2015.py @@ -0,0 +1,5 @@ +"""Fixes for rcm GERICS-REMO2015 driven by ICHEC-EC-EARTH.""" +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + TimeLongName as BaseFix) + +Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/knmi_racmo22e.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/knmi_racmo22e.py new file mode 100644 index 0000000000..96fd620afe --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/knmi_racmo22e.py @@ -0,0 +1,5 @@ +"""Fixes for rcm KNMI-RACMO22E driven by ICHEC-EC-EARTH.""" +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + TimeLongName as BaseFix) + +Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/mohc_hadrem3_ga7_05.py new file mode 100644 index 0000000000..15b643aa36 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/mohc_hadrem3_ga7_05.py @@ -0,0 +1,7 @@ +"""Fixes for rcm MOHC-HadREM3-GA7-05 driven by ICHEC-EC-EARTH.""" +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + MOHCHadREM3GA705 as BaseFix) + +Tas = BaseFix + +Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/smhi_rca4.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/smhi_rca4.py new file mode 100644 index 0000000000..de1d8d6536 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/smhi_rca4.py @@ -0,0 +1,7 @@ +"""Fixes for rcm SMHI-RCA4 driven by ICHEC-EC-EARTH.""" +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + TimeLongName as BaseFix) + +Pr = BaseFix + +Tas = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/__init__.py b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/__init__.py new file mode 100644 index 0000000000..093969370f --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/__init__.py @@ -0,0 +1 @@ +"""Fixes for CORDEX data.""" diff --git a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py new file mode 100644 index 0000000000..51d645f591 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py @@ -0,0 +1,5 @@ +"""Fixes for rcm CLMcom-CCLM4-8-17 driven by MIROC-MIROC5.""" +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + CLMcomCCLM4817 as BaseFix) + +AllVars = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/gerics_remo2015.py b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/gerics_remo2015.py new file mode 100644 index 0000000000..9f75b595f9 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/gerics_remo2015.py @@ -0,0 +1,5 @@ +"""Fixes for rcm GERICS-REMO2015 driven by MIROC-MIROC5.""" +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + TimeLongName as BaseFix) + +Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/uhoh_wrf361h.py b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/uhoh_wrf361h.py new file mode 100644 index 0000000000..3767c653ea --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/uhoh_wrf361h.py @@ -0,0 +1,34 @@ +"""Fixes for rcm UHOH-WRF361H driven by MIROC-MIROC5.""" +import iris +from esmvalcore.cmor.fix import Fix + + +class Tas(Fix): + """Fixes for tas.""" + + def fix_metadata(self, cubes): + """Fix tas coordinates. + + Set height as an auxiliary coordinate instead + of as a dimensional coordinate. + + Parameters + ---------- + cubes : iris.cube.CubeList + Input cubes. + + Returns + ------- + iris.cube.CubeList + + """ + fixed_cubes = iris.cube.CubeList() + for cube in cubes: + height = cube.coord('height') + if isinstance(height, iris.coords.DimCoord): + iris.util.demote_dim_coord_to_aux_coord( + cube, + height + ) + fixed_cubes.append(iris.util.squeeze(cube)) + return fixed_cubes diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/__init__.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/__init__.py new file mode 100644 index 0000000000..093969370f --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/__init__.py @@ -0,0 +1 @@ +"""Fixes for CORDEX data.""" diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py new file mode 100644 index 0000000000..fbf5633d13 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py @@ -0,0 +1,25 @@ +"""Fixes for rcm DMI-HIRHAM driven by MOHC-HadGEM2.""" +from esmvalcore.cmor.fix import Fix + + +class Pr(Fix): + """Fixes for pr.""" + + def fix_metadata(self, cubes): + """Remove latitude and longitude attributes. + + Parameters + ---------- + cubes : iris.cube.CubeList + Input cubes. + + Returns + ------- + iris.cube.CubeList + + """ + for cube in cubes: + cube.coord('latitude').attributes = {} + cube.coord('longitude').attributes = {} + + return cubes diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/gerics_remo2015.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/gerics_remo2015.py new file mode 100644 index 0000000000..85bb2e352c --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/gerics_remo2015.py @@ -0,0 +1,7 @@ +"""Fixes for rcm GERICS-REMO2015 driven by MOHC-HadGEM2.""" +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + TimeLongName as BaseFix) + +Pr = BaseFix + +Tas = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py new file mode 100644 index 0000000000..004cbc412f --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py @@ -0,0 +1,7 @@ +"""Fixes for rcm MOHC-HadREM3-GA7-05 driven by MOHC-HadGEM2-ES.""" +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + MOHCHadREM3GA705 as BaseFix) + +Tas = BaseFix + +Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py new file mode 100644 index 0000000000..543df3670a --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py @@ -0,0 +1,7 @@ +"""Fixes for rcm SMHI-RCA4 driven by MOHC-HadGEM2-ES.""" +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + TimeLongName as BaseFix) + +Pr = BaseFix + +Tas = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/__init__.py b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/__init__.py new file mode 100644 index 0000000000..093969370f --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/__init__.py @@ -0,0 +1 @@ +"""Fixes for CORDEX data.""" diff --git a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py new file mode 100644 index 0000000000..9a92874b03 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py @@ -0,0 +1,7 @@ +"""Fixes for rcm ICTP-RegCM4-6 driven by MPI-M-MPI-ESM-LR.""" +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + TimeLongName as BaseFix) + +Pr = BaseFix + +Tas = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py new file mode 100644 index 0000000000..ab944112b3 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py @@ -0,0 +1,5 @@ +"""Fixes for rcm KNMI-RACMO22E driven by MPI-M-MPI-ESM-LR.""" +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + TimeLongName as BaseFix) + +Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py new file mode 100644 index 0000000000..0abfc3dff1 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py @@ -0,0 +1,7 @@ +"""Fixes for rcm MOHC-HadREM3-GA7-05 driven by MPI-M-MPI-ESM-LR.""" +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + MOHCHadREM3GA705 as BaseFix) + +Tas = BaseFix + +Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/__init__.py b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/__init__.py new file mode 100644 index 0000000000..093969370f --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/__init__.py @@ -0,0 +1 @@ +"""Fixes for CORDEX data.""" diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py new file mode 100644 index 0000000000..b1d118cd38 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py @@ -0,0 +1,5 @@ +"""Fixes for rcm GERICS-REMO2015 driven by NCC-NorESM1-M.""" +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + TimeLongName as BaseFix) + +Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/knmi_racmo22e.py b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/knmi_racmo22e.py new file mode 100644 index 0000000000..cc2440a2b8 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/knmi_racmo22e.py @@ -0,0 +1,5 @@ +"""Fixes for rcm KNMI-RACMO22E driven by NCC-NorESM1-M.""" +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + TimeLongName as BaseFix) + +Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/mohc_hadrem3_ga7_05.py new file mode 100644 index 0000000000..1aa2d11b1b --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/mohc_hadrem3_ga7_05.py @@ -0,0 +1,7 @@ +"""Fixes for rcm HadREM3-GA7-05 driven by NCC-NorESM1-M.""" +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + MOHCHadREM3GA705 as BaseFix) + +Tas = BaseFix + +Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/smhi_rca4.py b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/smhi_rca4.py new file mode 100644 index 0000000000..5c9059f74a --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/smhi_rca4.py @@ -0,0 +1,7 @@ +"""Fixes for rcm SMHI-RCA4 driven by NCC-NorESM1-M.""" +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + TimeLongName as BaseFix) + +Pr = BaseFix + +Tas = BaseFix diff --git a/esmvalcore/cmor/_fixes/fix.py b/esmvalcore/cmor/_fixes/fix.py index be94c4a774..f7239a7a33 100644 --- a/esmvalcore/cmor/_fixes/fix.py +++ b/esmvalcore/cmor/_fixes/fix.py @@ -157,10 +157,27 @@ def get_fixes(project, dataset, mip, short_name, extra_facets=None): extra_facets = {} fixes = [] - try: - fixes_module = importlib.import_module( - 'esmvalcore.cmor._fixes.{0}.{1}'.format(project, dataset)) + fixes_modules = [] + if project == 'cordex': + driver = extra_facets['driver'].replace('-', '_').lower() + extra_facets['dataset'] = dataset + try: + fixes_modules.append(importlib.import_module( + f'esmvalcore.cmor._fixes.{project}.{driver}.{dataset}' + )) + except ImportError: + pass + fixes_modules.append(importlib.import_module( + 'esmvalcore.cmor._fixes.cordex.cordex_fixes')) + else: + try: + fixes_modules.append(importlib.import_module( + f'esmvalcore.cmor._fixes.{project}.{dataset}')) + except ImportError: + pass + + for fixes_module in fixes_modules: classes = inspect.getmembers(fixes_module, inspect.isclass) classes = dict((name.lower(), value) for name, value in classes) for fix_name in (short_name, mip.lower(), 'allvars'): @@ -168,8 +185,7 @@ def get_fixes(project, dataset, mip, short_name, extra_facets=None): fixes.append(classes[fix_name](vardef, extra_facets)) except KeyError: pass - except ImportError: - pass + return fixes @staticmethod diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index 5712ecc22b..328e9ce374 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -1081,6 +1081,10 @@ def _get_cmor_checker(table, table, ', '.join(CMOR_TABLES))) cmor_table = CMOR_TABLES[table] + if table == 'CORDEX' and mip.endswith('hr'): + # CORDEX X-hourly tables define the mip + # as ending in 'h' instead of 'hr'. + mip = mip.replace('hr', 'h') var_info = cmor_table.get_variable(mip, short_name) if var_info is None: var_info = CMOR_TABLES['custom'].get_variable(mip, short_name) diff --git a/setup.py b/setup.py index 49daac53ca..698011b115 100755 --- a/setup.py +++ b/setup.py @@ -50,6 +50,7 @@ 'pillow', 'prov', 'psutil', + 'py-cordex', 'pybtex', 'pyyaml', 'requests', diff --git a/tests/integration/cmor/_fixes/cordex/__init__.py b/tests/integration/cmor/_fixes/cordex/__init__.py new file mode 100644 index 0000000000..7aed60134a --- /dev/null +++ b/tests/integration/cmor/_fixes/cordex/__init__.py @@ -0,0 +1 @@ +"""Integration tests for CORDEX fixes.""" diff --git a/tests/integration/cmor/_fixes/cordex/test_cnrm_cerfacs_cnrm_cm5.py b/tests/integration/cmor/_fixes/cordex/test_cnrm_cerfacs_cnrm_cm5.py new file mode 100644 index 0000000000..2b29c7185c --- /dev/null +++ b/tests/integration/cmor/_fixes/cordex/test_cnrm_cerfacs_cnrm_cm5.py @@ -0,0 +1,61 @@ +"""Tests for the fixes for driver CNRM-CERFACS-CNRM-CM5.""" +import iris +import pytest + +from esmvalcore.cmor._fixes.cordex.cnrm_cerfacs_cnrm_cm5 import cnrm_aladin63 +from esmvalcore.cmor.fix import Fix + + +@pytest.fixture +def cubes(): + correct_time_coord = iris.coords.DimCoord([0.0], + var_name='time', + standard_name='time', + long_name='time') + correct_height_coord = iris.coords.AuxCoord([2.0], + var_name='height') + wrong_height_coord = iris.coords.AuxCoord([10.0], + var_name='height') + correct_cube = iris.cube.Cube( + [10.0], + var_name='tas', + dim_coords_and_dims=[(correct_time_coord, 0)], + aux_coords_and_dims=[(correct_height_coord, ())] + ) + wrong_cube = iris.cube.Cube( + [10.0], + var_name='tas', + dim_coords_and_dims=[(correct_time_coord, 0)], + aux_coords_and_dims=[(wrong_height_coord, ())] + ) + return iris.cube.CubeList([correct_cube, wrong_cube]) + + +@pytest.mark.parametrize('short_name', ['pr', 'tas']) +def test_get_mohc_hadrem3ga705_fix(short_name): + fix = Fix.get_fixes( + 'CORDEX', + 'MOHC-HadREM3-GA7-05', + 'Amon', + short_name, + extra_facets={'driver': 'CNRM-CERFACS-CNRM-CM5'}) + assert isinstance(fix[0], Fix) + + +@pytest.mark.parametrize('short_name', ['pr', 'tas']) +def test_get_cnrm_aladin63_fix(short_name): + fix = Fix.get_fixes( + 'CORDEX', + 'CNRM-ALADIN63', + 'Amon', + short_name, + extra_facets={'driver': 'CNRM-CERFACS-CNRM-CM5'}) + assert isinstance(fix[0], Fix) + + +def test_cnrm_aladin63_height_fix(cubes): + fix = cnrm_aladin63.Tas(None) + out_cubes = fix.fix_metadata(cubes) + assert cubes is out_cubes + for cube in out_cubes: + assert cube.coord('height').points == 2.0 diff --git a/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py new file mode 100644 index 0000000000..a831d37b32 --- /dev/null +++ b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py @@ -0,0 +1,229 @@ +"""Tests for general CORDEX fixes.""" +import cordex as cx +import iris +import numpy as np +import pytest +from cf_units import Unit + +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + AllVars, + CLMcomCCLM4817, + MOHCHadREM3GA705, + TimeLongName, +) +from esmvalcore.exceptions import RecipeError + + +@pytest.fixture +def cubes(): + correct_time_coord = iris.coords.DimCoord([0.0], + var_name='time', + standard_name='time', + long_name='time') + wrong_time_coord = iris.coords.DimCoord([0.0], + var_name='time', + standard_name='time', + long_name='wrong') + correct_lat_coord = iris.coords.DimCoord([0.0, 1.0], + var_name='lat', + standard_name='latitude', + long_name='latitude') + wrong_lat_coord = iris.coords.DimCoord([0.0, 1.0], + var_name='latitudeCoord', + standard_name='latitude', + long_name='latitude') + correct_lon_coord = iris.coords.DimCoord([0.0], + var_name='lon', + standard_name='longitude', + long_name='longitude') + wrong_lon_coord = iris.coords.DimCoord([0.0], + var_name='longitudeCoord', + standard_name='longitude', + long_name='longitude') + correct_cube = iris.cube.Cube( + [[[10.0], [10.0]]], + var_name='tas', + dim_coords_and_dims=[ + (correct_time_coord, 0), + (correct_lat_coord, 1), + (correct_lon_coord, 2)], + ) + wrong_cube = iris.cube.Cube( + [[[10.0], [10.0]]], + var_name='tas', + dim_coords_and_dims=[ + (wrong_time_coord, 0), + (wrong_lat_coord, 1), + (wrong_lon_coord, 2)], + ) + return iris.cube.CubeList([correct_cube, wrong_cube]) + + +@pytest.fixture +def cordex_cubes(): + coord_system = iris.coord_systems.RotatedGeogCS( + grid_north_pole_latitude=39.25, + grid_north_pole_longitude=-162, + ) + time = iris.coords.DimCoord(np.arange(0, 3), + var_name='time', + standard_name='time') + + rlat = iris.coords.DimCoord(np.arange(0, 412), + var_name='rlat', + standard_name='grid_latitude', + coord_system=coord_system, + ) + rlon = iris.coords.DimCoord(np.arange(0, 424), + var_name='rlon', + standard_name='grid_longitude', + coord_system=coord_system, + ) + lat = iris.coords.AuxCoord(np.ones((412, 424)), + var_name='lat', + standard_name='latitude') + lon = iris.coords.AuxCoord(np.ones((412, 424)), + var_name='lon', + standard_name='longitude') + + cube = iris.cube.Cube( + np.ones((3, 412, 424)), + var_name='tas', + dim_coords_and_dims=[ + (time, 0), + (rlat, 1), + (rlon, 2)], + aux_coords_and_dims=[ + (lat, (1, 2)), + (lon, (1, 2)) + ] + + ) + return iris.cube.CubeList([cube]) + + +@pytest.mark.parametrize( + 'coord, var_name, long_name', + [ + ('time', 'time', 'time'), + ('latitude', 'lat', 'latitude'), + ('longitude', 'lon', 'longitude'), + ]) +def test_mohchadrem3ga705_fix_metadata(cubes, coord, var_name, long_name): + fix = MOHCHadREM3GA705(None) + out_cubes = fix.fix_metadata(cubes) + assert cubes is out_cubes + for cube in out_cubes: + assert cube.coord(standard_name=coord).var_name == var_name + assert cube.coord(standard_name=coord).long_name == long_name + + +def test_timelongname_fix_metadata(cubes): + fix = TimeLongName(None) + out_cubes = fix.fix_metadata(cubes) + assert cubes is out_cubes + for cube in out_cubes: + assert cube.coord('time').long_name == 'time' + + +def test_clmcomcclm4817_fix_metadata(cubes): + cubes[0].coord('time').units = Unit( + 'days since 1850-1-1 00:00:00', + calendar='proleptic_gregorian') + cubes[1].coord('time').units = Unit( + 'days since 1850-1-1 00:00:00', + calendar='standard') + for coord in cubes[1].coords(): + coord.points = coord.core_points().astype( + '>f8', casting='same_kind') + lat = cubes[1].coord('latitude') + lat.guess_bounds() + lat.bounds = lat.core_bounds().astype( + '>f4', casting='same_kind') + + fix = CLMcomCCLM4817(None) + out_cubes = fix.fix_metadata(cubes) + assert cubes is out_cubes + for cube in out_cubes: + assert cube.coord('time').units == Unit( + 'days since 1850-1-1 00:00:00', + calendar='proleptic_gregorian') + for coord in cube.coords(): + assert coord.points.dtype == np.float64 + + +def test_rotated_grid_fix(cordex_cubes): + fix = AllVars( + vardef=None, + extra_facets={ + 'domain': 'EUR-11', + 'dataset': 'DATASET', + 'driver': 'DRIVER' + } + ) + domain = cx.cordex_domain('EUR-11', add_vertices=True) + for cube in cordex_cubes: + for coord in ['rlat', 'rlon', 'lat', 'lon']: + cube_coord = cube.coord(var_name=coord) + cube_coord.points = domain[coord].data + 1e-6 + out_cubes = fix.fix_metadata(cordex_cubes) + assert cordex_cubes is out_cubes + for out_cube in out_cubes: + for coord in ['rlat', 'rlon', 'lat', 'lon']: + cube_coord = out_cube.coord(var_name=coord) + domain_coord = domain[coord].data + np.testing.assert_array_equal( + cube_coord.points, domain_coord) + + +def test_rotated_grid_fix_error(cordex_cubes): + fix = AllVars( + vardef=None, + extra_facets={ + 'domain': 'EUR-11', + 'dataset': 'DATASET', + 'driver': 'DRIVER' + } + ) + msg = ("Differences between the original grid and the " + "standardised grid are above 10e-4 degrees.") + with pytest.raises(RecipeError) as exc: + fix.fix_metadata(cordex_cubes) + assert msg == exc.value.message + + +def test_lambert_grid_warning(cubes, caplog): + fix = AllVars( + vardef=None, + extra_facets={ + 'domain': 'EUR-11', + 'dataset': 'DATASET', + 'driver': 'DRIVER' + } + ) + for cube in cubes: + cube.coord_system = iris.coord_systems.LambertConformal + fix.fix_metadata(cubes) + msg = ("Support for CORDEX datasets in a Lambert Conformal " + "coordinate system is ongoing. Certain preprocessor " + "functions may fail.") + assert msg in caplog.text + + +def test_wrong_coord_system(cubes): + fix = AllVars( + vardef=None, + extra_facets={ + 'domain': 'EUR-11', + 'dataset': 'DATASET', + 'driver': 'DRIVER' + } + ) + for cube in cubes: + cube.coord_system = iris.coord_systems.AlbersEqualArea + msg = ("Coordinate system albers_conical_equal_area not supported in " + "CORDEX datasets. Must be rotated_latitude_longitude " + "or lambert_conformal_conic.") + with pytest.raises(RecipeError) as exc: + fix.fix_metadata(cubes) + assert msg == exc.value.message diff --git a/tests/integration/cmor/_fixes/cordex/test_ichec_ec_earth.py b/tests/integration/cmor/_fixes/cordex/test_ichec_ec_earth.py new file mode 100644 index 0000000000..09c345ba01 --- /dev/null +++ b/tests/integration/cmor/_fixes/cordex/test_ichec_ec_earth.py @@ -0,0 +1,46 @@ +"""Tests for the fixes for driver ICHEC-EC-Earth.""" +import pytest + +from esmvalcore.cmor.fix import Fix + + +def test_get_gerics_remo2015_fix(): + fix = Fix.get_fixes( + 'CORDEX', + 'GERICS-REMO2015', + 'Amon', + 'pr', + extra_facets={'driver': 'ICHEC-EC-Earth'}) + assert isinstance(fix[0], Fix) + + +def test_get_knmi_racmo22e_fix(): + fix = Fix.get_fixes( + 'CORDEX', + 'KNMI-RACMO22E', + 'Amon', + 'pr', + extra_facets={'driver': 'ICHEC-EC-Earth'}) + assert isinstance(fix[0], Fix) + + +@pytest.mark.parametrize('short_name', ['pr', 'tas']) +def test_get_mohc_hadrem3ga705_fix(short_name): + fix = Fix.get_fixes( + 'CORDEX', + 'MOHC-HadREM3-GA7-05', + 'Amon', + short_name, + extra_facets={'driver': 'ICHEC-EC-Earth'}) + assert isinstance(fix[0], Fix) + + +@pytest.mark.parametrize('short_name', ['pr', 'tas']) +def test_get_smhi_rca4_fix(short_name): + fix = Fix.get_fixes( + 'CORDEX', + 'SMHI-RCA4', + 'Amon', + short_name, + extra_facets={'driver': 'ICHEC-EC-Earth'}) + assert isinstance(fix[0], Fix) diff --git a/tests/integration/cmor/_fixes/cordex/test_miroc_miroc5.py b/tests/integration/cmor/_fixes/cordex/test_miroc_miroc5.py new file mode 100644 index 0000000000..a177498930 --- /dev/null +++ b/tests/integration/cmor/_fixes/cordex/test_miroc_miroc5.py @@ -0,0 +1,62 @@ +"""Tests for the fixes of AWI-CM-1-1-MR.""" +import pytest +import iris + +from esmvalcore.cmor._fixes.cordex.miroc_miroc5 import uhoh_wrf361h +from esmvalcore.cmor.fix import Fix + + +@pytest.fixture +def cubes(): + correct_time_coord = iris.coords.DimCoord([0.0, 1.0], + var_name='time', + standard_name='time', + long_name='time') + wrong_height_coord = iris.coords.DimCoord([2.0], + var_name='height') + wrong_cube = iris.cube.Cube( + [[10.0], [10.0]], + var_name='tas', + dim_coords_and_dims=[ + (correct_time_coord, 0), + (wrong_height_coord, 1)], + ) + return iris.cube.CubeList([wrong_cube]) + + +@pytest.mark.parametrize('short_name', ['pr', 'tas']) +def test_get_clmcom_cclm4_8_17fix(short_name): + fix = Fix.get_fixes( + 'CORDEX', + 'CLMCOM-CCLM4-8-17', + 'Amon', + short_name, + extra_facets={'driver': 'MIROC-MIROC5'}) + assert isinstance(fix[0], Fix) + + +def test_get_gerics_remo2015_fix(): + fix = Fix.get_fixes( + 'CORDEX', + 'GERICS-REMO2015', + 'Amon', + 'pr', + extra_facets={'driver': 'MIROC-MIROC5'}) + assert isinstance(fix[0], Fix) + + +def test_get_uhoh_wrf361h_fix(): + fix = Fix.get_fixes( + 'CORDEX', + 'UHOH-WRF361H', + 'Amon', + 'tas', + extra_facets={'driver': 'MIROC-MIROC5'}) + assert isinstance(fix[0], Fix) + + +def test_uhoh_wrf361h_height_fix(cubes): + fix = uhoh_wrf361h.Tas(None) + out_cubes = fix.fix_metadata(cubes) + for cube in out_cubes: + assert cube.ndim == 1 diff --git a/tests/integration/cmor/_fixes/cordex/test_mohc_hadgem2_es.py b/tests/integration/cmor/_fixes/cordex/test_mohc_hadgem2_es.py new file mode 100644 index 0000000000..408fe4d5a6 --- /dev/null +++ b/tests/integration/cmor/_fixes/cordex/test_mohc_hadgem2_es.py @@ -0,0 +1,105 @@ +"""Tests for the fixes for driver MOHC-HadGEM2-ES.""" +import iris +import pytest + +from esmvalcore.cmor._fixes.cordex.mohc_hadgem2_es import dmi_hirham5 +from esmvalcore.cmor.fix import Fix + + +@pytest.fixture +def cubes(): + correct_time_coord = iris.coords.DimCoord([0.0], + var_name='time', + standard_name='time', + long_name='time') + wrong_time_coord = iris.coords.DimCoord([0.0], + var_name='time', + standard_name='time', + long_name='wrong') + correct_lat_coord = iris.coords.DimCoord([0.0], + var_name='lat', + standard_name='latitude', + long_name='latitude') + wrong_lat_coord = iris.coords.DimCoord([0.0], + var_name='latitudeCoord', + standard_name='latitude', + long_name='latitude', + attributes={'wrong': 'attr'}) + correct_lon_coord = iris.coords.DimCoord([0.0], + var_name='lon', + standard_name='longitude', + long_name='longitude') + wrong_lon_coord = iris.coords.DimCoord([0.0], + var_name='longitudeCoord', + standard_name='longitude', + long_name='longitude', + attributes={'wrong': 'attr'}) + correct_cube = iris.cube.Cube( + [[[10.0]]], + var_name='tas', + dim_coords_and_dims=[ + (correct_time_coord, 0), + (correct_lat_coord, 1), + (correct_lon_coord, 2)], + ) + wrong_cube = iris.cube.Cube( + [[[10.0]]], + var_name='tas', + dim_coords_and_dims=[ + (wrong_time_coord, 0), + (wrong_lat_coord, 1), + (wrong_lon_coord, 2)], + ) + return iris.cube.CubeList([correct_cube, wrong_cube]) + + +def test_get_dmi_hirham5_fix(): + fix = Fix.get_fixes( + 'CORDEX', + 'DMI-HIRHAM5', + 'Amon', + 'pr', + extra_facets={'driver': 'MOHC-HadGEM2-ES'}) + assert isinstance(fix[0], Fix) + + +@pytest.mark.parametrize('short_name', ['pr', 'tas']) +def test_get_gerics_remo2015_fix(short_name): + fix = Fix.get_fixes( + 'CORDEX', + 'GERICS-REMO2015', + 'Amon', + short_name, + extra_facets={'driver': 'MOHC-HadGEM2-ES'}) + assert isinstance(fix[0], Fix) + + +@pytest.mark.parametrize('short_name', ['pr', 'tas']) +def test_get_mohc_hadrem3ga705_fix(short_name): + fix = Fix.get_fixes( + 'CORDEX', + 'MOHC-HadREM3-GA7-05', + 'Amon', + short_name, + extra_facets={'driver': 'MOHC-HadGEM2-ES'}) + assert isinstance(fix[0], Fix) + + +@pytest.mark.parametrize('short_name', ['pr', 'tas']) +def test_get_smhi_rca4_fix(short_name): + fix = Fix.get_fixes( + 'CORDEX', + 'SMHI-RCA4', + 'Amon', + short_name, + extra_facets={'driver': 'MOHC-HadGEM2-ES'}) + assert isinstance(fix[0], Fix) + + +def test_dmi_hirham5_fix(cubes): + fix = dmi_hirham5.Pr(None) + out_cubes = fix.fix_metadata(cubes) + assert cubes is out_cubes + for cube in out_cubes: + assert cube.coord('latitude').attributes == {} + assert cube.coord('longitude').attributes == {} diff --git a/tests/integration/cmor/_fixes/cordex/test_mpi_m_mpi_esm_lr.py b/tests/integration/cmor/_fixes/cordex/test_mpi_m_mpi_esm_lr.py new file mode 100644 index 0000000000..30c813297f --- /dev/null +++ b/tests/integration/cmor/_fixes/cordex/test_mpi_m_mpi_esm_lr.py @@ -0,0 +1,36 @@ +"""Tests for the fixes of driver MPI-M-MPI-ESM-LR.""" +import pytest + +from esmvalcore.cmor.fix import Fix + + +@pytest.mark.parametrize('short_name', ['pr', 'tas']) +def test_get_ictp_regcm4_6_fix(short_name): + fix = Fix.get_fixes( + 'CORDEX', + 'ICTP-REGCM4-6', + 'Amon', + short_name, + extra_facets={'driver': 'MPI-M-MPI-ESM-LR'}) + assert isinstance(fix[0], Fix) + + +def test_get_knmi_racmo22e_fix(): + fix = Fix.get_fixes( + 'CORDEX', + 'KNMI-RACMO22E', + 'Amon', + 'pr', + extra_facets={'driver': 'MPI-M-MPI-ESM-LR'}) + assert isinstance(fix[0], Fix) + + +@pytest.mark.parametrize('short_name', ['pr', 'tas']) +def test_get_mohc_hadrem3ga705_fix(short_name): + fix = Fix.get_fixes( + 'CORDEX', + 'MOHC-HadREM3-GA7-05', + 'Amon', + short_name, + extra_facets={'driver': 'MPI-M-MPI-ESM-LR'}) + assert isinstance(fix[0], Fix) diff --git a/tests/integration/cmor/_fixes/cordex/test_ncc_noresm1_m.py b/tests/integration/cmor/_fixes/cordex/test_ncc_noresm1_m.py new file mode 100644 index 0000000000..e071c8d7ea --- /dev/null +++ b/tests/integration/cmor/_fixes/cordex/test_ncc_noresm1_m.py @@ -0,0 +1,46 @@ +"""Tests for the fixes of driver NCC-NorESM1-M.""" +import pytest + +from esmvalcore.cmor.fix import Fix + + +def test_get_gerics_remo2015_fix(): + fix = Fix.get_fixes( + 'CORDEX', + 'GERICS-REMO2015', + 'Amon', + 'pr', + extra_facets={'driver': 'NCC-NorESM1-M'}) + assert isinstance(fix[0], Fix) + + +def test_get_knmi_racmo22e_fix(): + fix = Fix.get_fixes( + 'CORDEX', + 'KNMI-RACMO22E', + 'Amon', + 'pr', + extra_facets={'driver': 'NCC-NorESM1-M'}) + assert isinstance(fix[0], Fix) + + +@pytest.mark.parametrize('short_name', ['pr', 'tas']) +def test_get_mohc_hadrem3ga705_fix(short_name): + fix = Fix.get_fixes( + 'CORDEX', + 'MOHC-HadREM3-GA7-05', + 'Amon', + short_name, + extra_facets={'driver': 'NCC-NorESM1-M'}) + assert isinstance(fix[0], Fix) + + +@pytest.mark.parametrize('short_name', ['pr', 'tas']) +def test_get_smhi_rca4_fix(short_name): + fix = Fix.get_fixes( + 'CORDEX', + 'SMHI-RCA4', + 'Amon', + short_name, + extra_facets={'driver': 'NCC-NorESM1-M'}) + assert isinstance(fix[0], Fix) diff --git a/tests/integration/cmor/_fixes/test_fix.py b/tests/integration/cmor/_fixes/test_fix.py index 5ffc4ca868..16629549fe 100644 --- a/tests/integration/cmor/_fixes/test_fix.py +++ b/tests/integration/cmor/_fixes/test_fix.py @@ -12,6 +12,9 @@ from esmvalcore.cmor._fixes.cmip5.canesm2 import FgCo2 from esmvalcore.cmor._fixes.cmip5.cesm1_bgc import Gpp from esmvalcore.cmor._fixes.cmip6.cesm2 import Omon, Tos +from esmvalcore.cmor._fixes.cordex.cnrm_cerfacs_cnrm_cm5.cnrm_aladin63 import ( + Tas) +from esmvalcore.cmor._fixes.cordex.cordex_fixes import AllVars from esmvalcore.cmor.fix import Fix @@ -32,6 +35,26 @@ def test_get_fix_case_insensitive(self): self.assertListEqual( Fix.get_fixes('CMIP5', 'CanESM2', 'Amon', 'fgCo2'), [FgCo2(None)]) + def test_get_fix_cordex(self): + self.assertListEqual( + Fix.get_fixes( + 'CORDEX', + 'CNRM-ALADIN63', + 'Amon', + 'tas', + extra_facets={'driver': 'CNRM-CERFACS-CNRM-CM5'}), + [Tas(None), AllVars(None)]) + + def test_get_grid_fix_cordex(self): + self.assertListEqual( + Fix.get_fixes( + 'CORDEX', + 'CNRM-ALADIN53', + 'Amon', + 'tas', + extra_facets={'driver': 'CNRM-CERFACS-CNRM-CM5'}), + [AllVars(None)]) + def test_get_fixes_with_replace(self): self.assertListEqual(Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'ch4'), [Ch4(None)]) diff --git a/tests/unit/cmor/test_cmor_check.py b/tests/unit/cmor/test_cmor_check.py index fe965b5981..4e17f9ca99 100644 --- a/tests/unit/cmor/test_cmor_check.py +++ b/tests/unit/cmor/test_cmor_check.py @@ -11,7 +11,9 @@ import numpy as np from cf_units import Unit -from esmvalcore.cmor.check import CheckLevels, CMORCheck, CMORCheckError +from esmvalcore.cmor.check import ( + CheckLevels, CMORCheck, + CMORCheckError, _get_cmor_checker) class VariableInfoMock: @@ -1076,6 +1078,12 @@ def test_no_time_bounds(self): guessed_bounds = self.cube.coord('time').bounds assert guessed_bounds is None + def test_hr_mip_cordex(self): + """Test hourly CORDEX tables are found.""" + checker = _get_cmor_checker('CORDEX', '3hr', 'tas', '3hr') + assert checker(self.cube)._cmor_var.short_name == 'tas' + assert checker(self.cube)._cmor_var.frequency == '3hr' + def _check_fails_on_data(self): checker = CMORCheck(self.cube, self.var_info) checker.check_metadata() From da3b8644ef883ef236a2e72c8ddd3bbe6f2aad20 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 21 Dec 2022 10:29:31 +0000 Subject: [PATCH 7/8] [Condalock] Update Linux condalock file (#1881) Co-authored-by: valeriupredoi --- conda-linux-64.lock | 123 +++++++++++++++++++++++++------------------- 1 file changed, 69 insertions(+), 54 deletions(-) diff --git a/conda-linux-64.lock b/conda-linux-64.lock index 1fe61c661c..7bf2f8ff4c 100644 --- a/conda-linux-64.lock +++ b/conda-linux-64.lock @@ -1,6 +1,6 @@ # Generated by conda-lock. # platform: linux-64 -# input_hash: 153b7f6d9833c25e981b17319838235a50f9fd0c93fdaa09952a3919c2a09afc +# input_hash: a605e61eb7466781ff4b2ace6566da3866ccc2d98b9a60973588344048043ac1 @EXPLICIT https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2#d7c89558ba9fa0495403155b64376d81 https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2022.12.7-ha878542_0.conda#ff9f73d45c4a07d6f424495288a26080 @@ -10,9 +10,9 @@ https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77 https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-hab24e00_0.tar.bz2#19410c3df09dfb12d1206132a1d357c5 https://conda.anaconda.org/conda-forge/noarch/kernel-headers_linux-64-2.6.32-he073ed8_15.tar.bz2#5dd5127afd710f91f6a75821bac0a4f0 https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.39-hcc3a1bd_1.conda#737be0d34c22d24432049ab7a3214de4 -https://conda.anaconda.org/conda-forge/linux-64/libgcc-devel_linux-64-10.4.0-hd38fd1e_19.tar.bz2#b41d6540a78ba2518655eebcb0e41e20 +https://conda.anaconda.org/conda-forge/linux-64/libgcc-devel_linux-64-11.3.0-h210ce93_19.tar.bz2#9b7bdb0b42ce4e4670d32bfe0532b56a https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-12.2.0-h337968e_19.tar.bz2#164b4b1acaedc47ee7e658ae6b308ca3 -https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-devel_linux-64-10.4.0-hd38fd1e_19.tar.bz2#9367571bf3218f968a47c010618a9715 +https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-devel_linux-64-11.3.0-h210ce93_19.tar.bz2#8aee006c0662f551f3acef9a7077a5b9 https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-12.2.0-h46fd767_19.tar.bz2#1030b1f38c129f2634eae026f704fe60 https://conda.anaconda.org/conda-forge/linux-64/mpi-1.0-mpich.tar.bz2#c1fcff3417b5a22bbc4cf6e8c23648cf https://conda.anaconda.org/conda-forge/noarch/poppler-data-0.4.11-hd8ed1ab_0.tar.bz2#abc27381c4f005da588cffa1f76403ee @@ -49,7 +49,8 @@ https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2# https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.17-h166bdaf_0.tar.bz2#b62b52da46c39ee2bc3c162ac7f1804d https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.0-h7f98852_0.tar.bz2#39b1328babf85c7c3a61636d9cd50206 https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.21-pthreads_h78a6416_3.tar.bz2#8c5963a49b6035c40646a763293fbb35 -https://conda.anaconda.org/conda-forge/linux-64/libsanitizer-10.4.0-h5246dfb_19.tar.bz2#b068ad132a509367bc9e5a200a639429 +https://conda.anaconda.org/conda-forge/linux-64/libsanitizer-11.3.0-h239ccf8_19.tar.bz2#d17fd55aed84ab6592c5419b6600501c +https://conda.anaconda.org/conda-forge/linux-64/libspatialindex-1.9.3-h9c3ff4c_4.tar.bz2#d87fbe9c0ff589e802ff13872980bfd9 https://conda.anaconda.org/conda-forge/linux-64/libtool-2.4.6-h9c3ff4c_1008.tar.bz2#16e143a1ed4b4fd169536373957f6fee https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.32.1-h7f98852_1000.tar.bz2#772d69f030955d9646d3d0eaf21d859d https://conda.anaconda.org/conda-forge/linux-64/libwebp-base-1.2.4-h166bdaf_0.tar.bz2#ac2ccf7323d21f2994e4d1f5da664f37 @@ -73,7 +74,7 @@ https://conda.anaconda.org/conda-forge/linux-64/xorg-xproto-7.0.31-h7f98852_1007 https://conda.anaconda.org/conda-forge/linux-64/xxhash-0.8.0-h7f98852_3.tar.bz2#52402c791f35e414e704b7a113f99605 https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2#2161070d867d1b1204ea749c8eec4ef0 https://conda.anaconda.org/conda-forge/linux-64/yaml-0.2.5-h7f98852_2.tar.bz2#4cb3ad778ec2d5a7acbdf254eb1c42ae -https://conda.anaconda.org/conda-forge/linux-64/gcc_impl_linux-64-10.4.0-h5231bdf_19.tar.bz2#a086547de4cee874e72d5a43230372ec +https://conda.anaconda.org/conda-forge/linux-64/gcc_impl_linux-64-11.3.0-hab1b70f_19.tar.bz2#89ac16d36e66ccb9ca5d34c9217e5799 https://conda.anaconda.org/conda-forge/linux-64/libblas-3.9.0-16_linux64_openblas.tar.bz2#d9b7a8639171f6c6fa0a983edabcfe2b https://conda.anaconda.org/conda-forge/linux-64/libbrotlidec-1.0.9-h166bdaf_8.tar.bz2#4ae4d7795d33e02bd20f6b23d91caf82 https://conda.anaconda.org/conda-forge/linux-64/libbrotlienc-1.0.9-h166bdaf_8.tar.bz2#04bac51ba35ea023dc48af73c1c88c25 @@ -93,20 +94,20 @@ https://conda.anaconda.org/conda-forge/linux-64/udunits2-2.2.28-hc3e0081_0.tar.b https://conda.anaconda.org/conda-forge/linux-64/xorg-libsm-1.2.3-hd9c2040_1000.tar.bz2#9e856f78d5c80d5a78f61e72d1d473a3 https://conda.anaconda.org/conda-forge/linux-64/zlib-1.2.13-h166bdaf_4.tar.bz2#4b11e365c0275b808be78b30f904e295 https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.2-h6239696_4.tar.bz2#adcf0be7897e73e312bd24353b613f74 -https://conda.anaconda.org/conda-forge/linux-64/blosc-1.21.2-hafa529b_0.conda#9fc20ab886b80d1029b828ef7ee79a35 +https://conda.anaconda.org/conda-forge/linux-64/blosc-1.21.3-hafa529b_0.conda#bcf0664a2dbbbb86cbd4c1e6ff10ddd6 https://conda.anaconda.org/conda-forge/linux-64/boost-cpp-1.78.0-h75c5d50_1.tar.bz2#accc1f1ca33809bbf9ad067a0a69e236 https://conda.anaconda.org/conda-forge/linux-64/brotli-bin-1.0.9-h166bdaf_8.tar.bz2#e5613f2bc717e9945840ff474419b8e4 https://conda.anaconda.org/conda-forge/linux-64/freetype-2.12.1-hca18f0e_1.conda#e1232042de76d24539a436d37597eb06 -https://conda.anaconda.org/conda-forge/linux-64/gcc-10.4.0-hb92f740_11.tar.bz2#492fd2006232e01ddcf85994f3d9bdac -https://conda.anaconda.org/conda-forge/linux-64/gcc_linux-64-10.4.0-h9215b83_11.tar.bz2#8ec7a24818e75cd2975e6fe785ad18eb -https://conda.anaconda.org/conda-forge/linux-64/gfortran_impl_linux-64-10.4.0-h7d168d2_19.tar.bz2#2d598895087101a581a617221b815ec2 -https://conda.anaconda.org/conda-forge/linux-64/gxx_impl_linux-64-10.4.0-h5231bdf_19.tar.bz2#de8c00c5162b819c3e8a7f64ed32baf1 +https://conda.anaconda.org/conda-forge/linux-64/gcc-11.3.0-h02d0930_11.tar.bz2#6037ebe5f1e3054519ce78b11eec9cd4 +https://conda.anaconda.org/conda-forge/linux-64/gcc_linux-64-11.3.0-he6f903b_11.tar.bz2#25f76cb82e483ce96d118b9edffd12c9 +https://conda.anaconda.org/conda-forge/linux-64/gfortran_impl_linux-64-11.3.0-he34c6f7_19.tar.bz2#3de873ee757f1a2e583416a3583f84c4 +https://conda.anaconda.org/conda-forge/linux-64/gxx_impl_linux-64-11.3.0-hab1b70f_19.tar.bz2#b73564a352e64bb5f2c9bfd3cd6dd127 https://conda.anaconda.org/conda-forge/linux-64/hdf4-4.2.15-h9772cbc_5.tar.bz2#ee08782aff2ff9b3291c967fa6bc7336 -https://conda.anaconda.org/conda-forge/linux-64/krb5-1.19.3-h08a2579_0.tar.bz2#d25e05e7ee0e302b52d24491db4891eb +https://conda.anaconda.org/conda-forge/linux-64/krb5-1.20.1-h81ceb04_0.conda#89a41adce7106749573d883b2f657d78 https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.9.0-16_linux64_openblas.tar.bz2#20bae26d0a1db73f758fc3754cab4719 https://conda.anaconda.org/conda-forge/linux-64/libglib-2.74.1-h606061b_1.tar.bz2#ed5349aa96776e00b34eccecf4a948fe https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-16_linux64_openblas.tar.bz2#955d993f41f9354bf753d29864ea20ad -https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.4.0-h55922b4_4.tar.bz2#901791f0ec7cddc8714e76e273013a91 +https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.4.0-h82bc61c_5.conda#e712a63a21f9db647982971dc121cdcf https://conda.anaconda.org/conda-forge/linux-64/libxslt-1.1.37-h873f0b0_0.tar.bz2#ed0d77d947ddeb974892de8df7224d12 https://conda.anaconda.org/conda-forge/linux-64/nss-3.82-he02c5a1_0.conda#f8d7f11d19e4cb2207eab159fd4c0152 https://conda.anaconda.org/conda-forge/linux-64/python-3.10.8-h4a9ceb5_0_cpython.conda#be2a6d78752c2ab85f360ce37d2c64e2 @@ -119,7 +120,7 @@ https://conda.anaconda.org/conda-forge/linux-64/atk-1.0-2.38.0-hd4edc92_1.tar.bz https://conda.anaconda.org/conda-forge/noarch/attrs-22.1.0-pyh71513ae_1.tar.bz2#6d3ccbc56256204925bfa8378722792f https://conda.anaconda.org/conda-forge/linux-64/backports.zoneinfo-0.2.1-py310hff52083_7.tar.bz2#02d7c823f5e6fd4bbe5562c612465aed https://conda.anaconda.org/conda-forge/linux-64/brotli-1.0.9-h166bdaf_8.tar.bz2#2ff08978892a3e8b954397c461f18418 -https://conda.anaconda.org/conda-forge/linux-64/c-compiler-1.5.1-h166bdaf_0.tar.bz2#0667d7da14e682c9d07968601f6233ef +https://conda.anaconda.org/conda-forge/linux-64/c-compiler-1.5.2-h0b41bf4_0.conda#69afb4e35be6366c2c1f9ed7f49bc3e6 https://conda.anaconda.org/conda-forge/noarch/certifi-2022.12.7-pyhd8ed1ab_0.conda#fb9addc3db06e56abe03e0e9f21a63e6 https://conda.anaconda.org/conda-forge/noarch/cfgv-3.3.1-pyhd8ed1ab_0.tar.bz2#ebb5f5f7dc4f1a3780ef7ea7738db08c https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-2.1.1-pyhd8ed1ab_0.tar.bz2#c1d5b294fbf9a795dec349a6f4d8be8e @@ -140,11 +141,11 @@ https://conda.anaconda.org/conda-forge/linux-64/fontconfig-2.14.1-hc2a2eb6_0.tar https://conda.anaconda.org/conda-forge/noarch/fsspec-2022.11.0-pyhd8ed1ab_0.tar.bz2#eb919f2119a6db5d0192f9e9c3711572 https://conda.anaconda.org/conda-forge/linux-64/gdk-pixbuf-2.42.8-hff1cb4f_1.tar.bz2#a61c6312192e7c9de71548a6706a21e6 https://conda.anaconda.org/conda-forge/noarch/geographiclib-1.52-pyhd8ed1ab_0.tar.bz2#6880e7100ebae550a33ce26663316d85 -https://conda.anaconda.org/conda-forge/linux-64/gfortran-10.4.0-h0c96582_11.tar.bz2#9a22e19ae1d372f19a6514a4442f7917 -https://conda.anaconda.org/conda-forge/linux-64/gfortran_linux-64-10.4.0-h69d5af5_11.tar.bz2#7d42e71ff8a9f51b7a206ee35a742ce1 +https://conda.anaconda.org/conda-forge/linux-64/gfortran-11.3.0-ha859ce3_11.tar.bz2#9a6a0c6fc4d192fddc7347a0ca31a329 +https://conda.anaconda.org/conda-forge/linux-64/gfortran_linux-64-11.3.0-h3c55166_11.tar.bz2#f70b169eb69320d71f193758b7df67e8 https://conda.anaconda.org/conda-forge/linux-64/gts-0.7.6-h64030ff_2.tar.bz2#112eb9b5b93f0c02e59aea4fd1967363 -https://conda.anaconda.org/conda-forge/linux-64/gxx-10.4.0-hb92f740_11.tar.bz2#a286961cd68f7d36f4ece4578042567c -https://conda.anaconda.org/conda-forge/linux-64/gxx_linux-64-10.4.0-h6e491c6_11.tar.bz2#842f0029666e37e929cbd1e7614f5862 +https://conda.anaconda.org/conda-forge/linux-64/gxx-11.3.0-h02d0930_11.tar.bz2#e47dd4b4e577f03bb6aab18f48be5419 +https://conda.anaconda.org/conda-forge/linux-64/gxx_linux-64-11.3.0-hc203a17_11.tar.bz2#15fbc9079f191d468403639a6515652c https://conda.anaconda.org/conda-forge/noarch/heapdict-1.0.1-py_0.tar.bz2#77242bfb1e74a627fb06319b5a2d3b95 https://conda.anaconda.org/conda-forge/linux-64/humanfriendly-10.0-py310hff52083_4.tar.bz2#43bd27c73e9e3b0bc508217ae409798f https://conda.anaconda.org/conda-forge/noarch/idna-3.4-pyhd8ed1ab_0.tar.bz2#34272b248891bddccc64479f9a7fffed @@ -154,12 +155,12 @@ https://conda.anaconda.org/conda-forge/noarch/itsdangerous-2.1.2-pyhd8ed1ab_0.ta https://conda.anaconda.org/conda-forge/linux-64/kiwisolver-1.4.4-py310hbf28c38_1.tar.bz2#ad5647e517ba68e2868ef2e6e6ff7723 https://conda.anaconda.org/conda-forge/linux-64/lazy-object-proxy-1.8.0-py310h5764c6d_0.tar.bz2#7c3f704b65f1689d6ba31a79791d2ce1 https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.14-h6ed2654_0.tar.bz2#dcc588839de1445d90995a0a2c4f3a39 -https://conda.anaconda.org/conda-forge/linux-64/libcurl-7.86.0-h2283fc2_1.tar.bz2#fdca8cd67ec2676f90a70ac73a32538b +https://conda.anaconda.org/conda-forge/linux-64/libcurl-7.86.0-hdc1c0ab_2.conda#c4c21da8fc657a6c5e426b8cbad36cc2 https://conda.anaconda.org/conda-forge/linux-64/libkml-1.3.0-h37653c0_1015.tar.bz2#37d3747dd24d604f63d2610910576e63 -https://conda.anaconda.org/conda-forge/linux-64/libpq-15.1-h67c24c5_1.conda#e1389a8d9a907133b3e6483c2807d243 +https://conda.anaconda.org/conda-forge/linux-64/libpq-15.1-hb675445_2.conda#509f08b3789d9e7e9a72871491ae08e2 https://conda.anaconda.org/conda-forge/linux-64/libwebp-1.2.4-h522a892_0.tar.bz2#802e43f480122a85ae6a34c1909f8f98 https://conda.anaconda.org/conda-forge/noarch/locket-1.0.0-pyhd8ed1ab_0.tar.bz2#91e27ef3d05cc772ce627e51cff111c4 -https://conda.anaconda.org/conda-forge/linux-64/lxml-4.9.1-py310ha00c094_1.tar.bz2#4b7ed16f7db1eea5b53442aeab2d3b9e +https://conda.anaconda.org/conda-forge/linux-64/lxml-4.9.2-py310hbdc0903_0.conda#543906a26651f10c6180ca71fc4d48f2 https://conda.anaconda.org/conda-forge/linux-64/lz4-4.0.2-py310h5d5e884_0.tar.bz2#cda615ff0531107b6f83f14ea0a2e7f1 https://conda.anaconda.org/conda-forge/linux-64/markupsafe-2.1.1-py310h5764c6d_2.tar.bz2#2d7028ea2a77f909931e1a173d952261 https://conda.anaconda.org/conda-forge/noarch/mccabe-0.6.1-py_1.tar.bz2#a326cb400c1ccd91789f3e7d02124d61 @@ -168,7 +169,7 @@ https://conda.anaconda.org/conda-forge/linux-64/msgpack-python-1.0.4-py310hbf28c https://conda.anaconda.org/conda-forge/noarch/munkres-1.1.4-pyh9f0ad1d_0.tar.bz2#2ba8498c1018c1e9c61eb99b973dfe19 https://conda.anaconda.org/conda-forge/linux-64/mypy_extensions-0.4.3-py310hff52083_6.tar.bz2#58a812a2f042e6f8c027736c6464d2dd https://conda.anaconda.org/conda-forge/noarch/networkx-2.8.8-pyhd8ed1ab_0.tar.bz2#bb45ff9deddb045331fd039949f39650 -https://conda.anaconda.org/conda-forge/linux-64/numpy-1.23.5-py310h53a5b5f_0.conda#3b114b1559def8bad228fec544ac1812 +https://conda.anaconda.org/conda-forge/linux-64/numpy-1.24.0-py310h08bbf29_0.conda#d14a8960a052bd82cca0542a9ed15784 https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.0-h7d73246_1.tar.bz2#a11b4df9271a8d7917686725aa04c8f2 https://conda.anaconda.org/conda-forge/noarch/packaging-22.0-pyhd8ed1ab_0.conda#0e8e1bd93998978fc3125522266d12db https://conda.anaconda.org/conda-forge/noarch/pathspec-0.10.3-pyhd8ed1ab_0.conda#0f7d2186dd12ef3277e70fd85f9ff6a7 @@ -183,10 +184,11 @@ https://conda.anaconda.org/conda-forge/noarch/pyparsing-3.0.9-pyhd8ed1ab_0.tar.b https://conda.anaconda.org/conda-forge/noarch/pyshp-2.3.1-pyhd8ed1ab_0.tar.bz2#92a889dc236a5197612bc85bee6d7174 https://conda.anaconda.org/conda-forge/noarch/pysocks-1.7.1-pyha2e5f31_6.tar.bz2#2a7de29fb590ca14b5243c4c812c8025 https://conda.anaconda.org/conda-forge/linux-64/python-xxhash-3.0.0-py310h5764c6d_2.tar.bz2#cce72b32ccc346ed166fc85071854a86 -https://conda.anaconda.org/conda-forge/noarch/pytz-2022.6-pyhd8ed1ab_0.tar.bz2#b1f26ad83328e486910ef7f6e81dc061 +https://conda.anaconda.org/conda-forge/noarch/pytz-2022.7-pyhd8ed1ab_0.conda#c8d7e34ca76d6ecc03b84bedfd99d689 https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0-py310h5764c6d_5.tar.bz2#9e68d2ff6d98737c855b65f48dd3c597 +https://conda.anaconda.org/conda-forge/linux-64/rtree-1.0.1-py310hbdcdc62_1.tar.bz2#49ad4035b71bbf7289ac1523f8023ebe https://conda.anaconda.org/conda-forge/noarch/setoptconf-tmp-0.3.1-pyhd8ed1ab_0.tar.bz2#af3e36d4effb85b9b9f93cd1db0963df -https://conda.anaconda.org/conda-forge/noarch/setuptools-65.5.1-pyhd8ed1ab_0.tar.bz2#cfb8dc4d9d285ca5fb1177b9dd450e33 +https://conda.anaconda.org/conda-forge/noarch/setuptools-65.6.3-pyhd8ed1ab_0.conda#9600fc9524d3f821e6a6d58c52f5bf5a https://conda.anaconda.org/conda-forge/noarch/six-1.16.0-pyh6c4a22f_0.tar.bz2#e5f25f8dbc060e9a8d912e432202afc2 https://conda.anaconda.org/conda-forge/noarch/snowballstemmer-2.2.0-pyhd8ed1ab_0.tar.bz2#4d22a9315e78c6827f806065957d566e https://conda.anaconda.org/conda-forge/noarch/sortedcontainers-2.4.0-pyhd8ed1ab_0.tar.bz2#6d6552722448103793743dabfbda532d @@ -199,6 +201,7 @@ https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-serializinghtml-1.1. https://conda.anaconda.org/conda-forge/noarch/sqlparse-0.4.3-pyhd8ed1ab_0.tar.bz2#d008e81907166f6dacea7e34292e79c6 https://conda.anaconda.org/conda-forge/noarch/tblib-1.7.0-pyhd8ed1ab_0.tar.bz2#3d4afc31302aa7be471feb6be048ed76 https://conda.anaconda.org/conda-forge/noarch/termcolor-2.1.1-pyhd8ed1ab_0.conda#974d8b697fe5c069bfeb113e633314e5 +https://conda.anaconda.org/conda-forge/noarch/threadpoolctl-3.1.0-pyh8a188c0_0.tar.bz2#a2995ee828f65687ac5b1e71a2ab1e0c https://conda.anaconda.org/conda-forge/noarch/toml-0.10.2-pyhd8ed1ab_0.tar.bz2#f832c45a477c78bebd107098db465095 https://conda.anaconda.org/conda-forge/noarch/tomli-2.0.1-pyhd8ed1ab_0.tar.bz2#5844808ffab9ebdb694585b50ba02a96 https://conda.anaconda.org/conda-forge/noarch/toolz-0.12.0-pyhd8ed1ab_0.tar.bz2#92facfec94bc02d6ccf42e7173831a36 @@ -208,6 +211,7 @@ https://conda.anaconda.org/conda-forge/noarch/types-pyyaml-6.0.12.2-pyhd8ed1ab_0 https://conda.anaconda.org/conda-forge/noarch/types-urllib3-1.26.25.4-pyhd8ed1ab_0.tar.bz2#ca86be78a819654cf93d81649600437f https://conda.anaconda.org/conda-forge/noarch/typing-3.10.0.0-pyhd8ed1ab_0.tar.bz2#e6573ac68718f17b9d4f5c8eda3190f2 https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.4.0-pyha770c72_0.tar.bz2#2d93b130d148d7fc77e583677792fc6a +https://conda.anaconda.org/conda-forge/linux-64/ujson-5.5.0-py310hd8f1fbe_1.tar.bz2#9394053c95c67ecb37c7a755f473b0f5 https://conda.anaconda.org/conda-forge/linux-64/unicodedata2-15.0.0-py310h5764c6d_0.tar.bz2#e972c5a1f472561cf4a91962cb01f4b4 https://conda.anaconda.org/conda-forge/noarch/untokenize-0.1.1-py_0.tar.bz2#1447ead40f2a01733a9c8dfc32988375 https://conda.anaconda.org/conda-forge/noarch/webencodings-0.5.1-py_1.tar.bz2#3563be4c5611a44210d9ba0c16113136 @@ -216,35 +220,37 @@ https://conda.anaconda.org/conda-forge/noarch/wheel-0.38.4-pyhd8ed1ab_0.tar.bz2# https://conda.anaconda.org/conda-forge/linux-64/wrapt-1.14.1-py310h5764c6d_1.tar.bz2#49c8664940fe2f6124193ba3cc32ca20 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.4-h7f98852_1.tar.bz2#536cc5db4d0a3ba0630541aec064b5e4 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.10-h7f98852_1003.tar.bz2#f59c1242cc1dd93e72c2ee2b360979eb +https://conda.anaconda.org/conda-forge/noarch/xyzservices-2022.9.0-pyhd8ed1ab_0.tar.bz2#0c0e2e24aa2e9ee3dd99c611b1098047 https://conda.anaconda.org/conda-forge/noarch/yapf-0.32.0-pyhd8ed1ab_0.tar.bz2#177cba0b4bdfacad5c5fbb0ed31504c4 https://conda.anaconda.org/conda-forge/noarch/zipp-3.11.0-pyhd8ed1ab_0.conda#09b5b885341697137879a4f039a9e5a1 -https://conda.anaconda.org/conda-forge/noarch/asgiref-3.5.2-pyhd8ed1ab_0.tar.bz2#cea70257c4c605a4417632ba6cb3a20a +https://conda.anaconda.org/conda-forge/noarch/asgiref-3.6.0-pyhd8ed1ab_0.conda#4437fc8d76835df622027fe9ae7da997 https://conda.anaconda.org/conda-forge/linux-64/astroid-2.12.13-py310hff52083_0.conda#02ef25c493a7c0854c20443cf5a92c8e https://conda.anaconda.org/conda-forge/noarch/babel-2.11.0-pyhd8ed1ab_0.tar.bz2#2ea70fde8d581ba9425a761609eed6ba https://conda.anaconda.org/conda-forge/linux-64/cairo-1.16.0-ha61ee94_1014.tar.bz2#d1a88f3ed5b52e1024b80d4bcd26a7a0 https://conda.anaconda.org/conda-forge/noarch/cattrs-22.2.0-pyhd8ed1ab_0.tar.bz2#5dacf4d924ae284579288e378b1f5943 -https://conda.anaconda.org/conda-forge/linux-64/cffi-1.15.1-py310h255011f_2.tar.bz2#6bb8063dd08f9724c18744b0e040cfe2 +https://conda.anaconda.org/conda-forge/linux-64/cffi-1.15.1-py310h255011f_3.conda#800596144bb613cd7ac58b80900ce835 https://conda.anaconda.org/conda-forge/linux-64/cfitsio-4.2.0-hd9d235c_0.conda#8c57a9adbafd87f5eff842abde599cb4 https://conda.anaconda.org/conda-forge/linux-64/cftime-1.6.2-py310hde88566_1.tar.bz2#94ce7a76b0c912279f6958e0b6b21d2b https://conda.anaconda.org/conda-forge/noarch/click-plugins-1.1.1-py_0.tar.bz2#4fd2c6b53934bd7d96d1f3fdaf99b79f https://conda.anaconda.org/conda-forge/noarch/cligj-0.7.2-pyhd8ed1ab_1.tar.bz2#a29b7c141d6b2de4bb67788a5f107734 https://conda.anaconda.org/conda-forge/linux-64/contourpy-1.0.6-py310hbf28c38_0.tar.bz2#c5b1699e390d30b680dd93a2b251062b -https://conda.anaconda.org/conda-forge/linux-64/coverage-6.5.0-py310h5764c6d_1.tar.bz2#feb57771b1d5179cd2fb1a06fed17326 -https://conda.anaconda.org/conda-forge/linux-64/curl-7.86.0-h2283fc2_1.tar.bz2#9d4149760567cb232691cce2d8ccc21f -https://conda.anaconda.org/conda-forge/linux-64/cxx-compiler-1.5.1-h924138e_0.tar.bz2#45830a0730fee6c23551878c5f05a219 +https://conda.anaconda.org/conda-forge/linux-64/coverage-7.0.0-py310h1fa729e_0.conda#e83191fb54f9e90e5b95b6265972d0bf +https://conda.anaconda.org/conda-forge/linux-64/curl-7.86.0-hdc1c0ab_2.conda#3c96ba0c180e389621c4fd62d06c151b +https://conda.anaconda.org/conda-forge/linux-64/cxx-compiler-1.5.2-hf52228f_0.conda#6b3b19e359824b97df7145c8c878c8be https://conda.anaconda.org/conda-forge/linux-64/cytoolz-0.12.0-py310h5764c6d_1.tar.bz2#fd18cd597d23b2b5ddde23bd5b7aec32 -https://conda.anaconda.org/conda-forge/noarch/docformatter-1.5.0-pyhd8ed1ab_0.tar.bz2#0f65c1ff31bdb3b4fc0efbc6f9a145a9 +https://conda.anaconda.org/conda-forge/noarch/docformatter-1.5.1-pyhd8ed1ab_0.conda#7a8356601d0a66eb83fb5178bde841bf https://conda.anaconda.org/conda-forge/noarch/fire-0.4.0-pyh44b312d_0.tar.bz2#0b83cbdfb7b4067ebb051292d360d7a6 https://conda.anaconda.org/conda-forge/linux-64/fonttools-4.38.0-py310h5764c6d_1.tar.bz2#12ebe92a8a578bc903bd844744f4d040 -https://conda.anaconda.org/conda-forge/linux-64/fortran-compiler-1.5.1-h2a4ca65_0.tar.bz2#4851e61ed9676cee9e50136f2a373302 +https://conda.anaconda.org/conda-forge/linux-64/fortran-compiler-1.5.2-hdb1a99f_0.conda#265323e1bd53709aeb739c9b1794b398 https://conda.anaconda.org/conda-forge/noarch/geopy-2.3.0-pyhd8ed1ab_0.tar.bz2#529faeecd6eee3a3b782566ddf05ce92 https://conda.anaconda.org/conda-forge/linux-64/hdf5-1.12.2-mpi_mpich_h5d83325_0.tar.bz2#6b5c2d276f306df759cfbdb0f41c4db9 https://conda.anaconda.org/conda-forge/noarch/html5lib-1.1-pyh9f0ad1d_0.tar.bz2#b2355343d6315c892543200231d7154a -https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-5.1.0-pyha770c72_0.conda#46a62e35b9ae515cf0e49afc7fe0e7ef +https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-5.2.0-pyha770c72_0.conda#2e8eec7a08bcf85547a3778225f00634 https://conda.anaconda.org/conda-forge/noarch/importlib_resources-5.10.1-pyhd8ed1ab_0.conda#db5d88c84c769798bf4b2f6776446b32 https://conda.anaconda.org/conda-forge/noarch/isodate-0.6.1-pyhd8ed1ab_0.tar.bz2#4a62c93c1b5c0b920508ae3fd285eaf5 -https://conda.anaconda.org/conda-forge/noarch/isort-5.10.1-pyhd8ed1ab_0.tar.bz2#83df9ffefe1cc01d9e60405bf871bfdb +https://conda.anaconda.org/conda-forge/noarch/isort-5.11.3-pyhd8ed1ab_0.conda#df79686d95d8dd88eda0422711e5e7e3 https://conda.anaconda.org/conda-forge/noarch/jinja2-3.1.2-pyhd8ed1ab_1.tar.bz2#c8490ed5c70966d232fdd389d0dbed37 +https://conda.anaconda.org/conda-forge/noarch/joblib-1.2.0-pyhd8ed1ab_0.tar.bz2#7583652522d71ad78ba536bba06940eb https://conda.anaconda.org/conda-forge/noarch/latexcodec-2.0.1-pyh9f0ad1d_0.tar.bz2#8d67904973263afd2985ba56aa2d6bb4 https://conda.anaconda.org/conda-forge/linux-64/libgd-2.3.3-h18fbbfe_3.tar.bz2#ea9758cf553476ddf75c789fdd239dc5 https://conda.anaconda.org/conda-forge/noarch/logilab-common-1.7.3-py_0.tar.bz2#6eafcdf39a7eb90b6d951cfff59e8d3b @@ -255,7 +261,7 @@ https://conda.anaconda.org/conda-forge/noarch/nodeenv-1.7.0-pyhd8ed1ab_0.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/partd-1.3.0-pyhd8ed1ab_0.tar.bz2#af8c82d121e63082926062d61d9abb54 https://conda.anaconda.org/conda-forge/linux-64/pillow-9.2.0-py310h454ad03_3.tar.bz2#eb354ff791f505b1d6f13f776359d88e https://conda.anaconda.org/conda-forge/noarch/pip-22.3.1-pyhd8ed1ab_0.tar.bz2#da66f2851b9836d3a7c5190082a45f7d -https://conda.anaconda.org/conda-forge/linux-64/postgresql-15.1-ha105346_1.conda#81cfa38baa2a8741f0566f8815fef4e3 +https://conda.anaconda.org/conda-forge/linux-64/postgresql-15.1-h3248436_2.conda#4d2f2c04f8bdb6a9cb0d6e8d90b1f907 https://conda.anaconda.org/conda-forge/linux-64/proj-9.1.0-h93bde94_0.tar.bz2#255c7204dda39747c3ba380d28b026d7 https://conda.anaconda.org/conda-forge/noarch/pydocstyle-6.1.1-pyhd8ed1ab_0.tar.bz2#e417954a987d382b3142886726ab3aad https://conda.anaconda.org/conda-forge/noarch/pygments-2.13.0-pyhd8ed1ab_0.tar.bz2#9f478e8eedd301008b5f395bad0caaed @@ -263,7 +269,7 @@ https://conda.anaconda.org/conda-forge/noarch/pytest-7.2.0-pyhd8ed1ab_2.tar.bz2# https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.8.2-pyhd8ed1ab_0.tar.bz2#dd999d1cc9f79e67dbb855c8924c7984 https://conda.anaconda.org/conda-forge/linux-64/python-stratify-0.2.post0-py310hde88566_3.tar.bz2#0b686f306a76fba9a61e7019f854321f https://conda.anaconda.org/conda-forge/linux-64/scipy-1.9.3-py310hdfbd76f_2.tar.bz2#0582a434d03f6b06d5defbb142c96f4f -https://conda.anaconda.org/conda-forge/linux-64/shapely-1.8.5-py310h5b266fc_2.tar.bz2#c4a3707d6a630facb6cf7ed8e0d37326 +https://conda.anaconda.org/conda-forge/linux-64/shapely-2.0.0-py310h8b84c32_0.conda#823009371d9b961c83cdb9aa80e1e6e7 https://conda.anaconda.org/conda-forge/noarch/tomlkit-0.11.6-pyha770c72_0.tar.bz2#471bf9e605820b59988e830382b8d654 https://conda.anaconda.org/conda-forge/noarch/types-requests-2.28.11.5-pyhd8ed1ab_0.tar.bz2#bcd389a385a8b6bd866238f1a16d13cf https://conda.anaconda.org/conda-forge/noarch/url-normalize-1.4.3-pyhd8ed1ab_0.tar.bz2#7c4076e494f0efe76705154ac9302ba6 @@ -273,25 +279,25 @@ https://conda.anaconda.org/conda-forge/noarch/yamale-4.0.4-pyh6c4a22f_0.tar.bz2# https://conda.anaconda.org/conda-forge/noarch/yamllint-1.28.0-pyhd8ed1ab_0.tar.bz2#f28e487a994b504cc5fdb6cb5e6a0bf9 https://conda.anaconda.org/conda-forge/noarch/zict-2.2.0-pyhd8ed1ab_0.tar.bz2#cd563d01df94e51f968645dbf3b310b0 https://conda.anaconda.org/conda-forge/noarch/bokeh-2.4.3-pyhd8ed1ab_3.tar.bz2#e4c6e6d99add99cede5328d811cacb21 +https://conda.anaconda.org/conda-forge/noarch/branca-0.6.0-pyhd8ed1ab_0.tar.bz2#f4cc65697763ef8c2f7555f1ec355a6b https://conda.anaconda.org/conda-forge/linux-64/brotlipy-0.7.0-py310h5764c6d_1005.tar.bz2#87669c3468dff637bbd0363bc0f895cf https://conda.anaconda.org/conda-forge/linux-64/cf-units-3.1.1-py310hde88566_2.tar.bz2#7433944046deda7775c5b1f7e0b6fe18 -https://conda.anaconda.org/conda-forge/linux-64/compilers-1.5.1-ha770c72_0.tar.bz2#8a0ff3c519396696bbe9ca786606372f +https://conda.anaconda.org/conda-forge/linux-64/compilers-1.5.2-ha770c72_0.conda#f95226244ee1c487cf53272f971323f4 https://conda.anaconda.org/conda-forge/linux-64/cryptography-38.0.4-py310h600f1e7_0.conda#f999dcc21fe27ad97a8afcfa590daa14 -https://conda.anaconda.org/conda-forge/noarch/dask-core-2022.12.0-pyhd8ed1ab_0.conda#3a0f020d07998e1ae711df071f97fc19 +https://conda.anaconda.org/conda-forge/noarch/dask-core-2022.12.1-pyhd8ed1ab_0.conda#f12878f9839c72f3d51af02fb10da43d https://conda.anaconda.org/conda-forge/noarch/django-4.1.4-pyhd8ed1ab_0.conda#14f8b26ca366987ec53655f6a7a6cba0 https://conda.anaconda.org/conda-forge/noarch/flake8-4.0.1-pyhd8ed1ab_2.tar.bz2#a824bd55ce47e9c637427f730c651231 https://conda.anaconda.org/conda-forge/linux-64/geotiff-1.7.1-ha76d385_4.tar.bz2#6a613710a0f19aba3a5dfe83bf1c1c0f -https://conda.anaconda.org/conda-forge/linux-64/harfbuzz-5.3.0-h418a68e_0.tar.bz2#888056bd4b12e110b10d4d1f29161c5e +https://conda.anaconda.org/conda-forge/linux-64/harfbuzz-6.0.0-h8e241bc_0.conda#448fe40d2fed88ccf4d9ded37cbb2b38 https://conda.anaconda.org/conda-forge/linux-64/kealib-1.5.0-ha7026e8_0.conda#c948b920f45fd81a2dde8b1ab514cc84 -https://conda.anaconda.org/conda-forge/linux-64/libdap4-3.20.6-hd7c4107_2.tar.bz2#c265ae57e3acdc891f3e2b93cf6784f5 https://conda.anaconda.org/conda-forge/linux-64/libnetcdf-4.8.1-mpi_mpich_hcd871d9_6.tar.bz2#6cdc429ed22edb566ac4308f3da6916d https://conda.anaconda.org/conda-forge/linux-64/libspatialite-5.0.1-h7c8129e_22.tar.bz2#23abed7562ad969493b89ad0e5f5c521 https://conda.anaconda.org/conda-forge/linux-64/matplotlib-base-3.6.2-py310h8d5ebf3_0.tar.bz2#da51ddb20c0f99d672eb756c3abf27e7 https://conda.anaconda.org/conda-forge/linux-64/pandas-1.5.2-py310h769672d_0.conda#bc363997d22f3b058fb17f1e89d4c96f -https://conda.anaconda.org/conda-forge/linux-64/poppler-22.11.0-h92391eb_0.tar.bz2#833285a4a9d03a93f3e3e5f31b319de8 +https://conda.anaconda.org/conda-forge/linux-64/poppler-22.12.0-h92391eb_0.conda#7ad6d858f5615f9b0e9e4bd60395ea75 https://conda.anaconda.org/conda-forge/noarch/pybtex-0.24.0-pyhd8ed1ab_2.tar.bz2#2099b86a7399c44c0c61cdb6de6915ba -https://conda.anaconda.org/conda-forge/noarch/pylint-2.15.8-pyhd8ed1ab_0.conda#26a62404bbfc93452c4d22aa2cda6f08 -https://conda.anaconda.org/conda-forge/linux-64/pyproj-3.4.0-py310hb1338dc_2.tar.bz2#e1648c222911ad7559d62831e4bc447c +https://conda.anaconda.org/conda-forge/noarch/pylint-2.15.9-pyhd8ed1ab_0.conda#5bfc68e1b8f0883f793ae0acedc88770 +https://conda.anaconda.org/conda-forge/linux-64/pyproj-3.4.1-py310hfc24d34_0.conda#c126f81b5cea6b2d4a64d0744249a26f https://conda.anaconda.org/conda-forge/noarch/pytest-cov-4.0.0-pyhd8ed1ab_0.tar.bz2#c9e3f8bfdb9bfc34aa1836a6ed4b25d7 https://conda.anaconda.org/conda-forge/noarch/pytest-env-0.6.2-py_0.tar.bz2#2fe6beb5d2d87cf252a2575043919696 https://conda.anaconda.org/conda-forge/noarch/pytest-metadata-2.0.4-pyhd8ed1ab_0.tar.bz2#7ac02a65917993d38ca1bfd7b87208e4 @@ -300,21 +306,26 @@ https://conda.anaconda.org/conda-forge/noarch/pytest-mypy-0.8.0-pyhd8ed1ab_0.tar https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.1.0-pyhd8ed1ab_0.conda#e82f8fb903d7c4a59c77954759c341f9 https://conda.anaconda.org/conda-forge/noarch/rdflib-6.2.0-pyhd8ed1ab_0.tar.bz2#b9acd5fbaf467f7447746b1ecac50e83 https://conda.anaconda.org/conda-forge/linux-64/requirements-detector-0.7-py310hff52083_3.tar.bz2#0cd9f8972da2c8ad624676d47643805f -https://conda.anaconda.org/conda-forge/linux-64/tiledb-2.11.3-h3f4058f_1.tar.bz2#4ae02ca7d1da6e2b4e3005108df36812 +https://conda.anaconda.org/conda-forge/linux-64/scikit-learn-1.2.0-py310h209a8ca_0.conda#5a08a1f004445ee9bf58261feb16cc18 +https://conda.anaconda.org/conda-forge/linux-64/tiledb-2.13.0-h3f4058f_0.conda#b7009881353045d2bdae62ec0dd04bc0 https://conda.anaconda.org/conda-forge/linux-64/ukkonen-1.0.1-py310hbf28c38_3.tar.bz2#703ff1ac7d1b27fb5944b8052b5d1edb -https://conda.anaconda.org/conda-forge/linux-64/cartopy-0.21.0-py310h83f2385_3.tar.bz2#4ec35f7eebe4221c1c00fdd6540db4dc +https://conda.anaconda.org/conda-forge/linux-64/cartopy-0.21.1-py310hcb7e713_0.conda#bd14eaad9bbf54b78e48ecb8b644fcf6 https://conda.anaconda.org/conda-forge/noarch/flake8-polyfill-1.0.2-py_0.tar.bz2#a53db35e3d07f0af2eccd59c2a00bffe -https://conda.anaconda.org/conda-forge/noarch/identify-2.5.9-pyhd8ed1ab_0.conda#e7ecbbb61a37daed2a13de43d35d5282 -https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.5.3-h867e046_8.conda#4d3739bedfe7a58cb9095ec6b24f7544 +https://conda.anaconda.org/conda-forge/noarch/geopandas-base-0.12.2-pyha770c72_0.conda#cf04d066b97dfe698f0d01ebf4af6163 +https://conda.anaconda.org/conda-forge/noarch/identify-2.5.11-pyhd8ed1ab_0.conda#20d9fee9fd73a480ad7f264c07b4fcf8 +https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.6.1-hf2b5f72_1.conda#fc4c5716fac9e7f242223dca042e1045 +https://conda.anaconda.org/conda-forge/noarch/mapclassify-2.4.3-pyhd8ed1ab_0.tar.bz2#908bbfb54da154042c5cbda77b37a3d1 https://conda.anaconda.org/conda-forge/noarch/nc-time-axis-1.4.1-pyhd8ed1ab_0.tar.bz2#281b58948bf60a2582de9e548bcc5369 https://conda.anaconda.org/conda-forge/linux-64/netcdf-fortran-4.6.0-mpi_mpich_hd09bd1e_1.tar.bz2#0b69750bb937cab0db14f6bcef6fd787 https://conda.anaconda.org/conda-forge/linux-64/netcdf4-1.6.0-nompi_py310h0a86a1f_103.conda#7f69695b684f2595d9ba1ce26d693b7d -https://conda.anaconda.org/conda-forge/linux-64/pango-1.50.12-h382ae3d_0.conda#627bea5af786dbd8013ef26127d8115a +https://conda.anaconda.org/conda-forge/linux-64/pango-1.50.12-hd33c08f_1.conda#667dc93c913f0156e1237032e3a22046 +https://conda.anaconda.org/conda-forge/linux-64/parallelio-2.5.9-mpi_mpich_h50e6f33_101.conda#87fac13c80750b8be35b0a32bb965bbe https://conda.anaconda.org/conda-forge/noarch/pylint-plugin-utils-0.7-pyhd8ed1ab_0.tar.bz2#1657976383aee04dbb3ae3bdf654bb58 https://conda.anaconda.org/conda-forge/noarch/pyopenssl-22.1.0-pyhd8ed1ab_0.tar.bz2#fbfa0a180d48c800f922a10a114a8632 https://conda.anaconda.org/conda-forge/noarch/pytest-html-3.2.0-pyhd8ed1ab_1.tar.bz2#d5c7a941dfbceaab4b172a56d7918eb0 -https://conda.anaconda.org/conda-forge/linux-64/esmf-8.2.0-mpi_mpich_h5a1934d_102.tar.bz2#bb8bdfa5e3e9e3f6ec861f05cd2ad441 -https://conda.anaconda.org/conda-forge/linux-64/gdal-3.5.3-py310hc1b7723_8.conda#bcdf3bbfc55928afc900a92a70f745d5 +https://conda.anaconda.org/conda-forge/noarch/xarray-2022.12.0-pyhd8ed1ab_0.conda#fd0b8bb7c8df27e09fdee6c893f2ba9d +https://conda.anaconda.org/conda-forge/linux-64/esmf-8.3.1-mpi_mpich_h5a1934d_101.tar.bz2#ac4bfd5bdb0a5b4b99ee383fd0c8995c +https://conda.anaconda.org/conda-forge/linux-64/gdal-3.6.1-py310hc1b7723_1.conda#27f46e0d58cddd2b36520c7453a9ddca https://conda.anaconda.org/conda-forge/linux-64/gtk2-2.24.33-h90689f9_2.tar.bz2#957a0255ab58aaf394a91725d73ab422 https://conda.anaconda.org/conda-forge/noarch/iris-3.4.0-pyhd8ed1ab_0.conda#6a6c17cfff8564de41d2a952c97bc2e8 https://conda.anaconda.org/conda-forge/linux-64/librsvg-2.54.4-h7abd40a_0.tar.bz2#921e53675ed5ea352f022b79abab076a @@ -325,18 +336,22 @@ https://conda.anaconda.org/conda-forge/noarch/pylint-celery-0.3-py_1.tar.bz2#e29 https://conda.anaconda.org/conda-forge/noarch/pylint-django-2.5.3-pyhd8ed1ab_0.tar.bz2#00d8853fb1f87195722ea6a582cc9b56 https://conda.anaconda.org/conda-forge/noarch/pylint-flask-0.6-py_0.tar.bz2#5a9afd3d0a61b08d59eed70fab859c1b https://conda.anaconda.org/conda-forge/noarch/urllib3-1.26.13-pyhd8ed1ab_0.conda#3078ef2359efd6ecadbc7e085c5e0592 -https://conda.anaconda.org/conda-forge/noarch/distributed-2022.12.0-pyhd8ed1ab_0.conda#2c934253dc579d314ac2a1f43b4be15c -https://conda.anaconda.org/conda-forge/linux-64/esmpy-8.2.0-mpi_mpich_py310hd9c82d4_101.tar.bz2#0333d51ee594be40f50b157ac6f27b5a -https://conda.anaconda.org/conda-forge/linux-64/fiona-1.8.22-py310h60a68a4_2.tar.bz2#fc3bb0d986e4ede4ddedb62ae2d99c55 +https://conda.anaconda.org/conda-forge/noarch/distributed-2022.12.1-pyhd8ed1ab_0.conda#63cf20ed7b5205cf4c31aadee4c7fd2e +https://conda.anaconda.org/conda-forge/linux-64/esmpy-8.3.1-mpi_mpich_py310h515c5ea_100.conda#ad531847b7cea3df5c63e0b7f2388e7c +https://conda.anaconda.org/conda-forge/linux-64/fiona-1.8.22-py310ha325b7b_5.conda#4dbdf48d4712e8906595291f38423eff https://conda.anaconda.org/conda-forge/linux-64/graphviz-6.0.2-h99bc08f_0.conda#8f247587d1520a2bbc6f79a821b74c07 https://conda.anaconda.org/conda-forge/noarch/prospector-1.7.7-pyhd8ed1ab_0.tar.bz2#01010f8ea38d650158703a581e51b979 https://conda.anaconda.org/conda-forge/noarch/requests-2.28.1-pyhd8ed1ab_1.tar.bz2#089382ee0e2dc2eae33a04cc3c2bddb0 -https://conda.anaconda.org/conda-forge/noarch/dask-2022.12.0-pyhd8ed1ab_0.conda#db0f9c381d65054afdb6ae1e547d8b2e +https://conda.anaconda.org/conda-forge/noarch/dask-2022.12.1-pyhd8ed1ab_0.conda#5861b97a50edcd9c1332d21f2995eca1 +https://conda.anaconda.org/conda-forge/noarch/folium-0.14.0-pyhd8ed1ab_0.conda#48c8bb19df0d0268f1a9d30ffc56c5b0 +https://conda.anaconda.org/conda-forge/noarch/pooch-1.6.0-pyhd8ed1ab_0.tar.bz2#6429e1d1091c51f626b5dcfdd38bf429 https://conda.anaconda.org/conda-forge/linux-64/pydot-1.4.2-py310hff52083_3.tar.bz2#45231e3f8fa29b6cea52e2cfe9b47a22 -https://conda.anaconda.org/conda-forge/noarch/requests-cache-0.9.6-pyhd8ed1ab_0.tar.bz2#276b447a966e8f3d3daecb3c5a156330 +https://conda.anaconda.org/conda-forge/noarch/requests-cache-0.9.7-pyhd8ed1ab_0.conda#399c98b39eb6bdc4b1d40b7611a6d384 https://conda.anaconda.org/conda-forge/noarch/sphinx-5.3.0-pyhd8ed1ab_0.tar.bz2#f9e1fcfe235d655900bfeb6aee426472 https://conda.anaconda.org/conda-forge/noarch/autodocsumm-0.2.6-pyhd8ed1ab_0.tar.bz2#4409dd7e06a62c3b2aa9e96782c49c6d https://conda.anaconda.org/conda-forge/noarch/esgf-pyclient-0.3.1-pyh1a96a4e_2.tar.bz2#64068564a9c2932bf30e9b4ec567927d +https://conda.anaconda.org/conda-forge/noarch/geopandas-0.12.2-pyhd8ed1ab_0.conda#ee3b330f13297f5839d46e1ca3e57d56 https://conda.anaconda.org/conda-forge/noarch/iris-esmf-regrid-0.5.0-pyhd8ed1ab_0.tar.bz2#8c698d13c8d6b5497e0e0df2a0459be0 https://conda.anaconda.org/conda-forge/noarch/prov-2.0.0-pyhd3deb0d_0.tar.bz2#aa9b3ad140f6c0668c646f32e20ccf82 -https://conda.anaconda.org/conda-forge/noarch/sphinx_rtd_theme-1.1.1-pyhd8ed1ab_0.tar.bz2#2471715dba64124de7a42093e550ec6c +https://conda.anaconda.org/conda-forge/noarch/sphinx_rtd_theme-1.1.1-pyha770c72_1.conda#a8d25c9077767faf05148421a04874f6 +https://conda.anaconda.org/conda-forge/noarch/py-cordex-0.4.1-pyhd8ed1ab_1.tar.bz2#f50eef5248660f6e2e02aea4ed2977f5 From 31328ede42dda9438df1c3f8cc2a06651a881ed5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 31 Dec 2022 10:35:00 +0000 Subject: [PATCH 8/8] [Condalock] Update Linux condalock file (#1882) Co-authored-by: valeriupredoi --- conda-linux-64.lock | 74 +++++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 36 deletions(-) diff --git a/conda-linux-64.lock b/conda-linux-64.lock index 7bf2f8ff4c..59e576d73f 100644 --- a/conda-linux-64.lock +++ b/conda-linux-64.lock @@ -42,6 +42,7 @@ https://conda.anaconda.org/conda-forge/linux-64/jpeg-9e-h166bdaf_2.tar.bz2#ee8b8 https://conda.anaconda.org/conda-forge/linux-64/json-c-0.16-hc379101_0.tar.bz2#0e2bca6857cb73acec30387fef7c3142 https://conda.anaconda.org/conda-forge/linux-64/keyutils-1.6.1-h166bdaf_0.tar.bz2#30186d27e2c9fa62b45fb1476b7200e3 https://conda.anaconda.org/conda-forge/linux-64/lerc-4.0.0-h27087fc_0.tar.bz2#76bbff344f0134279f225174e9064c8f +https://conda.anaconda.org/conda-forge/linux-64/libaec-1.0.6-h9c3ff4c_0.tar.bz2#c77f5e4e418fa47d699d6afa54c5d444 https://conda.anaconda.org/conda-forge/linux-64/libbrotlicommon-1.0.9-h166bdaf_8.tar.bz2#9194c9bf9428035a05352d031462eae4 https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.14-h166bdaf_0.tar.bz2#fc84a0446e4e4fb882e78d786cfb9734 https://conda.anaconda.org/conda-forge/linux-64/libev-4.33-h516909a_1.tar.bz2#6f8720dff19e17ce5d48cfe7f3d2f0a3 @@ -51,7 +52,7 @@ https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.0-h7f98852_0.tar.bz2# https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.21-pthreads_h78a6416_3.tar.bz2#8c5963a49b6035c40646a763293fbb35 https://conda.anaconda.org/conda-forge/linux-64/libsanitizer-11.3.0-h239ccf8_19.tar.bz2#d17fd55aed84ab6592c5419b6600501c https://conda.anaconda.org/conda-forge/linux-64/libspatialindex-1.9.3-h9c3ff4c_4.tar.bz2#d87fbe9c0ff589e802ff13872980bfd9 -https://conda.anaconda.org/conda-forge/linux-64/libtool-2.4.6-h9c3ff4c_1008.tar.bz2#16e143a1ed4b4fd169536373957f6fee +https://conda.anaconda.org/conda-forge/linux-64/libtool-2.4.7-h27087fc_0.conda#f204c8ba400ec475452737094fb81d52 https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.32.1-h7f98852_1000.tar.bz2#772d69f030955d9646d3d0eaf21d859d https://conda.anaconda.org/conda-forge/linux-64/libwebp-base-1.2.4-h166bdaf_0.tar.bz2#ac2ccf7323d21f2994e4d1f5da664f37 https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.2.13-h166bdaf_4.tar.bz2#f3f9de449d32ca9b9c66a22863c96f41 @@ -107,7 +108,7 @@ https://conda.anaconda.org/conda-forge/linux-64/krb5-1.20.1-h81ceb04_0.conda#89a https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.9.0-16_linux64_openblas.tar.bz2#20bae26d0a1db73f758fc3754cab4719 https://conda.anaconda.org/conda-forge/linux-64/libglib-2.74.1-h606061b_1.tar.bz2#ed5349aa96776e00b34eccecf4a948fe https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-16_linux64_openblas.tar.bz2#955d993f41f9354bf753d29864ea20ad -https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.4.0-h82bc61c_5.conda#e712a63a21f9db647982971dc121cdcf +https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.5.0-h82bc61c_0.conda#a01611c54334d783847879ee40109657 https://conda.anaconda.org/conda-forge/linux-64/libxslt-1.1.37-h873f0b0_0.tar.bz2#ed0d77d947ddeb974892de8df7224d12 https://conda.anaconda.org/conda-forge/linux-64/nss-3.82-he02c5a1_0.conda#f8d7f11d19e4cb2207eab159fd4c0152 https://conda.anaconda.org/conda-forge/linux-64/python-3.10.8-h4a9ceb5_0_cpython.conda#be2a6d78752c2ab85f360ce37d2c64e2 @@ -117,7 +118,7 @@ https://conda.anaconda.org/conda-forge/noarch/alabaster-0.7.12-py_0.tar.bz2#2489 https://conda.anaconda.org/conda-forge/linux-64/antlr-python-runtime-4.7.2-py310hff52083_1003.tar.bz2#8324f8fff866055d4b32eb25e091fe31 https://conda.anaconda.org/conda-forge/noarch/appdirs-1.4.4-pyh9f0ad1d_0.tar.bz2#5f095bc6454094e96f146491fd03633b https://conda.anaconda.org/conda-forge/linux-64/atk-1.0-2.38.0-hd4edc92_1.tar.bz2#6c72ec3e660a51736913ef6ea68c454b -https://conda.anaconda.org/conda-forge/noarch/attrs-22.1.0-pyh71513ae_1.tar.bz2#6d3ccbc56256204925bfa8378722792f +https://conda.anaconda.org/conda-forge/noarch/attrs-22.2.0-pyh71513ae_0.conda#8b76db7818a4e401ed4486c4c1635cd9 https://conda.anaconda.org/conda-forge/linux-64/backports.zoneinfo-0.2.1-py310hff52083_7.tar.bz2#02d7c823f5e6fd4bbe5562c612465aed https://conda.anaconda.org/conda-forge/linux-64/brotli-1.0.9-h166bdaf_8.tar.bz2#2ff08978892a3e8b954397c461f18418 https://conda.anaconda.org/conda-forge/linux-64/c-compiler-1.5.2-h0b41bf4_0.conda#69afb4e35be6366c2c1f9ed7f49bc3e6 @@ -134,12 +135,12 @@ https://conda.anaconda.org/conda-forge/noarch/dill-0.3.6-pyhd8ed1ab_1.tar.bz2#88 https://conda.anaconda.org/conda-forge/noarch/distlib-0.3.6-pyhd8ed1ab_0.tar.bz2#b65b4d50dbd2d50fa0aeac367ec9eed7 https://conda.anaconda.org/conda-forge/linux-64/docutils-0.17.1-py310hff52083_3.tar.bz2#785160da087cf1d70e989afbb761f01c https://conda.anaconda.org/conda-forge/noarch/dodgy-0.2.1-py_0.tar.bz2#62a69d073f7446c90f417b0787122f5b -https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.0.4-pyhd8ed1ab_0.tar.bz2#e0734d1f12de77f9daca98bda3428733 +https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.1.0-pyhd8ed1ab_0.conda#a385c3e8968b4cf8fbc426ace915fd1a https://conda.anaconda.org/conda-forge/noarch/execnet-1.9.0-pyhd8ed1ab_0.tar.bz2#0e521f7a5e60d508b121d38b04874fb2 -https://conda.anaconda.org/conda-forge/noarch/filelock-3.8.2-pyhd8ed1ab_0.conda#0f09c2bc17ddd8732be8e5b99297c7ce +https://conda.anaconda.org/conda-forge/noarch/filelock-3.9.0-pyhd8ed1ab_0.conda#1addc115923d646ca19ed90edc413506 https://conda.anaconda.org/conda-forge/linux-64/fontconfig-2.14.1-hc2a2eb6_0.tar.bz2#78415f0180a8d9c5bcc47889e00d5fb1 https://conda.anaconda.org/conda-forge/noarch/fsspec-2022.11.0-pyhd8ed1ab_0.tar.bz2#eb919f2119a6db5d0192f9e9c3711572 -https://conda.anaconda.org/conda-forge/linux-64/gdk-pixbuf-2.42.8-hff1cb4f_1.tar.bz2#a61c6312192e7c9de71548a6706a21e6 +https://conda.anaconda.org/conda-forge/linux-64/gdk-pixbuf-2.42.8-h9fd3ed7_2.conda#34c6b05df6edd95835f63b3ddfa76dc5 https://conda.anaconda.org/conda-forge/noarch/geographiclib-1.52-pyhd8ed1ab_0.tar.bz2#6880e7100ebae550a33ce26663316d85 https://conda.anaconda.org/conda-forge/linux-64/gfortran-11.3.0-ha859ce3_11.tar.bz2#9a6a0c6fc4d192fddc7347a0ca31a329 https://conda.anaconda.org/conda-forge/linux-64/gfortran_linux-64-11.3.0-h3c55166_11.tar.bz2#f70b169eb69320d71f193758b7df67e8 @@ -154,14 +155,14 @@ https://conda.anaconda.org/conda-forge/noarch/iniconfig-1.1.1-pyh9f0ad1d_0.tar.b https://conda.anaconda.org/conda-forge/noarch/itsdangerous-2.1.2-pyhd8ed1ab_0.tar.bz2#3c3de74912f11d2b590184f03c7cd09b https://conda.anaconda.org/conda-forge/linux-64/kiwisolver-1.4.4-py310hbf28c38_1.tar.bz2#ad5647e517ba68e2868ef2e6e6ff7723 https://conda.anaconda.org/conda-forge/linux-64/lazy-object-proxy-1.8.0-py310h5764c6d_0.tar.bz2#7c3f704b65f1689d6ba31a79791d2ce1 -https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.14-h6ed2654_0.tar.bz2#dcc588839de1445d90995a0a2c4f3a39 -https://conda.anaconda.org/conda-forge/linux-64/libcurl-7.86.0-hdc1c0ab_2.conda#c4c21da8fc657a6c5e426b8cbad36cc2 +https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.14-hfd0df8a_1.conda#c2566c2ea5f153ddd6bf4acaf7547d97 +https://conda.anaconda.org/conda-forge/linux-64/libcurl-7.87.0-hdc1c0ab_0.conda#bc302fa1cf8eda15c60f669b7524a320 https://conda.anaconda.org/conda-forge/linux-64/libkml-1.3.0-h37653c0_1015.tar.bz2#37d3747dd24d604f63d2610910576e63 https://conda.anaconda.org/conda-forge/linux-64/libpq-15.1-hb675445_2.conda#509f08b3789d9e7e9a72871491ae08e2 -https://conda.anaconda.org/conda-forge/linux-64/libwebp-1.2.4-h522a892_0.tar.bz2#802e43f480122a85ae6a34c1909f8f98 +https://conda.anaconda.org/conda-forge/linux-64/libwebp-1.2.4-h1daa5a0_1.conda#77003f63d1763c1e6569a02c1742c9f4 https://conda.anaconda.org/conda-forge/noarch/locket-1.0.0-pyhd8ed1ab_0.tar.bz2#91e27ef3d05cc772ce627e51cff111c4 https://conda.anaconda.org/conda-forge/linux-64/lxml-4.9.2-py310hbdc0903_0.conda#543906a26651f10c6180ca71fc4d48f2 -https://conda.anaconda.org/conda-forge/linux-64/lz4-4.0.2-py310h5d5e884_0.tar.bz2#cda615ff0531107b6f83f14ea0a2e7f1 +https://conda.anaconda.org/conda-forge/linux-64/lz4-4.2.0-py310h0cfdcf0_0.conda#5ee9fdbb5a4a8f426fbcf7f7e10b931f https://conda.anaconda.org/conda-forge/linux-64/markupsafe-2.1.1-py310h5764c6d_2.tar.bz2#2d7028ea2a77f909931e1a173d952261 https://conda.anaconda.org/conda-forge/noarch/mccabe-0.6.1-py_1.tar.bz2#a326cb400c1ccd91789f3e7d02124d61 https://conda.anaconda.org/conda-forge/linux-64/mpi4py-3.1.4-py310h37cc914_0.tar.bz2#98d598d9178d7f3091212c61c0be693c @@ -169,11 +170,10 @@ https://conda.anaconda.org/conda-forge/linux-64/msgpack-python-1.0.4-py310hbf28c https://conda.anaconda.org/conda-forge/noarch/munkres-1.1.4-pyh9f0ad1d_0.tar.bz2#2ba8498c1018c1e9c61eb99b973dfe19 https://conda.anaconda.org/conda-forge/linux-64/mypy_extensions-0.4.3-py310hff52083_6.tar.bz2#58a812a2f042e6f8c027736c6464d2dd https://conda.anaconda.org/conda-forge/noarch/networkx-2.8.8-pyhd8ed1ab_0.tar.bz2#bb45ff9deddb045331fd039949f39650 -https://conda.anaconda.org/conda-forge/linux-64/numpy-1.24.0-py310h08bbf29_0.conda#d14a8960a052bd82cca0542a9ed15784 -https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.0-h7d73246_1.tar.bz2#a11b4df9271a8d7917686725aa04c8f2 +https://conda.anaconda.org/conda-forge/linux-64/numpy-1.24.1-py310h08bbf29_0.conda#0d1f2e988c8810be90ffe441a303090a +https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.0-hfec8fc6_2.conda#5ce6a42505c6e9e6151c54c3ec8d68ea https://conda.anaconda.org/conda-forge/noarch/packaging-22.0-pyhd8ed1ab_0.conda#0e8e1bd93998978fc3125522266d12db https://conda.anaconda.org/conda-forge/noarch/pathspec-0.10.3-pyhd8ed1ab_0.conda#0f7d2186dd12ef3277e70fd85f9ff6a7 -https://conda.anaconda.org/conda-forge/noarch/platformdirs-2.6.0-pyhd8ed1ab_0.conda#b1b2ab02d1ece1719f7fa002ad4bc70d https://conda.anaconda.org/conda-forge/noarch/pluggy-1.0.0-pyhd8ed1ab_5.tar.bz2#7d301a0d25f424d96175f810935f0da9 https://conda.anaconda.org/conda-forge/linux-64/psutil-5.9.4-py310h5764c6d_0.tar.bz2#c3c55664e9becc48e6a652e2b641961f https://conda.anaconda.org/conda-forge/noarch/py-1.11.0-pyh6c4a22f_0.tar.bz2#b4613d7e7a493916d867842a6a148054 @@ -234,8 +234,8 @@ https://conda.anaconda.org/conda-forge/linux-64/cftime-1.6.2-py310hde88566_1.tar https://conda.anaconda.org/conda-forge/noarch/click-plugins-1.1.1-py_0.tar.bz2#4fd2c6b53934bd7d96d1f3fdaf99b79f https://conda.anaconda.org/conda-forge/noarch/cligj-0.7.2-pyhd8ed1ab_1.tar.bz2#a29b7c141d6b2de4bb67788a5f107734 https://conda.anaconda.org/conda-forge/linux-64/contourpy-1.0.6-py310hbf28c38_0.tar.bz2#c5b1699e390d30b680dd93a2b251062b -https://conda.anaconda.org/conda-forge/linux-64/coverage-7.0.0-py310h1fa729e_0.conda#e83191fb54f9e90e5b95b6265972d0bf -https://conda.anaconda.org/conda-forge/linux-64/curl-7.86.0-hdc1c0ab_2.conda#3c96ba0c180e389621c4fd62d06c151b +https://conda.anaconda.org/conda-forge/linux-64/coverage-7.0.1-py310h1fa729e_0.conda#a00880abc45b82b2cfee326cfc3cef81 +https://conda.anaconda.org/conda-forge/linux-64/curl-7.87.0-hdc1c0ab_0.conda#b14123ca479b9473d7f7395b0fd25c97 https://conda.anaconda.org/conda-forge/linux-64/cxx-compiler-1.5.2-hf52228f_0.conda#6b3b19e359824b97df7145c8c878c8be https://conda.anaconda.org/conda-forge/linux-64/cytoolz-0.12.0-py310h5764c6d_1.tar.bz2#fd18cd597d23b2b5ddde23bd5b7aec32 https://conda.anaconda.org/conda-forge/noarch/docformatter-1.5.1-pyhd8ed1ab_0.conda#7a8356601d0a66eb83fb5178bde841bf @@ -243,26 +243,26 @@ https://conda.anaconda.org/conda-forge/noarch/fire-0.4.0-pyh44b312d_0.tar.bz2#0b https://conda.anaconda.org/conda-forge/linux-64/fonttools-4.38.0-py310h5764c6d_1.tar.bz2#12ebe92a8a578bc903bd844744f4d040 https://conda.anaconda.org/conda-forge/linux-64/fortran-compiler-1.5.2-hdb1a99f_0.conda#265323e1bd53709aeb739c9b1794b398 https://conda.anaconda.org/conda-forge/noarch/geopy-2.3.0-pyhd8ed1ab_0.tar.bz2#529faeecd6eee3a3b782566ddf05ce92 -https://conda.anaconda.org/conda-forge/linux-64/hdf5-1.12.2-mpi_mpich_h5d83325_0.tar.bz2#6b5c2d276f306df759cfbdb0f41c4db9 +https://conda.anaconda.org/conda-forge/linux-64/hdf5-1.12.2-mpi_mpich_h5d83325_1.conda#811c4d55cf17b42336ffa314239717b0 https://conda.anaconda.org/conda-forge/noarch/html5lib-1.1-pyh9f0ad1d_0.tar.bz2#b2355343d6315c892543200231d7154a https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-5.2.0-pyha770c72_0.conda#2e8eec7a08bcf85547a3778225f00634 -https://conda.anaconda.org/conda-forge/noarch/importlib_resources-5.10.1-pyhd8ed1ab_0.conda#db5d88c84c769798bf4b2f6776446b32 +https://conda.anaconda.org/conda-forge/noarch/importlib_resources-5.10.1-pyhd8ed1ab_1.conda#5b48ca365913b08b819884bc21d4fb56 https://conda.anaconda.org/conda-forge/noarch/isodate-0.6.1-pyhd8ed1ab_0.tar.bz2#4a62c93c1b5c0b920508ae3fd285eaf5 -https://conda.anaconda.org/conda-forge/noarch/isort-5.11.3-pyhd8ed1ab_0.conda#df79686d95d8dd88eda0422711e5e7e3 +https://conda.anaconda.org/conda-forge/noarch/isort-5.11.4-pyhd8ed1ab_0.conda#cc1909c46e375f8d4e3eb9902b21fa6a https://conda.anaconda.org/conda-forge/noarch/jinja2-3.1.2-pyhd8ed1ab_1.tar.bz2#c8490ed5c70966d232fdd389d0dbed37 https://conda.anaconda.org/conda-forge/noarch/joblib-1.2.0-pyhd8ed1ab_0.tar.bz2#7583652522d71ad78ba536bba06940eb https://conda.anaconda.org/conda-forge/noarch/latexcodec-2.0.1-pyh9f0ad1d_0.tar.bz2#8d67904973263afd2985ba56aa2d6bb4 -https://conda.anaconda.org/conda-forge/linux-64/libgd-2.3.3-h18fbbfe_3.tar.bz2#ea9758cf553476ddf75c789fdd239dc5 +https://conda.anaconda.org/conda-forge/linux-64/libgd-2.3.3-h5aea950_4.conda#82ef57611ace65b59db35a9687264572 https://conda.anaconda.org/conda-forge/noarch/logilab-common-1.7.3-py_0.tar.bz2#6eafcdf39a7eb90b6d951cfff59e8d3b https://conda.anaconda.org/conda-forge/noarch/munch-2.5.0-py_0.tar.bz2#31d9e9be500e25ff0050bc9f57a6bcd7 https://conda.anaconda.org/conda-forge/linux-64/mypy-0.991-py310h5764c6d_0.tar.bz2#aaa26cf19bd0903cfed601a4da61ec25 https://conda.anaconda.org/conda-forge/noarch/nested-lookup-0.2.25-pyhd8ed1ab_1.tar.bz2#2f59daeb14581d41b1e2dda0895933b2 https://conda.anaconda.org/conda-forge/noarch/nodeenv-1.7.0-pyhd8ed1ab_0.tar.bz2#fbe1182f650c04513046d6894046cd6c https://conda.anaconda.org/conda-forge/noarch/partd-1.3.0-pyhd8ed1ab_0.tar.bz2#af8c82d121e63082926062d61d9abb54 -https://conda.anaconda.org/conda-forge/linux-64/pillow-9.2.0-py310h454ad03_3.tar.bz2#eb354ff791f505b1d6f13f776359d88e +https://conda.anaconda.org/conda-forge/linux-64/pillow-9.2.0-py310h023d228_4.conda#303776988a91771e6b60b8291d153553 https://conda.anaconda.org/conda-forge/noarch/pip-22.3.1-pyhd8ed1ab_0.tar.bz2#da66f2851b9836d3a7c5190082a45f7d https://conda.anaconda.org/conda-forge/linux-64/postgresql-15.1-h3248436_2.conda#4d2f2c04f8bdb6a9cb0d6e8d90b1f907 -https://conda.anaconda.org/conda-forge/linux-64/proj-9.1.0-h93bde94_0.tar.bz2#255c7204dda39747c3ba380d28b026d7 +https://conda.anaconda.org/conda-forge/linux-64/proj-9.1.0-h8ffa02c_1.conda#ed901e1f5c504b144b31f015c6702634 https://conda.anaconda.org/conda-forge/noarch/pydocstyle-6.1.1-pyhd8ed1ab_0.tar.bz2#e417954a987d382b3142886726ab3aad https://conda.anaconda.org/conda-forge/noarch/pygments-2.13.0-pyhd8ed1ab_0.tar.bz2#9f478e8eedd301008b5f395bad0caaed https://conda.anaconda.org/conda-forge/noarch/pytest-7.2.0-pyhd8ed1ab_2.tar.bz2#ac82c7aebc282e6ac0450fca012ca78c @@ -271,9 +271,9 @@ https://conda.anaconda.org/conda-forge/linux-64/python-stratify-0.2.post0-py310h https://conda.anaconda.org/conda-forge/linux-64/scipy-1.9.3-py310hdfbd76f_2.tar.bz2#0582a434d03f6b06d5defbb142c96f4f https://conda.anaconda.org/conda-forge/linux-64/shapely-2.0.0-py310h8b84c32_0.conda#823009371d9b961c83cdb9aa80e1e6e7 https://conda.anaconda.org/conda-forge/noarch/tomlkit-0.11.6-pyha770c72_0.tar.bz2#471bf9e605820b59988e830382b8d654 -https://conda.anaconda.org/conda-forge/noarch/types-requests-2.28.11.5-pyhd8ed1ab_0.tar.bz2#bcd389a385a8b6bd866238f1a16d13cf +https://conda.anaconda.org/conda-forge/noarch/types-requests-2.28.11.7-pyhd8ed1ab_0.conda#6ae8b42b925b9ed5185345faf82ea39e +https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.4.0-hd8ed1ab_0.tar.bz2#be969210b61b897775a0de63cd9e9026 https://conda.anaconda.org/conda-forge/noarch/url-normalize-1.4.3-pyhd8ed1ab_0.tar.bz2#7c4076e494f0efe76705154ac9302ba6 -https://conda.anaconda.org/conda-forge/linux-64/virtualenv-20.17.1-py310hff52083_0.conda#d26ee3f6561669ec1f118d6d3404e42a https://conda.anaconda.org/conda-forge/linux-64/xerces-c-3.2.4-h55805fa_1.tar.bz2#d127dc8efe24033b306180939e51e6af https://conda.anaconda.org/conda-forge/noarch/yamale-4.0.4-pyh6c4a22f_0.tar.bz2#cc9f59f147740d88679bf1bd94dbe588 https://conda.anaconda.org/conda-forge/noarch/yamllint-1.28.0-pyhd8ed1ab_0.tar.bz2#f28e487a994b504cc5fdb6cb5e6a0bf9 @@ -287,16 +287,16 @@ https://conda.anaconda.org/conda-forge/linux-64/cryptography-38.0.4-py310h600f1e https://conda.anaconda.org/conda-forge/noarch/dask-core-2022.12.1-pyhd8ed1ab_0.conda#f12878f9839c72f3d51af02fb10da43d https://conda.anaconda.org/conda-forge/noarch/django-4.1.4-pyhd8ed1ab_0.conda#14f8b26ca366987ec53655f6a7a6cba0 https://conda.anaconda.org/conda-forge/noarch/flake8-4.0.1-pyhd8ed1ab_2.tar.bz2#a824bd55ce47e9c637427f730c651231 -https://conda.anaconda.org/conda-forge/linux-64/geotiff-1.7.1-ha76d385_4.tar.bz2#6a613710a0f19aba3a5dfe83bf1c1c0f +https://conda.anaconda.org/conda-forge/linux-64/geotiff-1.7.1-h7157cca_5.conda#dfb1b96aee336c345c1833cf8f486acc https://conda.anaconda.org/conda-forge/linux-64/harfbuzz-6.0.0-h8e241bc_0.conda#448fe40d2fed88ccf4d9ded37cbb2b38 https://conda.anaconda.org/conda-forge/linux-64/kealib-1.5.0-ha7026e8_0.conda#c948b920f45fd81a2dde8b1ab514cc84 https://conda.anaconda.org/conda-forge/linux-64/libnetcdf-4.8.1-mpi_mpich_hcd871d9_6.tar.bz2#6cdc429ed22edb566ac4308f3da6916d https://conda.anaconda.org/conda-forge/linux-64/libspatialite-5.0.1-h7c8129e_22.tar.bz2#23abed7562ad969493b89ad0e5f5c521 https://conda.anaconda.org/conda-forge/linux-64/matplotlib-base-3.6.2-py310h8d5ebf3_0.tar.bz2#da51ddb20c0f99d672eb756c3abf27e7 https://conda.anaconda.org/conda-forge/linux-64/pandas-1.5.2-py310h769672d_0.conda#bc363997d22f3b058fb17f1e89d4c96f -https://conda.anaconda.org/conda-forge/linux-64/poppler-22.12.0-h92391eb_0.conda#7ad6d858f5615f9b0e9e4bd60395ea75 +https://conda.anaconda.org/conda-forge/noarch/platformdirs-2.6.2-pyhd8ed1ab_0.conda#0b4cc3f8181b0d8446eb5387d7848a54 +https://conda.anaconda.org/conda-forge/linux-64/poppler-22.12.0-h091648b_1.conda#25cfe805ac16b3bed648f5376bc389f0 https://conda.anaconda.org/conda-forge/noarch/pybtex-0.24.0-pyhd8ed1ab_2.tar.bz2#2099b86a7399c44c0c61cdb6de6915ba -https://conda.anaconda.org/conda-forge/noarch/pylint-2.15.9-pyhd8ed1ab_0.conda#5bfc68e1b8f0883f793ae0acedc88770 https://conda.anaconda.org/conda-forge/linux-64/pyproj-3.4.1-py310hfc24d34_0.conda#c126f81b5cea6b2d4a64d0744249a26f https://conda.anaconda.org/conda-forge/noarch/pytest-cov-4.0.0-pyhd8ed1ab_0.tar.bz2#c9e3f8bfdb9bfc34aa1836a6ed4b25d7 https://conda.anaconda.org/conda-forge/noarch/pytest-env-0.6.2-py_0.tar.bz2#2fe6beb5d2d87cf252a2575043919696 @@ -307,44 +307,46 @@ https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.1.0-pyhd8ed1ab_0.co https://conda.anaconda.org/conda-forge/noarch/rdflib-6.2.0-pyhd8ed1ab_0.tar.bz2#b9acd5fbaf467f7447746b1ecac50e83 https://conda.anaconda.org/conda-forge/linux-64/requirements-detector-0.7-py310hff52083_3.tar.bz2#0cd9f8972da2c8ad624676d47643805f https://conda.anaconda.org/conda-forge/linux-64/scikit-learn-1.2.0-py310h209a8ca_0.conda#5a08a1f004445ee9bf58261feb16cc18 -https://conda.anaconda.org/conda-forge/linux-64/tiledb-2.13.0-h3f4058f_0.conda#b7009881353045d2bdae62ec0dd04bc0 +https://conda.anaconda.org/conda-forge/linux-64/tiledb-2.13.0-hd532e3d_1.conda#d7655c5bdd7a9b47a49d0029f72773da https://conda.anaconda.org/conda-forge/linux-64/ukkonen-1.0.1-py310hbf28c38_3.tar.bz2#703ff1ac7d1b27fb5944b8052b5d1edb https://conda.anaconda.org/conda-forge/linux-64/cartopy-0.21.1-py310hcb7e713_0.conda#bd14eaad9bbf54b78e48ecb8b644fcf6 https://conda.anaconda.org/conda-forge/noarch/flake8-polyfill-1.0.2-py_0.tar.bz2#a53db35e3d07f0af2eccd59c2a00bffe https://conda.anaconda.org/conda-forge/noarch/geopandas-base-0.12.2-pyha770c72_0.conda#cf04d066b97dfe698f0d01ebf4af6163 https://conda.anaconda.org/conda-forge/noarch/identify-2.5.11-pyhd8ed1ab_0.conda#20d9fee9fd73a480ad7f264c07b4fcf8 -https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.6.1-hf2b5f72_1.conda#fc4c5716fac9e7f242223dca042e1045 +https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.6.1-he31f7c0_2.conda#e48802f38ae5d01141b5bc4c8259797d https://conda.anaconda.org/conda-forge/noarch/mapclassify-2.4.3-pyhd8ed1ab_0.tar.bz2#908bbfb54da154042c5cbda77b37a3d1 https://conda.anaconda.org/conda-forge/noarch/nc-time-axis-1.4.1-pyhd8ed1ab_0.tar.bz2#281b58948bf60a2582de9e548bcc5369 https://conda.anaconda.org/conda-forge/linux-64/netcdf-fortran-4.6.0-mpi_mpich_hd09bd1e_1.tar.bz2#0b69750bb937cab0db14f6bcef6fd787 https://conda.anaconda.org/conda-forge/linux-64/netcdf4-1.6.0-nompi_py310h0a86a1f_103.conda#7f69695b684f2595d9ba1ce26d693b7d https://conda.anaconda.org/conda-forge/linux-64/pango-1.50.12-hd33c08f_1.conda#667dc93c913f0156e1237032e3a22046 https://conda.anaconda.org/conda-forge/linux-64/parallelio-2.5.9-mpi_mpich_h50e6f33_101.conda#87fac13c80750b8be35b0a32bb965bbe -https://conda.anaconda.org/conda-forge/noarch/pylint-plugin-utils-0.7-pyhd8ed1ab_0.tar.bz2#1657976383aee04dbb3ae3bdf654bb58 +https://conda.anaconda.org/conda-forge/noarch/pylint-2.15.9-pyhd8ed1ab_0.conda#5bfc68e1b8f0883f793ae0acedc88770 https://conda.anaconda.org/conda-forge/noarch/pyopenssl-22.1.0-pyhd8ed1ab_0.tar.bz2#fbfa0a180d48c800f922a10a114a8632 https://conda.anaconda.org/conda-forge/noarch/pytest-html-3.2.0-pyhd8ed1ab_1.tar.bz2#d5c7a941dfbceaab4b172a56d7918eb0 +https://conda.anaconda.org/conda-forge/linux-64/virtualenv-20.17.1-py310hff52083_0.conda#d26ee3f6561669ec1f118d6d3404e42a https://conda.anaconda.org/conda-forge/noarch/xarray-2022.12.0-pyhd8ed1ab_0.conda#fd0b8bb7c8df27e09fdee6c893f2ba9d https://conda.anaconda.org/conda-forge/linux-64/esmf-8.3.1-mpi_mpich_h5a1934d_101.tar.bz2#ac4bfd5bdb0a5b4b99ee383fd0c8995c -https://conda.anaconda.org/conda-forge/linux-64/gdal-3.6.1-py310hc1b7723_1.conda#27f46e0d58cddd2b36520c7453a9ddca +https://conda.anaconda.org/conda-forge/linux-64/gdal-3.6.1-py310hc1b7723_2.conda#6682a2c089232996ba981161cc2cc456 https://conda.anaconda.org/conda-forge/linux-64/gtk2-2.24.33-h90689f9_2.tar.bz2#957a0255ab58aaf394a91725d73ab422 https://conda.anaconda.org/conda-forge/noarch/iris-3.4.0-pyhd8ed1ab_0.conda#6a6c17cfff8564de41d2a952c97bc2e8 https://conda.anaconda.org/conda-forge/linux-64/librsvg-2.54.4-h7abd40a_0.tar.bz2#921e53675ed5ea352f022b79abab076a https://conda.anaconda.org/conda-forge/noarch/myproxyclient-2.1.0-pyhd8ed1ab_2.tar.bz2#363b0816e411feb0df925d4f224f026a https://conda.anaconda.org/conda-forge/noarch/pep8-naming-0.10.0-pyh9f0ad1d_0.tar.bz2#b3c5536e4f9f58a4b16adb6f1e11732d -https://conda.anaconda.org/conda-forge/linux-64/pre-commit-2.20.0-py310hff52083_1.tar.bz2#8c151d720f9fe3b9962efe71fc10b07b -https://conda.anaconda.org/conda-forge/noarch/pylint-celery-0.3-py_1.tar.bz2#e29456a611a62d3f26105a2f9c68f759 -https://conda.anaconda.org/conda-forge/noarch/pylint-django-2.5.3-pyhd8ed1ab_0.tar.bz2#00d8853fb1f87195722ea6a582cc9b56 -https://conda.anaconda.org/conda-forge/noarch/pylint-flask-0.6-py_0.tar.bz2#5a9afd3d0a61b08d59eed70fab859c1b +https://conda.anaconda.org/conda-forge/linux-64/pre-commit-2.21.0-py310hff52083_0.conda#41b6a707f04268b028c497d346a97693 +https://conda.anaconda.org/conda-forge/noarch/pylint-plugin-utils-0.7-pyhd8ed1ab_0.tar.bz2#1657976383aee04dbb3ae3bdf654bb58 https://conda.anaconda.org/conda-forge/noarch/urllib3-1.26.13-pyhd8ed1ab_0.conda#3078ef2359efd6ecadbc7e085c5e0592 https://conda.anaconda.org/conda-forge/noarch/distributed-2022.12.1-pyhd8ed1ab_0.conda#63cf20ed7b5205cf4c31aadee4c7fd2e https://conda.anaconda.org/conda-forge/linux-64/esmpy-8.3.1-mpi_mpich_py310h515c5ea_100.conda#ad531847b7cea3df5c63e0b7f2388e7c https://conda.anaconda.org/conda-forge/linux-64/fiona-1.8.22-py310ha325b7b_5.conda#4dbdf48d4712e8906595291f38423eff -https://conda.anaconda.org/conda-forge/linux-64/graphviz-6.0.2-h99bc08f_0.conda#8f247587d1520a2bbc6f79a821b74c07 -https://conda.anaconda.org/conda-forge/noarch/prospector-1.7.7-pyhd8ed1ab_0.tar.bz2#01010f8ea38d650158703a581e51b979 +https://conda.anaconda.org/conda-forge/linux-64/graphviz-7.0.5-h2e5815a_0.conda#96bf06b24d74a5bf826485e9032c9312 +https://conda.anaconda.org/conda-forge/noarch/pylint-celery-0.3-py_1.tar.bz2#e29456a611a62d3f26105a2f9c68f759 +https://conda.anaconda.org/conda-forge/noarch/pylint-django-2.5.3-pyhd8ed1ab_0.tar.bz2#00d8853fb1f87195722ea6a582cc9b56 +https://conda.anaconda.org/conda-forge/noarch/pylint-flask-0.6-py_0.tar.bz2#5a9afd3d0a61b08d59eed70fab859c1b https://conda.anaconda.org/conda-forge/noarch/requests-2.28.1-pyhd8ed1ab_1.tar.bz2#089382ee0e2dc2eae33a04cc3c2bddb0 https://conda.anaconda.org/conda-forge/noarch/dask-2022.12.1-pyhd8ed1ab_0.conda#5861b97a50edcd9c1332d21f2995eca1 https://conda.anaconda.org/conda-forge/noarch/folium-0.14.0-pyhd8ed1ab_0.conda#48c8bb19df0d0268f1a9d30ffc56c5b0 https://conda.anaconda.org/conda-forge/noarch/pooch-1.6.0-pyhd8ed1ab_0.tar.bz2#6429e1d1091c51f626b5dcfdd38bf429 +https://conda.anaconda.org/conda-forge/noarch/prospector-1.7.7-pyhd8ed1ab_0.tar.bz2#01010f8ea38d650158703a581e51b979 https://conda.anaconda.org/conda-forge/linux-64/pydot-1.4.2-py310hff52083_3.tar.bz2#45231e3f8fa29b6cea52e2cfe9b47a22 https://conda.anaconda.org/conda-forge/noarch/requests-cache-0.9.7-pyhd8ed1ab_0.conda#399c98b39eb6bdc4b1d40b7611a6d384 https://conda.anaconda.org/conda-forge/noarch/sphinx-5.3.0-pyhd8ed1ab_0.tar.bz2#f9e1fcfe235d655900bfeb6aee426472