diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index 5ec6ca5aa8..02de1aa0a0 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -180,22 +180,85 @@ To get an overview on data fixes and how to implement new ones, please go to Fx variables as cell measures or ancillary variables ==================================================== -The following preprocessor may require the use of ``fx_variables`` -to be able to perform the computations: +The following preprocessors may require the use of ``fx_variables`` to be able +to perform the computations: + +============================================================== ===================== +Preprocessor Default fx variables +============================================================== ===================== +:ref:`area_statistics` ``areacella``, ``areacello`` +:ref:`mask_landsea` ``sftlf``, ``sftof`` +:ref:`mask_landseaice` ``sftgif`` +:ref:`volume_statistics` ``volcello`` +:ref:`weighting_landsea_fraction` ``sftlf``, ``sftof`` +============================================================== ===================== + +If no ``fx_variables`` are specified for these preprocessors, the fx variables +in the second column are used. If given, the ``fx_variables`` argument +specifies the fx variables that the user wishes to input to the corresponding +preprocessor function. The user may specify these by simply adding the names of +the variables, e.g., - - ``area_statistics`` - - ``mask_landsea`` - - ``mask_landseaice`` - - ``volume_statistics`` - - ``weighting_landsea_fraction`` +.. code-block:: yaml + + fx_variables: + areacello: + volcello: + +or by additionally specifying further keys that are used to define the fx +datasets, e.g., + +.. code-block:: yaml + + fx_variables: + areacello: + mip: Ofx + exp: piControl + volcello: + mip: Omon + +This might be useful to select fx files from a specific ``mip`` table or from a +specific ``exp`` in case not all experiments provide the fx variable. + +Alternatively, the ``fx_variables`` argument can also be specified as a list: + +.. code-block:: yaml + + fx_variables: ['areacello', 'volcello'] + +or as a list of dictionaries: + +.. code-block:: yaml -The preprocessor step ``add_fx_variables`` loads the required ``fx_variables``, -checks them against CMOR standards and adds them either as ``cell_measure`` -or ``ancillary_variable`` inside the cube data. This ensures that the -defined preprocessor chain is applied to both ``variables`` and ``fx_variables``. + fx_variables: [{'short_name': 'areacello', 'mip': 'Ofx', 'exp': 'piControl'}, {'short_name': 'volcello', 'mip': 'Omon'}] + +The recipe parser will automatically find the data files that are associated +with these variables and pass them to the function for loading and processing. + +If ``mip`` is not given, ESMValTool will search for the fx variable in all +available tables of the specified project. + +.. warning:: + Some fx variables exist in more than one table (e.g., ``volcello`` exists in + CMIP6's ``Odec``, ``Ofx``, ``Omon``, and ``Oyr`` tables; ``sftgif`` exists + in CMIP6's ``fx``, ``IyrAnt`` and ``IyrGre``, and ``LImon`` tables). If (for + a given dataset) fx files are found in more than one table, ``mip`` needs to + be specified, otherwise an error is raised. + +Internally, the required ``fx_variables`` are automatically loaded by the +preprocessor step ``add_fx_variables`` which also checks them against CMOR +standards and adds them either as ``cell_measure`` (see `CF conventions on cell +measures +`_ +and :class:`iris.coords.CellMeasure`) or ``ancillary_variable`` (see `CF +conventions on ancillary variables +`_ +and :class:`iris.coords.AncillaryVariable`) inside the cube data. This ensures +that the defined preprocessor chain is applied to both ``variables`` and +``fx_variables``. Note that when calling steps that require ``fx_variables`` inside diagnostic -scripts, the variables are expected to contain the required ``cell_measures`` or +scripts, the variables are expected to contain the required ``cell_measures`` or ``ancillary_variables``. If missing, they can be added using the following functions: .. code-block:: @@ -209,7 +272,7 @@ scripts, the variables are expected to contain the required ``cell_measures`` or cube_with_ancillary_sftlf = add_ancillary_variable(cube, sftlf_cube) cube_with_ancillary_sftgif = add_ancillary_variable(cube, sftgif_cube) - + Details on the arguments needed for each step can be found in the following sections. .. _Vertical interpolation: @@ -351,8 +414,8 @@ is for example useful for climate models which do not offer land/sea fraction files. This arguments also accepts the special dataset specifiers ``reference_dataset`` and ``alternative_dataset``. -Optionally you can specify your own custom fx variable to be used in cases when e.g. a certain -experiment is preferred for fx data retrieval: +Optionally you can specify your own custom fx variable to be used in cases when +e.g. a certain experiment is preferred for fx data retrieval: .. code-block:: yaml @@ -361,7 +424,7 @@ experiment is preferred for fx data retrieval: weighting_landsea_fraction: area_type: land exclude: ['CanESM2', 'reference_dataset'] - fx_variables: + fx_variables: sftlf: exp: piControl sftof: @@ -377,10 +440,13 @@ or alternatively: area_type: land exclude: ['CanESM2', 'reference_dataset'] fx_variables: [ - {'short_name': 'sftlf', 'exp': 'piControl'}, + {'short_name': 'sftlf', 'exp': 'piControl'}, {'short_name': 'sftof', 'exp': 'piControl'} ] +More details on the argument ``fx_variables`` and its default values are given +in :ref:`Fx variables as cell measures or ancillary variables`. + See also :func:`esmvalcore.preprocessor.weighting_landsea_fraction`. @@ -426,11 +492,6 @@ To mask out a certain domain (e.g., sea) in the preprocessor, and requires only one argument: ``mask_out``: either ``land`` or ``sea``. -The preprocessor automatically retrieves the corresponding mask (``fx: stfof`` -in this case) and applies it so that sea-covered grid cells are set to -missing. Conversely, it retrieves the ``fx: sftlf`` mask when land needs to be -masked out, respectively. - Optionally you can specify your own custom fx variable to be used in cases when e.g. a certain experiment is preferred for fx data retrieval. Note that it is possible to specify as many tags for the fx variable as required: @@ -442,8 +503,8 @@ for the fx variable as required: landmask: mask_landsea: mask_out: sea - fx_variables: - sftlf: + fx_variables: + sftlf: exp: piControl sftof: exp: piControl @@ -458,10 +519,13 @@ or alternatively: mask_landsea: mask_out: sea fx_variables: [ - {'short_name': 'sftlf', 'exp': 'piControl'}, + {'short_name': 'sftlf', 'exp': 'piControl'}, {'short_name': 'sftof', 'exp': 'piControl', 'ensemble': 'r2i1p1f1'} ] +More details on the argument ``fx_variables`` and its default values are given +in :ref:`Fx variables as cell measures or ancillary variables`. + If the corresponding fx file is not found (which is the case for some models and almost all observational datasets), the preprocessor attempts to mask the data using Natural Earth mask files (that are @@ -471,6 +535,8 @@ land and glaciated areas and 50m for ocean masks). See also :func:`esmvalcore.preprocessor.mask_landsea`. +.. _ice masking: + Ice masking ----------- @@ -487,11 +553,8 @@ losing generality. To mask ice out, ``mask_landseaice`` can be used: and requires only one argument: ``mask_out``: either ``landsea`` or ``ice``. -As in the case of ``mask_landsea``, the preprocessor automatically retrieves -the ``fx_variables: [sftgif]`` mask. - -Optionally you can specify your own custom fx variable to be used in cases when e.g. a certain -experiment is preferred for fx data retrieval: +Optionally you can specify your own custom fx variable to be used in cases when +e.g. a certain experiment is preferred for fx data retrieval: .. code-block:: yaml @@ -500,7 +563,7 @@ experiment is preferred for fx data retrieval: landseaicemask: mask_landseaice: mask_out: sea - fx_variables: + fx_variables: sftgif: exp: piControl @@ -514,6 +577,9 @@ or alternatively: mask_out: sea fx_variables: [{'short_name': 'sftgif', 'exp': 'piControl'}] +More details on the argument ``fx_variables`` and its default values are given +in :ref:`Fx variables as cell measures or ancillary variables`. + See also :func:`esmvalcore.preprocessor.mask_landseaice`. Glaciated masking @@ -1021,7 +1087,7 @@ See also :func:`esmvalcore.preprocessor.decadal_statistics`. ---------------------- This function produces statistics for the whole dataset. It can produce scalars -(if the full period is chosen) or daily, monthly or seasonal statics. +(if the full period is chosen) or daily, monthly or seasonal statistics. Parameters: * operator: operation to apply. Accepted values are 'mean', 'median', @@ -1406,6 +1472,8 @@ argument: See also :func:`esmvalcore.preprocessor.meridional_means`. +.. _area_statistics: + ``area_statistics`` ------------------- @@ -1420,40 +1488,9 @@ Note that this function is applied over the entire dataset. If only a specific region, depth layer or time period is required, then those regions need to be removed using other preprocessor operations in advance. -The ``fx_variables`` argument specifies the fx variables that the user wishes to input to the function; -the user may specify it calling the variables e.g. - -.. code-block:: yaml - - fx_variables: - areacello: - volcello: - -or calling the variables and adding specific variable parameters (the key-value pair may be as specific -as a CMOR variable can permit): - -.. code-block:: yaml - - fx_variables: - areacello: - mip: Omon - volcello: - mip: fx - -Alternatively, the ``fx_variables`` argument can also be specified as a list: - -.. code-block:: yaml - - fx_variables: ['areacello', 'volcello'] - -or as a list of dictionaries: - -.. code-block:: yaml - - fx_variables: [{'short_name': 'areacello', 'mip': 'Omon'}, {'short_name': 'volcello', 'mip': 'fx'}] - -The recipe parser will automatically find the data files that are associated with these -variables and pass them to the function for loading and processing. +The optional ``fx_variables`` argument specifies the fx variables that the user +wishes to input to the function. More details on this are given in :ref:`Fx +variables as cell measures or ancillary variables`. See also :func:`esmvalcore.preprocessor.area_statistics`. @@ -1486,6 +1523,8 @@ as the Iris cube. That is, if the cube has `z`-coordinate as negative, then See also :func:`esmvalcore.preprocessor.extract_volume`. +.. _volume_statistics: + ``volume_statistics`` --------------------- @@ -1496,42 +1535,10 @@ This function takes the argument: ``operator``, which defines the operation to apply over the volume. No depth coordinate is required as this is determined by Iris. This function -works best when the ``fx_variables`` provide the cell volume. - -The ``fx_variables`` argument specifies the fx variables that the user wishes to input to the function; -the user may specify it calling the variables e.g. - -.. code-block:: yaml - - fx_variables: - areacello: - volcello: - -or calling the variables and adding specific variable parameters (the key-value pair may be as specific -as a CMOR variable can permit): - -.. code-block:: yaml - - fx_variables: - areacello: - mip: Omon - volcello: - mip: fx - -Alternatively, the ``fx_variables`` argument can also be specified as a list: - -.. code-block:: yaml - - fx_variables: ['areacello', 'volcello'] - -or as a list of dictionaries: - -.. code-block:: yaml - - fx_variables: [{'short_name': 'areacello', 'mip': 'Omon'}, {'short_name': 'volcello', 'mip': 'fx'}] - -The recipe parser will automatically find the data files that are associated with these -variables and pass them to the function for loading and processing. +works best when the ``fx_variables`` provide the cell volume. The optional +``fx_variables`` argument specifies the fx variables that the user wishes to +input to the function. More details on this are given in :ref:`Fx variables as +cell measures or ancillary variables`. See also :func:`esmvalcore.preprocessor.volume_statistics`. diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index fcf06502f4..10b4691591 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -349,26 +349,59 @@ def _add_fxvar_keys(fx_info, variable): return fx_variable -def _search_fx_mip(tables, found_mip, variable, fx_info, config_user): - fx_files = None - for mip in tables: - fx_cmor = tables[mip].get(fx_info['short_name']) - if fx_cmor: - found_mip = True - fx_info['mip'] = mip - fx_info = _add_fxvar_keys(fx_info, variable) - logger.debug("For fx variable '%s', found table '%s'", - fx_info['short_name'], mip) - fx_files = _get_input_files(fx_info, config_user)[0] - if fx_files: - logger.debug("Found fx variables '%s':\n%s", - fx_info['short_name'], pformat(fx_files)) - return found_mip, fx_info, fx_files +def _search_fx_mip(tables, variable, fx_info, config_user): + """Search mip for fx variable.""" + # Get all mips that offer that specific fx variable + mips_with_fx_var = [] + for (mip, table) in tables.items(): + if fx_info['short_name'] in table: + mips_with_fx_var.append(mip) + + # List is empty -> no table includes the fx variable + if not mips_with_fx_var: + raise RecipeError( + f"Requested fx variable '{fx_info['short_name']}' not available " + f"in any CMOR table for '{variable['project']}'") + + # Iterate through all possible mips and check if files are available; in + # case of ambiguity raise an error + fx_files_for_mips = {} + for mip in mips_with_fx_var: + fx_info['mip'] = mip + fx_info = _add_fxvar_keys(fx_info, variable) + logger.debug("For fx variable '%s', found table '%s'", + fx_info['short_name'], mip) + fx_files = _get_input_files(fx_info, config_user)[0] + if fx_files: + logger.debug("Found fx variables '%s':\n%s", + fx_info['short_name'], pformat(fx_files)) + fx_files_for_mips[mip] = fx_files + + # Dict contains more than one element -> ambiguity + if len(fx_files_for_mips) > 1: + raise RecipeError( + f"Requested fx variable '{fx_info['short_name']}' for dataset " + f"'{variable['dataset']}' of project '{variable['project']}' is " + f"available in more than one CMOR table for " + f"'{variable['project']}': {sorted(list(fx_files_for_mips))}") + + # Dict is empty -> no files found -> handled at later stage + if not fx_files_for_mips: + fx_info['mip'] = variable['mip'] + fx_files = [] + + # Dict contains one element -> ok + else: + mip = list(fx_files_for_mips)[0] + fx_info['mip'] = mip + fx_info = _add_fxvar_keys(fx_info, variable) + fx_files = fx_files_for_mips[mip] + + return fx_info, fx_files def _get_fx_files(variable, fx_info, config_user): """Get fx files (searching all possible mips).""" - # assemble info from master variable var_project = variable['project'] # check if project in config-developer @@ -380,31 +413,31 @@ def _get_fx_files(variable, fx_info, config_user): f"a '{var_project}' project in config-developer.") project_tables = CMOR_TABLES[var_project].tables - # force only the mip declared by user - found_mip = False + # If mip is not given, search all available tables. If the variable is not + # found or files are available in more than one table, raise error if not fx_info['mip']: - found_mip, fx_info, fx_files = _search_fx_mip(project_tables, - found_mip, variable, - fx_info, config_user) + fx_info, fx_files = _search_fx_mip(project_tables, variable, fx_info, + config_user) else: - fx_cmor = project_tables[fx_info['mip']].get(fx_info['short_name']) - if fx_cmor: - found_mip = True - fx_info = _add_fxvar_keys(fx_info, variable) - fx_files = _get_input_files(fx_info, config_user)[0] - - # If fx variable was not found in any table, raise exception - if not found_mip: - raise RecipeError( - f"Requested fx variable '{fx_info['short_name']}' " - f"not available in any CMOR table for '{var_project}'") + mip = fx_info['mip'] + if mip not in project_tables: + raise RecipeError( + f"Requested mip table '{mip}' for fx variable " + f"'{fx_info['short_name']}' not available for project " + f"'{var_project}'") + if fx_info['short_name'] not in project_tables[mip]: + raise RecipeError( + f"fx variable '{fx_info['short_name']}' not available in CMOR " + f"table '{mip}' for '{var_project}'") + fx_info = _add_fxvar_keys(fx_info, variable) + fx_files = _get_input_files(fx_info, config_user)[0] - # flag a warning + # Flag a warning if no files are found if not fx_files: logger.warning("Missing data for fx variable '%s'", fx_info['short_name']) - # allow for empty lists corrected for by NE masks + # If frequency = fx, only allow a single file if fx_files: if fx_info['frequency'] == 'fx': fx_files = fx_files[0] @@ -470,7 +503,6 @@ def _fx_list_to_dict(fx_vars): def _update_fx_settings(settings, variable, config_user): """Update fx settings depending on the needed method.""" - # get fx variables either from user defined attribute or fixed def _get_fx_vars_from_attribute(step_settings, step_name): user_fx_vars = step_settings.get('fx_variables') diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py index 75af43bacb..5f7bb2ce58 100644 --- a/tests/integration/test_recipe.py +++ b/tests/integration/test_recipe.py @@ -10,7 +10,6 @@ from PIL import Image import esmvalcore -from esmvalcore import __version__ as current_version from esmvalcore._config import TAGS from esmvalcore._recipe import TASKSEP, read_recipe_file from esmvalcore._recipe_checks import RecipeError @@ -296,6 +295,10 @@ def find_files(_, filenames): return [] if 'sftlf' in filename: return [] + if 'IyrAnt_' in filename: + return [] + if 'IyrGre_' in filename: + return [] return _get_filenames(tmp_path, filenames, tracking_id) monkeypatch.setattr(esmvalcore._data_finder, 'find_files', find_files) @@ -2010,8 +2013,8 @@ def test_user_defined_fxvar(tmp_path, patched_datafinder, config_user): settings = product.settings['mask_landseaice'] assert len(settings) == 1 assert settings['mask_out'] == 'sea' - assert '_fx_' in fx_variables['sftlf']['filename'] - assert '_piControl_' in fx_variables['sftlf']['filename'] + assert '_fx_' in fx_variables['sftgif']['filename'] + assert '_piControl_' in fx_variables['sftgif']['filename'] # volume statistics settings = product.settings['volume_statistics'] @@ -2143,10 +2146,8 @@ def test_landmask_no_fx(tmp_path, patched_failing_datafinder, config_user): assert not any(fx_variables) -# TODO remove after fixing test below -def test_fx_vars_mip_change_cmip6_incomplete(tmp_path, - patched_datafinder, - config_user): +def test_fx_vars_fixed_mip_cmip6(tmp_path, patched_datafinder, config_user): + """Test fx variables with given mips.""" TAGS.set_tag_values(TAGS_FOR_TESTING) content = dedent(""" @@ -2155,16 +2156,11 @@ def test_fx_vars_mip_change_cmip6_incomplete(tmp_path, area_statistics: operator: mean fx_variables: - areacella: - ensemble: r2i1p1f1 - areacello: - clayfrac: - sftlf: sftgif: mip: fx - sftof: - mask_landsea: - mask_out: sea + volcello: + ensemble: r2i1p1f1 + mip: Ofx diagnostics: diagnostic_name: @@ -2196,26 +2192,18 @@ def test_fx_vars_mip_change_cmip6_incomplete(tmp_path, settings = product.settings['area_statistics'] assert len(settings) == 1 assert settings['operator'] == 'mean' + + # Check add_fx_variables fx_variables = product.settings['add_fx_variables']['fx_variables'] assert isinstance(fx_variables, dict) - assert len(fx_variables) == 6 - assert '_fx_' in fx_variables['areacella']['filename'] - assert '_r2i1p1f1_' in fx_variables['areacella']['filename'] - assert '_Ofx_' in fx_variables['areacello']['filename'] - assert '_Efx_' in fx_variables['clayfrac']['filename'] - assert '_fx_' in fx_variables['sftlf']['filename'] - assert '_Ofx_' in fx_variables['sftof']['filename'] - - # Check mask_landsea - assert 'mask_landsea' in product.settings - settings = product.settings['mask_landsea'] - assert len(settings) == 1 - assert settings['mask_out'] == 'sea' + assert len(fx_variables) == 2 + assert '_fx_' in fx_variables['sftgif']['filename'] + assert '_r2i1p1f1_' in fx_variables['volcello']['filename'] + assert '_Ofx_' in fx_variables['volcello']['filename'] -@pytest.mark.skipif(current_version < "2.4.0", - reason="github.com/ESMValGroup/ESMValCore/issues/1159") -def test_fx_vars_mip_change_cmip6(tmp_path, patched_datafinder, config_user): +def test_fx_vars_invalid_mip_cmip6(tmp_path, patched_datafinder, config_user): + """Test fx variables with invalid mip.""" TAGS.set_tag_values(TAGS_FOR_TESTING) content = dedent(""" @@ -2225,15 +2213,7 @@ def test_fx_vars_mip_change_cmip6(tmp_path, patched_datafinder, config_user): operator: mean fx_variables: areacella: - ensemble: r2i1p1f1 - areacello: - clayfrac: - sftlf: - sftgif: - mip: fx - sftof: - mask_landsea: - mask_out: sea + mip: INVALID diagnostics: diagnostic_name: @@ -2251,55 +2231,65 @@ def test_fx_vars_mip_change_cmip6(tmp_path, patched_datafinder, config_user): - {dataset: CanESM5} scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + msg = ("Requested mip table 'INVALID' for fx variable 'areacella' not " + "available for project 'CMIP6'") + with pytest.raises(RecipeError) as rec_err_exp: + get_recipe(tmp_path, content, config_user) + assert str(rec_err_exp.value) == INITIALIZATION_ERROR_MSG + assert msg in rec_err_exp.value.failed_tasks[0].message - # Check generated tasks - assert len(recipe.tasks) == 1 - task = recipe.tasks.pop() - assert task.name == 'diagnostic_name' + TASKSEP + 'tas' - assert len(task.products) == 1 - product = task.products.pop() - # Check area_statistics - assert 'area_statistics' in product.settings - settings = product.settings['area_statistics'] - assert len(settings) == 1 - assert settings['operator'] == 'mean' - fx_variables = product.settings['add_fx_variables']['fx_variables'] - assert isinstance(fx_variables, dict) - assert len(fx_variables) == 6 - assert '_fx_' in fx_variables['areacella']['filename'] - assert '_r2i1p1f1_' in fx_variables['areacella']['filename'] - assert '_Ofx_' in fx_variables['areacello']['filename'] - assert '_Efx_' in fx_variables['clayfrac']['filename'] - assert '_fx_' in fx_variables['sftlf']['filename'] - assert '_fx_' in fx_variables['sftgif']['filename'] # fails, see skipif - assert '_Ofx_' in fx_variables['sftof']['filename'] +def test_fx_vars_invalid_mip_for_var_cmip6(tmp_path, patched_datafinder, + config_user): + """Test fx variables with invalid mip for variable.""" + TAGS.set_tag_values(TAGS_FOR_TESTING) - # Check mask_landsea - assert 'mask_landsea' in product.settings - settings = product.settings['mask_landsea'] - assert len(settings) == 1 - assert settings['mask_out'] == 'sea' + content = dedent(""" + preprocessors: + preproc: + area_statistics: + operator: mean + fx_variables: + areacella: + mip: Lmon + diagnostics: + diagnostic_name: + variables: + tas: + preprocessor: preproc + project: CMIP6 + mip: Amon + exp: historical + start_year: 2000 + end_year: 2005 + ensemble: r1i1p1f1 + grid: gn + additional_datasets: + - {dataset: CanESM5} + scripts: null + """) + msg = ("fx variable 'areacella' not available in CMOR table 'Lmon' for " + "'CMIP6'") + with pytest.raises(RecipeError) as rec_err_exp: + get_recipe(tmp_path, content, config_user) + assert str(rec_err_exp.value) == INITIALIZATION_ERROR_MSG + assert msg in rec_err_exp.value.failed_tasks[0].message + + +def test_fx_vars_mip_search_cmip6(tmp_path, patched_datafinder, config_user): + """Test mip tables search for different fx variables.""" + TAGS.set_tag_values(TAGS_FOR_TESTING) -# TODO remove when you fix test below -def test_fx_list_mip_change_cmip6_incomplete(tmp_path, - patched_datafinder, - config_user): content = dedent(""" preprocessors: preproc: area_statistics: operator: mean - fx_variables: [ - 'areacella', - 'areacello', - 'clayfrac', - 'sftlf', - 'sftgif', - 'sftof', - ] + fx_variables: + areacella: + areacello: + clayfrac: mask_landsea: mask_out: sea @@ -2333,25 +2323,26 @@ def test_fx_list_mip_change_cmip6_incomplete(tmp_path, settings = product.settings['area_statistics'] assert len(settings) == 1 assert settings['operator'] == 'mean' + + # Check mask_landsea + assert 'mask_landsea' in product.settings + settings = product.settings['mask_landsea'] + assert len(settings) == 1 + assert settings['mask_out'] == 'sea' + + # Check add_fx_variables fx_variables = product.settings['add_fx_variables']['fx_variables'] assert isinstance(fx_variables, dict) - assert len(fx_variables) == 6 + assert len(fx_variables) == 5 assert '_fx_' in fx_variables['areacella']['filename'] assert '_Ofx_' in fx_variables['areacello']['filename'] assert '_Efx_' in fx_variables['clayfrac']['filename'] assert '_fx_' in fx_variables['sftlf']['filename'] assert '_Ofx_' in fx_variables['sftof']['filename'] - # Check mask_landsea - assert 'mask_landsea' in product.settings - settings = product.settings['mask_landsea'] - assert len(settings) == 1 - assert settings['mask_out'] == 'sea' - -@pytest.mark.skipif(current_version < "2.4.0", - reason="github.com/ESMValGroup/ESMValCore/issues/1159") -def test_fx_list_mip_change_cmip6(tmp_path, patched_datafinder, config_user): +def test_fx_list_mip_search_cmip6(tmp_path, patched_datafinder, config_user): + """Test mip tables search for list of different fx variables.""" content = dedent(""" preprocessors: preproc: @@ -2362,11 +2353,8 @@ def test_fx_list_mip_change_cmip6(tmp_path, patched_datafinder, config_user): 'areacello', 'clayfrac', 'sftlf', - 'sftgif', 'sftof', ] - mask_landsea: - mask_out: sea diagnostics: diagnostic_name: @@ -2398,22 +2386,17 @@ def test_fx_list_mip_change_cmip6(tmp_path, patched_datafinder, config_user): settings = product.settings['area_statistics'] assert len(settings) == 1 assert settings['operator'] == 'mean' + + # Check add_fx_variables fx_variables = product.settings['add_fx_variables']['fx_variables'] assert isinstance(fx_variables, dict) - assert len(fx_variables) == 6 + assert len(fx_variables) == 5 assert '_fx_' in fx_variables['areacella']['filename'] assert '_Ofx_' in fx_variables['areacello']['filename'] assert '_Efx_' in fx_variables['clayfrac']['filename'] assert '_fx_' in fx_variables['sftlf']['filename'] - assert '_fx_' in fx_variables['sftgif']['filename'] # fails, see skipif assert '_Ofx_' in fx_variables['sftof']['filename'] - # Check mask_landsea - assert 'mask_landsea' in product.settings - settings = product.settings['mask_landsea'] - assert len(settings) == 1 - assert settings['mask_out'] == 'sea' - def test_fx_vars_volcello_in_ofx_cmip6(tmp_path, patched_datafinder, config_user): @@ -2743,6 +2726,7 @@ def test_wrong_project(tmp_path, patched_datafinder, config_user): def test_invalid_fx_var_cmip6(tmp_path, patched_datafinder, config_user): + """Test that error is raised for invalid fx variable.""" TAGS.set_tag_values(TAGS_FOR_TESTING) content = dedent(""" @@ -2778,6 +2762,100 @@ def test_invalid_fx_var_cmip6(tmp_path, patched_datafinder, config_user): assert msg in rec_err_exp.value.failed_tasks[0].message +def test_ambiguous_fx_var_cmip6(tmp_path, patched_datafinder, config_user): + """Test that error is raised for fx files available in multiple mips.""" + TAGS.set_tag_values(TAGS_FOR_TESTING) + + content = dedent(""" + preprocessors: + preproc: + area_statistics: + operator: mean + fx_variables: + volcello: + + diagnostics: + diagnostic_name: + variables: + tas: + preprocessor: preproc + project: CMIP6 + mip: Amon + exp: historical + start_year: 2000 + end_year: 2005 + ensemble: r1i1p1f1 + grid: gn + additional_datasets: + - {dataset: CanESM5} + scripts: null + """) + msg = ("Requested fx variable 'volcello' for dataset 'CanESM5' of project " + "'CMIP6' is available in more than one CMOR table for 'CMIP6': " + "['Odec', 'Ofx', 'Omon', 'Oyr']") + with pytest.raises(RecipeError) as rec_err_exp: + get_recipe(tmp_path, content, config_user) + assert str(rec_err_exp.value) == INITIALIZATION_ERROR_MSG + assert msg in rec_err_exp.value.failed_tasks[0].message + + +def test_unique_fx_var_in_multiple_mips_cmip6(tmp_path, + patched_failing_datafinder, + config_user): + """Test that no error is raised for fx files available in one mip.""" + TAGS.set_tag_values(TAGS_FOR_TESTING) + + content = dedent(""" + preprocessors: + preproc: + area_statistics: + operator: mean + fx_variables: + sftgif: + + diagnostics: + diagnostic_name: + variables: + tas: + preprocessor: preproc + project: CMIP6 + mip: Amon + exp: historical + start_year: 2000 + end_year: 2005 + ensemble: r1i1p1f1 + grid: gn + additional_datasets: + - {dataset: CanESM5} + scripts: null + """) + recipe = get_recipe(tmp_path, content, config_user) + + # Check generated tasks + assert len(recipe.tasks) == 1 + task = recipe.tasks.pop() + assert task.name == 'diagnostic_name' + TASKSEP + 'tas' + assert len(task.products) == 1 + product = task.products.pop() + + # Check area_statistics + assert 'area_statistics' in product.settings + settings = product.settings['area_statistics'] + assert len(settings) == 1 + assert settings['operator'] == 'mean' + + # Check add_fx_variables + # Due to failing datafinder, only files in LImon are found even though + # sftgif is available in the tables fx, IyrAnt, IyrGre and LImon + fx_variables = product.settings['add_fx_variables']['fx_variables'] + assert isinstance(fx_variables, dict) + assert len(fx_variables) == 1 + sftgif_files = fx_variables['sftgif']['filename'] + assert isinstance(sftgif_files, list) + assert len(sftgif_files) == 1 + assert'_LImon_' in sftgif_files[0] + + def test_multimodel_mask(tmp_path, patched_datafinder, config_user): """Test ``mask_multimodel``.""" content = dedent("""