From 5bd32d702a35cf54d5398dfdcb78b91125e590bd Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 21 Dec 2020 12:50:56 +0100 Subject: [PATCH 1/5] First attempt at a better error messages when data is missing --- esmvalcore/_recipe.py | 128 +++++++++++++++++++++-------------- esmvalcore/_recipe_checks.py | 19 ++++-- 2 files changed, 88 insertions(+), 59 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index c1100139e6..a1d5fd5207 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -442,21 +442,18 @@ def _update_fx_files(step_name, settings, variable, config_user, fx_vars): if not fx_vars: return - fx_vars = [ - _get_fx_file(variable, fxvar, config_user) - for fxvar in fx_vars - ] + fx_vars = [_get_fx_file(variable, fxvar, config_user) for fxvar in fx_vars] fx_dict = {fx_var[1]['short_name']: fx_var[0] for fx_var in fx_vars} settings['fx_variables'] = fx_dict logger.info('Using fx_files: %s for variable %s during step %s', - pformat(settings['fx_variables']), - variable['short_name'], + pformat(settings['fx_variables']), variable['short_name'], step_name) def _update_fx_settings(settings, variable, config_user): """Update fx settings depending on the needed method.""" + # get fx variables either from user defined attribute or fixed def _get_fx_vars_from_attribute(step_settings, step_name): user_fx_vars = step_settings.get('fx_variables') @@ -467,8 +464,8 @@ def _get_fx_vars_from_attribute(step_settings, step_name): user_fx_vars.append('sftof') elif step_name == 'mask_landseaice': user_fx_vars = ['sftgif'] - elif step_name in ('area_statistics', - 'volume_statistics', 'zonal_statistics'): + elif step_name in ('area_statistics', 'volume_statistics', + 'zonal_statistics'): user_fx_vars = [] return user_fx_vars @@ -480,8 +477,8 @@ def _get_fx_vars_from_attribute(step_settings, step_name): for step_name, step_settings in settings.items(): if step_name in fx_steps: fx_vars = _get_fx_vars_from_attribute(step_settings, step_name) - _update_fx_files(step_name, step_settings, - variable, config_user, fx_vars) + _update_fx_files(step_name, step_settings, variable, config_user, + fx_vars) def _read_attributes(filename): @@ -663,16 +660,12 @@ def get_matching(attributes): return grouped_products -def _get_preprocessor_products(variables, - profile, - order, - ancestor_products, - config_user): - """ - Get preprocessor product definitions for a set of datasets. +def _get_preprocessor_products(variables, profile, order, ancestor_products, + config_user, name): + """Get preprocessor product definitions for a set of datasets. - It updates recipe settings as needed by various preprocessors - and sets the correct ancestry. + It updates recipe settings as needed by various preprocessors and + sets the correct ancestry. """ products = set() for variable in variables: @@ -683,7 +676,7 @@ def _get_preprocessor_products(variables, grouped_ancestors = _match_products(ancestor_products, variables) else: grouped_ancestors = {} - + missing_vars = set() for variable in variables: settings = _get_default_settings( variable, @@ -692,30 +685,41 @@ def _get_preprocessor_products(variables, ) _apply_preprocessor_profile(settings, profile) _update_multi_dataset_settings(variable, settings) - _update_target_levels( - variable=variable, - variables=variables, - settings=settings, - config_user=config_user, - ) + try: + _update_target_levels( + variable=variable, + variables=variables, + settings=settings, + config_user=config_user, + ) + except RecipeError as ex: + missing_vars.add(ex.message) _update_extract_shape(settings, config_user) _update_weighting_settings(settings, variable) _update_fx_settings(settings=settings, variable=variable, config_user=config_user) - _update_target_grid( - variable=variable, - variables=variables, - settings=settings, - config_user=config_user, - ) + try: + _update_target_grid( + variable=variable, + variables=variables, + settings=settings, + config_user=config_user, + ) + except RecipeError as ex: + missing_vars.add(ex.message) _update_regrid_time(variable, settings) ancestors = grouped_ancestors.get(variable['filename']) if not ancestors: - ancestors = _get_ancestors(variable, config_user) - if config_user.get('skip-nonexistent') and not ancestors: - logger.info("Skipping: no data found for %s", variable) - continue + try: + ancestors = _get_ancestors(variable, config_user) + except RecipeError as ex: + if config_user.get('skip-nonexistent') and not ancestors: + logger.info("Skipping: %s", ex.message) + continue + else: + missing_vars.add(ex.message) + continue product = PreprocessorFile( attributes=variable, settings=settings, @@ -723,6 +727,11 @@ def _get_preprocessor_products(variables, ) products.add(product) + if missing_vars: + separator = "\n- " + raise RecipeError(f'Missing data for preprocessor {name}:{separator}' + f'{separator.join(sorted(missing_vars))}') + _update_statistic_settings(products, order, config_user['preproc_dir']) for product in products: @@ -751,7 +760,9 @@ def _get_single_preprocessor_task(variables, profile=profile, order=order, ancestor_products=ancestor_products, - config_user=config_user) + config_user=config_user, + name=name, + ) if not products: raise RecipeError( @@ -921,7 +932,13 @@ def __init__(self, raw_recipe['diagnostics'], raw_recipe.get('datasets', [])) self.entity = self._initialize_provenance( raw_recipe.get('documentation', {})) - self.tasks = self.initialize_tasks() if initialize_tasks else None + try: + self.tasks = self.initialize_tasks() if initialize_tasks else None + except RecipeError as ex: + logger.error(ex.message) + for task in ex.failed_tasks: + logger.error(task.message) + raise RecipeError('Some tasks could not be initalized') @staticmethod def _need_ncl(raw_diagnostics): @@ -985,8 +1002,7 @@ def _initialize_datasets(raw_datasets): @staticmethod def _expand_ensemble(variables): - """ - Expand ensemble members to multiple datasets. + """Expand ensemble members to multiple datasets. Expansion only supports ensembles defined as strings, not lists. """ @@ -1263,6 +1279,7 @@ def initialize_tasks(self): tasks = set() priority = 0 + failed_tasks = [] for diagnostic_name, diagnostic in self.diagnostics.items(): logger.info("Creating tasks for diagnostic %s", diagnostic_name) @@ -1270,17 +1287,21 @@ def initialize_tasks(self): for variable_group in diagnostic['preprocessor_output']: task_name = diagnostic_name + TASKSEP + variable_group logger.info("Creating preprocessor task %s", task_name) - task = _get_preprocessor_task( - variables=diagnostic['preprocessor_output'] - [variable_group], - profiles=self._preprocessors, - config_user=self._cfg, - task_name=task_name, - ) - for task0 in task.flatten(): - task0.priority = priority - tasks.add(task) - priority += 1 + try: + task = _get_preprocessor_task( + variables=diagnostic['preprocessor_output'] + [variable_group], + profiles=self._preprocessors, + config_user=self._cfg, + task_name=task_name, + ) + except RecipeError as ex: + failed_tasks.append(ex) + else: + for task0 in task.flatten(): + task0.priority = priority + tasks.add(task) + priority += 1 # Create diagnostic tasks for script_name, script_cfg in diagnostic['scripts'].items(): @@ -1295,7 +1316,10 @@ def initialize_tasks(self): task.priority = priority tasks.add(task) priority += 1 - + if failed_tasks: + ex = RecipeError('Could not create all tasks') + ex.failed_tasks = failed_tasks + raise ex check.tasks_valid(tasks) # Resolve diagnostic ancestors diff --git a/esmvalcore/_recipe_checks.py b/esmvalcore/_recipe_checks.py index 4072c52f01..3fcfe1d512 100644 --- a/esmvalcore/_recipe_checks.py +++ b/esmvalcore/_recipe_checks.py @@ -2,21 +2,26 @@ import itertools import logging import os +import re import subprocess from shutil import which -import re import yamale from ._data_finder import get_start_end_year from ._task import get_flattened_tasks -from .preprocessor import PreprocessingTask, TIME_PREPROCESSORS +from .preprocessor import TIME_PREPROCESSORS, PreprocessingTask logger = logging.getLogger(__name__) class RecipeError(Exception): """Recipe contains an error.""" + def __init__(self, msg): + self.message = msg + + def __str__(self): + return self.message def ncl_version(): @@ -116,7 +121,8 @@ def data_availability(input_files, var, dirnames, filenames): "Looked for files matching %s, but did not find any existing " "input directory", filenames) logger.error("Set 'log_level' to 'debug' to get more information") - raise RecipeError("Missing data") + raise RecipeError( + f"Missing data for {var['alias']}: {var['short_name']}") # check time avail only for non-fx variables if var['frequency'] == 'fx': @@ -185,10 +191,9 @@ def valid_multimodel_statistic(statistic): """Check that `statistic` is a valid argument for multimodel stats.""" valid_names = ["mean", "median", "std", "min", "max"] valid_patterns = [r"^(p\d{1,2})(\.\d*)?$"] - if not (statistic in valid_names or - re.match(r'|'.join(valid_patterns), statistic)): + if not (statistic in valid_names + or re.match(r'|'.join(valid_patterns), statistic)): raise RecipeError( "Invalid value encountered for `statistic` in preprocessor " f"`multi_model_statistics`. Valid values are {valid_names} " - f"or patterns matching {valid_patterns}. Got '{statistic}.'" - ) + f"or patterns matching {valid_patterns}. Got '{statistic}.'") From 0d8355306a725a58bf4d19b52c96b9dafcab41bf Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 21 Dec 2020 13:45:34 +0100 Subject: [PATCH 2/5] Fix tests --- tests/integration/test_recipe_checks.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_recipe_checks.py b/tests/integration/test_recipe_checks.py index dc984afe5b..3eb2626a28 100644 --- a/tests/integration/test_recipe_checks.py +++ b/tests/integration/test_recipe_checks.py @@ -5,7 +5,6 @@ import esmvalcore._recipe_checks as check - ERR_ALL = 'Looked for files matching%s' ERR_D = ('Looked for files in %s, but did not find any file pattern to match ' 'against') @@ -18,6 +17,7 @@ 'short_name': 'tas', 'start_year': 2020, 'end_year': 2025, + 'alias': 'alias', } FX_VAR = { 'filename': 'a/b.nc', @@ -33,15 +33,13 @@ 'a/b/c_20250101-20251231', ] - DATA_AVAILABILITY_DATA = [ (FILES, dict(VAR), None), (FILES, dict(FX_VAR), None), (FILES[:-1], dict(VAR), ERR_RANGE.format('2025', FILES[:-1])), (FILES[:-2], dict(VAR), ERR_RANGE.format('2024, 2025', FILES[:-2])), - ([FILES[1]] + [FILES[3]], dict(VAR), ERR_RANGE.format( - '2024, 2025, 2020, 2022', [FILES[1]] + [FILES[3]])), - + ([FILES[1]] + [FILES[3]], dict(VAR), + ERR_RANGE.format('2024, 2025, 2020, 2022', [FILES[1]] + [FILES[3]])), ] @@ -90,12 +88,13 @@ def test_data_availability_no_data(mock_logger, dirnames, filenames, error): 'short_name': 'tas', 'start_year': 2020, 'end_year': 2025, + 'alias': 'alias', } error_first = ('No input files found for variable %s', var_no_filename) error_last = ("Set 'log_level' to 'debug' to get more information", ) with pytest.raises(check.RecipeError) as rec_err: check.data_availability([], var, dirnames, filenames) - assert str(rec_err.value) == 'Missing data' + assert str(rec_err.value) == 'Missing data for alias: tas' if error is None: assert mock_logger.error.call_count == 2 errors = [error_first, error_last] From f577a2d1cbb7e0cf63101b8d5ef00569fb017f78 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 7 Jan 2021 13:19:27 +0100 Subject: [PATCH 3/5] Fix codacy issues --- esmvalcore/_recipe.py | 5 ++--- esmvalcore/_recipe_checks.py | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index b240cefdf0..525bd524a9 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -716,10 +716,9 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, except RecipeError as ex: if config_user.get('skip-nonexistent') and not ancestors: logger.info("Skipping: %s", ex.message) - continue else: missing_vars.add(ex.message) - continue + continue product = PreprocessorFile( attributes=variable, settings=settings, @@ -938,7 +937,7 @@ def __init__(self, logger.error(ex.message) for task in ex.failed_tasks: logger.error(task.message) - raise RecipeError('Some tasks could not be initalized') + raise RecipeError('Some tasks could not be initalized') from ex @staticmethod def _need_ncl(raw_diagnostics): diff --git a/esmvalcore/_recipe_checks.py b/esmvalcore/_recipe_checks.py index 3fcfe1d512..2c7b479c53 100644 --- a/esmvalcore/_recipe_checks.py +++ b/esmvalcore/_recipe_checks.py @@ -18,6 +18,7 @@ class RecipeError(Exception): """Recipe contains an error.""" def __init__(self, msg): + super().__init__(self) self.message = msg def __str__(self): From 33f6fd6abd9db3cbe448abd2eece72ba63d16403 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 26 Jan 2021 12:22:14 +0100 Subject: [PATCH 4/5] Refactor exception handling and fix tests --- esmvalcore/_recipe.py | 4 +- esmvalcore/_recipe_checks.py | 1 + tests/integration/test_recipe.py | 111 ++++++++++++++++--------------- 3 files changed, 59 insertions(+), 57 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index df1514ef39..acfac37de5 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -932,7 +932,7 @@ def __init__(self, logger.error(ex.message) for task in ex.failed_tasks: logger.error(task.message) - raise RecipeError('Some tasks could not be initalized') from ex + raise @staticmethod def _need_ncl(raw_diagnostics): @@ -1316,7 +1316,7 @@ def initialize_tasks(self): priority += 1 if failed_tasks: ex = RecipeError('Could not create all tasks') - ex.failed_tasks = failed_tasks + ex.failed_tasks.extend(failed_tasks) raise ex check.tasks_valid(tasks) diff --git a/esmvalcore/_recipe_checks.py b/esmvalcore/_recipe_checks.py index 2c7b479c53..06268ac12e 100644 --- a/esmvalcore/_recipe_checks.py +++ b/esmvalcore/_recipe_checks.py @@ -20,6 +20,7 @@ class RecipeError(Exception): def __init__(self, msg): super().__init__(self) self.message = msg + self.failed_tasks = [] def __str__(self): return self.message diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py index 7b1c04eb1b..b389c16acd 100644 --- a/tests/integration/test_recipe.py +++ b/tests/integration/test_recipe.py @@ -59,6 +59,8 @@ 'save', ) +INITIALIZATION_ERROR_MSG = 'Could not create all tasks' + @pytest.fixture def config_user(tmp_path): @@ -214,18 +216,22 @@ def find_files(_, filenames): @pytest.fixture def patched_tas_derivation(monkeypatch): - def get_required(short_name, _): if short_name != 'tas': assert False required = [ - {'short_name': 'pr'}, - {'short_name': 'areacella', 'mip': 'fx', 'optional': True}, + { + 'short_name': 'pr' + }, + { + 'short_name': 'areacella', + 'mip': 'fx', + 'optional': True + }, ] return required - monkeypatch.setattr( - esmvalcore._recipe, 'get_required', get_required) + monkeypatch.setattr(esmvalcore._recipe, 'get_required', get_required) DEFAULT_DOCUMENTATION = dedent(""" @@ -274,14 +280,13 @@ def test_recipe_no_datasets(tmp_path, config_user): end_year: 2002 scripts: null """) - exc_message = ( - "You have not specified any dataset " - "or additional_dataset groups for variable " - "{'preprocessor': 'preprocessor_name', 'project': 'CMIP5'," - " 'mip': 'Amon', 'exp': 'historical', 'ensemble': 'r1i1p1'" - ", 'start_year': 1999, 'end_year': 2002, 'variable_group':" - " 'ta', 'short_name': 'ta', 'diagnostic': " - "'diagnostic_name'} Exiting.") + exc_message = ("You have not specified any dataset " + "or additional_dataset groups for variable " + "{'preprocessor': 'preprocessor_name', 'project': 'CMIP5'," + " 'mip': 'Amon', 'exp': 'historical', 'ensemble': 'r1i1p1'" + ", 'start_year': 1999, 'end_year': 2002, 'variable_group':" + " 'ta', 'short_name': 'ta', 'diagnostic': " + "'diagnostic_name'} Exiting.") with pytest.raises(RecipeError) as exc: get_recipe(tmp_path, content, config_user) assert str(exc.value) == exc_message @@ -402,7 +407,8 @@ def test_fx_preproc_error(tmp_path, patched_datafinder, config_user): "permitted on fx vars, please remove them from recipe") with pytest.raises(Exception) as rec_err_exp: get_recipe(tmp_path, content, config_user) - assert str(rec_err_exp.value) == msg + assert str(rec_err_exp.value) == INITIALIZATION_ERROR_MSG + assert str(rec_err_exp.value.failed_tasks[0].message) == msg def test_default_preprocessor(tmp_path, patched_datafinder, config_user): @@ -503,9 +509,8 @@ def test_default_fx_preprocessor(tmp_path, patched_datafinder, config_user): preproc_dir = os.path.dirname(product.filename) assert preproc_dir.startswith(str(tmp_path)) - fix_dir = os.path.join( - preproc_dir, - 'CMIP5_CanESM2_fx_historical_r0i0p0_sftlf_fixed') + fix_dir = os.path.join(preproc_dir, + 'CMIP5_CanESM2_fx_historical_r0i0p0_sftlf_fixed') defaults = { 'load': { @@ -724,8 +729,7 @@ def test_cmip6_variable_autocomplete(tmp_path, patched_datafinder, assert variable[key] == reference[key] -def test_simple_cordex_recipe(tmp_path, patched_datafinder, - config_user): +def test_simple_cordex_recipe(tmp_path, patched_datafinder, config_user): """Test simple CORDEX recipe.""" content = dedent(""" diagnostics: @@ -954,24 +958,24 @@ def test_custom_preproc_order(tmp_path, patched_datafinder, config_user): elif task.name == 'diagnostic_name/chl_empty_custom': assert len(task.products) == 1 product = list(task.products)[0] - assert set(product.settings.keys()) == set( - DEFAULT_PREPROCESSOR_STEPS) + assert set( + product.settings.keys()) == set(DEFAULT_PREPROCESSOR_STEPS) elif task.name == 'diagnostic_name/chl_with_extract_time': assert len(task.products) == 1 product = list(task.products)[0] steps = set(DEFAULT_PREPROCESSOR_STEPS + tuple(['extract_time'])) assert set(product.settings.keys()) == steps assert product.settings['extract_time'] == { - 'start_year': 2001, - 'start_month': 3, - 'start_day': 14, - 'end_year': 2002, - 'end_month': 6, - 'end_day': 28, + 'start_year': 2001, + 'start_month': 3, + 'start_day': 14, + 'end_year': 2002, + 'end_month': 6, + 'end_day': 28, } assert product.settings['clip_start_end_year'] == { - 'start_year': 2000, - 'end_year': 2005, + 'start_year': 2000, + 'end_year': 2005, } else: assert False, f"invalid task {task.name}" @@ -1129,8 +1133,7 @@ def test_derive_with_fx_ohc(tmp_path, patched_datafinder, config_user): assert ancestor_product.filename in all_product_files -def test_derive_with_fx_ohc_fail(tmp_path, - patched_failing_datafinder, +def test_derive_with_fx_ohc_fail(tmp_path, patched_failing_datafinder, config_user): content = dedent(""" diagnostics: @@ -1156,10 +1159,8 @@ def test_derive_with_fx_ohc_fail(tmp_path, get_recipe(tmp_path, content, config_user) -def test_derive_with_optional_var(tmp_path, - patched_datafinder, - patched_tas_derivation, - config_user): +def test_derive_with_optional_var(tmp_path, patched_datafinder, + patched_tas_derivation, config_user): content = dedent(""" diagnostics: diagnostic_name: @@ -1197,8 +1198,7 @@ def test_derive_with_optional_var(tmp_path, # Check ancestors assert len(task.ancestors) == 2 - assert task.ancestors[0].name == ( - 'diagnostic_name/tas_derive_input_pr') + assert task.ancestors[0].name == ('diagnostic_name/tas_derive_input_pr') assert task.ancestors[1].name == ( 'diagnostic_name/tas_derive_input_areacella') for ancestor_product in task.ancestors[0].products: @@ -1209,10 +1209,8 @@ def test_derive_with_optional_var(tmp_path, assert ancestor_product.filename in all_product_files -def test_derive_with_optional_var_nodata(tmp_path, - patched_failing_datafinder, - patched_tas_derivation, - config_user): +def test_derive_with_optional_var_nodata(tmp_path, patched_failing_datafinder, + patched_tas_derivation, config_user): content = dedent(""" diagnostics: diagnostic_name: @@ -1250,8 +1248,7 @@ def test_derive_with_optional_var_nodata(tmp_path, # Check ancestors assert len(task.ancestors) == 1 - assert task.ancestors[0].name == ( - 'diagnostic_name/tas_derive_input_pr') + assert task.ancestors[0].name == ('diagnostic_name/tas_derive_input_pr') for ancestor_product in task.ancestors[0].products: assert ancestor_product.attributes['short_name'] == 'pr' assert ancestor_product.filename in all_product_files @@ -1331,10 +1328,10 @@ def simulate_diagnostic_run(diagnostic_task): def test_diagnostic_task_provenance( - tmp_path, - patched_datafinder, - monkeypatch, - config_user, + tmp_path, + patched_datafinder, + monkeypatch, + config_user, ): monkeypatch.setattr(esmvalcore._config, 'TAGS', TAGS) monkeypatch.setattr(esmvalcore._recipe, 'TAGS', TAGS) @@ -1614,8 +1611,10 @@ def test_extract_shape_raises(tmp_path, patched_datafinder, config_user, with pytest.raises(RecipeError) as exc: get_recipe(tmp_path, content, config_user) - assert 'extract_shape' in str(exc.value) - assert invalid_arg in str(exc.value) + + assert str(exc.value) == INITIALIZATION_ERROR_MSG + assert 'extract_shape' in exc.value.failed_tasks[0].message + assert invalid_arg in exc.value.failed_tasks[0].message def test_weighting_landsea_fraction(tmp_path, patched_datafinder, config_user): @@ -1799,7 +1798,8 @@ def test_weighting_landsea_fraction_exclude_fail(tmp_path, patched_datafinder, """) with pytest.raises(RecipeError) as exc_info: get_recipe(tmp_path, content, config_user) - assert str(exc_info.value) == ( + assert str(exc_info.value) == INITIALIZATION_ERROR_MSG + assert str(exc_info.value.failed_tasks[0].message) == ( 'Preprocessor landfrac_weighting uses alternative_dataset, but ' 'alternative_dataset is not defined for variable gpp of diagnostic ' 'diagnostic_name') @@ -2356,12 +2356,12 @@ def test_wrong_project(tmp_path, patched_datafinder, config_user): - {dataset: CanESM2} scripts: null """) - msg = ( - "Unable to load CMOR table (project) 'CMIP7' for variable 'tos' " - "with mip 'Omon'") + msg = ("Unable to load CMOR table (project) 'CMIP7' for variable 'tos' " + "with mip 'Omon'") with pytest.raises(RecipeError) as wrong_proj: get_recipe(tmp_path, content, config_user) - assert str(wrong_proj.value) == msg + assert str(wrong_proj.value) == INITIALIZATION_ERROR_MSG + assert str(wrong_proj.value.failed_tasks[0].message) == msg def test_invalid_fx_var_cmip6(tmp_path, patched_datafinder, config_user): @@ -2395,4 +2395,5 @@ def test_invalid_fx_var_cmip6(tmp_path, patched_datafinder, config_user): "'fx'-related CMOR table") with pytest.raises(RecipeError) as rec_err_exp: get_recipe(tmp_path, content, config_user) - assert msg in str(rec_err_exp.value) + assert str(rec_err_exp.value) == INITIALIZATION_ERROR_MSG + assert msg in rec_err_exp.value.failed_tasks[0].message From 2387da63e73d62527e650a0e2c506d30d36231c0 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 26 Jan 2021 12:40:20 +0100 Subject: [PATCH 5/5] Remove some codacy warnings --- esmvalcore/_recipe.py | 36 +++++++++++++++++++++--------------- esmvalcore/_recipe_checks.py | 1 + 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index acfac37de5..3a393d088c 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -689,21 +689,8 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, ) except RecipeError as ex: missing_vars.add(ex.message) - _update_extract_shape(settings, config_user) - _update_weighting_settings(settings, variable) - _update_fx_settings(settings=settings, - variable=variable, - config_user=config_user) - try: - _update_target_grid( - variable=variable, - variables=variables, - settings=settings, - config_user=config_user, - ) - except RecipeError as ex: - missing_vars.add(ex.message) - _update_regrid_time(variable, settings) + _update_preproc_functions(settings, config_user, variable, variables, + missing_vars) ancestors = grouped_ancestors.get(variable['filename']) if not ancestors: try: @@ -734,6 +721,25 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, return products +def _update_preproc_functions(settings, config_user, variable, variables, + missing_vars): + _update_extract_shape(settings, config_user) + _update_weighting_settings(settings, variable) + _update_fx_settings(settings=settings, + variable=variable, + config_user=config_user) + try: + _update_target_grid( + variable=variable, + variables=variables, + settings=settings, + config_user=config_user, + ) + except RecipeError as ex: + missing_vars.add(ex.message) + _update_regrid_time(variable, settings) + + def _get_single_preprocessor_task(variables, profile, config_user, diff --git a/esmvalcore/_recipe_checks.py b/esmvalcore/_recipe_checks.py index 06268ac12e..9bf96586de 100644 --- a/esmvalcore/_recipe_checks.py +++ b/esmvalcore/_recipe_checks.py @@ -23,6 +23,7 @@ def __init__(self, msg): self.failed_tasks = [] def __str__(self): + """Return message string.""" return self.message