diff --git a/pyproject.toml b/pyproject.toml index 051b95fa..f70deb41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,6 @@ classifiers = [ "Programming Language :: Python :: 3.12", ] dependencies = [ - # TODO definitely don't need all of these "cdsapi", "cytoolz", "fastparquet", @@ -37,7 +36,7 @@ dependencies = [ "matplotlib", "netcdf4", "netcomp@ git+https://github.com/barneydobson/NetComp.git", - "networkx", + "networkx>=3", "numpy", "osmnx", "pandas", @@ -90,6 +89,9 @@ select = ["D", "E", "F", "I"] # pydocstyle, pycodestyle, Pyflakes, isort [tool.ruff.pydocstyle] convention = "google" +[tool.ruff.lint.isort] +required-imports = ["from __future__ import annotations"] + [tool.codespell] skip = "swmmanywhere/defs/iso_converter.yml,*.inp" ignore-words-list = "gage,gages" diff --git a/swmmanywhere/__init__.py b/swmmanywhere/__init__.py index dde32fd0..d3431c22 100644 --- a/swmmanywhere/__init__.py +++ b/swmmanywhere/__init__.py @@ -1,2 +1,4 @@ """The main module for MyProject.""" +from __future__ import annotations + __version__ = "0.1.0" diff --git a/swmmanywhere/defs/basic_drainage_all_bits.inp b/swmmanywhere/defs/basic_drainage_all_bits.inp index 80059277..a03ec8cb 100644 --- a/swmmanywhere/defs/basic_drainage_all_bits.inp +++ b/swmmanywhere/defs/basic_drainage_all_bits.inp @@ -11,12 +11,12 @@ MIN_SLOPE 0 ALLOW_PONDING NO SKIP_STEADY_STATE NO -START_DATE 10/03/2020 +START_DATE 01/01/2000 START_TIME 00:00:00 -REPORT_START_DATE 10/03/2020 +REPORT_START_DATE 01/01/2000 REPORT_START_TIME 00:00:00 -END_DATE 10/08/2020 -END_TIME 00:00:00 +END_DATE 01/02/2000 +END_TIME 23:59:00 SWEEP_START 1/1 SWEEP_END 12/31 DRY_DAYS 0 diff --git a/swmmanywhere/defs/schema.yml b/swmmanywhere/defs/schema.yml index 3653df92..ace1e858 100644 --- a/swmmanywhere/defs/schema.yml +++ b/swmmanywhere/defs/schema.yml @@ -4,6 +4,7 @@ properties: project: {type: string} bbox: {type: array, items: {type: number}, minItems: 4, maxItems: 4} api_keys: {type: string} + model_number: {type: integer} run_settings: type: object properties: @@ -17,7 +18,7 @@ properties: real: type: ['object', 'null'] properties: - inp: {type: string} + inp: {type: ['string', 'null']} graph: {type: string} subcatchments: {type: string} results: {type: ['string', 'null']} @@ -30,4 +31,6 @@ properties: metric_list: {type: array, items: {type: string}} address_overrides: {type: ['object', 'null']} parameter_overrides: {type: ['object', 'null']} + parameters_to_sample: {type: ['array', 'null']} + sample_magnitude: {type: ['integer', 'null']} required: [base_dir, project, bbox, api_keys, graphfcn_list] \ No newline at end of file diff --git a/swmmanywhere/defs/storm.dat b/swmmanywhere/defs/storm.dat index 4b450941..97ed2c07 100644 --- a/swmmanywhere/defs/storm.dat +++ b/swmmanywhere/defs/storm.dat @@ -1,144 +1,5 @@ -1 2020 10 03 00 00 0.06383868499999999 -1 2020 10 03 01 00 0.041554296 -1 2020 10 03 02 00 0.05283641 -1 2020 10 03 03 00 0.06305583499999999 -1 2020 10 03 04 00 0.0631444 -1 2020 10 03 05 00 0.025762885 -1 2020 10 03 06 00 0.015543459999999999 -1 2020 10 03 07 00 0.002143075 -1 2020 10 03 08 00 0.0005809343000000001 -1 2020 10 03 09 00 0.00029047078 -1 2020 10 03 10 00 7.275957999999999e-09 -1 2020 10 03 11 00 7.275957999999999e-09 -1 2020 10 03 12 00 7.275957999999999e-09 -1 2020 10 03 13 00 7.275957999999999e-09 -1 2020 10 03 14 00 7.275957999999999e-09 -1 2020 10 03 15 00 7.275957999999999e-09 -1 2020 10 03 16 00 7.275957999999999e-09 -1 2020 10 03 17 00 7.275957999999999e-09 -1 2020 10 03 18 00 0.013442892 -1 2020 10 03 19 00 0.05766097 -1 2020 10 03 20 00 0.040084262 -1 2020 10 03 21 00 0.0140769625 -1 2020 10 03 22 00 0.012433353 -1 2020 10 03 23 00 0.0035387275 -1 2020 10 04 00 00 0.006939299 -1 2020 10 04 01 00 0.004810397500000001 -1 2020 10 04 02 00 0.0047572685 -1 2020 10 04 03 00 0.007219139699999999 -1 2020 10 04 04 00 0.0034714249 -1 2020 10 04 05 00 0.0043428226 -1 2020 10 04 06 00 0.006737391200000001 -1 2020 10 04 07 00 7.275957999999999e-09 -1 2020 10 04 08 00 7.275957999999999e-09 -1 2020 10 04 09 00 7.275957999999999e-09 -1 2020 10 04 10 00 0.0031667878 -1 2020 10 04 11 00 0.011328164 -1 2020 10 04 12 00 0.024292849999999998 -1 2020 10 04 13 00 0.020297179 -1 2020 10 04 14 00 0.022971587999999998 -1 2020 10 04 15 00 0.023254965000000002 -1 2020 10 04 16 00 0.026450084999999998 -1 2020 10 04 17 00 0.02760841 -1 2020 10 04 18 00 0.03214959 -1 2020 10 04 19 00 0.13477959 -1 2020 10 04 20 00 0.08938194000000001 -1 2020 10 04 21 00 0.09367871 -1 2020 10 04 22 00 0.07757912 -1 2020 10 04 23 00 0.038086422 -1 2020 10 05 00 00 0.0710649 -1 2020 10 05 01 00 0.025242174 -1 2020 10 05 02 00 0.019326595 -1 2020 10 05 03 00 0.055535613 -1 2020 10 05 04 00 0.025847905 -1 2020 10 05 05 00 0.012900928 -1 2020 10 05 06 00 0.0010201766 -1 2020 10 05 07 00 0.00010981603 -1 2020 10 05 08 00 7.275957999999999e-09 -1 2020 10 05 09 00 7.275957999999999e-09 -1 2020 10 05 10 00 7.275957999999999e-09 -1 2020 10 05 11 00 0.0005809343000000001 -1 2020 10 05 12 00 0.0005809343000000001 -1 2020 10 05 13 00 0.0026885828 -1 2020 10 05 14 00 0.009766023 -1 2020 10 05 15 00 0.03863193 -1 2020 10 05 16 00 0.018175357 -1 2020 10 05 17 00 0.0071270406 -1 2020 10 05 18 00 0.00036840356 -1 2020 10 05 19 00 0.06358718399999999 -1 2020 10 05 20 00 0.035277407999999996 -1 2020 10 05 21 00 0.057979774 -1 2020 10 05 22 00 0.06011222 -1 2020 10 05 23 00 0.06900684 -1 2020 10 06 00 00 0.07822027 -1 2020 10 06 01 00 0.15135737999999999 -1 2020 10 06 02 00 0.107465195 -1 2020 10 06 03 00 0.08602388 -1 2020 10 06 04 00 0.06740928 -1 2020 10 06 05 00 0.0111085465 -1 2020 10 06 06 00 0.0130497065 -1 2020 10 06 07 00 0.0067621877 -1 2020 10 06 08 00 0.0020120133 -1 2020 10 06 09 00 0.0049308364 -1 2020 10 06 10 00 0.008625414999999999 -1 2020 10 06 11 00 0.010024611000000001 -1 2020 10 06 12 00 0.019184903000000003 -1 2020 10 06 13 00 0.015600133 -1 2020 10 06 14 00 0.014664976 -1 2020 10 06 15 00 0.028685259999999997 -1 2020 10 06 16 00 0.099735975 -1 2020 10 06 17 00 0.05485196 -1 2020 10 06 18 00 0.06388118999999999 -1 2020 10 06 19 00 0.08494349 -1 2020 10 06 20 00 0.23213516 -1 2020 10 06 21 00 0.09611933 -1 2020 10 06 22 00 0.14322790000000002 -1 2020 10 06 23 00 0.03817852 -1 2020 10 07 00 00 0.014767706 -1 2020 10 07 01 00 0.017080797999999998 -1 2020 10 07 02 00 0.012560871 -1 2020 10 07 03 00 0.020651401 -1 2020 10 07 04 00 0.02854711 -1 2020 10 07 05 00 0.028164540000000002 -1 2020 10 07 06 00 0.020105894 -1 2020 10 07 07 00 0.00052780524 -1 2020 10 07 08 00 0.0014239921999999999 -1 2020 10 07 09 00 0.0008253555 -1 2020 10 07 10 00 0.007109331 -1 2020 10 07 11 00 0.06388118999999999 -1 2020 10 07 12 00 0.039464365 -1 2020 10 07 13 00 0.025624737 -1 2020 10 07 14 00 0.06524496 -1 2020 10 07 15 00 0.08188297 -1 2020 10 07 16 00 0.0747276 -1 2020 10 07 17 00 0.04946063 -1 2020 10 07 18 00 0.08798275 -1 2020 10 07 19 00 0.019656029 -1 2020 10 07 20 00 0.005929752700000001 -1 2020 10 07 21 00 0.0019057406 -1 2020 10 07 22 00 0.0037654318000000003 -1 2020 10 07 23 00 0.0055011405 -1 2020 10 08 00 00 0.010166303 -1 2020 10 08 01 00 0.010956223999999999 -1 2020 10 08 02 00 0.0071553804 -1 2020 10 08 03 00 0.0034112018 -1 2020 10 08 04 00 0.00032943353 -1 2020 10 08 05 00 7.275957999999999e-09 -1 2020 10 08 06 00 7.275957999999999e-09 -1 2020 10 08 07 00 7.275957999999999e-09 -1 2020 10 08 08 00 7.275957999999999e-09 -1 2020 10 08 09 00 7.275957999999999e-09 -1 2020 10 08 10 00 7.275957999999999e-09 -1 2020 10 08 11 00 7.275957999999999e-09 -1 2020 10 08 12 00 7.275957999999999e-09 -1 2020 10 08 13 00 7.275957999999999e-09 -1 2020 10 08 14 00 0.0019942962999999997 -1 2020 10 08 15 00 0.017066632 -1 2020 10 08 16 00 0.1024281 -1 2020 10 08 17 00 0.09838992 -1 2020 10 08 18 00 0.04274804 -1 2020 10 08 19 00 0.013393306 -1 2020 10 08 20 00 0.005975802 -1 2020 10 08 21 00 0.0024016626 -1 2020 10 08 22 00 0.01663802 -1 2020 10 08 23 00 0.048688416 +;File: "storm.dat" +1 2000 01 01 00 00 0.0 +1 2000 01 01 00 05 28 +1 2000 01 01 00 10 32 +1 2000 01 01 00 15 3 \ No newline at end of file diff --git a/swmmanywhere/geospatial_utilities.py b/swmmanywhere/geospatial_utilities.py index 47af36d1..712e775a 100644 --- a/swmmanywhere/geospatial_utilities.py +++ b/swmmanywhere/geospatial_utilities.py @@ -3,6 +3,8 @@ A module containing functions to perform a variety of geospatial operations, such as reprojecting coordinates and handling raster data. """ +from __future__ import annotations + import itertools import json import math @@ -25,6 +27,7 @@ from scipy.interpolate import RegularGridInterpolator from shapely import geometry as sgeom from shapely import ops as sops +from shapely.errors import GEOSException from shapely.strtree import STRtree from tqdm import tqdm @@ -655,8 +658,13 @@ def derive_rc(polys_gdf: gpd.GeoDataFrame, building_footprints[['geometry']], how='union') result = gpd.overlay(polys_gdf, result) - - dissolved_result = result.dissolve(by='id').reset_index() + try: + dissolved_result = result.dissolve(by='id').reset_index() + except GEOSException: + # Temporary fix for bug: + # https://github.com/ImperialCollegeLondon/SWMManywhere/issues/115 + result['geometry'] = result['geometry'].simplify(0.1) + dissolved_result = result.dissolve(by='id').reset_index() dissolved_result['impervious_area'] = dissolved_result.geometry.area polys_gdf = pd.merge(polys_gdf, dissolved_result[['id','impervious_area']], diff --git a/swmmanywhere/graph_utilities.py b/swmmanywhere/graph_utilities.py index a2262d71..db89f43f 100644 --- a/swmmanywhere/graph_utilities.py +++ b/swmmanywhere/graph_utilities.py @@ -3,6 +3,8 @@ A module to contain graphfcns, the graphfcn registry object, and other graph utilities (such as save/load functions). """ +from __future__ import annotations + import json import os import tempfile @@ -695,11 +697,12 @@ def __call__(self, G: nx.Graph, bounds[w][1] = max(bounds[w][1], d.get(w, -np.Inf)) G = G.copy() + eps = np.finfo(float).eps for u, v, d in G.edges(data=True): total_weight = 0 for attr, bds in bounds.items(): # Normalise - weight = (d[attr] - bds[0]) / (bds[1] - bds[0]) + weight = max((d[attr] - bds[0]) / (bds[1] - bds[0]), eps) # Exponent weight = weight ** getattr(topology_derivation,f'{attr}_exponent') # Scaling @@ -845,7 +848,7 @@ def __call__(self, G: nx.Graph, # Check for negative cycles if nx.negative_edge_cycle(G, weight = 'weight'): - raise ValueError('Graph contains negative cycle') + logger.warning('Graph contains negative cycle') # Initialize the dictionary with infinity for all nodes shortest_paths = {node: float('inf') for node in G.nodes} diff --git a/swmmanywhere/logging.py b/swmmanywhere/logging.py index 0ad1a655..1906886e 100644 --- a/swmmanywhere/logging.py +++ b/swmmanywhere/logging.py @@ -10,6 +10,8 @@ >>> logger.add("file.log") # Add a log file # doctest: +SKIP >>> os.environ["SWMMANYWHERE_VERBOSE"] = "false" # Disable logging """ +from __future__ import annotations + import os import sys diff --git a/swmmanywhere/metric_utilities.py b/swmmanywhere/metric_utilities.py index 02ecde5d..fb2406a6 100644 --- a/swmmanywhere/metric_utilities.py +++ b/swmmanywhere/metric_utilities.py @@ -3,10 +3,11 @@ A module for metrics, the metrics registry object and utilities for calculating metrics (such as NSE or timeseries data alignment) used in SWMManywhere. """ +from __future__ import annotations + from collections import defaultdict -from inspect import signature from itertools import product -from typing import Callable, Optional +from typing import Callable, Optional, get_type_hints import cytoolz.curried as tlz import geopandas as gpd @@ -18,11 +19,18 @@ import shapely from scipy import stats +from swmmanywhere.logging import logger from swmmanywhere.parameters import MetricEvaluation class MetricRegistry(dict): """Registry object.""" + def _log_completion(self, func): + def _wrapper(*args, **kwargs): + result = func(*args, **kwargs) + logger.info(f'{func.__name__} completed') + return result + return _wrapper def register(self, func: Callable) -> Callable: """Register a metric.""" @@ -37,17 +45,20 @@ def register(self, func: Callable) -> Callable: "real_G": nx.Graph, "metric_evaluation": MetricEvaluation} - sig = signature(func) - for param, obj in sig.parameters.items(): - if param == 'kwargs': + # Use get_type_hints to resolve annotations, + # considering 'from __future__ import annotations' + type_hints = get_type_hints(func) + + for param, annotation in type_hints.items(): + if param in ('kwargs', 'return'): continue if param not in allowable_params: raise ValueError(f"{param} of {func.__name__} not allowed.") - if obj.annotation != allowable_params[param]: + if annotation != allowable_params[param]: raise ValueError(f"""{param} of {func.__name__} should be of type {allowable_params[param]}, not - {obj.__class__}.""") - self[func.__name__] = func + {annotation}.""") + self[func.__name__] = self._log_completion(func) return func def __getattr__(self, name): @@ -102,8 +113,8 @@ def iterate_metrics(synthetic_results: pd.DataFrame, def extract_var(df: pd.DataFrame, var: str) -> pd.DataFrame: """Extract var from a dataframe.""" - df_ = df.loc[df.variable == var] - df_['duration'] = (df_.date - \ + df_ = df.loc[df.variable == var].copy() + df_.loc[:,'duration'] = (df_.date - \ df_.date.min()).dt.total_seconds() return df_ @@ -111,24 +122,33 @@ def align_calc_nse(synthetic_results: pd.DataFrame, real_results: pd.DataFrame, variable: str, syn_ids: list, - real_ids: list) -> float: + real_ids: list) -> float | None: """Align and calculate NSE. Align the synthetic and real data and calculate the Nash-Sutcliffe efficiency (NSE) of the variable over time. In cases where the synthetic data is does not overlap the real data, the value is interpolated. """ + synthetic_results = synthetic_results.copy() + real_results = real_results.copy() + # Format dates synthetic_results['date'] = pd.to_datetime(synthetic_results['date']) real_results['date'] = pd.to_datetime(real_results['date']) + # Help alignment + synthetic_results["id"] = synthetic_results["id"].astype(str) + real_results["id"] = real_results["id"].astype(str) + syn_ids = [str(x) for x in syn_ids] + real_ids = [str(x) for x in real_ids] + # Extract data syn_data = extract_var(synthetic_results, variable) - syn_data = syn_data.loc[syn_data.id.isin(syn_ids)] + syn_data = syn_data.loc[syn_data["id"].isin(syn_ids)] syn_data = syn_data.groupby('date').value.sum() real_data = extract_var(real_results, variable) - real_data = real_data.loc[real_data.id.isin(real_ids)] + real_data = real_data.loc[real_data["id"].isin(real_ids)] real_data = real_data.groupby('date').value.sum() # Align data @@ -176,12 +196,14 @@ def create_subgraph(G: nx.Graph, return SG def nse(y: np.ndarray, - yhat: np.ndarray) -> float: + yhat: np.ndarray) -> float | None: """Calculate Nash-Sutcliffe efficiency (NSE).""" + if np.std(y) == 0: + return np.inf return 1 - np.sum((y - yhat)**2) / np.sum((y - np.mean(y))**2) def median_nse_by_group(results: pd.DataFrame, - gb_key: str) -> float: + gb_key: str) -> float | None: """Median NSE by group. Calculate the median Nash-Sutcliffe efficiency (NSE) of a variable over time @@ -203,7 +225,9 @@ def median_nse_by_group(results: pd.DataFrame, .groupby(gb_key) .apply(lambda x: nse(x.value_real, x.value_sim)) .median() - ) + ) + if not np.isfinite(val): + return None return val @@ -284,7 +308,7 @@ def dominant_outlet(G: nx.DiGraph, # Identify the outlet with the highest flow outlet_flows = results.loc[(results.variable == 'flow') & - (results.id.isin(outlet_arcs))] + (results["id"].isin(outlet_arcs))] max_outlet_arc = outlet_flows.groupby('id').value.median().idxmax() max_outlet = [v for u,v,d in G.edges(data=True) if d['id'] == max_outlet_arc][0] @@ -346,15 +370,20 @@ def align_by_shape(var, real_results = extract_var(real_results, var) synthetic_results = extract_var(synthetic_results, var) + # Format to help alignment + real_results["id"] = real_results["id"].astype(str) + synthetic_results["id"] = synthetic_results["id"].astype(str) + real_joined["id"] = real_joined["id"].astype(str) + synthetic_joined["id"] = synthetic_joined["id"].astype(str) + + # Align data synthetic_results = pd.merge(synthetic_results, synthetic_joined[['id','sub_id']], - left_on='object', - right_on = 'id') + on='id') real_results = pd.merge(real_results, real_joined[['id','sub_id']], - left_on='object', - right_on = 'id') + on='id') results = pd.merge(real_results[['date','sub_id','value']], synthetic_results[['date','sub_id','value']], @@ -513,7 +542,7 @@ def outlet_nse_flow(synthetic_G: nx.Graph, real_G: nx.Graph, real_results: pd.DataFrame, real_subs: gpd.GeoDataFrame, - **kwargs) -> float: + **kwargs) -> float | None: """Outlet NSE flow. Calculate the Nash-Sutcliffe efficiency (NSE) of flow over time, where flow @@ -542,7 +571,7 @@ def outlet_nse_flooding(synthetic_G: nx.Graph, real_G: nx.Graph, real_results: pd.DataFrame, real_subs: gpd.GeoDataFrame, - **kwargs) -> float: + **kwargs) -> float | None: """Outlet NSE flooding. Calculate the Nash-Sutcliffe efficiency (NSE) of flooding over time, where @@ -684,7 +713,7 @@ def subcatchment_nse_flooding(synthetic_G: nx.Graph, synthetic_results: pd.DataFrame, real_results: pd.DataFrame, real_subs: gpd.GeoDataFrame, - **kwargs) -> float: + **kwargs) -> float | None: """Subcatchment NSE flooding. Classify synthetic nodes to real subcatchments and calculate the NSE of @@ -707,7 +736,7 @@ def grid_nse_flooding(synthetic_G: nx.Graph, real_results: pd.DataFrame, real_subs: gpd.GeoDataFrame, metric_evaluation: MetricEvaluation, - **kwargs) -> float: + **kwargs) -> float | None: """Grid NSE flooding. Classify synthetic nodes to a grid and calculate the NSE of diff --git a/swmmanywhere/misc/debug_topology.py b/swmmanywhere/misc/debug_topology.py index 2593779a..f7b32722 100644 --- a/swmmanywhere/misc/debug_topology.py +++ b/swmmanywhere/misc/debug_topology.py @@ -3,6 +3,8 @@ @author: Barnaby Dobson """ +from __future__ import annotations + from pathlib import Path from time import time diff --git a/swmmanywhere/paper/experimenter.py b/swmmanywhere/paper/experimenter.py new file mode 100644 index 00000000..88760139 --- /dev/null +++ b/swmmanywhere/paper/experimenter.py @@ -0,0 +1,246 @@ +"""The experimenter module is used to sample and run SWMManywhere. + +This module is designed to be run in parallel as a jobarray. It generates +parameter samples and runs the SWMManywhere model for each sample. The results +are saved to a csv file in a results directory. +""" +from __future__ import annotations + +import argparse +import os +from collections import defaultdict +from pathlib import Path + +import pandas as pd +import toolz as tlz +from SALib.sample import sobol + +# Set the number of threads to 1 to avoid conflicts with parallel processing +# for pysheds (at least I think that is what is happening) +os.environ['NUMBA_NUM_THREADS'] = '1' +os.environ['OMP_NUM_THREADS'] = '1' + +from swmmanywhere import swmmanywhere # noqa: E402 +from swmmanywhere.logging import logger # noqa: E402 +from swmmanywhere.parameters import get_full_parameters_flat # noqa: E402 + +os.environ['SWMMANYWHERE_VERBOSE'] = "true" + +def formulate_salib_problem(parameters_to_select: + list[str | dict] | None = None) -> dict: + """Formulate a SALib problem for a sensitivity analysis. + + Args: + parameters_to_select (list, optional): List of parameters to include in + the analysis, if a list entry is a dictionary, the value is the + bounds, otherwise the bounds are taken from the parameters file. + Defaults to None. + + Returns: + dict: A dictionary containing the problem formulation. + """ + # Set as empty by default + parameters_to_select = [] if parameters_to_select is None else parameters_to_select + + # Get all parameters schema + parameters = get_full_parameters_flat() + names = [] + bounds = [] + dists = [] + groups = [] + + for parameter in parameters_to_select: + if isinstance(parameter, dict): + bound = next(iter(parameter.values())) + parameter = next(iter(parameter)) + else: + bound = [parameters[parameter]['minimum'], + parameters[parameter]['maximum']] + + names.append(parameter) + bounds.append(bound) + dists.append(parameters[parameter].get('dist', 'unif')) + groups.append(parameters[parameter]['category']) + return {'num_vars': len(names), 'names': names, 'bounds': bounds, + 'dists': dists, 'groups': groups} + +def generate_samples(N: int | None = None, + parameters_to_select: list[str | dict] = [], + seed: int = 1, + groups: bool = False, + calc_second_order: bool = True) -> list[dict]: + """Generate samples for a sensitivity analysis. + + Args: + N (int, optional): Number of samples to generate. Defaults to None. + parameters_to_select (list, optional): List of parameters to include in + the analysis, if a list entry is a dictionary, the value is the + bounds, otherwise the bounds are taken from the parameters file. + Defaults to []. + seed (int, optional): Random seed. Defaults to 1. + groups (bool, optional): Whether to sample by group, True, or by + parameter, False (significantly changes how many samples are taken). + Defaults to False. + calc_second_order (bool, optional): Whether to calculate second order + indices. Defaults to True. + + Returns: + list: A list of dictionaries containing the parameter values. + """ + problem = formulate_salib_problem(parameters_to_select) + + if N is None: + N = 2 ** (problem['num_vars'] - 1) + + # If we are not grouping, we need to remove the groups from the problem to + # pass to SAlib, but we retain the groups information for the output + # regardless + problem_ = problem.copy() + + if not groups: + del problem_['groups'] + + # Sample + param_values = sobol.sample(problem_, + N, + calc_second_order=calc_second_order, + seed = seed) + # Store samples + X = [ + {'param': y, 'value': z, 'iter': ix, 'group': x} + for ix, params in enumerate(param_values) + for x, y, z in zip(problem['groups'], problem['names'], params, strict=True) + ] + return X + +def process_parameters(jobid: int, + nproc: int | None, + config_base: dict) -> tuple[dict[int, dict], Path]: + """Generate and run parameter samples for the sensitivity analysis. + + This function generates parameter samples and runs the swmmanywhere model + for each sample. It is designed to be run in parallel as a jobarray. + + Args: + jobid (int): The job id. + nproc (int | None): The number of processors to use. If None, the number + of samples is used (i.e., only one model is simulated). + config_base (dict): The base configuration dictionary. + + Returns: + dict[dict]: A dict (keys as models) of dictionaries containing the results. + Path: The path to the inp file. + """ + # Generate samples + X = generate_samples(parameters_to_select=config_base['parameters_to_sample'], + N=2**config_base['sample_magnitude']) + + df = pd.DataFrame(X) + gb = df.groupby('iter') + + flooding_results = {} + nproc = nproc if nproc is not None else len(X) + + # Assign jobs based on jobid + job_iter = tlz.partition_all(nproc, range(len(X))) + for _ in range(jobid + 1): + job_idx = next(job_iter, None) + + if job_idx is None: + raise ValueError(f"Jobid {jobid} is required.") + + config = config_base.copy() + + # Iterate over the samples, running the model when the jobid matches the + # processor number + for ix in job_idx: + config = config_base.copy() + params_ = gb.get_group(ix) + + # Update the parameters + overrides: dict = defaultdict(dict) + for grp, param, val in params_[["group", + "param", + "value"]].itertuples(index=False, + name=None): + if grp not in overrides: + overrides[grp] = {} + overrides[grp][param] = val + config['parameter_overrides'].update(overrides) + + # Run the model + config['model_number'] = ix + address, metrics = swmmanywhere.swmmanywhere(config) + + if metrics is None: + raise ValueError(f"Model run {ix} failed.") + + # Save the results + flooding_results[ix] = {'iter': ix, + **metrics, + **params_.set_index('param').value.to_dict()} + return flooding_results, address + +def save_results(jobid: int, results: dict[int, dict], address: Path) -> None: + """Save the results of the sensitivity analysis. + + A results directory is created in the addresses.bbox directory, and the + results are saved to a csv file there, labelled by jobid. + + Args: + jobid (int): The job id. + results (dict[str, dict]): A list of dictionaries containing the results. + address (Path): The path to the inp file + """ + results_fid = address.parent.parent / 'results' + results_fid.mkdir(parents=True, exist_ok=True) + fid_flooding = results_fid / f'{jobid}_metrics.csv' + df = pd.DataFrame(results).T + df['jobid'] = jobid + df.to_csv(fid_flooding, index=False) + +def parse_arguments() -> tuple[int, int | None, Path]: + """Parse the command line arguments. + + Returns: + tuple: A tuple containing the job id, number of processors, and the + configuration file path. + """ + parser = argparse.ArgumentParser(description='Process command line arguments.') + parser.add_argument('--jobid', + type=int, + default=1, + help='Job ID') + parser.add_argument('--nproc', + type=int, + default=None, + help='Number of processors') + parser.add_argument('--config_path', + type=Path, + default=Path(__file__).parent.parent.parent / 'tests' /\ + 'test_data' / 'demo_config_sa.yml', + help='Configuration file path') + + args = parser.parse_args() + + return args.jobid, args.nproc, args.config_path + +if __name__ == '__main__': + # Get args + jobid, nproc, config_path = parse_arguments() + + # Set up logging + logger.add(config_path.parent / f'experimenter_{jobid}.log') + + # Load the configuration + config_base = swmmanywhere.load_config(config_path) + + # Ensure the parameter overrides are set, since these are the way the + # sampled parameter values are implemented + config_base['parameter_overrides'] = config_base.get('parameter_overrides') or {} + + # Sample and run + flooding_results, address = process_parameters(jobid, nproc, config_base) + + # Save the results + save_results(jobid, flooding_results, address) diff --git a/swmmanywhere/paper/submit_icl_example b/swmmanywhere/paper/submit_icl_example new file mode 100644 index 00000000..212f0880 --- /dev/null +++ b/swmmanywhere/paper/submit_icl_example @@ -0,0 +1,22 @@ +#PBS -lselect=1:ncpus=1:mem=20gb +#PBS -lwalltime=01:00:00 +#PBS -J 0-2 + +# subjob's index within the array + +## All subjobs run independently of one another + + +# Load modules for any applications + +module load anaconda3/personal + +source activate swmmanywhere + +# Change to the submission directory + +cd $PBS_O_WORKDIR + +# Run program, passing the index of this subjob within the array + +python experimenter.py $PBS_ARRAY_INDEX 7000 /rds/general/user/bdobson/ephemeral/swmmanywhere/cranbrook/cranbrook_hpc.yml diff --git a/swmmanywhere/parameters.py b/swmmanywhere/parameters.py index b2d70c8d..32724c13 100644 --- a/swmmanywhere/parameters.py +++ b/swmmanywhere/parameters.py @@ -1,4 +1,5 @@ """Parameters and file paths module for SWMManywhere.""" +from __future__ import annotations from pathlib import Path @@ -16,6 +17,20 @@ def get_full_parameters(): "metric_evaluation": MetricEvaluation() } +def get_full_parameters_flat(): + """Get the full set of parameters in a flat format.""" + parameters = get_full_parameters() + # Flatten + # parameters_flat = {k : {**y, **{'category' : cat}} + # for cat,v in parameters.items() + # for k, y in v.model_json_schema()['properties'].items()} + parameters_flat = {} + for cat, v in parameters.items(): + for k, y in v.schema()['properties'].items(): + parameters_flat[k] = {**y, **{'category' : cat}} + + return parameters_flat + class SubcatchmentDerivation(BaseModel): """Parameters for subcatchment derivation.""" lane_width: float = Field(default = 3.5, diff --git a/swmmanywhere/post_processing.py b/swmmanywhere/post_processing.py index 3f1ab9c5..655a9457 100644 --- a/swmmanywhere/post_processing.py +++ b/swmmanywhere/post_processing.py @@ -3,6 +3,8 @@ A module containing functions to format and write processed data into SWMM .inp files. """ +from __future__ import annotations + import re import shutil from pathlib import Path diff --git a/swmmanywhere/prepare_data.py b/swmmanywhere/prepare_data.py index 79439fc1..2c04a1ab 100644 --- a/swmmanywhere/prepare_data.py +++ b/swmmanywhere/prepare_data.py @@ -2,6 +2,7 @@ A module to download data needed for SWMManywhere. """ +from __future__ import annotations import shutil from pathlib import Path diff --git a/swmmanywhere/preprocessing.py b/swmmanywhere/preprocessing.py index 95de336d..b4aae488 100644 --- a/swmmanywhere/preprocessing.py +++ b/swmmanywhere/preprocessing.py @@ -4,6 +4,7 @@ for graphfcns, and some other utilities (such as creating a project folder structure or create the starting graph from rivers/streets). """ +from __future__ import annotations import json import tempfile @@ -93,7 +94,8 @@ def get_next_bbox_number(bbox: tuple[float, float, float, float], def create_project_structure(bbox: tuple[float, float, float, float], project: str, - base_dir: Path): + base_dir: Path, + model_number: int | None = None): """Create the project directory structure. Create the project, bbox, national, model and download directories within @@ -104,6 +106,9 @@ def create_project_structure(bbox: tuple[float, float, float, float], the format (minx, miny, maxx, maxy). project (str): Name of the project. base_dir (Path): Path to the base directory. + model_number (int | None): Model number, if not provided it will use a + number that is one higher than the highest number that exists for + that bbox. Returns: Addresses: Class containing the addresses of the directories. @@ -122,14 +127,19 @@ def create_project_structure(bbox: tuple[float, float, float, float], addresses.bbox_number = bbox_number addresses.bbox.mkdir(parents=True, exist_ok=True) bounding_box_info = {"bbox": bbox, "project": project} - with open(addresses.bbox / 'bounding_box_info.json', 'w') as info_file: - json.dump(bounding_box_info, info_file, indent=2) + if not (addresses.bbox / 'bounding_box_info.json').exists(): + with open(addresses.bbox / 'bounding_box_info.json', 'w') as info_file: + json.dump(bounding_box_info, info_file, indent=2) # Create downloads directory addresses.download.mkdir(parents=True, exist_ok=True) # Create model directory - addresses.model_number = next_directory('model', addresses.bbox) + if not model_number: + addresses.model_number = next_directory('model', addresses.bbox) + else: + addresses.model_number = model_number + addresses.model.mkdir(parents=True, exist_ok=True) return addresses diff --git a/swmmanywhere/swmmanywhere.py b/swmmanywhere/swmmanywhere.py index 5d9decb1..052fb687 100644 --- a/swmmanywhere/swmmanywhere.py +++ b/swmmanywhere/swmmanywhere.py @@ -1,4 +1,6 @@ """The main SWMManywhere module to generate and run a synthetic network.""" +from __future__ import annotations + import os from pathlib import Path @@ -9,33 +11,36 @@ import yaml import swmmanywhere.geospatial_utilities as go -from swmmanywhere import preprocessing +from swmmanywhere import parameters, preprocessing from swmmanywhere.graph_utilities import iterate_graphfcns, load_graph, save_graph from swmmanywhere.logging import logger from swmmanywhere.metric_utilities import iterate_metrics -from swmmanywhere.parameters import get_full_parameters from swmmanywhere.post_processing import synthetic_write -def swmmanywhere(config: dict): +def swmmanywhere(config: dict) -> tuple[Path, dict | None]: """Run SWMManywhere processes. This function runs the SWMManywhere processes, including downloading data, preprocessing the graphfcns, running the model, and comparing the results - to real data using metrics. + to real data using metrics. The function will always return the path to + the generated .inp file. If real data (either a results file or the .inp, + as well as graph, and subcatchments) is provided, the function will also + return the metrics comparing the synthetic network with the real. Args: config (dict): The loaded config as a dict. Returns: - pd.DataFrame: A DataFrame containing the results. + tuple[Path, dict | None]: The address of generated .inp and metrics. """ # Create the project structure addresses = preprocessing.create_project_structure(config['bbox'], - config['project'], - config['base_dir'] - ) - + config['project'], + config['base_dir'], + config.get('model_number',None) + ) + for key, val in config.get('address_overrides', {}).items(): setattr(addresses, key, val) @@ -53,15 +58,15 @@ def swmmanywhere(config: dict): G = preprocessing.create_starting_graph(addresses) # Load the parameters and perform any manual overrides - parameters = get_full_parameters() + params = parameters.get_full_parameters() for category, overrides in config.get('parameter_overrides', {}).items(): for key, val in overrides.items(): - setattr(parameters[category], key, val) + setattr(params[category], key, val) # Iterate the graph functions G = iterate_graphfcns(G, config['graphfcn_list'], - parameters, + params, addresses) # Save the final graph @@ -103,7 +108,7 @@ def swmmanywhere(config: dict): gpd.read_file(config['real']['subcatchments']), load_graph(config['real']['graph']), config['metric_list'], - parameters['metric_evaluation']) + params['metric_evaluation']) return addresses.inp, metrics @@ -165,6 +170,42 @@ def check_real_network_paths(config: dict): return config +def check_parameters_to_sample(config: dict): + """Check the parameters to sample in the config. + + Args: + config (dict): The configuration. + + Raises: + ValueError: If a parameter to sample is not in the parameters + dictionary. + """ + params = parameters.get_full_parameters_flat() + for param in config.get('parameters_to_sample',{}): + # If the parameter is a dictionary, the values are bounds, all we are + # checking here is that the parameter exists, we only need the first + # entry. + if isinstance(param, dict): + if len(param) > 1: + raise ValueError("""If providing new bounds in the config, a dict + of len 1 is required, where the key is the + parameter to change and the values are + (new_lower_bound, new_upper_bound).""") + param = list(param.keys())[0] + + # Check that the parameter is available + if param not in params: + raise ValueError(f"{param} not found in parameters dictionary.") + + # Check that the parameter is sample-able + required_attrs = set(['minimum', 'maximum', 'default', 'category']) + correct_attrs = required_attrs.intersection(params[param]) + missing_attrs = required_attrs.difference(correct_attrs) + if any(missing_attrs): + raise ValueError(f"{param} missing {missing_attrs} so cannot be sampled.") + + return config + def load_config(config_path: Path): """Load, validate, and convert Paths in a configuration file. @@ -195,6 +236,9 @@ def load_config(config_path: Path): # Check real network paths config = check_real_network_paths(config) + # Check the parameters to sample + config = check_parameters_to_sample(config) + return config diff --git a/tests/__init__.py b/tests/__init__.py index 76d8514b..23406cab 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,4 +1,5 @@ """Unit tests for MyProject.""" +from __future__ import annotations from logging import getLogger diff --git a/tests/conftest.py b/tests/conftest.py index c523d703..d7168ad9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,6 @@ +from __future__ import annotations + + def pytest_collection_modifyitems(config, items): """Skip tests marked with downloads.""" if not config.getoption('markexpr', 'False'): diff --git a/tests/test_data/demo_config.yml b/tests/test_data/demo_config.yml index 10a5b698..5ac91fe8 100644 --- a/tests/test_data/demo_config.yml +++ b/tests/test_data/demo_config.yml @@ -59,4 +59,19 @@ parameter_overrides: - 1.5 - 1.95 - 3.0 -address_overrides: null \ No newline at end of file +address_overrides: null +parameters_to_sample: + - min_v: [0.5, 1.5] + - max_v + - max_fr + - precipitation + - outlet_length + - chahinian_slope_scaling + - length_scaling + - contributing_area_scaling + - chahinian_slope_exponent + - length_exponent + - contributing_area_exponent + - lane_width + - max_street_length +sample_magnitude: 9 \ No newline at end of file diff --git a/tests/test_experimenter.py b/tests/test_experimenter.py new file mode 100644 index 00000000..37c40bc6 --- /dev/null +++ b/tests/test_experimenter.py @@ -0,0 +1,44 @@ +"""Tests for the main experimenter.""" +from __future__ import annotations + +import numpy as np + +from swmmanywhere import parameters +from swmmanywhere.paper import experimenter + + +def assert_close(a: float, b: float, rtol: float = 1e-3) -> None: + """Assert that two floats are close.""" + assert np.isclose(a, b, rtol=rtol).all() + +def test_formulate_salib_problem(): + """Test the formulate_salib_problem function.""" + problem = experimenter.formulate_salib_problem([{'min_v' : [0.5,1.5]}, + 'max_v']) + assert problem['num_vars'] == 2 + max_v = parameters.HydraulicDesign().model_json_schema()['properties']['max_v'] + assert problem['names'] == ['min_v','max_v'] + assert problem['bounds'] == [[0.5,1.5], [max_v['minimum'],max_v['maximum']]] + +def test_generate_samples(): + """Test the generate_samples function.""" + samples = experimenter.generate_samples(N = 2, + parameters_to_select = ['min_v', + 'max_v', + 'chahinian_slope_scaling'], + seed = 1, + groups = False) + assert len(samples) == 48 + assert set([x['param'] for x in samples]) == {'min_v', + 'max_v', + 'chahinian_slope_scaling'} + assert_close(samples[0]['value'], 0.31093) + + samples = experimenter.generate_samples(N = 2, + parameters_to_select = ['min_v', + 'max_v', + 'chahinian_slope_scaling'], + seed = 1, + groups = True) + assert len(samples) == 36 + \ No newline at end of file diff --git a/tests/test_geospatial_utilities.py b/tests/test_geospatial_utilities.py index 6e6c048b..b158a367 100644 --- a/tests/test_geospatial_utilities.py +++ b/tests/test_geospatial_utilities.py @@ -3,6 +3,7 @@ @author: Barney """ +from __future__ import annotations import tempfile from pathlib import Path diff --git a/tests/test_graph_utilities.py b/tests/test_graph_utilities.py index 002a30f6..23528f64 100644 --- a/tests/test_graph_utilities.py +++ b/tests/test_graph_utilities.py @@ -3,8 +3,9 @@ @author: Barney """ +from __future__ import annotations + import math -import os import tempfile from pathlib import Path @@ -247,19 +248,23 @@ def test_iterate_graphfcns(): """Test the iterate_graphfcns function.""" G = load_graph(Path(__file__).parent / 'test_data' / 'graph_topo_derived.json') params = parameters.get_full_parameters() - addresses = parameters.FilePaths(base_dir = None, - project_name = None, - bbox_number = None, - model_number = None) - os.environ['SWMMANYWHERE_VERBOSE'] = "false" - G = iterate_graphfcns(G, - ['assign_id', - 'format_osmnx_lanes'], - params, - addresses) - for u, v, d in G.edges(data=True): - assert 'id' in d.keys() - assert 'width' in d.keys() + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + addresses = parameters.FilePaths(base_dir = None, + project_name = None, + bbox_number = None, + model_number = None) + # Needed if VERBOSE is on.. maybe I should turn it off at the top of + # each test, not sure + addresses.model = temp_path + G = iterate_graphfcns(G, + ['assign_id', + 'format_osmnx_lanes'], + params, + addresses) + for u, v, d in G.edges(data=True): + assert 'id' in d.keys() + assert 'width' in d.keys() def test_fix_geometries(): """Test the fix_geometries function.""" diff --git a/tests/test_logging.py b/tests/test_logging.py index 69695db9..fb1f212c 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -3,6 +3,8 @@ @author: Barney """ +from __future__ import annotations + import os from pathlib import Path from tempfile import NamedTemporaryFile diff --git a/tests/test_metric_utilities.py b/tests/test_metric_utilities.py index 180fc7b3..d5a6e1c2 100644 --- a/tests/test_metric_utilities.py +++ b/tests/test_metric_utilities.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from pathlib import Path import geopandas as gpd @@ -311,7 +313,8 @@ def test_design_params(): real_subs = subs, real_results = results, metric_list = design_results.keys(), - metric_evaluation = MetricEvaluation()) + metric_evaluation = MetricEvaluation() + ) for metric, val in metrics.items(): assert metric in design_results assert np.isclose(val, 0) @@ -328,7 +331,8 @@ def test_design_params(): real_subs = subs, real_results = results, metric_list = design_results.keys(), - metric_evaluation = MetricEvaluation()) + metric_evaluation = MetricEvaluation() + ) for metric, val in metrics.items(): assert metric in design_results @@ -375,51 +379,51 @@ def test_subcatchment_nse_flooding(): subs = get_subs() # Mock results - results = pd.DataFrame([{'object' : 4253560, + results = pd.DataFrame([{'id' : 4253560, 'variable' : 'flow', 'value' : 10, 'date' : pd.to_datetime('2021-01-01 00:00:00')}, - {'object' : 4253560, + {'id' : 4253560, 'variable' : 'flow', 'value' : 5, 'date' : pd.to_datetime('2021-01-01 00:00:05')}, - {'object' : 1696030874, + {'id' : 1696030874, 'variable' : 'flooding', 'value' : 4.5, 'date' : pd.to_datetime('2021-01-01 00:00:00')}, - {'object' : 770549936, + {'id' : 770549936, 'variable' : 'flooding', 'value' : 5, 'date' : pd.to_datetime('2021-01-01 00:00:00')}, - {'object' : 107736, + {'id' : 107736, 'variable' : 'flooding', 'value' : 10, 'date' : pd.to_datetime('2021-01-01 00:00:00')}, - {'object' : 107733, + {'id' : 107733, 'variable' : 'flooding', 'value' : 1, 'date' : pd.to_datetime('2021-01-01 00:00:00')}, - {'object' : 107737, + {'id' : 107737, 'variable' : 'flooding', 'value' : 2, 'date' : pd.to_datetime('2021-01-01 00:00:00')}, - {'object' : 1696030874, + {'id' : 1696030874, 'variable' : 'flooding', 'value' : 0, 'date' : pd.to_datetime('2021-01-01 00:00:05')}, - {'object' : 770549936, + {'id' : 770549936, 'variable' : 'flooding', 'value' : 5, 'date' : pd.to_datetime('2021-01-01 00:00:05')}, - {'object' : 107736, + {'id' : 107736, 'variable' : 'flooding', 'value' : 15, 'date' : pd.to_datetime('2021-01-01 00:00:05')}, - {'object' : 107733, + {'id' : 107733, 'variable' : 'flooding', 'value' : 2, 'date' : pd.to_datetime('2021-01-01 00:00:05')}, - {'object' : 107737, + {'id' : 107737, 'variable' : 'flooding', 'value' : 2, 'date' : pd.to_datetime('2021-01-01 00:00:05')}]) @@ -443,7 +447,7 @@ def test_subcatchment_nse_flooding(): G_ = nx.relabel_nodes(G_, mapping) results_ = results.copy() - results_.object = results_.object.replace(mapping) + results_.id = results_.id.replace(mapping) val = mu.metrics.subcatchment_nse_flooding(synthetic_G = G_, synthetic_results = results_, diff --git a/tests/test_post_processing.py b/tests/test_post_processing.py index 901395e4..68c414f7 100644 --- a/tests/test_post_processing.py +++ b/tests/test_post_processing.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import difflib import filecmp import shutil diff --git a/tests/test_prepare_data.py b/tests/test_prepare_data.py index 293999be..dca6fec3 100644 --- a/tests/test_prepare_data.py +++ b/tests/test_prepare_data.py @@ -6,6 +6,7 @@ pytest -m downloads """ +from __future__ import annotations import io import tempfile diff --git a/tests/test_preprocessing.py b/tests/test_preprocessing.py index fdaba97b..7f2bd75f 100644 --- a/tests/test_preprocessing.py +++ b/tests/test_preprocessing.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from pathlib import Path from swmmanywhere.parameters import FilePaths diff --git a/tests/test_swmmanywhere.py b/tests/test_swmmanywhere.py index 3eca430f..7dedd78c 100644 --- a/tests/test_swmmanywhere.py +++ b/tests/test_swmmanywhere.py @@ -1,4 +1,6 @@ """Tests for the main module.""" +from __future__ import annotations + import os import tempfile from pathlib import Path @@ -89,6 +91,8 @@ def test_swmmanywhere(): # Check metrics were calculated assert metrics is not None for key, val in metrics.items(): + if not val: + continue assert isinstance(val, float) assert set(metrics.keys()) == set(config['metric_list']) @@ -148,3 +152,33 @@ def test_load_config_schema_validation(): swmmanywhere.load_config(base_dir / 'test_config.yml') assert "null" in str(exc_info.value) +def test_check_parameters_to_sample(): + """Test the check_parameters_to_sample validation.""" + with tempfile.TemporaryDirectory() as temp_dir: + test_data_dir = Path(__file__).parent / 'test_data' + defs_dir = Path(__file__).parent.parent / 'swmmanywhere' / 'defs' + base_dir = Path(temp_dir) + + # Load the config + with (test_data_dir / 'demo_config.yml').open('r') as f: + config = yaml.safe_load(f) + + # Correct and avoid filevalidation errors + config['real'] = None + + # Fill with unused paths to avoid filevalidation errors + config['base_dir'] = str(defs_dir / 'storm.dat') + config['api_keys'] = str(defs_dir / 'storm.dat') + + # Make an edit that should fail + config['parameters_to_sample'] = ['not_a_parameter'] + + with open(base_dir / 'test_config.yml', 'w') as f: + yaml.dump(config, f) + + # Test parameter validation + with pytest.raises(ValueError) as exc_info: + swmmanywhere.load_config(base_dir / 'test_config.yml') + assert "not_a_parameter" in str(exc_info.value) + +