From d4ce15c77cbbda1d6f0e5a01d392c1295d4cdad7 Mon Sep 17 00:00:00 2001 From: Dobson Date: Mon, 29 Apr 2024 14:00:15 +0100 Subject: [PATCH 01/28] Update geospatial_utilities.py avoid double counting --- swmmanywhere/geospatial_utilities.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/swmmanywhere/geospatial_utilities.py b/swmmanywhere/geospatial_utilities.py index 6aa4f7b6..4ef37513 100644 --- a/swmmanywhere/geospatial_utilities.py +++ b/swmmanywhere/geospatial_utilities.py @@ -734,9 +734,12 @@ def derive_rc(subcatchments: gpd.GeoDataFrame, """ # Map buffered streets and buildings to subcatchments subcat_tree = subcatchments.sindex - impervious = gpd.overlay(streetcover[['geometry']], - building_footprints[['geometry']], - how='union') + impervious = gpd.GeoDataFrame( + pd.concat([building_footprints[['geometry']], + streetcover[['geometry']]]), + crs = building_footprints.crs + ) + impervious = impervious.dissolve() bf_pidx, sb_pidx = subcat_tree.query(impervious.geometry, predicate='intersects') sb_idx = subcatchments.iloc[sb_pidx].index From 7e20d5c2a52f6e92d9ead3a1026da9b0098fadbf Mon Sep 17 00:00:00 2001 From: Dobson Date: Mon, 29 Apr 2024 14:46:13 +0100 Subject: [PATCH 02/28] Update geospatial_utilities.py Improve efficiency of area calc --- swmmanywhere/geospatial_utilities.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/swmmanywhere/geospatial_utilities.py b/swmmanywhere/geospatial_utilities.py index 4ef37513..e3600c99 100644 --- a/swmmanywhere/geospatial_utilities.py +++ b/swmmanywhere/geospatial_utilities.py @@ -701,12 +701,6 @@ def remove_(mp): return remove_zero_area_subareas(mp, removed_subareas) polys_gdf['width'] = polys_gdf['area'].div(np.pi).pow(0.5) return polys_gdf -def _intersection_area(gdf1: gpd.GeoDataFrame, gdf2: gpd.GeoDataFrame)-> np.array: - return shapely.area( - shapely.intersection( - gdf1.geometry.to_numpy(), - gdf2.geometry.to_numpy())) - def derive_rc(subcatchments: gpd.GeoDataFrame, building_footprints: gpd.GeoDataFrame, streetcover: gpd.GeoDataFrame) -> gpd.GeoDataFrame: @@ -730,7 +724,7 @@ def derive_rc(subcatchments: gpd.GeoDataFrame, 'geometry', 'area', 'id', 'impervious_area', and 'rc'. Author: - @cheginit + @cheginit, @barneydobson """ # Map buffered streets and buildings to subcatchments subcat_tree = subcatchments.sindex @@ -739,7 +733,6 @@ def derive_rc(subcatchments: gpd.GeoDataFrame, streetcover[['geometry']]]), crs = building_footprints.crs ) - impervious = impervious.dissolve() bf_pidx, sb_pidx = subcat_tree.query(impervious.geometry, predicate='intersects') sb_idx = subcatchments.iloc[sb_pidx].index @@ -747,18 +740,24 @@ def derive_rc(subcatchments: gpd.GeoDataFrame, # Calculate impervious area and runoff coefficient (rc) subcatchments["impervious_area"] = 0.0 - # Calculate all intersection-impervious areas - intersection_area = _intersection_area(subcatchments.iloc[sb_pidx], - impervious.iloc[bf_pidx]) + # Calculate all intersection-impervious geometries + intersection_area = shapely.intersection( + subcatchments.iloc[sb_pidx].geometry.to_numpy(), + impervious.iloc[bf_pidx].geometry.to_numpy()) # Indicate which catchment each intersection is part of intersections = pd.DataFrame([{'sb_idx': ix, - 'impervious_area': ia} + 'impervious_geometry': ia} for ix, ia in zip(sb_idx, intersection_area)] ) # Aggregate by catchment - areas = intersections.groupby('sb_idx').sum() + areas = ( + intersections + .groupby('sb_idx') + .apply(shapely.ops.unary_union) + .apply(shapely.area) + ) # Store as impervious area in subcatchments subcatchments["impervious_area"] = 0 From 7d325690cb1d2cfbebd445e42f3d14cc834f95db Mon Sep 17 00:00:00 2001 From: Dobson Date: Mon, 29 Apr 2024 14:50:11 +0100 Subject: [PATCH 03/28] Add different network types --- swmmanywhere/graph_utilities.py | 10 +++++++++- swmmanywhere/parameters.py | 8 +++++++- swmmanywhere/prepare_data.py | 8 +++++--- swmmanywhere/preprocessing.py | 14 +++++++++++++- 4 files changed, 34 insertions(+), 6 deletions(-) diff --git a/swmmanywhere/graph_utilities.py b/swmmanywhere/graph_utilities.py index d4ba74a2..16c9ad40 100644 --- a/swmmanywhere/graph_utilities.py +++ b/swmmanywhere/graph_utilities.py @@ -286,6 +286,11 @@ def __call__(self, """ edges_to_remove = set() for u, v, keys, data in G.edges(data=True,keys = True): + if data.get('network_type','drive') \ + not in topology_derivation.allowable_networks: + + edges_to_remove.add((u, v, keys)) + continue for omit in topology_derivation.omit_edges: if data.get('highway', None) == omit: # Check whether the 'highway' property is 'omit' @@ -325,7 +330,10 @@ def __call__(self, G = G.copy() lines = [] for u, v, data in G.edges(data=True): - lanes = data.get('lanes',1) + if data.get('network_type','drive') == 'drive': + lanes = data.get('lanes',1) + else: + lanes = 0 if isinstance(lanes, list): lanes = sum([float(x) for x in lanes]) else: diff --git a/swmmanywhere/parameters.py b/swmmanywhere/parameters.py index 3c8a3aa1..cb46bcca 100644 --- a/swmmanywhere/parameters.py +++ b/swmmanywhere/parameters.py @@ -73,6 +73,11 @@ class OutletDerivation(BaseModel): class TopologyDerivation(BaseModel): """Parameters for topology derivation.""" + allowable_networks: list = Field(default = ['drive', 'walk'], + min_items = 1, + unit = "-", + description = "OSM networks to consider") + weights: list = Field(default = ['chahinian_slope', 'chahinian_angle', 'length', @@ -84,7 +89,8 @@ class TopologyDerivation(BaseModel): omit_edges: list = Field(default = ['motorway', 'motorway_link', 'bridge', - 'tunnel'], + 'tunnel', + 'corridor'], min_items = 1, unit = "-", description = "OSM paths pipes are not allowed under") diff --git a/swmmanywhere/prepare_data.py b/swmmanywhere/prepare_data.py index e87b4703..c05b2566 100644 --- a/swmmanywhere/prepare_data.py +++ b/swmmanywhere/prepare_data.py @@ -89,7 +89,8 @@ def download_buildings(file_address: Path, logger.error(f"Error downloading data. Status code: {response.status_code}") return response.status_code -def download_street(bbox: tuple[float, float, float, float]) -> nx.MultiDiGraph: +def download_street(bbox: tuple[float, float, float, float], + network_type = 'drive') -> nx.MultiDiGraph: """Get street network within a bounding box using OSMNX. [CREDIT: Taher Cheghini busn_estimator package] @@ -97,6 +98,7 @@ def download_street(bbox: tuple[float, float, float, float]) -> nx.MultiDiGraph: Args: bbox (tuple[float, float, float, float]): Bounding box as tuple in form of (west, south, east, north) at EPSG:4326. + network_type (str, optional): Type of street network. Defaults to 'drive'. Returns: nx.MultiDiGraph: Street network with type drive and @@ -105,8 +107,8 @@ def download_street(bbox: tuple[float, float, float, float]) -> nx.MultiDiGraph: west, south, east, north = bbox bbox = (north, south, east, west) # not sure why osmnx uses this order graph = ox.graph_from_bbox( - bbox = bbox, network_type="drive", truncate_by_edge=True - ) + bbox = bbox, network_type=network_type, truncate_by_edge=True + ) return cast("nx.MultiDiGraph", graph) def download_river(bbox: tuple[float, float, float, float]) -> nx.MultiDiGraph: diff --git a/swmmanywhere/preprocessing.py b/swmmanywhere/preprocessing.py index b4aae488..d780fb8b 100644 --- a/swmmanywhere/preprocessing.py +++ b/swmmanywhere/preprocessing.py @@ -226,10 +226,22 @@ def prepare_street(bbox: tuple[float, float, float, float], if addresses.street.exists(): return logger.info(f'downloading street network to {addresses.street}') - street_network = prepare_data.download_street(bbox) + street_network = prepare_data.download_street(bbox, network_type='drive') + nx.set_edge_attributes(street_network, 'drive', 'network_type') + + # Download walk network to enable pipes along walkways + walk_network = prepare_data.download_street(bbox, network_type='walk') + nx.set_edge_attributes(walk_network, 'walk', 'network_type') + + # Combine streets and walkways (use street_network as first arg so that + # parameters from street_network are used where there are conflicts) + street_network = nx.compose(walk_network, street_network) + + # Reproject graph street_network = go.reproject_graph(street_network, source_crs, target_crs) + gu.save_graph(street_network, addresses.street) def prepare_river(bbox: tuple[float, float, float, float], From 8a893285706622fdf8d9fad9ac6a31ad95b0c878 Mon Sep 17 00:00:00 2001 From: Dobson Date: Mon, 29 Apr 2024 15:18:30 +0100 Subject: [PATCH 04/28] improve node merging --- swmmanywhere/geospatial_utilities.py | 19 +++++++++++++++---- swmmanywhere/parameters.py | 2 +- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/swmmanywhere/geospatial_utilities.py b/swmmanywhere/geospatial_utilities.py index e3600c99..c1184cf2 100644 --- a/swmmanywhere/geospatial_utilities.py +++ b/swmmanywhere/geospatial_utilities.py @@ -946,12 +946,23 @@ def merge_points(coordinates: list[tuple[float, float]], # Merge pairs into families of points that are all nearby families: list = [] + for pair in pairs: - for family in families: - if pair[0] in family or pair[1] in family: - family.update(pair) - break + matched_families = [family for family in families + if pair[0] in family or pair[1] in family] + + if matched_families: + # Merge all matched families and add the current pair + new_family = set(pair) + for family in matched_families: + new_family.update(family) + + # Remove the old families and add the newly formed one + for family in matched_families: + families.remove(family) + families.append(new_family) else: + # No matching family found, so create a new one families.append(set(pair)) # Create a mapping of the original point to the merged point diff --git a/swmmanywhere/parameters.py b/swmmanywhere/parameters.py index cb46bcca..ce78d63b 100644 --- a/swmmanywhere/parameters.py +++ b/swmmanywhere/parameters.py @@ -51,7 +51,7 @@ class SubcatchmentDerivation(BaseModel): unit = "m", description = "Distance to split streets into segments.") - node_merge_distance: float = Field(default = 30, + node_merge_distance: float = Field(default = 10, ge = 1, le = 40, unit = 'm', From d8cddb1f4a7e265ccbe2c7ccc2f73d54b4b15d7b Mon Sep 17 00:00:00 2001 From: Dobson Date: Mon, 29 Apr 2024 15:41:57 +0100 Subject: [PATCH 05/28] Update test_swmmanywhere.py widen to prevent outlet nonsense --- tests/test_swmmanywhere.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_swmmanywhere.py b/tests/test_swmmanywhere.py index 7dedd78c..c043f172 100644 --- a/tests/test_swmmanywhere.py +++ b/tests/test_swmmanywhere.py @@ -54,7 +54,7 @@ def test_swmmanywhere(): # Set some test values base_dir = Path(temp_dir) config['base_dir'] = str(base_dir) - config['bbox'] = [0.05428,51.55847,0.07193,51.56726] + config['bbox'] = [0.05677,51.55656,0.07193,51.56726] config['address_overrides'] = { 'building': str(test_data_dir / 'building.geoparquet'), 'precipitation': str(defs_dir / 'storm.dat') From 2ccf95c574517e5ec28c3749a1ceff796d304aad Mon Sep 17 00:00:00 2001 From: Dobson Date: Mon, 29 Apr 2024 15:47:37 +0100 Subject: [PATCH 06/28] Update graph_utilities.py more verbosity --- swmmanywhere/graph_utilities.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/swmmanywhere/graph_utilities.py b/swmmanywhere/graph_utilities.py index 16c9ad40..f4d62923 100644 --- a/swmmanywhere/graph_utilities.py +++ b/swmmanywhere/graph_utilities.py @@ -185,6 +185,12 @@ def iterate_graphfcns(G: nx.Graph, logger.info(f"graphfcn: {function} completed.") if verbose: save_graph(G, addresses.model / f"{function}_graph.json") + go.graph_to_geojson(G, + addresses.model / f"{function}_nodes.geojson", + addresses.model / f"{function}_edges.geojson", + G.graph['crs'] + ) + return G @register_graphfcn @@ -1114,6 +1120,11 @@ def __call__(self, G: nx.Graph, G.remove_node(node) del paths[node], shortest_paths[node] + if len(G.nodes) == 0: + raise ValueError("""No nodes with path to outlet, consider + broadening bounding box or removing trim_to_outlet + from config graphfcn_list""") + edges_to_keep: set = set() for path in paths.values(): # Assign outlet From 52293e5647f6648f4880871fd9822e4cc2a937b6 Mon Sep 17 00:00:00 2001 From: Dobson Date: Mon, 29 Apr 2024 15:56:58 +0100 Subject: [PATCH 07/28] Update parameters.py typo --- swmmanywhere/parameters.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/swmmanywhere/parameters.py b/swmmanywhere/parameters.py index ce78d63b..0693785b 100644 --- a/swmmanywhere/parameters.py +++ b/swmmanywhere/parameters.py @@ -201,8 +201,8 @@ class HydraulicDesign(BaseModel): class MetricEvaluation(BaseModel): """Parameters for metric evaluation.""" grid_scale: float = Field(default = 100, - le = 10, - ge = 5000, + le = 5000, + ge = 10, unit = "m", description = "Scale of the grid for metric evaluation") From e6d6fde544ee00aa3d0456ef63ff0c15296827ff Mon Sep 17 00:00:00 2001 From: Dobson Date: Mon, 29 Apr 2024 15:59:48 +0100 Subject: [PATCH 08/28] Update experimenter.py typo --- swmmanywhere/paper/experimenter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/swmmanywhere/paper/experimenter.py b/swmmanywhere/paper/experimenter.py index 2c01743e..0486ee0c 100644 --- a/swmmanywhere/paper/experimenter.py +++ b/swmmanywhere/paper/experimenter.py @@ -206,7 +206,7 @@ def parse_arguments() -> tuple[int, int | None, Path]: parser = argparse.ArgumentParser(description='Process command line arguments.') parser.add_argument('--jobid', type=int, - default=1, + default=0, help='Job ID') parser.add_argument('--nproc', type=int, @@ -215,7 +215,7 @@ def parse_arguments() -> tuple[int, int | None, Path]: parser.add_argument('--config_path', type=Path, default=Path(__file__).parent.parent.parent / 'tests' /\ - 'test_data' / 'demo_config_sa.yml', + 'test_data' / 'demo_config.yml', help='Configuration file path') args = parser.parse_args() From 725bcfe9bce13454a2841abf294f69321cbb199d Mon Sep 17 00:00:00 2001 From: Dobson Date: Mon, 29 Apr 2024 16:05:59 +0100 Subject: [PATCH 09/28] Update prepare_data.py deprecation error --- swmmanywhere/prepare_data.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/swmmanywhere/prepare_data.py b/swmmanywhere/prepare_data.py index c05b2566..266c664f 100644 --- a/swmmanywhere/prepare_data.py +++ b/swmmanywhere/prepare_data.py @@ -124,10 +124,7 @@ def download_river(bbox: tuple[float, float, float, float]) -> nx.MultiDiGraph: """ west, south, east, north = bbox graph = ox.graph_from_bbox( - north, - south, - east, - west, + bbox = (north, south, east, west), truncate_by_edge=True, custom_filter='["waterway"]') From 85f69599b1e849c002739c1883e867f40c06bd26 Mon Sep 17 00:00:00 2001 From: Dobson Date: Mon, 29 Apr 2024 16:08:47 +0100 Subject: [PATCH 10/28] Update geospatial_utilities.py make float --- swmmanywhere/geospatial_utilities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swmmanywhere/geospatial_utilities.py b/swmmanywhere/geospatial_utilities.py index c1184cf2..53fdb3ea 100644 --- a/swmmanywhere/geospatial_utilities.py +++ b/swmmanywhere/geospatial_utilities.py @@ -760,7 +760,7 @@ def derive_rc(subcatchments: gpd.GeoDataFrame, ) # Store as impervious area in subcatchments - subcatchments["impervious_area"] = 0 + subcatchments["impervious_area"] = 0.0 subcatchments.loc[areas.index, "impervious_area"] = areas subcatchments["rc"] = subcatchments["impervious_area"] / \ subcatchments.geometry.area * 100 From 92324d38060d740adf02070b570ed7180310e7bc Mon Sep 17 00:00:00 2001 From: Dobson Date: Mon, 29 Apr 2024 16:45:11 +0100 Subject: [PATCH 11/28] Update graph_utilities.py merge only street nodes --- swmmanywhere/graph_utilities.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/swmmanywhere/graph_utilities.py b/swmmanywhere/graph_utilities.py index f4d62923..a5dfecc3 100644 --- a/swmmanywhere/graph_utilities.py +++ b/swmmanywhere/graph_utilities.py @@ -511,16 +511,22 @@ def __call__(self, """ G = G.copy() + # Separate out streets + street_edges = [(u, v, k) for u, v, k, d in G.edges(data=True, keys=True) + if d.get('edge_type','street') == 'street'] + streets = G.edge_subgraph(street_edges).copy() + # Identify nodes that are within threshold of each other - mapping = go.merge_points([(d['x'], d['y']) for u,d in G.nodes(data=True)], + mapping = go.merge_points([(d['x'], d['y']) + for u,d in streets.nodes(data=True)], subcatchment_derivation.node_merge_distance) # Get indexes of node names - node_indices = {ix: node for ix, node in enumerate(G.nodes)} + node_indices = {ix: node for ix, node in enumerate(streets.nodes)} # Create a mapping of old node names to new node names node_names = {} - for ix, node in enumerate(G.nodes): + for ix, node in enumerate(streets.nodes): if ix in mapping: # If the node is in the mapping, then it is mapped and # given the new coordinate (all nodes in a mapping family must From 7f9ef7366e9774e3e0f110eee16d357985afc229 Mon Sep 17 00:00:00 2001 From: Dobson Date: Tue, 30 Apr 2024 09:16:13 +0100 Subject: [PATCH 12/28] Update graph_utilities.py Fix new double_directed --- swmmanywhere/graph_utilities.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/swmmanywhere/graph_utilities.py b/swmmanywhere/graph_utilities.py index a5dfecc3..2f5980dc 100644 --- a/swmmanywhere/graph_utilities.py +++ b/swmmanywhere/graph_utilities.py @@ -385,8 +385,22 @@ def __call__(self, G: nx.Graph, **kwargs) -> nx.Graph: Returns: G (nx.Graph): A graph """ + # Convert to directed G_new = G.copy() G_new = nx.MultiDiGraph(G_new) + + # MultiDiGraph adds edges in both directions, but rivers (and geometries) + # are only in one direction. So we remove the reverse edges and add them + # back in with the correct geometry. + # This assumes that 'id' is of format 'start-end' (see assign_id) + arcs_to_remove = [(u,v) for u,v,d in G_new.edges(data=True) + if f'{u}-{v}' != d.get('id')] + + # Remove the reverse edges + for u, v in arcs_to_remove: + G_new.remove_edge(u, v) + + # Add in reversed edges for streets only and with geometry for u, v, data in G.edges(data=True): include = data.get('edge_type', True) if isinstance(include, str): From f073eafc8044f42d85f949da21b3dc6ac7ea4aa8 Mon Sep 17 00:00:00 2001 From: Dobson Date: Tue, 30 Apr 2024 09:54:58 +0100 Subject: [PATCH 13/28] Update graph_utilities.py another bug --- swmmanywhere/graph_utilities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swmmanywhere/graph_utilities.py b/swmmanywhere/graph_utilities.py index 2f5980dc..34a13de2 100644 --- a/swmmanywhere/graph_utilities.py +++ b/swmmanywhere/graph_utilities.py @@ -405,7 +405,7 @@ def __call__(self, G: nx.Graph, **kwargs) -> nx.Graph: include = data.get('edge_type', True) if isinstance(include, str): include = include == 'street' - if ((v, u) not in G.edges) & include: + if ((v, u) not in G_new.edges) & include: reverse_data = data.copy() reverse_data['id'] = f"{data['id']}.reversed" new_geometry = shapely.LineString( From ceb7c7d1102074f68e6d943ab60ef705c8fe984b Mon Sep 17 00:00:00 2001 From: Dobson Date: Tue, 30 Apr 2024 14:44:50 +0100 Subject: [PATCH 14/28] Update experimenter.py --- swmmanywhere/paper/experimenter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/swmmanywhere/paper/experimenter.py b/swmmanywhere/paper/experimenter.py index 0486ee0c..7047bd3c 100644 --- a/swmmanywhere/paper/experimenter.py +++ b/swmmanywhere/paper/experimenter.py @@ -137,6 +137,7 @@ def process_parameters(jobid: int, df = pd.DataFrame(X) gb = df.groupby('iter') n_iter = len(gb) + logger.info(f"{n_iter} samples created") flooding_results = {} nproc = nproc if nproc is not None else n_iter From 755ad752ec23517a7d36b946a949243211555a10 Mon Sep 17 00:00:00 2001 From: Dobson Date: Wed, 1 May 2024 14:57:14 +0100 Subject: [PATCH 15/28] Update experimenter.py --- swmmanywhere/paper/experimenter.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/swmmanywhere/paper/experimenter.py b/swmmanywhere/paper/experimenter.py index 7047bd3c..33e4e8af 100644 --- a/swmmanywhere/paper/experimenter.py +++ b/swmmanywhere/paper/experimenter.py @@ -168,11 +168,16 @@ def process_parameters(jobid: int, # Run the model config['model_number'] = ix logger.info(f"Running swmmanywhere for model {ix}") - address, metrics = swmmanywhere.swmmanywhere(config) + try: + # Code that might throw an exception + address, metrics = swmmanywhere.swmmanywhere(config) + if metrics is None: + raise ValueError(f"Model run {ix} failed.") + except Exception as e: + # Print the error message + print(f"An error occurred: {e}") + metrics = {metric: None for metric in config_base['metric_list']} - if metrics is None: - raise ValueError(f"Model run {ix} failed.") - # Save the results flooding_results[ix] = {'iter': ix, **metrics, From 1524ce84c07fd6d31056e7ba56cd37bbb3c2011e Mon Sep 17 00:00:00 2001 From: Dobson Date: Fri, 3 May 2024 16:20:18 +0100 Subject: [PATCH 16/28] Remove trim --- swmmanywhere/geospatial_utilities.py | 49 ---------------------- swmmanywhere/graph_utilities.py | 63 ---------------------------- tests/test_graph_utilities.py | 13 ------ 3 files changed, 125 deletions(-) diff --git a/swmmanywhere/geospatial_utilities.py b/swmmanywhere/geospatial_utilities.py index 53fdb3ea..82ac735e 100644 --- a/swmmanywhere/geospatial_utilities.py +++ b/swmmanywhere/geospatial_utilities.py @@ -31,8 +31,6 @@ from shapely.strtree import STRtree from tqdm import tqdm -from swmmanywhere.logging import logger - os.environ['NUMBA_NUM_THREADS'] = '1' import pyflwdir # noqa: E402 import pysheds # noqa: E402 @@ -880,53 +878,6 @@ def graph_to_geojson(graph: nx.Graph, with fid.open('w') as output_file: json.dump(geojson, output_file, indent=2) -def trim_touching_polygons(polygons: gpd.GeoDataFrame, - fid: Path, - trim: bool = False) -> gpd.GeoDataFrame: - """Trim touching polygons in a GeoDataFrame. - - Args: - polygons (gpd.GeoDataFrame): A GeoDataFrame containing polygons with - columns: 'geometry', 'area', and 'id'. - fid (Path): Filepath to the elevation DEM. - - trim (bool, optional): Whether to trim polygons that touch the edge of - the DEM or just to warn a user that they do touch the edge. - Defaults to False. - - Returns: - gpd.GeoDataFrame: A GeoDataFrame containing polygons with no touching - polygons. - """ - # Get elevation boundary - with rst.open(fid) as src: - image = src.read(1) # Read the first band - nodata = src.nodata - transform = src.transform - crs = src.crs - resolution = abs(transform.a) - - # Mask elevation with data - data_mask = (image != nodata) - image[data_mask] = 1 - mask_shapes = features.shapes(image, mask=data_mask, transform=transform) - - # Convert shapes to GeoDataFrame - geoms = [sgeom.Polygon(geom['coordinates'][0]) for geom, value in mask_shapes] - - # Create GeoDataFrame from the list of geometries - dem_outline = gpd.GeoDataFrame({'geometry': geoms}, crs=crs).exterior - - ind = polygons.geometry.exterior.buffer(resolution + 1).apply( - lambda x: x.intersects(dem_outline)).values - if ind.sum() != 0: - logger.warning("""Some catchments touch the edge of the elevation DEM, - inspect the outputs and check whether the area of - interest has been included, otherwise widen your bbox""") - if trim: - trimmed_gdf = polygons.loc[~ind] - return trimmed_gdf - return polygons def merge_points(coordinates: list[tuple[float, float]], threshold: float)-> dict: diff --git a/swmmanywhere/graph_utilities.py b/swmmanywhere/graph_utilities.py index 8bc08f6e..f8754dc1 100644 --- a/swmmanywhere/graph_utilities.py +++ b/swmmanywhere/graph_utilities.py @@ -669,69 +669,6 @@ def __call__(self, G: nx.Graph, # Set edge attributes nx.set_edge_attributes(G, edge_attributes, 'contributing_area') return G - -@register_graphfcn -class trim_to_outlets(BaseGraphFunction, - required_edge_attributes = ['edge_type'] # i.e., 'outlet' - ): - """trim_to_outlets class.""" - def __call__(self, - G: nx.Graph, - addresses: parameters.FilePaths, - **kwargs) -> nx.Graph: - """Trim the graph to the outlets. - - This function trims the graph to the hydrological catchments that - drains to the outlets. Nodes that are not in the catchment of any - outlets are removed. The outlets are the edges with the 'outlet' - edge_type attribute. - - Args: - G (nx.Graph): A graph - addresses (parameters.FilePaths): An FilePaths parameter object - **kwargs: Additional keyword arguments are ignored. - - Returns: - G (nx.Graph): A graph - """ - G = G.copy() - graph_ = G.graph.copy() - - # Create a graph of outlets - outlets = {v : G.nodes[v] for u,v, data in G.edges(data=True) - if data.get('edge_type', None) == 'outlet'} - if not outlets: - raise ValueError("No outlets found in the graph.") - outlet_graph = nx.Graph() - outlet_graph.add_nodes_from(outlets) - outlet_graph.graph = graph_ - nx.set_node_attributes(outlet_graph, outlets) - - # Derive outlet subcatchments - outlet_catchments = go.derive_subcatchments(outlet_graph, - addresses.elevation, - method = 'pyflwdir') - - # Check whether the outlet catchments are touching the edge of - # the elevation data. trim=False retains these catchments while - # trim=True would remove them. - outlet_catchments = go.trim_touching_polygons(outlet_catchments, - addresses.elevation, - trim = False) - - # Keep only nodes within subcatchments - nodes_gdf = gpd.GeoDataFrame(G.nodes, - geometry = gpd.points_from_xy( - [d['x'] for n,d in G.nodes(data=True)], - [d['y'] for n,d in G.nodes(data=True)]), - crs = G.graph['crs'], - columns = ['id']) - keep_nodes = gpd.sjoin(nodes_gdf, - outlet_catchments[['geometry']], - predicate = 'intersects').id - G = G.subgraph(keep_nodes).copy() - G.graph = graph_ - return G @register_graphfcn class set_elevation(BaseGraphFunction, diff --git a/tests/test_graph_utilities.py b/tests/test_graph_utilities.py index d485b701..cbd3b0a1 100644 --- a/tests/test_graph_utilities.py +++ b/tests/test_graph_utilities.py @@ -340,19 +340,6 @@ def test_fix_geometries(): # Check that the edge geometry now matches the node coordinates assert G_fixed.get_edge_data(107733, 25472373,0)['geometry'].coords[0] == \ (G_fixed.nodes[107733]['x'], G_fixed.nodes[107733]['y']) - -def test_trim_to_outlets(): - """Test the trim_to_outlets function.""" - G, _ = load_street_network() - elev_fid = Path(__file__).parent / 'test_data' / 'elevation.tif' - G.edges[107738, 21392086,0]['edge_type'] = 'outlet' - addresses = parameters.FilePaths(base_dir = None, - project_name = None, - bbox_number = None, - model_number = None) - addresses.elevation = elev_fid - G_ = gu.trim_to_outlets(G,addresses) - assert set(G_.nodes) == set([21392086]) def almost_equal(a, b, tol=1e-6): """Check if two numbers are almost equal.""" From 15586da95ce5356cda1a060a3b1b6a92b4fcec1b Mon Sep 17 00:00:00 2001 From: Dobson Date: Fri, 3 May 2024 16:25:05 +0100 Subject: [PATCH 17/28] Update demo_config.yml --- tests/test_data/demo_config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_data/demo_config.yml b/tests/test_data/demo_config.yml index 796a4797..c9d166b5 100644 --- a/tests/test_data/demo_config.yml +++ b/tests/test_data/demo_config.yml @@ -31,7 +31,6 @@ graphfcn_list: - set_chahinian_angle # Transform edge angles to more sensible angle for weights - calculate_weights # Calculate weights for each edge - identify_outlets # Identify potential street->river outlets - - trim_to_outlets # Remove nodes that don't fall in any outlets' catchments - derive_topology # Shortest path to give network topology - pipe_by_pipe # Design pipe diameters and depths - fix_geometries # Ensure geometries present before printing From b347dbf40c46d760a4f71d40a3a088492e09270f Mon Sep 17 00:00:00 2001 From: Dobson Date: Wed, 8 May 2024 16:33:29 +0100 Subject: [PATCH 18/28] Update graph_utilities.py remove old change --- swmmanywhere/graph_utilities.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/swmmanywhere/graph_utilities.py b/swmmanywhere/graph_utilities.py index f8754dc1..c15a0016 100644 --- a/swmmanywhere/graph_utilities.py +++ b/swmmanywhere/graph_utilities.py @@ -913,28 +913,6 @@ def __call__(self, # Copy graph to run shortest path on G_ = G.copy() - if not matched_outlets: - # In cases of e.g., an area with no rivers to discharge into or too - # small a buffer - - # Identify the lowest elevation node - lowest_elevation_node = min(G.nodes, - key = lambda x: G.nodes[x]['surface_elevation']) - - # Create a dummy river to discharge into - dummy_river = {'id' : 'dummy_river', - 'x' : G.nodes[lowest_elevation_node]['x'] + 1, - 'y' : G.nodes[lowest_elevation_node]['y'] + 1} - G_.add_node('dummy_river') - nx.set_node_attributes(G_, {'dummy_river' : dummy_river}) - - # Update function's dicts - matched_outlets = {'dummy_river' : lowest_elevation_node} - river_points['dummy_river'] = shapely.Point(dummy_river['x'], - dummy_river['y']) - - logger.warning('No outlets found, using lowest elevation node as outlet') - # Add edges between the paired river and street nodes for river_id, street_id in matched_outlets.items(): # TODO instead use a weight based on the distance between the two nodes From 7485b792c15b6e2c2d3fef38c50c3165a7a8179e Mon Sep 17 00:00:00 2001 From: Dobson Date: Wed, 8 May 2024 16:38:13 +0100 Subject: [PATCH 19/28] Revert "Update graph_utilities.py" This reverts commit b347dbf40c46d760a4f71d40a3a088492e09270f. --- swmmanywhere/graph_utilities.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/swmmanywhere/graph_utilities.py b/swmmanywhere/graph_utilities.py index c15a0016..f8754dc1 100644 --- a/swmmanywhere/graph_utilities.py +++ b/swmmanywhere/graph_utilities.py @@ -913,6 +913,28 @@ def __call__(self, # Copy graph to run shortest path on G_ = G.copy() + if not matched_outlets: + # In cases of e.g., an area with no rivers to discharge into or too + # small a buffer + + # Identify the lowest elevation node + lowest_elevation_node = min(G.nodes, + key = lambda x: G.nodes[x]['surface_elevation']) + + # Create a dummy river to discharge into + dummy_river = {'id' : 'dummy_river', + 'x' : G.nodes[lowest_elevation_node]['x'] + 1, + 'y' : G.nodes[lowest_elevation_node]['y'] + 1} + G_.add_node('dummy_river') + nx.set_node_attributes(G_, {'dummy_river' : dummy_river}) + + # Update function's dicts + matched_outlets = {'dummy_river' : lowest_elevation_node} + river_points['dummy_river'] = shapely.Point(dummy_river['x'], + dummy_river['y']) + + logger.warning('No outlets found, using lowest elevation node as outlet') + # Add edges between the paired river and street nodes for river_id, street_id in matched_outlets.items(): # TODO instead use a weight based on the distance between the two nodes From f0b4c177fa5063954f3f1e86c834ca426cdbcce1 Mon Sep 17 00:00:00 2001 From: Dobson Date: Thu, 9 May 2024 09:32:50 +0100 Subject: [PATCH 20/28] remove change from another branch --- swmmanywhere/graph_utilities.py | 22 ---------------------- tests/test_graph_utilities.py | 20 -------------------- 2 files changed, 42 deletions(-) diff --git a/swmmanywhere/graph_utilities.py b/swmmanywhere/graph_utilities.py index f8754dc1..c15a0016 100644 --- a/swmmanywhere/graph_utilities.py +++ b/swmmanywhere/graph_utilities.py @@ -913,28 +913,6 @@ def __call__(self, # Copy graph to run shortest path on G_ = G.copy() - if not matched_outlets: - # In cases of e.g., an area with no rivers to discharge into or too - # small a buffer - - # Identify the lowest elevation node - lowest_elevation_node = min(G.nodes, - key = lambda x: G.nodes[x]['surface_elevation']) - - # Create a dummy river to discharge into - dummy_river = {'id' : 'dummy_river', - 'x' : G.nodes[lowest_elevation_node]['x'] + 1, - 'y' : G.nodes[lowest_elevation_node]['y'] + 1} - G_.add_node('dummy_river') - nx.set_node_attributes(G_, {'dummy_river' : dummy_river}) - - # Update function's dicts - matched_outlets = {'dummy_river' : lowest_elevation_node} - river_points['dummy_river'] = shapely.Point(dummy_river['x'], - dummy_river['y']) - - logger.warning('No outlets found, using lowest elevation node as outlet') - # Add edges between the paired river and street nodes for river_id, street_id in matched_outlets.items(): # TODO instead use a weight based on the distance between the two nodes diff --git a/tests/test_graph_utilities.py b/tests/test_graph_utilities.py index cbd3b0a1..62b0b159 100644 --- a/tests/test_graph_utilities.py +++ b/tests/test_graph_utilities.py @@ -177,26 +177,6 @@ def test_calculate_weights(): for u, v, data in G.edges(data=True): assert 'weight' in data.keys() assert math.isfinite(data['weight']) - -def test_identify_outlets_no_river(): - """Test the identify_outlets in the no river case.""" - G, _ = load_street_network() - G = gu.assign_id(G) - G = gu.double_directed(G) - elev_fid = Path(__file__).parent / 'test_data' / 'elevation.tif' - addresses = parameters.FilePaths(base_dir = None, - project_name = None, - bbox_number = None, - model_number = None) - addresses.elevation = elev_fid - G = gu.set_elevation(G, addresses) - for ix, (u,v,d) in enumerate(G.edges(data=True)): - d['edge_type'] = 'street' - d['weight'] = ix - params = parameters.OutletDerivation() - G = gu.identify_outlets(G, params) - outlets = [(u,v,d) for u,v,d in G.edges(data=True) if d['edge_type'] == 'outlet'] - assert len(outlets) == 1 def test_identify_outlets_and_derive_topology(): """Test the identify_outlets and derive_topology functions.""" From 87fc54dbb0b3bd760ba54e2c573b01168194f3cf Mon Sep 17 00:00:00 2001 From: Dobson Date: Thu, 9 May 2024 09:34:48 +0100 Subject: [PATCH 21/28] Update graph_utilities.py --- swmmanywhere/graph_utilities.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/swmmanywhere/graph_utilities.py b/swmmanywhere/graph_utilities.py index c15a0016..9d23166f 100644 --- a/swmmanywhere/graph_utilities.py +++ b/swmmanywhere/graph_utilities.py @@ -1045,11 +1045,6 @@ def __call__(self, G: nx.Graph, G.remove_node(node) del paths[node], shortest_paths[node] - if len(G.nodes) == 0: - raise ValueError("""No nodes with path to outlet, consider - broadening bounding box or removing trim_to_outlet - from config graphfcn_list""") - edges_to_keep: set = set() for path in paths.values(): # Assign outlet From 725249fa705c9a31d65d7d7d0061a4dd4b8b57a3 Mon Sep 17 00:00:00 2001 From: Dobson Date: Thu, 9 May 2024 09:36:19 +0100 Subject: [PATCH 22/28] Update experimenter.py --- swmmanywhere/paper/experimenter.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/swmmanywhere/paper/experimenter.py b/swmmanywhere/paper/experimenter.py index 33e4e8af..aae3e03a 100644 --- a/swmmanywhere/paper/experimenter.py +++ b/swmmanywhere/paper/experimenter.py @@ -168,15 +168,10 @@ def process_parameters(jobid: int, # Run the model config['model_number'] = ix logger.info(f"Running swmmanywhere for model {ix}") - try: - # Code that might throw an exception - address, metrics = swmmanywhere.swmmanywhere(config) - if metrics is None: - raise ValueError(f"Model run {ix} failed.") - except Exception as e: - # Print the error message - print(f"An error occurred: {e}") - metrics = {metric: None for metric in config_base['metric_list']} + + address, metrics = swmmanywhere.swmmanywhere(config) + if metrics is None: + raise ValueError(f"Model run {ix} failed.") # Save the results flooding_results[ix] = {'iter': ix, From 4ab65c76d7ab83564a3edfe3f374942a4a7c0438 Mon Sep 17 00:00:00 2001 From: Dobson Date: Thu, 9 May 2024 09:51:49 +0100 Subject: [PATCH 23/28] Iterate over netework_types --- swmmanywhere/parameters.py | 2 +- swmmanywhere/preprocessing.py | 42 +++++++++++++++++++++++++---------- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/swmmanywhere/parameters.py b/swmmanywhere/parameters.py index 0693785b..b0c487a2 100644 --- a/swmmanywhere/parameters.py +++ b/swmmanywhere/parameters.py @@ -73,7 +73,7 @@ class OutletDerivation(BaseModel): class TopologyDerivation(BaseModel): """Parameters for topology derivation.""" - allowable_networks: list = Field(default = ['drive', 'walk'], + allowable_networks: list = Field(default = ['walk', 'drive'], min_items = 1, unit = "-", description = "OSM networks to consider") diff --git a/swmmanywhere/preprocessing.py b/swmmanywhere/preprocessing.py index d780fb8b..3f14b9c4 100644 --- a/swmmanywhere/preprocessing.py +++ b/swmmanywhere/preprocessing.py @@ -221,21 +221,39 @@ def prepare_building(bbox: tuple[float, float, float, float], def prepare_street(bbox: tuple[float, float, float, float], addresses: parameters.FilePaths, target_crs: str, - source_crs: str = 'EPSG:4326'): - """Download and reproject street graph.""" + source_crs: str = 'EPSG:4326', + network_types = ['drive']): + """Download and reproject street graph. + + Download the street graph within the bbox and reproject it to the UTM zone. + The street graph is downloaded for all network types in network_types. The + street graph is saved to the addresses.street directory. + + Args: + bbox (tuple[float, float, float, float]): Bounding box coordinates in + the format (minx, miny, maxx, maxy) in EPSG:4326. + addresses (FilePaths): Class containing the addresses of the directories. + target_crs (str): Target CRS to reproject the graph to. + source_crs (str): Source CRS of the graph. + network_types (list): List of network types to download. For duplicate + edges, nx.compose_all selects the attributes in priority of last to + first. In likelihood, you want to ensure that the last network in + the list is `drive`, so as to retain information about `lanes`, + which is needed to calculate impervious area. + """ if addresses.street.exists(): return - logger.info(f'downloading street network to {addresses.street}') + logger.info(f'downloading network to {addresses.street}') + if network_types[-1] != 'drive': + logger.warning("""The last network type should be `drive` to retain + `lanes` attribute, needed to calculate impervious area.""") + networks = [] + for network_type in network_types: + network = prepare_data.download_street(bbox, network_type=network_type) + nx.set_edge_attributes(network, network_type, 'network_type') + networks.append(network) + street_network = nx.compose_all(networks) street_network = prepare_data.download_street(bbox, network_type='drive') - nx.set_edge_attributes(street_network, 'drive', 'network_type') - - # Download walk network to enable pipes along walkways - walk_network = prepare_data.download_street(bbox, network_type='walk') - nx.set_edge_attributes(walk_network, 'walk', 'network_type') - - # Combine streets and walkways (use street_network as first arg so that - # parameters from street_network are used where there are conflicts) - street_network = nx.compose(walk_network, street_network) # Reproject graph street_network = go.reproject_graph(street_network, From 200cd4f67f48ae7eb1658fa105b798e45168e19b Mon Sep 17 00:00:00 2001 From: Dobson Date: Thu, 9 May 2024 10:01:41 +0100 Subject: [PATCH 24/28] allowable networks made more flexible --- swmmanywhere/graph_utilities.py | 5 ----- swmmanywhere/preprocessing.py | 6 ++++-- swmmanywhere/swmmanywhere.py | 21 +++++++++++---------- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/swmmanywhere/graph_utilities.py b/swmmanywhere/graph_utilities.py index 9d23166f..20fa1dfa 100644 --- a/swmmanywhere/graph_utilities.py +++ b/swmmanywhere/graph_utilities.py @@ -291,11 +291,6 @@ def __call__(self, """ edges_to_remove = set() for u, v, keys, data in G.edges(data=True,keys = True): - if data.get('network_type','drive') \ - not in topology_derivation.allowable_networks: - - edges_to_remove.add((u, v, keys)) - continue for omit in topology_derivation.omit_edges: if data.get('highway', None) == omit: # Check whether the 'highway' property is 'omit' diff --git a/swmmanywhere/preprocessing.py b/swmmanywhere/preprocessing.py index 3f14b9c4..dfd91638 100644 --- a/swmmanywhere/preprocessing.py +++ b/swmmanywhere/preprocessing.py @@ -278,7 +278,8 @@ def prepare_river(bbox: tuple[float, float, float, float], def run_downloads(bbox: tuple[float, float, float, float], addresses: parameters.FilePaths, - api_keys: dict[str, str]): + api_keys: dict[str, str], + network_types = ['drive']): """Run the data downloads. Run the precipitation, elevation, building, street and river network @@ -290,6 +291,7 @@ def run_downloads(bbox: tuple[float, float, float, float], the format (minx, miny, maxx, maxy) in EPSG:4326. addresses (FilePaths): Class containing the addresses of the directories. api_keys (dict): Dictionary containing the API keys. + network_types (list): List of network types to download. """ target_crs = go.get_utm_epsg(bbox[0], bbox[1]) @@ -303,7 +305,7 @@ def run_downloads(bbox: tuple[float, float, float, float], prepare_building(bbox, addresses, target_crs) # Download street network data - prepare_street(bbox, addresses, target_crs) + prepare_street(bbox, addresses, target_crs, network_types=network_types) # Download river network data prepare_river(bbox, addresses, target_crs) diff --git a/swmmanywhere/swmmanywhere.py b/swmmanywhere/swmmanywhere.py index eb84d837..4b26cfea 100644 --- a/swmmanywhere/swmmanywhere.py +++ b/swmmanywhere/swmmanywhere.py @@ -45,13 +45,21 @@ def swmmanywhere(config: dict) -> tuple[Path, dict | None]: for key, val in config.get('address_overrides', {}).items(): setattr(addresses, key, val) + # Load the parameters and perform any manual overrides + logger.info("Loading and setting parameters.") + params = parameters.get_full_parameters() + for category, overrides in config.get('parameter_overrides', {}).items(): + for key, val in overrides.items(): + setattr(params[category], key, val) + # Run downloads logger.info("Running downloads.") api_keys = yaml.safe_load(config['api_keys'].open('r')) preprocessing.run_downloads(config['bbox'], - addresses, - api_keys - ) + addresses, + api_keys, + network_types = params['topology_derivation'].allowable_networks + ) # Identify the starting graph logger.info("Iterating graphs.") @@ -60,13 +68,6 @@ def swmmanywhere(config: dict) -> tuple[Path, dict | None]: else: G = preprocessing.create_starting_graph(addresses) - # Load the parameters and perform any manual overrides - logger.info("Loading and setting parameters.") - params = parameters.get_full_parameters() - for category, overrides in config.get('parameter_overrides', {}).items(): - for key, val in overrides.items(): - setattr(params[category], key, val) - # Iterate the graph functions logger.info("Iterating graph functions.") G = iterate_graphfcns(G, From 123e5b9ee28a23a3bd096dbb6cdbd3d1a3e5361c Mon Sep 17 00:00:00 2001 From: Dobson Date: Fri, 10 May 2024 11:38:58 +0100 Subject: [PATCH 25/28] Update preprocessing.py fix merge --- swmmanywhere/preprocessing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/swmmanywhere/preprocessing.py b/swmmanywhere/preprocessing.py index dfd91638..e56ed0fa 100644 --- a/swmmanywhere/preprocessing.py +++ b/swmmanywhere/preprocessing.py @@ -253,7 +253,6 @@ def prepare_street(bbox: tuple[float, float, float, float], nx.set_edge_attributes(network, network_type, 'network_type') networks.append(network) street_network = nx.compose_all(networks) - street_network = prepare_data.download_street(bbox, network_type='drive') # Reproject graph street_network = go.reproject_graph(street_network, From f9459fb71d4ad224178a080eea875ac0b46ffaab Mon Sep 17 00:00:00 2001 From: Dobson Date: Fri, 10 May 2024 13:38:08 +0100 Subject: [PATCH 26/28] Update docs/naming --- swmmanywhere/graph_utilities.py | 8 ++++++-- tests/test_data/demo_config.yml | 4 ++-- tests/test_graph_utilities.py | 6 +++--- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/swmmanywhere/graph_utilities.py b/swmmanywhere/graph_utilities.py index 41ebf688..76b8dc78 100644 --- a/swmmanywhere/graph_utilities.py +++ b/swmmanywhere/graph_utilities.py @@ -317,6 +317,10 @@ def __call__(self, **kwargs) -> nx.Graph: """Format the lanes attribute of each edge and calculates width. + Only the `drive` network is assumed to contribute to impervious area and + so others `network_types` have lanes set to 0. If no `network_type` is + present, the edge is assumed to be of type `drive`. + Args: G (nx.Graph): A graph subcatchment_derivation (parameters.SubcatchmentDerivation): A @@ -485,7 +489,7 @@ def __call__(self, ) @register_graphfcn -class merge_nodes(BaseGraphFunction): +class merge_street_nodes(BaseGraphFunction): """merge_nodes class.""" def __call__(self, G: nx.Graph, @@ -493,7 +497,7 @@ def __call__(self, **kwargs) -> nx.Graph: """Merge nodes that are close together. - This function merges nodes that are within a certain distance of each + Merges `street` nodes that are within a certain distance of each other. The distance is specified in the `node_merge_distance` attribute of the `subcatchment_derivation` parameter. The merged nodes are given the same coordinates, and the graph is relabeled with nx.relabel_nodes. diff --git a/tests/test_data/demo_config.yml b/tests/test_data/demo_config.yml index 805777e8..92c1ac6c 100644 --- a/tests/test_data/demo_config.yml +++ b/tests/test_data/demo_config.yml @@ -20,8 +20,8 @@ graphfcn_list: - remove_parallel_edges # Remove parallel edges retaining the shorter one - to_undirected # Convert graph to undirected to facilitate cleanup - split_long_edges # Set a maximum edge length - - merge_nodes # Merge nodes that are too close together - - assign_id # Remove duplicates arising from merge_nodes + - merge_street_nodes # Merge street nodes that are too close together + - assign_id # Remove duplicates arising from merge_street_nodes - clip_to_catchments # Clip graph to catchment subbasins - calculate_contributing_area # Calculate runoff coefficient - set_elevation # Set node elevation from DEM diff --git a/tests/test_graph_utilities.py b/tests/test_graph_utilities.py index c3f3acf8..5967f09a 100644 --- a/tests/test_graph_utilities.py +++ b/tests/test_graph_utilities.py @@ -327,12 +327,12 @@ def almost_equal(a, b, tol=1e-6): """Check if two numbers are almost equal.""" return abs(a-b) < tol -def test_merge_nodes(): - """Test the merge_nodes function.""" +def test_merge_street_nodes(): + """Test the merge_street_nodes function.""" G, _ = load_street_network() subcatchment_derivation = parameters.SubcatchmentDerivation( node_merge_distance = 20) - G_ = gu.merge_nodes(G, subcatchment_derivation) + G_ = gu.merge_street_nodes(G, subcatchment_derivation) assert not set([107736,266325461,2623975694,32925453]).intersection(G_.nodes) assert almost_equal(G_.nodes[25510321]['x'], 700445.0112082) From 7625ddd998abd68c9f3bd35909d8fbb7b98379cd Mon Sep 17 00:00:00 2001 From: Dobson Date: Fri, 10 May 2024 13:44:12 +0100 Subject: [PATCH 27/28] Update preprocessing.py --- swmmanywhere/preprocessing.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/swmmanywhere/preprocessing.py b/swmmanywhere/preprocessing.py index e56ed0fa..f0dcdc7f 100644 --- a/swmmanywhere/preprocessing.py +++ b/swmmanywhere/preprocessing.py @@ -244,9 +244,12 @@ def prepare_street(bbox: tuple[float, float, float, float], if addresses.street.exists(): return logger.info(f'downloading network to {addresses.street}') - if network_types[-1] != 'drive': + if 'drive' in network_types and network_types[-1] != 'drive': logger.warning("""The last network type should be `drive` to retain - `lanes` attribute, needed to calculate impervious area.""") + `lanes` attribute, needed to calculate impervious area. + Moving it to the last position.""") + network_types.pop("drive") + network_types.append("drive") networks = [] for network_type in network_types: network = prepare_data.download_street(bbox, network_type=network_type) From cfbe635d68ff23689068c2d873cb5d4c425e56c9 Mon Sep 17 00:00:00 2001 From: Dobson Date: Mon, 13 May 2024 16:15:48 +0100 Subject: [PATCH 28/28] Revert "remove change from another branch" This reverts commit f0b4c177fa5063954f3f1e86c834ca426cdbcce1. --- swmmanywhere/graph_utilities.py | 22 ++++++++++++++++++++++ tests/test_graph_utilities.py | 20 ++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/swmmanywhere/graph_utilities.py b/swmmanywhere/graph_utilities.py index 76b8dc78..8a73a26b 100644 --- a/swmmanywhere/graph_utilities.py +++ b/swmmanywhere/graph_utilities.py @@ -1050,6 +1050,28 @@ def __call__(self, # Copy graph to run shortest path on G_ = G.copy() + if not matched_outlets: + # In cases of e.g., an area with no rivers to discharge into or too + # small a buffer + + # Identify the lowest elevation node + lowest_elevation_node = min(G.nodes, + key = lambda x: G.nodes[x]['surface_elevation']) + + # Create a dummy river to discharge into + dummy_river = {'id' : 'dummy_river', + 'x' : G.nodes[lowest_elevation_node]['x'] + 1, + 'y' : G.nodes[lowest_elevation_node]['y'] + 1} + G_.add_node('dummy_river') + nx.set_node_attributes(G_, {'dummy_river' : dummy_river}) + + # Update function's dicts + matched_outlets = {'dummy_river' : lowest_elevation_node} + river_points['dummy_river'] = shapely.Point(dummy_river['x'], + dummy_river['y']) + + logger.warning('No outlets found, using lowest elevation node as outlet') + # Add edges between the paired river and street nodes for river_id, street_id in matched_outlets.items(): # TODO instead use a weight based on the distance between the two nodes diff --git a/tests/test_graph_utilities.py b/tests/test_graph_utilities.py index 5967f09a..0936b8d3 100644 --- a/tests/test_graph_utilities.py +++ b/tests/test_graph_utilities.py @@ -179,6 +179,26 @@ def test_calculate_weights(): for u, v, data in G.edges(data=True): assert 'weight' in data.keys() assert math.isfinite(data['weight']) + +def test_identify_outlets_no_river(): + """Test the identify_outlets in the no river case.""" + G, _ = load_street_network() + G = gu.assign_id(G) + G = gu.double_directed(G) + elev_fid = Path(__file__).parent / 'test_data' / 'elevation.tif' + addresses = parameters.FilePaths(base_dir = None, + project_name = None, + bbox_number = None, + model_number = None) + addresses.elevation = elev_fid + G = gu.set_elevation(G, addresses) + for ix, (u,v,d) in enumerate(G.edges(data=True)): + d['edge_type'] = 'street' + d['weight'] = ix + params = parameters.OutletDerivation() + G = gu.identify_outlets(G, params) + outlets = [(u,v,d) for u,v,d in G.edges(data=True) if d['edge_type'] == 'outlet'] + assert len(outlets) == 1 def test_identify_outlets_and_derive_topology(): """Test the identify_outlets and derive_topology functions."""