From 1170e357f4bd068c40e37e709e06795ed061d450 Mon Sep 17 00:00:00 2001 From: Dobson Date: Fri, 18 Oct 2024 09:45:21 +0100 Subject: [PATCH 01/36] first attempt --- docs/index.md | 1 + docs/notebooks/extended_demo.py | 83 +++++++++++++++++++++++++++++++++ mkdocs.yml | 3 +- pyproject.toml | 1 + 4 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 docs/notebooks/extended_demo.py diff --git a/docs/index.md b/docs/index.md index dcdf0525..1dae00ec 100644 --- a/docs/index.md +++ b/docs/index.md @@ -9,6 +9,7 @@ derive a synthetic urban drainage network anywhere in the world. - [Quickstart](quickstart.md) - Guides: - [Configuration file](config_guide.md) + - [Extended demo](./notebooks/extended_demo.py) - [Graph functions](graphfcns_guide.md) - [Metrics guide](metrics_guide.md) - [Contributing](CONTRIBUTING.md) diff --git a/docs/notebooks/extended_demo.py b/docs/notebooks/extended_demo.py new file mode 100644 index 00000000..c0c0fe14 --- /dev/null +++ b/docs/notebooks/extended_demo.py @@ -0,0 +1,83 @@ +"""A demo.""" + +# %% [markdown] +# # Extended Demo +# Note - this script can also be opened in interactive Python if you wanted to +# play around. On the GitHub it is in [docs/notebooks]. To run this on your +# local machine, you will need to install the optional dependencies for `doc`: +# +# `pip install swmmanywhere[doc]` +# +# %% [markdown] +# ## Introduction +# This script demonstrates a simple use case of `SWMManywhere`, building on the +# [quickstart](quickstart.md) example, but including plotting and alterations. +# +# Since this is a notebook, we will define [`config`](config_guide.md) as a +# dictionary rather than a `yaml` file, but the same principles apply. +# +# ## Initial run +# +# Here we will run the [quickstart](quickstart.md) configuration, keeping +# everything in a temporary directory. +# %% +# Imports +from __future__ import annotations + +import tempfile +from pathlib import Path + +import folium +import geopandas as gpd + +from swmmanywhere.swmmanywhere import swmmanywhere + +# Create temporary directory +temp_dir = tempfile.TemporaryDirectory() +base_dir = Path(temp_dir.name) + +# Define minimum viable config +config = { + "base_dir": base_dir, + "project": "my_first_swmm", + "bbox": [1.52740, 42.50524, 1.54273, 42.51259], + "address_overrides": { + "elevation": Path( + r"C:\Users\bdobson\Downloads\test\my_first_swmm\bbox_1\download\elevation.tif" + ) + }, +} + +# Run SWMManywhere +outputs = swmmanywhere(config) + +# Verify the output +model_file = outputs[0] +if not model_file.exists(): + raise FileNotFoundError(f"Model file not created: {model_file}") + +# %% [markdown] +# ## Plotting output +# +# If you do not have a real UDM, the majority of your interpretation will be +# around the synthesised `nodes` and `edges`. These are +# created in the same directory as the `model_file``. +# +# Let's have a look at these +# %% + +# Load and inspect results +nodes = gpd.read_file(model_file.parent / "nodes.geoparquet") +edges = gpd.read_file(model_file.parent / "edges.geoparquet") + +# Convert to EPSG 4326 for plotting +nodes = nodes.to_crs(4326) +edges = edges.to_crs(4326) + +# Create a folium map and add the nodes and edges +m = folium.Map(location=[nodes.y.mean(), nodes.x.mean()], zoom_start=16) +folium.GeoJson(nodes).add_to(m) +folium.GeoJson(edges).add_to(m) + +# Display the map +m diff --git a/mkdocs.yml b/mkdocs.yml index 7d02f737..7b9680d1 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -15,7 +15,7 @@ extra_css: plugins: - mkdocstrings - mkdocs-jupyter: - execute: false + execute: true - search - include-markdown @@ -44,6 +44,7 @@ nav: - Quickstart: quickstart.md - Guides: - Configuration guide: config_guide.md + - Extended demo: ./notebooks/extended_demo.py - Graph functions: graphfcns_guide.md - Metrics guide: metrics_guide.md - Contributing: CONTRIBUTING.md diff --git a/pyproject.toml b/pyproject.toml index 665f451b..1c220409 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,6 +72,7 @@ optional-dependencies.dev = [ "ruff", ] optional-dependencies.doc = [ + "folium", "mkdocs", "mkdocs-include-markdown-plugin", "mkdocs-jupyter", From 4205538bd8ec0b8c94735bbc5ca2d1ea45fad00d Mon Sep 17 00:00:00 2001 From: Dobson Date: Fri, 18 Oct 2024 11:55:20 +0100 Subject: [PATCH 02/36] continue demo --- docs/notebooks/extended_demo.py | 183 +++++++++++++++++++++-- pyproject.toml | 1 + src/swmmanywhere/geospatial_utilities.py | 2 +- src/swmmanywhere/logging.py | 3 +- src/swmmanywhere/swmmanywhere.py | 5 + 5 files changed, 178 insertions(+), 16 deletions(-) diff --git a/docs/notebooks/extended_demo.py b/docs/notebooks/extended_demo.py index c0c0fe14..2a5c863b 100644 --- a/docs/notebooks/extended_demo.py +++ b/docs/notebooks/extended_demo.py @@ -1,3 +1,4 @@ +# %% """A demo.""" # %% [markdown] @@ -24,11 +25,15 @@ # Imports from __future__ import annotations +import base64 import tempfile +from io import BytesIO from pathlib import Path import folium import geopandas as gpd +import pandas as pd +from matplotlib import pyplot as plt from swmmanywhere.swmmanywhere import swmmanywhere @@ -56,28 +61,180 @@ if not model_file.exists(): raise FileNotFoundError(f"Model file not created: {model_file}") + # %% [markdown] # ## Plotting output # # If you do not have a real UDM, the majority of your interpretation will be # around the synthesised `nodes` and `edges`. These are -# created in the same directory as the `model_file``. +# created in the same directory as the `model_file`. Let's have a look at them. +# %% +# Create a folium map and add the nodes and edges +def basic_map(model_dir): + # Load and inspect results + nodes = gpd.read_file(model_dir / "nodes.geoparquet") + edges = gpd.read_file(model_dir / "edges.geoparquet") + + + # Convert to EPSG 4326 for plotting + nodes = nodes.to_crs(4326) + edges = edges.to_crs(4326) + + m = folium.Map(location=[nodes.geometry.y.mean(), nodes.geometry.x.mean()], zoom_start=16) + folium.GeoJson(edges, color='black',weight=1).add_to(m) + folium.GeoJson(nodes, marker=folium.CircleMarker(radius = 3, # Radius in metres + weight = 0, #outline weight + fill_color = 'black', + fill_opacity = 1)).add_to(m) + + # Display the map + return m + +basic_map(model_file.parent) + +# %% [markdown] +# OK, it's done something! Though perhaps we're not super satisfied with the output. +# +# ## Customising outputs # -# Let's have a look at these +# Some things stick out on first glance: +# - Probably we do not need pipes in the hills to the South, these seem to be along pedestrian routes, which can be adjusted with the `allowable_networks` parameter. +# - We will also remove any types under the `omit_edges` entry, here you can specify to not allow pipes to cross bridges, tunnels, motorways, etc., however, this is such a small area we probably don't want to restrict things so much. +# - The density of points seems a bit extreme, ultimately we'd like to walk around and figure out where the manholes are, but for now we can reduce density by increasing `node_merge_distance`. +# Let's just demonstrate that using the [`parameter_overrides` functionality](config_guide.md/#changing-parameters). + # %% +config['parameter_overrides'] = {'topology_derivation' : {'allowable_networks' : ['drive'], 'omit_edges' : []}, 'subcatchment_derivation' : {'node_merge_distance':15 }} +outputs = swmmanywhere(config) +basic_map(outputs[0].parent) -# Load and inspect results -nodes = gpd.read_file(model_file.parent / "nodes.geoparquet") -edges = gpd.read_file(model_file.parent / "edges.geoparquet") +# %% [markdown] +# OK that clearly helped, although we have appear to have stranded pipes along *Carrer de la Grella*, presumably due to some mistake in the OSM specifying that it is connected via a pedestrian route. We won't remedy this in the tutorial, but you can manually provide your [`starting_graph`](config_guide.md/#change-starting_graph) via the configuration file to address such mitakes. +# +# More importantly we can see some distinctive unconnected network in the South West. What is going on there? To explain this we will have to turn on verbosity to print the intermediate files used in model derivation. +# +# To do this with a command line call we simply add the flag `--verbose=True`. -# Convert to EPSG 4326 for plotting -nodes = nodes.to_crs(4326) -edges = edges.to_crs(4326) +# %% +# Make verbose +from swmmanywhere import logging +logging.set_verbose(True) # Set verbosity -# Create a folium map and add the nodes and edges -m = folium.Map(location=[nodes.y.mean(), nodes.x.mean()], zoom_start=16) -folium.GeoJson(nodes).add_to(m) -folium.GeoJson(edges).add_to(m) +# Run again +outputs = swmmanywhere(config) +model_dir = outputs[0].parent +m = basic_map(model_dir) -# Display the map +# %% [markdown] +# OK that's a lot of information! We can see `swmmanywhere` iterating through the various graph functions and a variety of other messages. However, the reason we are currently interested in this is because the files associated with each step are saved when `verbose=True`. +# +# We will load a file called `subbasins` and add it to the map. + +# %% +subbasins = gpd.read_file(model_dir / "subbasins.geoparquet") +folium.GeoJson(subbasins,fill_opacity=0, color='blue',weight=2).add_to(m) m + +# %% [markdown] +# Although this can be customised, the default behaviour of `swmmanywhere` is to not allow edges to cross hydrological subbasins. It is now super clear why these unconnected networks have appeared, and are ultimately due to the underlying DEM. If you did desperately care about these streets, then you should probably widen your bounding box. +# +# ## Plotting results +# +# Because we have run the model with `verbose=True` we will also see that a new `results` file has appeared, which contains all of the simulation results from SWMM. + +# %% +df = pd.read_parquet(model_dir / "results.parquet") +df.head() + +# %% [markdown] +# `results` contains all simulation results in long format, with `flooding` at nodes and `flow` at edges. We will plot a random `flow`. + +# %% +floods = df.loc[df.variable == 'flooding'] +flows = df.loc[df.variable == 'flow'] +flows.loc[flows.id == flows.iloc[0].id].set_index('date').value.plot(ylabel='flow (m3/s)') + + +# %% [markdown] +# Since folium is super clever, we can make these clickable on our map - and now you can inspect your results in a much more elegant way than the SWMM GUI. + +# %% +# Create a folium map and add the nodes and edges +def clickable_map(model_dir): + # Load and inspect results + nodes = gpd.read_file(model_dir / "nodes.geoparquet") + edges = gpd.read_file(model_dir / "edges.geoparquet") + df = pd.read_parquet(model_dir / "results.parquet") + df.id = df.id.astype(str) + floods = df.loc[df.variable == 'flooding'].groupby('id') + flows = df.loc[df.variable == 'flow'].groupby('id') + + + # Convert to EPSG 4326 for plotting + nodes = nodes.to_crs(4326).set_index('id') + edges = edges.to_crs(4326).set_index('id') + + # Create map + m = folium.Map(location=[nodes.geometry.y.mean(), nodes.geometry.x.mean()], zoom_start=16) + + # Add nodes + for node, row in nodes.iterrows(): + grp = floods.get_group(str(node)) + grp.set_index('date').value.plot(ylabel='flooding (m3)', title = node) + img = BytesIO() + f = plt.gcf() + f.savefig(img, format="png",dpi=94) + plt.close(f) + img.seek(0) + img_base64 = base64.b64encode(img.read()).decode() + img_html = f'' + folium.CircleMarker( + [nodes.loc[node].geometry.y, nodes.loc[node].geometry.x], + color="black", + radius=5, + weight=0, + fill_color='black', + fill_opacity=1, + popup=folium.Popup(img_html, max_width=450), + ).add_to(m) + + # Add edges + for edge, row in edges.iterrows(): + grp = flows.get_group(str(edge)) + grp.set_index('date').value.plot(ylabel='flow (m3/s)', title = edge) + img = BytesIO() + f = plt.gcf() + f.savefig(img, format="png",dpi=94) + plt.close(f) + img.seek(0) + img_base64 = base64.b64encode(img.read()).decode() + img_html = f'' + folium.PolyLine( + [[c[1],c[0]] for c in row.geometry.coords], + color="black", + weight=5, + popup=folium.Popup(img_html, max_width=450), + ).add_to(m) + return m + +clickable_map(model_dir) + +# %% +from io import BytesIO +from matplotlib import pyplot as plt +import base64 + +# %% +nodes = gpd.read_file(model_dir / "nodes.geoparquet") +nodes + +# %% +df = pd.read_parquet(model_dir / "results.parquet") +df.id = df.id.astype(str) +floods = df.loc[df.variable == 'flooding'].groupby('id') +floods.get_group('174') + +# %% +floods + +# %% diff --git a/pyproject.toml b/pyproject.toml index 1c220409..3325cf29 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,6 +73,7 @@ optional-dependencies.dev = [ ] optional-dependencies.doc = [ "folium", + "matplotlib", "mkdocs", "mkdocs-include-markdown-plugin", "mkdocs-jupyter", diff --git a/src/swmmanywhere/geospatial_utilities.py b/src/swmmanywhere/geospatial_utilities.py index b23253e8..5e29599b 100644 --- a/src/swmmanywhere/geospatial_utilities.py +++ b/src/swmmanywhere/geospatial_utilities.py @@ -646,7 +646,7 @@ def flwdir_whitebox(fid: Path) -> np.array: temp_path, wbt_args, save_dir=temp_path, - verbose=verbose(), + verbose=False, wbt_root=temp_path / "WBT", zip_path=fid.parent / "whiteboxtools_binaries.zip", max_procs=1, diff --git a/src/swmmanywhere/logging.py b/src/swmmanywhere/logging.py index 21335726..31574d25 100644 --- a/src/swmmanywhere/logging.py +++ b/src/swmmanywhere/logging.py @@ -35,7 +35,6 @@ def dynamic_filter(record): """A dynamic filter.""" return verbose() - def get_logger() -> loguru.logger: """Get a logger.""" logger = loguru.logger @@ -78,4 +77,4 @@ def new_add(sink, **kwargs): # Replace the logger's add method with new_add -logger.add = new_add +logger.add = new_add \ No newline at end of file diff --git a/src/swmmanywhere/swmmanywhere.py b/src/swmmanywhere/swmmanywhere.py index b68e503a..b8dfaa17 100644 --- a/src/swmmanywhere/swmmanywhere.py +++ b/src/swmmanywhere/swmmanywhere.py @@ -105,6 +105,11 @@ def swmmanywhere(config: dict) -> tuple[Path, dict | None]: logger.info(f"Setting {category} {key} to {val}") setattr(params[category], key, val) + # If `allowable_networks` has been changed, force a redownload of street graph. + if 'allowable_networks' in config.get("parameter_overrides", {}).get("topology_derivation", {}): + logger.info("Allowable networks have been changed, removing old street graph.") + addresses.bbox_paths.street.unlink(missing_ok=True) + # Run downloads logger.info("Running downloads.") preprocessing.run_downloads( From 3b725210ddf18e358e512043c0d64ec86e53c10e Mon Sep 17 00:00:00 2001 From: Dobson Date: Fri, 18 Oct 2024 12:01:18 +0100 Subject: [PATCH 03/36] finalise --- docs/notebooks/extended_demo.py | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/docs/notebooks/extended_demo.py b/docs/notebooks/extended_demo.py index 2a5c863b..453c0220 100644 --- a/docs/notebooks/extended_demo.py +++ b/docs/notebooks/extended_demo.py @@ -191,7 +191,7 @@ def clickable_map(model_dir): folium.CircleMarker( [nodes.loc[node].geometry.y, nodes.loc[node].geometry.x], color="black", - radius=5, + radius=3, weight=0, fill_color='black', fill_opacity=1, @@ -212,29 +212,12 @@ def clickable_map(model_dir): folium.PolyLine( [[c[1],c[0]] for c in row.geometry.coords], color="black", - weight=5, + weight=2, popup=folium.Popup(img_html, max_width=450), ).add_to(m) return m clickable_map(model_dir) -# %% -from io import BytesIO -from matplotlib import pyplot as plt -import base64 - -# %% -nodes = gpd.read_file(model_dir / "nodes.geoparquet") -nodes - -# %% -df = pd.read_parquet(model_dir / "results.parquet") -df.id = df.id.astype(str) -floods = df.loc[df.variable == 'flooding'].groupby('id') -floods.get_group('174') - -# %% -floods - -# %% +# %% [markdown] +# If we explore around, clicking on edges, we can see that flows are often looking sensible, though we can definitely some areas that have been hampered by our starting street graph (e.g., *Carrer dels Canals*). The first suggestion here would be to widen your bounding box, however, if you want to make more sophisticated customisations then your probably want to learn about [graph functions](graphfcns_guide.md) From 40273ba6dcf9260ea4da3519334dc59871b91f0a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 18 Oct 2024 11:21:45 +0000 Subject: [PATCH 04/36] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/notebooks/extended_demo.py | 74 +++++++++++++++++++------------- src/swmmanywhere/logging.py | 3 +- src/swmmanywhere/swmmanywhere.py | 4 +- 3 files changed, 50 insertions(+), 31 deletions(-) diff --git a/docs/notebooks/extended_demo.py b/docs/notebooks/extended_demo.py index 453c0220..3867c154 100644 --- a/docs/notebooks/extended_demo.py +++ b/docs/notebooks/extended_demo.py @@ -74,22 +74,29 @@ def basic_map(model_dir): # Load and inspect results nodes = gpd.read_file(model_dir / "nodes.geoparquet") edges = gpd.read_file(model_dir / "edges.geoparquet") - # Convert to EPSG 4326 for plotting nodes = nodes.to_crs(4326) edges = edges.to_crs(4326) - - m = folium.Map(location=[nodes.geometry.y.mean(), nodes.geometry.x.mean()], zoom_start=16) - folium.GeoJson(edges, color='black',weight=1).add_to(m) - folium.GeoJson(nodes, marker=folium.CircleMarker(radius = 3, # Radius in metres - weight = 0, #outline weight - fill_color = 'black', - fill_opacity = 1)).add_to(m) - + + m = folium.Map( + location=[nodes.geometry.y.mean(), nodes.geometry.x.mean()], zoom_start=16 + ) + folium.GeoJson(edges, color="black", weight=1).add_to(m) + folium.GeoJson( + nodes, + marker=folium.CircleMarker( + radius=3, # Radius in metres + weight=0, # outline weight + fill_color="black", + fill_opacity=1, + ), + ).add_to(m) + # Display the map return m + basic_map(model_file.parent) # %% [markdown] @@ -104,12 +111,15 @@ def basic_map(model_dir): # Let's just demonstrate that using the [`parameter_overrides` functionality](config_guide.md/#changing-parameters). # %% -config['parameter_overrides'] = {'topology_derivation' : {'allowable_networks' : ['drive'], 'omit_edges' : []}, 'subcatchment_derivation' : {'node_merge_distance':15 }} +config["parameter_overrides"] = { + "topology_derivation": {"allowable_networks": ["drive"], "omit_edges": []}, + "subcatchment_derivation": {"node_merge_distance": 15}, +} outputs = swmmanywhere(config) basic_map(outputs[0].parent) # %% [markdown] -# OK that clearly helped, although we have appear to have stranded pipes along *Carrer de la Grella*, presumably due to some mistake in the OSM specifying that it is connected via a pedestrian route. We won't remedy this in the tutorial, but you can manually provide your [`starting_graph`](config_guide.md/#change-starting_graph) via the configuration file to address such mitakes. +# OK that clearly helped, although we have appear to have stranded pipes along *Carrer de la Grella*, presumably due to some mistake in the OSM specifying that it is connected via a pedestrian route. We won't remedy this in the tutorial, but you can manually provide your [`starting_graph`](config_guide.md/#change-starting_graph) via the configuration file to address such mitakes. # # More importantly we can see some distinctive unconnected network in the South West. What is going on there? To explain this we will have to turn on verbosity to print the intermediate files used in model derivation. # @@ -118,7 +128,8 @@ def basic_map(model_dir): # %% # Make verbose from swmmanywhere import logging -logging.set_verbose(True) # Set verbosity + +logging.set_verbose(True) # Set verbosity # Run again outputs = swmmanywhere(config) @@ -126,13 +137,13 @@ def basic_map(model_dir): m = basic_map(model_dir) # %% [markdown] -# OK that's a lot of information! We can see `swmmanywhere` iterating through the various graph functions and a variety of other messages. However, the reason we are currently interested in this is because the files associated with each step are saved when `verbose=True`. +# OK that's a lot of information! We can see `swmmanywhere` iterating through the various graph functions and a variety of other messages. However, the reason we are currently interested in this is because the files associated with each step are saved when `verbose=True`. # # We will load a file called `subbasins` and add it to the map. # %% subbasins = gpd.read_file(model_dir / "subbasins.geoparquet") -folium.GeoJson(subbasins,fill_opacity=0, color='blue',weight=2).add_to(m) +folium.GeoJson(subbasins, fill_opacity=0, color="blue", weight=2).add_to(m) m # %% [markdown] @@ -150,14 +161,17 @@ def basic_map(model_dir): # `results` contains all simulation results in long format, with `flooding` at nodes and `flow` at edges. We will plot a random `flow`. # %% -floods = df.loc[df.variable == 'flooding'] -flows = df.loc[df.variable == 'flow'] -flows.loc[flows.id == flows.iloc[0].id].set_index('date').value.plot(ylabel='flow (m3/s)') +floods = df.loc[df.variable == "flooding"] +flows = df.loc[df.variable == "flow"] +flows.loc[flows.id == flows.iloc[0].id].set_index("date").value.plot( + ylabel="flow (m3/s)" +) # %% [markdown] # Since folium is super clever, we can make these clickable on our map - and now you can inspect your results in a much more elegant way than the SWMM GUI. + # %% # Create a folium map and add the nodes and edges def clickable_map(model_dir): @@ -166,24 +180,25 @@ def clickable_map(model_dir): edges = gpd.read_file(model_dir / "edges.geoparquet") df = pd.read_parquet(model_dir / "results.parquet") df.id = df.id.astype(str) - floods = df.loc[df.variable == 'flooding'].groupby('id') - flows = df.loc[df.variable == 'flow'].groupby('id') - + floods = df.loc[df.variable == "flooding"].groupby("id") + flows = df.loc[df.variable == "flow"].groupby("id") # Convert to EPSG 4326 for plotting - nodes = nodes.to_crs(4326).set_index('id') - edges = edges.to_crs(4326).set_index('id') + nodes = nodes.to_crs(4326).set_index("id") + edges = edges.to_crs(4326).set_index("id") # Create map - m = folium.Map(location=[nodes.geometry.y.mean(), nodes.geometry.x.mean()], zoom_start=16) + m = folium.Map( + location=[nodes.geometry.y.mean(), nodes.geometry.x.mean()], zoom_start=16 + ) # Add nodes for node, row in nodes.iterrows(): grp = floods.get_group(str(node)) - grp.set_index('date').value.plot(ylabel='flooding (m3)', title = node) + grp.set_index("date").value.plot(ylabel="flooding (m3)", title=node) img = BytesIO() f = plt.gcf() - f.savefig(img, format="png",dpi=94) + f.savefig(img, format="png", dpi=94) plt.close(f) img.seek(0) img_base64 = base64.b64encode(img.read()).decode() @@ -193,7 +208,7 @@ def clickable_map(model_dir): color="black", radius=3, weight=0, - fill_color='black', + fill_color="black", fill_opacity=1, popup=folium.Popup(img_html, max_width=450), ).add_to(m) @@ -201,22 +216,23 @@ def clickable_map(model_dir): # Add edges for edge, row in edges.iterrows(): grp = flows.get_group(str(edge)) - grp.set_index('date').value.plot(ylabel='flow (m3/s)', title = edge) + grp.set_index("date").value.plot(ylabel="flow (m3/s)", title=edge) img = BytesIO() f = plt.gcf() - f.savefig(img, format="png",dpi=94) + f.savefig(img, format="png", dpi=94) plt.close(f) img.seek(0) img_base64 = base64.b64encode(img.read()).decode() img_html = f'' folium.PolyLine( - [[c[1],c[0]] for c in row.geometry.coords], + [[c[1], c[0]] for c in row.geometry.coords], color="black", weight=2, popup=folium.Popup(img_html, max_width=450), ).add_to(m) return m + clickable_map(model_dir) # %% [markdown] diff --git a/src/swmmanywhere/logging.py b/src/swmmanywhere/logging.py index 31574d25..21335726 100644 --- a/src/swmmanywhere/logging.py +++ b/src/swmmanywhere/logging.py @@ -35,6 +35,7 @@ def dynamic_filter(record): """A dynamic filter.""" return verbose() + def get_logger() -> loguru.logger: """Get a logger.""" logger = loguru.logger @@ -77,4 +78,4 @@ def new_add(sink, **kwargs): # Replace the logger's add method with new_add -logger.add = new_add \ No newline at end of file +logger.add = new_add diff --git a/src/swmmanywhere/swmmanywhere.py b/src/swmmanywhere/swmmanywhere.py index b8dfaa17..c90b7de6 100644 --- a/src/swmmanywhere/swmmanywhere.py +++ b/src/swmmanywhere/swmmanywhere.py @@ -106,7 +106,9 @@ def swmmanywhere(config: dict) -> tuple[Path, dict | None]: setattr(params[category], key, val) # If `allowable_networks` has been changed, force a redownload of street graph. - if 'allowable_networks' in config.get("parameter_overrides", {}).get("topology_derivation", {}): + if "allowable_networks" in config.get("parameter_overrides", {}).get( + "topology_derivation", {} + ): logger.info("Allowable networks have been changed, removing old street graph.") addresses.bbox_paths.street.unlink(missing_ok=True) From 499c7c69037408bf54e7a621bf9245ed4bd74acb Mon Sep 17 00:00:00 2001 From: barneydobson Date: Fri, 18 Oct 2024 14:48:17 +0100 Subject: [PATCH 05/36] update link --- docs/config_guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/config_guide.md b/docs/config_guide.md index b0df8a1e..9a5688af 100644 --- a/docs/config_guide.md +++ b/docs/config_guide.md @@ -52,7 +52,7 @@ parameter_overrides: Note that we must provide the parameter category for the parameter that we are changing (`subcatchment_derivation` above). -As our SWMManywhere paper [link preprint] demonstrates, you can capture an enormously wide range of UDM behaviours through changing parameters. However, if your system is particularly unusual, or you are testing out new behaviours then you may need to adopt a more elaborate approach. +As our SWMManywhere paper [link preprint](https://doi.org/10.31223/X5GT5X) demonstrates, you can capture an enormously wide range of UDM behaviours through changing parameters. However, if your system is particularly unusual, or you are testing out new behaviours then you may need to adopt a more elaborate approach. ### Customise `graphfcns` From f7d0eca350c3cae5dc96b8ce40b50e745fd76ae7 Mon Sep 17 00:00:00 2001 From: barneydobson Date: Fri, 18 Oct 2024 15:03:26 +0100 Subject: [PATCH 06/36] update --- docs/notebooks/extended_demo.py | 80 +++++++++++++++++++++------------ pyproject.toml | 2 + 2 files changed, 53 insertions(+), 29 deletions(-) diff --git a/docs/notebooks/extended_demo.py b/docs/notebooks/extended_demo.py index 453c0220..db16898a 100644 --- a/docs/notebooks/extended_demo.py +++ b/docs/notebooks/extended_demo.py @@ -1,6 +1,3 @@ -# %% -"""A demo.""" - # %% [markdown] # # Extended Demo # Note - this script can also be opened in interactive Python if you wanted to @@ -46,11 +43,6 @@ "base_dir": base_dir, "project": "my_first_swmm", "bbox": [1.52740, 42.50524, 1.54273, 42.51259], - "address_overrides": { - "elevation": Path( - r"C:\Users\bdobson\Downloads\test\my_first_swmm\bbox_1\download\elevation.tif" - ) - }, } # Run SWMManywhere @@ -98,35 +90,60 @@ def basic_map(model_dir): # ## Customising outputs # # Some things stick out on first glance: -# - Probably we do not need pipes in the hills to the South, these seem to be along pedestrian routes, which can be adjusted with the `allowable_networks` parameter. -# - We will also remove any types under the `omit_edges` entry, here you can specify to not allow pipes to cross bridges, tunnels, motorways, etc., however, this is such a small area we probably don't want to restrict things so much. -# - The density of points seems a bit extreme, ultimately we'd like to walk around and figure out where the manholes are, but for now we can reduce density by increasing `node_merge_distance`. -# Let's just demonstrate that using the [`parameter_overrides` functionality](config_guide.md/#changing-parameters). - +# - Probably we do not need pipes in the hills to the South, these seem to be along +# pedestrian routes, which can be adjusted with the `allowable_networks` parameter. +# - We will also remove any types under the `omit_edges` entry, here you can specify +# to not allow pipes to cross bridges, tunnels, motorways, etc., however, this is +# such a small area we probably don't want to restrict things so much. +# - The density of points seems a bit extreme, ultimately we'd like to survey +# manholes locations, but for now we can reduce density by increasing +# `node_merge_distance`. +# +# Let's just demonstrate that using the +# [`parameter_overrides` functionality](https://imperialcollegelondon.github.io/SWMManywhere/config_guide/#changing-parameters). +# # %% config['parameter_overrides'] = {'topology_derivation' : {'allowable_networks' : ['drive'], 'omit_edges' : []}, 'subcatchment_derivation' : {'node_merge_distance':15 }} outputs = swmmanywhere(config) basic_map(outputs[0].parent) # %% [markdown] -# OK that clearly helped, although we have appear to have stranded pipes along *Carrer de la Grella*, presumably due to some mistake in the OSM specifying that it is connected via a pedestrian route. We won't remedy this in the tutorial, but you can manually provide your [`starting_graph`](config_guide.md/#change-starting_graph) via the configuration file to address such mitakes. +# OK that clearly helped, although we have appear to have stranded pipes along (e.g.,) +# *Carrer de la Grella*, presumably due to some mistake in the OSM specifying that it +# is connected via a pedestrian route. We won't remedy this in the tutorial, but you can +# manually provide your +# [`starting_graph`](https://imperialcollegelondon.github.io/SWMManywhere/config_guide/#change-starting_graph) +# via the configuration file to address such mitakes. # -# More importantly we can see some distinctive unconnected network in the South West. What is going on there? To explain this we will have to turn on verbosity to print the intermediate files used in model derivation. +# More importantly we can see some distinctive unconnected network in the South West. +# What is going on there? To explain this we will have to turn on verbosity to print the +# intermediate files used in model derivation. # -# To do this with a command line call we simply add the flag `--verbose=True`. +# To do this with a command line call we simply add the flag `--verbose=True`. +# Though in code we will have to import the `logging` module. # %% # Make verbose from swmmanywhere import logging logging.set_verbose(True) # Set verbosity - -# Run again +# Run again (wrapped in some html to limit the display) +output_html = """ +
+ 
+"""
 outputs = swmmanywhere(config)
+
+output_html += """
+
+
+""" model_dir = outputs[0].parent m = basic_map(model_dir) # %% [markdown] -# OK that's a lot of information! We can see `swmmanywhere` iterating through the various graph functions and a variety of other messages. However, the reason we are currently interested in this is because the files associated with each step are saved when `verbose=True`. +# That's a lot of information! However, the reason we are currently interested +# in this is because the files associated with +# each workflow step are saved when `verbose=True`. # # We will load a file called `subbasins` and add it to the map. @@ -136,11 +153,15 @@ def basic_map(model_dir): m # %% [markdown] -# Although this can be customised, the default behaviour of `swmmanywhere` is to not allow edges to cross hydrological subbasins. It is now super clear why these unconnected networks have appeared, and are ultimately due to the underlying DEM. If you did desperately care about these streets, then you should probably widen your bounding box. +# Although this can be customised, the default behaviour of `swmmanywhere` is to not +# allow edges to cross hydrological subbasins. It is now super clear why these unconnected +# networks have appeared, and are ultimately due to the underlying DEM. If you did +# desperately care about these streets, then you should probably widen your bounding box. # # ## Plotting results # -# Because we have run the model with `verbose=True` we will also see that a new `results` file has appeared, which contains all of the simulation results from SWMM. +# Because we have run the model with `verbose=True` we will also see that a new `results` +# file has appeared, which contains all of the simulation results from SWMM. # %% df = pd.read_parquet(model_dir / "results.parquet") @@ -150,7 +171,6 @@ def basic_map(model_dir): # `results` contains all simulation results in long format, with `flooding` at nodes and `flow` at edges. We will plot a random `flow`. # %% -floods = df.loc[df.variable == 'flooding'] flows = df.loc[df.variable == 'flow'] flows.loc[flows.id == flows.iloc[0].id].set_index('date').value.plot(ylabel='flow (m3/s)') @@ -180,9 +200,10 @@ def clickable_map(model_dir): # Add nodes for node, row in nodes.iterrows(): grp = floods.get_group(str(node)) - grp.set_index('date').value.plot(ylabel='flooding (m3)', title = node) + f,ax=plt.subplots(figsize=(4,3)) + grp.set_index('date').value.plot(ylabel='flooding (m3)', title = node,ax=ax) img = BytesIO() - f = plt.gcf() + f.tight_layout() f.savefig(img, format="png",dpi=94) plt.close(f) img.seek(0) @@ -195,15 +216,16 @@ def clickable_map(model_dir): weight=0, fill_color='black', fill_opacity=1, - popup=folium.Popup(img_html, max_width=450), + popup=folium.Popup(img_html), ).add_to(m) # Add edges for edge, row in edges.iterrows(): grp = flows.get_group(str(edge)) - grp.set_index('date').value.plot(ylabel='flow (m3/s)', title = edge) + f,ax=plt.subplots(figsize=(4,3)) + grp.set_index('date').value.plot(ylabel='flow (m3/s)', title = edge,ax=ax) img = BytesIO() - f = plt.gcf() + f.tight_layout() f.savefig(img, format="png",dpi=94) plt.close(f) img.seek(0) @@ -213,11 +235,11 @@ def clickable_map(model_dir): [[c[1],c[0]] for c in row.geometry.coords], color="black", weight=2, - popup=folium.Popup(img_html, max_width=450), + popup=folium.Popup(img_html), ).add_to(m) return m clickable_map(model_dir) # %% [markdown] -# If we explore around, clicking on edges, we can see that flows are often looking sensible, though we can definitely some areas that have been hampered by our starting street graph (e.g., *Carrer dels Canals*). The first suggestion here would be to widen your bounding box, however, if you want to make more sophisticated customisations then your probably want to learn about [graph functions](graphfcns_guide.md) +# If we explore around, clicking on edges, we can see that flows are often looking sensible, though we can definitely some areas that have been hampered by our starting street graph (e.g., *Carrer dels Canals*). The first suggestion here would be to widen your bounding box, however, if you want to make more sophisticated customisations then your probably want to learn about [graph functions](https://imperialcollegelondon.github.io/SWMManywhere/graphfcns_guide/). diff --git a/pyproject.toml b/pyproject.toml index 3325cf29..42edd360 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,6 +62,8 @@ dependencies = [ "xarray", ] optional-dependencies.dev = [ + "folium", + "matplotlib", "mypy", "pip-tools", "pre-commit", From be25578760c9cdfadd4bc42b327657bc7cfb3aae Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 18 Oct 2024 14:05:13 +0000 Subject: [PATCH 07/36] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/notebooks/extended_demo.py | 34 +++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/docs/notebooks/extended_demo.py b/docs/notebooks/extended_demo.py index 62a3db9b..1e39b810 100644 --- a/docs/notebooks/extended_demo.py +++ b/docs/notebooks/extended_demo.py @@ -103,10 +103,10 @@ def basic_map(model_dir): # to not allow pipes to cross bridges, tunnels, motorways, etc., however, this is # such a small area we probably don't want to restrict things so much. # - The density of points seems a bit extreme, ultimately we'd like to survey -# manholes locations, but for now we can reduce density by increasing +# manholes locations, but for now we can reduce density by increasing # `node_merge_distance`. # -# Let's just demonstrate that using the +# Let's just demonstrate that using the # [`parameter_overrides` functionality](https://imperialcollegelondon.github.io/SWMManywhere/config_guide/#changing-parameters). # # %% @@ -121,15 +121,15 @@ def basic_map(model_dir): # OK that clearly helped, although we have appear to have stranded pipes along (e.g.,) # *Carrer de la Grella*, presumably due to some mistake in the OSM specifying that it # is connected via a pedestrian route. We won't remedy this in the tutorial, but you can -# manually provide your -# [`starting_graph`](https://imperialcollegelondon.github.io/SWMManywhere/config_guide/#change-starting_graph) -# via the configuration file to address such mitakes. +# manually provide your +# [`starting_graph`](https://imperialcollegelondon.github.io/SWMManywhere/config_guide/#change-starting_graph) +# via the configuration file to address such mitakes. # # More importantly we can see some distinctive unconnected network in the South West. # What is going on there? To explain this we will have to turn on verbosity to print the # intermediate files used in model derivation. # -# To do this with a command line call we simply add the flag `--verbose=True`. +# To do this with a command line call we simply add the flag `--verbose=True`. # Though in code we will have to import the `logging` module. # %% @@ -149,9 +149,9 @@ def basic_map(model_dir): m = basic_map(model_dir) # %% [markdown] -# That's a lot of information! However, the reason we are currently interested +# That's a lot of information! However, the reason we are currently interested # in this is because the files associated with -# each workflow step are saved when `verbose=True`. +# each workflow step are saved when `verbose=True`. # # We will load a file called `subbasins` and add it to the map. @@ -179,8 +179,10 @@ def basic_map(model_dir): # `results` contains all simulation results in long format, with `flooding` at nodes and `flow` at edges. We will plot a random `flow`. # %% -flows = df.loc[df.variable == 'flow'] -flows.loc[flows.id == flows.iloc[0].id].set_index('date').value.plot(ylabel='flow (m3/s)') +flows = df.loc[df.variable == "flow"] +flows.loc[flows.id == flows.iloc[0].id].set_index("date").value.plot( + ylabel="flow (m3/s)" +) # %% [markdown] @@ -210,11 +212,11 @@ def clickable_map(model_dir): # Add nodes for node, row in nodes.iterrows(): grp = floods.get_group(str(node)) - f,ax=plt.subplots(figsize=(4,3)) - grp.set_index('date').value.plot(ylabel='flooding (m3)', title = node,ax=ax) + f, ax = plt.subplots(figsize=(4, 3)) + grp.set_index("date").value.plot(ylabel="flooding (m3)", title=node, ax=ax) img = BytesIO() f.tight_layout() - f.savefig(img, format="png",dpi=94) + f.savefig(img, format="png", dpi=94) plt.close(f) img.seek(0) img_base64 = base64.b64encode(img.read()).decode() @@ -232,11 +234,11 @@ def clickable_map(model_dir): # Add edges for edge, row in edges.iterrows(): grp = flows.get_group(str(edge)) - f,ax=plt.subplots(figsize=(4,3)) - grp.set_index('date').value.plot(ylabel='flow (m3/s)', title = edge,ax=ax) + f, ax = plt.subplots(figsize=(4, 3)) + grp.set_index("date").value.plot(ylabel="flow (m3/s)", title=edge, ax=ax) img = BytesIO() f.tight_layout() - f.savefig(img, format="png",dpi=94) + f.savefig(img, format="png", dpi=94) plt.close(f) img.seek(0) img_base64 = base64.b64encode(img.read()).decode() From ea0c2c39499724d3474b9c0fa50884245cd1d368 Mon Sep 17 00:00:00 2001 From: Dobson Date: Fri, 18 Oct 2024 15:37:40 +0100 Subject: [PATCH 08/36] fix --- docs/notebooks/custom.css | 6 ++++++ docs/notebooks/extended_demo.py | 36 ++++++++++++++++++--------------- mkdocs.yml | 1 + pyproject.toml | 3 ++- 4 files changed, 29 insertions(+), 17 deletions(-) create mode 100644 docs/notebooks/custom.css diff --git a/docs/notebooks/custom.css b/docs/notebooks/custom.css new file mode 100644 index 00000000..45a194e7 --- /dev/null +++ b/docs/notebooks/custom.css @@ -0,0 +1,6 @@ +.jp-OutputArea { + max-height: 300px; + overflow-y: auto; + border: 1px solid #ddd; + padding: 5px; +} \ No newline at end of file diff --git a/docs/notebooks/extended_demo.py b/docs/notebooks/extended_demo.py index 1e39b810..f8d62d9e 100644 --- a/docs/notebooks/extended_demo.py +++ b/docs/notebooks/extended_demo.py @@ -32,6 +32,7 @@ import pandas as pd from matplotlib import pyplot as plt +from swmmanywhere import logging from swmmanywhere.swmmanywhere import swmmanywhere # Create temporary directory @@ -63,6 +64,7 @@ # %% # Create a folium map and add the nodes and edges def basic_map(model_dir): + """Create a basic map with nodes and edges.""" # Load and inspect results nodes = gpd.read_file(model_dir / "nodes.geoparquet") edges = gpd.read_file(model_dir / "edges.geoparquet") @@ -123,7 +125,7 @@ def basic_map(model_dir): # is connected via a pedestrian route. We won't remedy this in the tutorial, but you can # manually provide your # [`starting_graph`](https://imperialcollegelondon.github.io/SWMManywhere/config_guide/#change-starting_graph) -# via the configuration file to address such mitakes. +# via the configuration file to address such mistakes. # # More importantly we can see some distinctive unconnected network in the South West. # What is going on there? To explain this we will have to turn on verbosity to print the @@ -134,17 +136,10 @@ def basic_map(model_dir): # %% # Make verbose -from swmmanywhere import logging - logging.set_verbose(True) # Set verbosity # Run again outputs = swmmanywhere(config) - -output_html += """ - - -""" model_dir = outputs[0].parent m = basic_map(model_dir) @@ -162,21 +157,23 @@ def basic_map(model_dir): # %% [markdown] # Although this can be customised, the default behaviour of `swmmanywhere` is to not -# allow edges to cross hydrological subbasins. It is now super clear why these unconnected -# networks have appeared, and are ultimately due to the underlying DEM. If you did -# desperately care about these streets, then you should probably widen your bounding box. +# allow edges to cross hydrological subbasins. It is now super clear why these +# unconnected networks have appeared, and are ultimately due to the underlying DEM. +# If you did desperately care about these streets, then you should probably +# widen your bounding box. # # ## Plotting results # -# Because we have run the model with `verbose=True` we will also see that a new `results` -# file has appeared, which contains all of the simulation results from SWMM. +# Because we have run the model with `verbose=True` we will also see that a new +# `results` file has appeared, which contains all of the simulation results from SWMM. # %% df = pd.read_parquet(model_dir / "results.parquet") df.head() # %% [markdown] -# `results` contains all simulation results in long format, with `flooding` at nodes and `flow` at edges. We will plot a random `flow`. +# `results` contains all simulation results in long format, with `flooding` at +# nodes and `flow` at edges. We will plot a random `flow`. # %% flows = df.loc[df.variable == "flow"] @@ -186,12 +183,14 @@ def basic_map(model_dir): # %% [markdown] -# Since folium is super clever, we can make these clickable on our map - and now you can inspect your results in a much more elegant way than the SWMM GUI. +# Since folium is super clever, we can make these clickable on our map - and +# now you can inspect your results in a much more elegant way than the SWMM GUI. # %% # Create a folium map and add the nodes and edges def clickable_map(model_dir): + """Create a clickable map with nodes, edges and results.""" # Load and inspect results nodes = gpd.read_file(model_dir / "nodes.geoparquet") edges = gpd.read_file(model_dir / "edges.geoparquet") @@ -255,4 +254,9 @@ def clickable_map(model_dir): clickable_map(model_dir) # %% [markdown] -# If we explore around, clicking on edges, we can see that flows are often looking sensible, though we can definitely some areas that have been hampered by our starting street graph (e.g., *Carrer dels Canals*). The first suggestion here would be to widen your bounding box, however, if you want to make more sophisticated customisations then your probably want to learn about [graph functions](https://imperialcollegelondon.github.io/SWMManywhere/graphfcns_guide/). +# If we explore around, clicking on edges, we can see that flows are often +# looking sensible, though we can definitely some areas that have been hampered +# by our starting street graph (e.g., *Carrer dels Canals*). The first +# suggestion here would be to widen your bounding box, however, if you want to +# make more sophisticated customisations then your probably want to learn about +# [graph functions](https://imperialcollegelondon.github.io/SWMManywhere/graphfcns_guide/). diff --git a/mkdocs.yml b/mkdocs.yml index 7b9680d1..3fc9b859 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -11,6 +11,7 @@ theme: extra_css: - stylesheets/extra.css + - docs/notebooks/custom.css plugins: - mkdocstrings diff --git a/pyproject.toml b/pyproject.toml index 42edd360..4be8c39a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -103,6 +103,7 @@ version-file = "_version.py" [tool.ruff] lint.select = [ "D", "E", "F", "I" ] # pydocstyle, pycodestyle, Pyflakes, isort +lint.per-file-ignores."docs/notebooks/*" = [ "D100" ] lint.per-file-ignores."src/netcomp/*" = [ "D", "F" ] # Ignore all checks for netcomp lint.per-file-ignores."tests/*" = [ "D100", "D104" ] lint.isort.known-first-party = [ "swmmanywhere", "netcomp" ] @@ -111,7 +112,7 @@ lint.pydocstyle.convention = "google" [tool.codespell] skip = "src/swmmanywhere/defs/iso_converter.yml,*.inp" -ignore-words-list = "gage,gages" +ignore-words-list = "gage,gages,Carrer" [tool.pytest.ini_options] addopts = "-v --cov=src/swmmanywhere --cov-report=xml --doctest-modules --ignore=src/swmmanywhere/logging.py" From e6dd828ebcbb16ccb8d4b988957a1085754d2efc Mon Sep 17 00:00:00 2001 From: Dobson Date: Fri, 18 Oct 2024 16:10:53 +0100 Subject: [PATCH 09/36] use custom css for scroll --- docs/custom.css | 15 +++++++++++++++ docs/notebooks/custom.css | 6 ------ mkdocs.yml | 8 ++------ src/swmmanywhere/geospatial_utilities.py | 2 +- 4 files changed, 18 insertions(+), 13 deletions(-) create mode 100644 docs/custom.css delete mode 100644 docs/notebooks/custom.css diff --git a/docs/custom.css b/docs/custom.css new file mode 100644 index 00000000..e68bbc3a --- /dev/null +++ b/docs/custom.css @@ -0,0 +1,15 @@ +:root { + --md-primary-fg-color: #00BCD4; + --md-accent-fg-color: #00BCD4; + } + +.jupyter-wrapper .jp-OutputArea { + max-height: 700px; + overflow-y: auto; + border: 1px solid #ddd; + padding: 5px; +} + +.jupyter-wrapper .jp-OutputArea-child { + display: block; /* Make sure child elements are block-level to avoid flex wrapping issues */ +} \ No newline at end of file diff --git a/docs/notebooks/custom.css b/docs/notebooks/custom.css deleted file mode 100644 index 45a194e7..00000000 --- a/docs/notebooks/custom.css +++ /dev/null @@ -1,6 +0,0 @@ -.jp-OutputArea { - max-height: 300px; - overflow-y: auto; - border: 1px solid #ddd; - padding: 5px; -} \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 3fc9b859..039abfc1 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -2,16 +2,12 @@ site_name: SWMManywhere docs theme: name: "material" - palette: - primary: 'cyan' + icon: repo: fontawesome/brands/github features: - content.code.copy - -extra_css: - - stylesheets/extra.css - - docs/notebooks/custom.css +extra_css: [custom.css] plugins: - mkdocstrings diff --git a/src/swmmanywhere/geospatial_utilities.py b/src/swmmanywhere/geospatial_utilities.py index 5e29599b..b23253e8 100644 --- a/src/swmmanywhere/geospatial_utilities.py +++ b/src/swmmanywhere/geospatial_utilities.py @@ -646,7 +646,7 @@ def flwdir_whitebox(fid: Path) -> np.array: temp_path, wbt_args, save_dir=temp_path, - verbose=False, + verbose=verbose(), wbt_root=temp_path / "WBT", zip_path=fid.parent / "whiteboxtools_binaries.zip", max_procs=1, From 3efd28ffe19ffb96922193a674e8599fb72fc73e Mon Sep 17 00:00:00 2001 From: Dobson Date: Fri, 18 Oct 2024 16:34:59 +0100 Subject: [PATCH 10/36] Update extended_demo.py fiddling --- docs/notebooks/extended_demo.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/notebooks/extended_demo.py b/docs/notebooks/extended_demo.py index f8d62d9e..1492d019 100644 --- a/docs/notebooks/extended_demo.py +++ b/docs/notebooks/extended_demo.py @@ -114,14 +114,15 @@ def basic_map(model_dir): # %% config["parameter_overrides"] = { "topology_derivation": {"allowable_networks": ["drive"], "omit_edges": []}, - "subcatchment_derivation": {"node_merge_distance": 15}, + "subcatchment_derivation": {"node_merge_distance": 12.5}, } outputs = swmmanywhere(config) basic_map(outputs[0].parent) # %% [markdown] -# OK that clearly helped, although we have appear to have stranded pipes along (e.g.,) -# *Carrer de la Grella*, presumably due to some mistake in the OSM specifying that it +# OK that clearly helped, although we have appear to have stranded pipes along (e.g., +# *Carrer de la Grella* in North West), presumably due to some mistake in the +# OSM specifying that it # is connected via a pedestrian route. We won't remedy this in the tutorial, but you can # manually provide your # [`starting_graph`](https://imperialcollegelondon.github.io/SWMManywhere/config_guide/#change-starting_graph) @@ -250,13 +251,16 @@ def clickable_map(model_dir): ).add_to(m) return m - +# Display the map clickable_map(model_dir) +# Clear temp dir +temp_dir.cleanup() + # %% [markdown] # If we explore around, clicking on edges, we can see that flows are often # looking sensible, though we can definitely some areas that have been hampered -# by our starting street graph (e.g., *Carrer dels Canals*). The first +# by our starting street graph (e.g., *Carrer dels Canals* in North West). The first # suggestion here would be to widen your bounding box, however, if you want to # make more sophisticated customisations then your probably want to learn about # [graph functions](https://imperialcollegelondon.github.io/SWMManywhere/graphfcns_guide/). From 8ac48f4b612b73152a24585ed3b9cfdddc9cfbba Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 18 Oct 2024 15:35:19 +0000 Subject: [PATCH 11/36] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/notebooks/extended_demo.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/notebooks/extended_demo.py b/docs/notebooks/extended_demo.py index 1492d019..17c69744 100644 --- a/docs/notebooks/extended_demo.py +++ b/docs/notebooks/extended_demo.py @@ -251,6 +251,7 @@ def clickable_map(model_dir): ).add_to(m) return m + # Display the map clickable_map(model_dir) From a5669c4b7d0202d8ba181dc3e40fa69def26d7ef Mon Sep 17 00:00:00 2001 From: Dobson Date: Fri, 18 Oct 2024 16:38:42 +0100 Subject: [PATCH 12/36] Update extended_demo.py couple more edits --- docs/notebooks/extended_demo.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/notebooks/extended_demo.py b/docs/notebooks/extended_demo.py index 1492d019..534a0ab8 100644 --- a/docs/notebooks/extended_demo.py +++ b/docs/notebooks/extended_demo.py @@ -133,7 +133,7 @@ def basic_map(model_dir): # intermediate files used in model derivation. # # To do this with a command line call we simply add the flag `--verbose=True`. -# Though in code we will have to import the `logging` module. +# Though in code we will have to use the `logging` module. # %% # Make verbose @@ -254,9 +254,6 @@ def clickable_map(model_dir): # Display the map clickable_map(model_dir) -# Clear temp dir -temp_dir.cleanup() - # %% [markdown] # If we explore around, clicking on edges, we can see that flows are often # looking sensible, though we can definitely some areas that have been hampered From a9b730b6f1b8add3996441e300604383838fa15a Mon Sep 17 00:00:00 2001 From: Dobson Date: Fri, 18 Oct 2024 16:43:08 +0100 Subject: [PATCH 13/36] Update extended_demo.py units --- docs/notebooks/extended_demo.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/notebooks/extended_demo.py b/docs/notebooks/extended_demo.py index 751301f5..f09d8975 100644 --- a/docs/notebooks/extended_demo.py +++ b/docs/notebooks/extended_demo.py @@ -179,7 +179,7 @@ def basic_map(model_dir): # %% flows = df.loc[df.variable == "flow"] flows.loc[flows.id == flows.iloc[0].id].set_index("date").value.plot( - ylabel="flow (m3/s)" + ylabel="flow (l/s)" ) @@ -213,7 +213,7 @@ def clickable_map(model_dir): for node, row in nodes.iterrows(): grp = floods.get_group(str(node)) f, ax = plt.subplots(figsize=(4, 3)) - grp.set_index("date").value.plot(ylabel="flooding (m3)", title=node, ax=ax) + grp.set_index("date").value.plot(ylabel="flooding (l)", title=node, ax=ax) img = BytesIO() f.tight_layout() f.savefig(img, format="png", dpi=94) @@ -235,7 +235,7 @@ def clickable_map(model_dir): for edge, row in edges.iterrows(): grp = flows.get_group(str(edge)) f, ax = plt.subplots(figsize=(4, 3)) - grp.set_index("date").value.plot(ylabel="flow (m3/s)", title=edge, ax=ax) + grp.set_index("date").value.plot(ylabel="flow (l/s)", title=edge, ax=ax) img = BytesIO() f.tight_layout() f.savefig(img, format="png", dpi=94) From 99cd081ba517fb4daee719988a82bede44d8c585 Mon Sep 17 00:00:00 2001 From: Dobson Date: Fri, 18 Oct 2024 16:45:40 +0100 Subject: [PATCH 14/36] Update extended_demo.py links --- docs/notebooks/extended_demo.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/notebooks/extended_demo.py b/docs/notebooks/extended_demo.py index f09d8975..ca5b9225 100644 --- a/docs/notebooks/extended_demo.py +++ b/docs/notebooks/extended_demo.py @@ -9,14 +9,14 @@ # %% [markdown] # ## Introduction # This script demonstrates a simple use case of `SWMManywhere`, building on the -# [quickstart](quickstart.md) example, but including plotting and alterations. +# [quickstart](..//quickstart.md) example, but including plotting and alterations. # -# Since this is a notebook, we will define [`config`](config_guide.md) as a +# Since this is a notebook, we will define [`config`](..//config_guide.md) as a # dictionary rather than a `yaml` file, but the same principles apply. # # ## Initial run # -# Here we will run the [quickstart](quickstart.md) configuration, keeping +# Here we will run the [quickstart](..//quickstart.md) configuration, keeping # everything in a temporary directory. # %% # Imports From e743eb19196bf15925cf3265bdd140b731f100a0 Mon Sep 17 00:00:00 2001 From: Dobson Date: Fri, 18 Oct 2024 17:02:49 +0100 Subject: [PATCH 15/36] Update extended_demo.py fix links --- docs/notebooks/extended_demo.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/notebooks/extended_demo.py b/docs/notebooks/extended_demo.py index ca5b9225..c7485f31 100644 --- a/docs/notebooks/extended_demo.py +++ b/docs/notebooks/extended_demo.py @@ -9,14 +9,15 @@ # %% [markdown] # ## Introduction # This script demonstrates a simple use case of `SWMManywhere`, building on the -# [quickstart](..//quickstart.md) example, but including plotting and alterations. +# [quickstart](https://imperialcollegelondon.github.io/SWMManywhere/quickstart/) +# example, but including plotting and alterations. # -# Since this is a notebook, we will define [`config`](..//config_guide.md) as a -# dictionary rather than a `yaml` file, but the same principles apply. +# Since this is a notebook, we will define [`config`](https://imperialcollegelondon.github.io/SWMManywhere/config_guide/) +# as a dictionary rather than a `yaml` file, but the same principles apply. # # ## Initial run # -# Here we will run the [quickstart](..//quickstart.md) configuration, keeping +# Here we will run the [quickstart](https://imperialcollegelondon.github.io/SWMManywhere/config_guide/) configuration, keeping # everything in a temporary directory. # %% # Imports @@ -114,7 +115,7 @@ def basic_map(model_dir): # %% config["parameter_overrides"] = { "topology_derivation": {"allowable_networks": ["drive"], "omit_edges": []}, - "subcatchment_derivation": {"node_merge_distance": 12.5}, + "subcatchment_derivation": {"node_merge_distance": 20}, } outputs = swmmanywhere(config) basic_map(outputs[0].parent) From 024575d8b385025e5770771a1f0a4dffa18f1345 Mon Sep 17 00:00:00 2001 From: barneydobson Date: Mon, 21 Oct 2024 11:41:25 +0100 Subject: [PATCH 16/36] update text and demo --- docs/notebooks/extended_demo.py | 95 +++++++++++++++++++++------------ 1 file changed, 62 insertions(+), 33 deletions(-) diff --git a/docs/notebooks/extended_demo.py b/docs/notebooks/extended_demo.py index c7485f31..9057d7e8 100644 --- a/docs/notebooks/extended_demo.py +++ b/docs/notebooks/extended_demo.py @@ -17,8 +17,8 @@ # # ## Initial run # -# Here we will run the [quickstart](https://imperialcollegelondon.github.io/SWMManywhere/config_guide/) configuration, keeping -# everything in a temporary directory. +# Here we will run the [quickstart](https://imperialcollegelondon.github.io/SWMManywhere/config_guide/) +# configuration, keeping everything in a temporary directory. # %% # Imports from __future__ import annotations @@ -33,7 +33,7 @@ import pandas as pd from matplotlib import pyplot as plt -from swmmanywhere import logging +from swmmanywhere.logging import set_verbose from swmmanywhere.swmmanywhere import swmmanywhere # Create temporary directory @@ -62,6 +62,8 @@ # If you do not have a real UDM, the majority of your interpretation will be # around the synthesised `nodes` and `edges`. These are # created in the same directory as the `model_file`. Let's have a look at them. +# Note that the `outfall` that each node drains to is specified in the `outfall` +# attribute, we will plot these in red and other nodes in black. # %% # Create a folium map and add the nodes and edges def basic_map(model_dir): @@ -74,12 +76,16 @@ def basic_map(model_dir): nodes = nodes.to_crs(4326) edges = edges.to_crs(4326) + # Identify outfalls + outfall = nodes.id == nodes.outfall + + # Plot on map m = folium.Map( location=[nodes.geometry.y.mean(), nodes.geometry.x.mean()], zoom_start=16 ) folium.GeoJson(edges, color="black", weight=1).add_to(m) folium.GeoJson( - nodes, + nodes.loc[~outfall], marker=folium.CircleMarker( radius=3, # Radius in metres weight=0, # outline weight @@ -88,6 +94,16 @@ def basic_map(model_dir): ), ).add_to(m) + folium.GeoJson( + nodes.loc[outfall], + marker=folium.CircleMarker( + radius=3, # Radius in metres + weight=0, # outline weight + fill_color="red", + fill_opacity=1, + ), + ).add_to(m) + # Display the map return m @@ -99,29 +115,36 @@ def basic_map(model_dir): # # ## Customising outputs # -# Some things stick out on first glance: +# Some things stick out on first glance, +# # - Probably we do not need pipes in the hills to the South, these seem to be along # pedestrian routes, which can be adjusted with the `allowable_networks` parameter. # - We will also remove any types under the `omit_edges` entry, here you can specify # to not allow pipes to cross bridges, tunnels, motorways, etc., however, this is # such a small area we probably don't want to restrict things so much. -# - The density of points seems a bit extreme, ultimately we'd like to survey -# manholes locations, but for now we can reduce density by increasing -# `node_merge_distance`. +# - Due to the apparent steep slopes, it seems like other topological factors, such +# as pipe length, are less important. We can adjust the `weights` parameter to +# include only slope, whose parameter name is `chahinian_slope`. +# - We have far too few outfalls, it seems implausible that so many riverside streets +# would not have outfalls. Furthermore, there are points that are quite far from the +# river that have been assigned as outfalls. We can reduce the `river_buffer_distance` +# to make nodes nearer the river more likely to be outfalls, but also reduce the +# `outfall_length` distance parameter to enable `swmmanywhere` to more freely select +# outfalls that are adjacent to the river. # # Let's just demonstrate that using the # [`parameter_overrides` functionality](https://imperialcollegelondon.github.io/SWMManywhere/config_guide/#changing-parameters). # # %% config["parameter_overrides"] = { - "topology_derivation": {"allowable_networks": ["drive"], "omit_edges": []}, - "subcatchment_derivation": {"node_merge_distance": 20}, + "topology_derivation": {"allowable_networks": ["drive"], "omit_edges": ["bridge"]}, + "outfall_derivation": {"outfall_length": 5, "river_buffer_distance": 30}, } outputs = swmmanywhere(config) basic_map(outputs[0].parent) # %% [markdown] -# OK that clearly helped, although we have appear to have stranded pipes along (e.g., +# OK that clearly helped, although we have appear to have stranded pipes (e.g., along # *Carrer de la Grella* in North West), presumably due to some mistake in the # OSM specifying that it # is connected via a pedestrian route. We won't remedy this in the tutorial, but you can @@ -134,11 +157,11 @@ def basic_map(model_dir): # intermediate files used in model derivation. # # To do this with a command line call we simply add the flag `--verbose=True`. -# Though in code we will have to use the `logging` module. +# Though in code we will have to use `set_verbose` from the `logging` module. # %% # Make verbose -logging.set_verbose(True) # Set verbosity +set_verbose(True) # Set verbosity # Run again outputs = swmmanywhere(config) @@ -210,11 +233,11 @@ def clickable_map(model_dir): location=[nodes.geometry.y.mean(), nodes.geometry.x.mean()], zoom_start=16 ) - # Add nodes - for node, row in nodes.iterrows(): - grp = floods.get_group(str(node)) + # Add edges + for edge, row in edges.iterrows(): + grp = flows.get_group(str(edge)) f, ax = plt.subplots(figsize=(4, 3)) - grp.set_index("date").value.plot(ylabel="flooding (l)", title=node, ax=ax) + grp.set_index("date").value.plot(ylabel="flow (l/s)", title=edge, ax=ax) img = BytesIO() f.tight_layout() f.savefig(img, format="png", dpi=94) @@ -222,21 +245,18 @@ def clickable_map(model_dir): img.seek(0) img_base64 = base64.b64encode(img.read()).decode() img_html = f'' - folium.CircleMarker( - [nodes.loc[node].geometry.y, nodes.loc[node].geometry.x], + folium.PolyLine( + [[c[1], c[0]] for c in row.geometry.coords], color="black", - radius=3, - weight=0, - fill_color="black", - fill_opacity=1, + weight=2, popup=folium.Popup(img_html), ).add_to(m) - # Add edges - for edge, row in edges.iterrows(): - grp = flows.get_group(str(edge)) + # Add nodes + for node, row in nodes.iterrows(): + grp = floods.get_group(str(node)) f, ax = plt.subplots(figsize=(4, 3)) - grp.set_index("date").value.plot(ylabel="flow (l/s)", title=edge, ax=ax) + grp.set_index("date").value.plot(ylabel="flooding (l)", title=node, ax=ax) img = BytesIO() f.tight_layout() f.savefig(img, format="png", dpi=94) @@ -244,12 +264,20 @@ def clickable_map(model_dir): img.seek(0) img_base64 = base64.b64encode(img.read()).decode() img_html = f'' - folium.PolyLine( - [[c[1], c[0]] for c in row.geometry.coords], - color="black", - weight=2, + if row.outfall == node: + color = "red" + else: + color = "black" + folium.CircleMarker( + [nodes.loc[node].geometry.y, nodes.loc[node].geometry.x], + color=color, + radius=3, + weight=0, + fill_color=color, + fill_opacity=1, popup=folium.Popup(img_html), ).add_to(m) + return m @@ -259,7 +287,8 @@ def clickable_map(model_dir): # %% [markdown] # If we explore around, clicking on edges, we can see that flows are often # looking sensible, though we can definitely some areas that have been hampered -# by our starting street graph (e.g., *Carrer dels Canals* in North West). The first -# suggestion here would be to widen your bounding box, however, if you want to +# by our starting street graph (e.g., the Western portion of *Carrer del Sant Andreu* +# in North West). The first suggestion here would be to examine the starting graph, +# however, if you want to # make more sophisticated customisations then your probably want to learn about # [graph functions](https://imperialcollegelondon.github.io/SWMManywhere/graphfcns_guide/). From dd195f0dcc58c222de08d30b40b1a3ebfdfc69d5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 21 Oct 2024 11:34:45 +0000 Subject: [PATCH 17/36] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/notebooks/extended_demo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/notebooks/extended_demo.py b/docs/notebooks/extended_demo.py index 7379b378..9057d7e8 100644 --- a/docs/notebooks/extended_demo.py +++ b/docs/notebooks/extended_demo.py @@ -291,4 +291,4 @@ def clickable_map(model_dir): # in North West). The first suggestion here would be to examine the starting graph, # however, if you want to # make more sophisticated customisations then your probably want to learn about -# [graph functions](https://imperialcollegelondon.github.io/SWMManywhere/graphfcns_guide/). \ No newline at end of file +# [graph functions](https://imperialcollegelondon.github.io/SWMManywhere/graphfcns_guide/). From 4e156cbb701dd322115021ec07074dc9515f1121 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Alonso=20=C3=81lvarez?= <6095790+dalonsoa@users.noreply.github.com> Date: Mon, 21 Oct 2024 16:04:50 +0100 Subject: [PATCH 18/36] Update ci_template.yml --- .github/workflows/ci_template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_template.yml b/.github/workflows/ci_template.yml index d57c392d..a3c731c2 100644 --- a/.github/workflows/ci_template.yml +++ b/.github/workflows/ci_template.yml @@ -30,7 +30,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies - run: pip install .[dev] + run: pip install -e .[dev] - name: Run tests run: pytest @@ -41,4 +41,4 @@ jobs: with: fail_ci_if_error: true token: ${{ secrets.codecov_token }} - verbose: true \ No newline at end of file + verbose: true From 545d59abe50df6853aefaf27ad51ab0927f8b65a Mon Sep 17 00:00:00 2001 From: barneydobson Date: Tue, 22 Oct 2024 09:48:26 +0100 Subject: [PATCH 19/36] Update docs/notebooks/extended_demo.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Diego Alonso Álvarez <6095790+dalonsoa@users.noreply.github.com> --- docs/notebooks/extended_demo.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/notebooks/extended_demo.py b/docs/notebooks/extended_demo.py index 9057d7e8..ce52e27e 100644 --- a/docs/notebooks/extended_demo.py +++ b/docs/notebooks/extended_demo.py @@ -137,8 +137,14 @@ def basic_map(model_dir): # # %% config["parameter_overrides"] = { - "topology_derivation": {"allowable_networks": ["drive"], "omit_edges": ["bridge"]}, - "outfall_derivation": {"outfall_length": 5, "river_buffer_distance": 30}, + "topology_derivation": { + "allowable_networks": ["drive"], + "omit_edges": ["bridge"], + }, + "outfall_derivation": { + "outfall_length": 5, + "river_buffer_distance": 30, + }, } outputs = swmmanywhere(config) basic_map(outputs[0].parent) From e87c6e1ef6bb8c09741dc91e48e3d3feb23e42a7 Mon Sep 17 00:00:00 2001 From: barneydobson Date: Tue, 22 Oct 2024 10:15:44 +0100 Subject: [PATCH 20/36] fix deprecation --- src/swmmanywhere/graphfcns/network_cleaning_graphfcns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swmmanywhere/graphfcns/network_cleaning_graphfcns.py b/src/swmmanywhere/graphfcns/network_cleaning_graphfcns.py index f6aaf1a3..e09b1054 100644 --- a/src/swmmanywhere/graphfcns/network_cleaning_graphfcns.py +++ b/src/swmmanywhere/graphfcns/network_cleaning_graphfcns.py @@ -77,7 +77,7 @@ def __call__(self, G: nx.Graph, **kwargs) -> nx.Graph: # Set the attribute (weight) used to determine which parallel edge to # retain. Could make this a parameter in parameters.py if needed. weight = "length" - graph = ox.get_digraph(G) + graph = ox.convert.to_digraph(G) _, _, attr_list = next(iter(graph.edges(data=True))) # type: ignore attr_list = cast("dict[str, Any]", attr_list) if weight not in attr_list: From 04eaca044b75ea2da21099cb8c0b38db1ade9c91 Mon Sep 17 00:00:00 2001 From: barneydobson Date: Tue, 22 Oct 2024 11:45:03 +0100 Subject: [PATCH 21/36] suppress message --- docs/notebooks/extended_demo.py | 5 +++-- src/swmmanywhere/geospatial_utilities.py | 26 ++++++++++++++++++++---- src/swmmanywhere/post_processing.py | 7 ++++++- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/docs/notebooks/extended_demo.py b/docs/notebooks/extended_demo.py index ce52e27e..b77b8c62 100644 --- a/docs/notebooks/extended_demo.py +++ b/docs/notebooks/extended_demo.py @@ -38,13 +38,14 @@ # Create temporary directory temp_dir = tempfile.TemporaryDirectory() -base_dir = Path(temp_dir.name) +base_dir = Path(r"C:\Users\darne\Documents\data\temp") -# Define minimum viable config +# Define minimum viable config (with shorter duration so better inspect results) config = { "base_dir": base_dir, "project": "my_first_swmm", "bbox": [1.52740, 42.50524, 1.54273, 42.51259], + "run_settings": {"duration" : 3600}, } # Run SWMManywhere diff --git a/src/swmmanywhere/geospatial_utilities.py b/src/swmmanywhere/geospatial_utilities.py index b23253e8..901e6858 100644 --- a/src/swmmanywhere/geospatial_utilities.py +++ b/src/swmmanywhere/geospatial_utilities.py @@ -883,6 +883,25 @@ def calculate_angle( return angle_degrees +def get_serializable_properties(base_props: dict, extra_data: dict) -> dict: + """Convert graph properties to serializable GeoJSON properties. + + Args: + base_props (dict): Base properties that must be included (e.g. id for nodes, u/v for edges) + extra_data (dict): Additional data to include if serializable + + Returns: + dict: Combined properties with only serializable values + """ + properties = base_props.copy() + for key, value in extra_data.items(): + # Check for basic GeoJSON-compatible types + if isinstance(value, (str, int, float, bool, type(None))): + properties[key] = value + else: + logger.warning(f"Skipping field {key}: unsupported type: {type(value)}") + + return properties def nodes_to_features(G: nx.Graph): """Convert a graph to a GeoJSON node feature collection. @@ -898,7 +917,7 @@ def nodes_to_features(G: nx.Graph): feature = { "type": "Feature", "geometry": sgeom.mapping(sgeom.Point(data["x"], data["y"])), - "properties": {"id": node, **data}, + "properties": get_serializable_properties({"id": node}, data), } features.append(feature) return features @@ -923,12 +942,11 @@ def edges_to_features(G: nx.Graph): feature = { "type": "Feature", "geometry": geom, - "properties": {"u": u, "v": v, **data}, + "properties": get_serializable_properties({"u": u, "v": v}, data), } features.append(feature) return features - def graph_to_geojson(graph: nx.Graph, fid_nodes: Path, fid_edges: Path, crs: str): """Write a graph to a GeoJSON file. @@ -941,7 +959,7 @@ def graph_to_geojson(graph: nx.Graph, fid_nodes: Path, fid_edges: Path, crs: str graph = graph.copy() nodes = nodes_to_features(graph) edges = edges_to_features(graph) - + for iterable, fid in zip([nodes, edges], [fid_nodes, fid_edges]): geojson = { "type": "FeatureCollection", diff --git a/src/swmmanywhere/post_processing.py b/src/swmmanywhere/post_processing.py index 4253833d..e59cb88e 100644 --- a/src/swmmanywhere/post_processing.py +++ b/src/swmmanywhere/post_processing.py @@ -6,8 +6,11 @@ from __future__ import annotations +import contextlib +import os import re import shutil +import sys from pathlib import Path from typing import Any, Literal @@ -37,7 +40,9 @@ def synthetic_write(addresses: FilePaths): # TODO these node/edge names are probably not good or extendible defulats # revisit once overall software architecture is more clear. nodes = gpd.read_file(addresses.model_paths.nodes) - edges = gpd.read_file(addresses.model_paths.edges) + with open(os.devnull, 'w') as devnull: + with contextlib.redirect_stderr(devnull): + edges = gpd.read_file(addresses.model_paths.edges) if addresses.model_paths.subcatchments.suffix == ".geoparquet": subs = gpd.read_parquet(addresses.model_paths.subcatchments) From 7ce31da127c217446df6cc07745dde3d3ed4e871 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2024 10:45:17 +0000 Subject: [PATCH 22/36] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/notebooks/extended_demo.py | 2 +- src/swmmanywhere/post_processing.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/notebooks/extended_demo.py b/docs/notebooks/extended_demo.py index b77b8c62..15866626 100644 --- a/docs/notebooks/extended_demo.py +++ b/docs/notebooks/extended_demo.py @@ -45,7 +45,7 @@ "base_dir": base_dir, "project": "my_first_swmm", "bbox": [1.52740, 42.50524, 1.54273, 42.51259], - "run_settings": {"duration" : 3600}, + "run_settings": {"duration": 3600}, } # Run SWMManywhere diff --git a/src/swmmanywhere/post_processing.py b/src/swmmanywhere/post_processing.py index e59cb88e..77b52b2b 100644 --- a/src/swmmanywhere/post_processing.py +++ b/src/swmmanywhere/post_processing.py @@ -10,7 +10,6 @@ import os import re import shutil -import sys from pathlib import Path from typing import Any, Literal @@ -40,7 +39,7 @@ def synthetic_write(addresses: FilePaths): # TODO these node/edge names are probably not good or extendible defulats # revisit once overall software architecture is more clear. nodes = gpd.read_file(addresses.model_paths.nodes) - with open(os.devnull, 'w') as devnull: + with open(os.devnull, "w") as devnull: with contextlib.redirect_stderr(devnull): edges = gpd.read_file(addresses.model_paths.edges) From cfe7f010fd7ca5b5cc757db28c7e6fdd250bcd7c Mon Sep 17 00:00:00 2001 From: barneydobson Date: Tue, 22 Oct 2024 11:50:03 +0100 Subject: [PATCH 23/36] suppress warning --- docs/notebooks/extended_demo.py | 2 +- src/swmmanywhere/geospatial_utilities.py | 26 ++++-------------------- src/swmmanywhere/post_processing.py | 3 +-- 3 files changed, 6 insertions(+), 25 deletions(-) diff --git a/docs/notebooks/extended_demo.py b/docs/notebooks/extended_demo.py index b77b8c62..15866626 100644 --- a/docs/notebooks/extended_demo.py +++ b/docs/notebooks/extended_demo.py @@ -45,7 +45,7 @@ "base_dir": base_dir, "project": "my_first_swmm", "bbox": [1.52740, 42.50524, 1.54273, 42.51259], - "run_settings": {"duration" : 3600}, + "run_settings": {"duration": 3600}, } # Run SWMManywhere diff --git a/src/swmmanywhere/geospatial_utilities.py b/src/swmmanywhere/geospatial_utilities.py index 901e6858..b23253e8 100644 --- a/src/swmmanywhere/geospatial_utilities.py +++ b/src/swmmanywhere/geospatial_utilities.py @@ -883,25 +883,6 @@ def calculate_angle( return angle_degrees -def get_serializable_properties(base_props: dict, extra_data: dict) -> dict: - """Convert graph properties to serializable GeoJSON properties. - - Args: - base_props (dict): Base properties that must be included (e.g. id for nodes, u/v for edges) - extra_data (dict): Additional data to include if serializable - - Returns: - dict: Combined properties with only serializable values - """ - properties = base_props.copy() - for key, value in extra_data.items(): - # Check for basic GeoJSON-compatible types - if isinstance(value, (str, int, float, bool, type(None))): - properties[key] = value - else: - logger.warning(f"Skipping field {key}: unsupported type: {type(value)}") - - return properties def nodes_to_features(G: nx.Graph): """Convert a graph to a GeoJSON node feature collection. @@ -917,7 +898,7 @@ def nodes_to_features(G: nx.Graph): feature = { "type": "Feature", "geometry": sgeom.mapping(sgeom.Point(data["x"], data["y"])), - "properties": get_serializable_properties({"id": node}, data), + "properties": {"id": node, **data}, } features.append(feature) return features @@ -942,11 +923,12 @@ def edges_to_features(G: nx.Graph): feature = { "type": "Feature", "geometry": geom, - "properties": get_serializable_properties({"u": u, "v": v}, data), + "properties": {"u": u, "v": v, **data}, } features.append(feature) return features + def graph_to_geojson(graph: nx.Graph, fid_nodes: Path, fid_edges: Path, crs: str): """Write a graph to a GeoJSON file. @@ -959,7 +941,7 @@ def graph_to_geojson(graph: nx.Graph, fid_nodes: Path, fid_edges: Path, crs: str graph = graph.copy() nodes = nodes_to_features(graph) edges = edges_to_features(graph) - + for iterable, fid in zip([nodes, edges], [fid_nodes, fid_edges]): geojson = { "type": "FeatureCollection", diff --git a/src/swmmanywhere/post_processing.py b/src/swmmanywhere/post_processing.py index e59cb88e..77b52b2b 100644 --- a/src/swmmanywhere/post_processing.py +++ b/src/swmmanywhere/post_processing.py @@ -10,7 +10,6 @@ import os import re import shutil -import sys from pathlib import Path from typing import Any, Literal @@ -40,7 +39,7 @@ def synthetic_write(addresses: FilePaths): # TODO these node/edge names are probably not good or extendible defulats # revisit once overall software architecture is more clear. nodes = gpd.read_file(addresses.model_paths.nodes) - with open(os.devnull, 'w') as devnull: + with open(os.devnull, "w") as devnull: with contextlib.redirect_stderr(devnull): edges = gpd.read_file(addresses.model_paths.edges) From 718078292807e2d698f47ea2ee9c1d2de97f799c Mon Sep 17 00:00:00 2001 From: barneydobson Date: Tue, 22 Oct 2024 11:54:27 +0100 Subject: [PATCH 24/36] remove debug --- docs/notebooks/extended_demo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/notebooks/extended_demo.py b/docs/notebooks/extended_demo.py index 15866626..e23775fa 100644 --- a/docs/notebooks/extended_demo.py +++ b/docs/notebooks/extended_demo.py @@ -38,7 +38,7 @@ # Create temporary directory temp_dir = tempfile.TemporaryDirectory() -base_dir = Path(r"C:\Users\darne\Documents\data\temp") +base_dir = Path(temp_dir.name) # Define minimum viable config (with shorter duration so better inspect results) config = { From 804b39f9a354de10757b7465e5dc3ce95aa1d821 Mon Sep 17 00:00:00 2001 From: barneydobson Date: Tue, 22 Oct 2024 12:04:54 +0100 Subject: [PATCH 25/36] Fix output printing --- src/swmmanywhere/post_processing.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/swmmanywhere/post_processing.py b/src/swmmanywhere/post_processing.py index 77b52b2b..2dd73bee 100644 --- a/src/swmmanywhere/post_processing.py +++ b/src/swmmanywhere/post_processing.py @@ -6,13 +6,12 @@ from __future__ import annotations -import contextlib -import os import re import shutil from pathlib import Path from typing import Any, Literal +from IPython.utils import io import geopandas as gpd import numpy as np import pandas as pd @@ -36,12 +35,10 @@ def synthetic_write(addresses: FilePaths): Args: addresses (FilePaths): A dictionary of file paths. """ - # TODO these node/edge names are probably not good or extendible defulats - # revisit once overall software architecture is more clear. nodes = gpd.read_file(addresses.model_paths.nodes) - with open(os.devnull, "w") as devnull: - with contextlib.redirect_stderr(devnull): - edges = gpd.read_file(addresses.model_paths.edges) + with io.capture_output() as captured: + # TODO: this is hacky, to be addressed when converted to SWMMIO. + edges = gpd.read_file(addresses.model_paths.edges) if addresses.model_paths.subcatchments.suffix == ".geoparquet": subs = gpd.read_parquet(addresses.model_paths.subcatchments) From d163af9202ea57d0c1ebb9fce73472ec771a36b6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2024 11:05:13 +0000 Subject: [PATCH 26/36] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/swmmanywhere/post_processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swmmanywhere/post_processing.py b/src/swmmanywhere/post_processing.py index 2dd73bee..4860502e 100644 --- a/src/swmmanywhere/post_processing.py +++ b/src/swmmanywhere/post_processing.py @@ -11,11 +11,11 @@ from pathlib import Path from typing import Any, Literal -from IPython.utils import io import geopandas as gpd import numpy as np import pandas as pd import yaml +from IPython.utils import io from swmmanywhere.filepaths import FilePaths from swmmanywhere.logging import logger From ac1afb818f571d64d62c05da333174a235c0ce6b Mon Sep 17 00:00:00 2001 From: barneydobson Date: Tue, 22 Oct 2024 12:08:37 +0100 Subject: [PATCH 27/36] give up --- src/swmmanywhere/post_processing.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/swmmanywhere/post_processing.py b/src/swmmanywhere/post_processing.py index 2dd73bee..82adfd58 100644 --- a/src/swmmanywhere/post_processing.py +++ b/src/swmmanywhere/post_processing.py @@ -11,7 +11,6 @@ from pathlib import Path from typing import Any, Literal -from IPython.utils import io import geopandas as gpd import numpy as np import pandas as pd @@ -36,9 +35,7 @@ def synthetic_write(addresses: FilePaths): addresses (FilePaths): A dictionary of file paths. """ nodes = gpd.read_file(addresses.model_paths.nodes) - with io.capture_output() as captured: - # TODO: this is hacky, to be addressed when converted to SWMMIO. - edges = gpd.read_file(addresses.model_paths.edges) + edges = gpd.read_file(addresses.model_paths.edges) if addresses.model_paths.subcatchments.suffix == ".geoparquet": subs = gpd.read_parquet(addresses.model_paths.subcatchments) From d09fb6434ecd16be1f8f1a7e69aeee2730adb769 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2024 11:09:21 +0000 Subject: [PATCH 28/36] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/swmmanywhere/post_processing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/swmmanywhere/post_processing.py b/src/swmmanywhere/post_processing.py index eb5b37c4..82adfd58 100644 --- a/src/swmmanywhere/post_processing.py +++ b/src/swmmanywhere/post_processing.py @@ -15,7 +15,6 @@ import numpy as np import pandas as pd import yaml -from IPython.utils import io from swmmanywhere.filepaths import FilePaths from swmmanywhere.logging import logger From f6a12abc95c55124f1a20172e12a9666536bb6cb Mon Sep 17 00:00:00 2001 From: barneydobson Date: Tue, 22 Oct 2024 15:38:48 +0100 Subject: [PATCH 29/36] refactor map --- docs/notebooks/extended_demo.py | 124 ++------------------------ pyproject.toml | 6 +- src/swmmanywhere/utilities.py | 148 ++++++++++++++++++++++++++++++++ 3 files changed, 156 insertions(+), 122 deletions(-) diff --git a/docs/notebooks/extended_demo.py b/docs/notebooks/extended_demo.py index e23775fa..d3f775e0 100644 --- a/docs/notebooks/extended_demo.py +++ b/docs/notebooks/extended_demo.py @@ -23,18 +23,16 @@ # Imports from __future__ import annotations -import base64 import tempfile -from io import BytesIO from pathlib import Path import folium import geopandas as gpd import pandas as pd -from matplotlib import pyplot as plt from swmmanywhere.logging import set_verbose from swmmanywhere.swmmanywhere import swmmanywhere +from swmmanywhere.utilities import map # Create temporary directory temp_dir = tempfile.TemporaryDirectory() @@ -67,49 +65,7 @@ # attribute, we will plot these in red and other nodes in black. # %% # Create a folium map and add the nodes and edges -def basic_map(model_dir): - """Create a basic map with nodes and edges.""" - # Load and inspect results - nodes = gpd.read_file(model_dir / "nodes.geoparquet") - edges = gpd.read_file(model_dir / "edges.geoparquet") - - # Convert to EPSG 4326 for plotting - nodes = nodes.to_crs(4326) - edges = edges.to_crs(4326) - - # Identify outfalls - outfall = nodes.id == nodes.outfall - - # Plot on map - m = folium.Map( - location=[nodes.geometry.y.mean(), nodes.geometry.x.mean()], zoom_start=16 - ) - folium.GeoJson(edges, color="black", weight=1).add_to(m) - folium.GeoJson( - nodes.loc[~outfall], - marker=folium.CircleMarker( - radius=3, # Radius in metres - weight=0, # outline weight - fill_color="black", - fill_opacity=1, - ), - ).add_to(m) - - folium.GeoJson( - nodes.loc[outfall], - marker=folium.CircleMarker( - radius=3, # Radius in metres - weight=0, # outline weight - fill_color="red", - fill_opacity=1, - ), - ).add_to(m) - - # Display the map - return m - - -basic_map(model_file.parent) +map(model_file.parent) # %% [markdown] # OK, it's done something! Though perhaps we're not super satisfied with the output. @@ -148,7 +104,7 @@ def basic_map(model_dir): }, } outputs = swmmanywhere(config) -basic_map(outputs[0].parent) +map(outputs[0].parent) # %% [markdown] # OK that clearly helped, although we have appear to have stranded pipes (e.g., along @@ -173,7 +129,7 @@ def basic_map(model_dir): # Run again outputs = swmmanywhere(config) model_dir = outputs[0].parent -m = basic_map(model_dir) +m = map(model_dir) # %% [markdown] # That's a lot of information! However, the reason we are currently interested @@ -217,79 +173,11 @@ def basic_map(model_dir): # %% [markdown] # Since folium is super clever, we can make these clickable on our map - and # now you can inspect your results in a much more elegant way than the SWMM GUI. +# Just click a node or link to view the flooding or flow timeseries! # %% -# Create a folium map and add the nodes and edges -def clickable_map(model_dir): - """Create a clickable map with nodes, edges and results.""" - # Load and inspect results - nodes = gpd.read_file(model_dir / "nodes.geoparquet") - edges = gpd.read_file(model_dir / "edges.geoparquet") - df = pd.read_parquet(model_dir / "results.parquet") - df.id = df.id.astype(str) - floods = df.loc[df.variable == "flooding"].groupby("id") - flows = df.loc[df.variable == "flow"].groupby("id") - - # Convert to EPSG 4326 for plotting - nodes = nodes.to_crs(4326).set_index("id") - edges = edges.to_crs(4326).set_index("id") - - # Create map - m = folium.Map( - location=[nodes.geometry.y.mean(), nodes.geometry.x.mean()], zoom_start=16 - ) - - # Add edges - for edge, row in edges.iterrows(): - grp = flows.get_group(str(edge)) - f, ax = plt.subplots(figsize=(4, 3)) - grp.set_index("date").value.plot(ylabel="flow (l/s)", title=edge, ax=ax) - img = BytesIO() - f.tight_layout() - f.savefig(img, format="png", dpi=94) - plt.close(f) - img.seek(0) - img_base64 = base64.b64encode(img.read()).decode() - img_html = f'' - folium.PolyLine( - [[c[1], c[0]] for c in row.geometry.coords], - color="black", - weight=2, - popup=folium.Popup(img_html), - ).add_to(m) - - # Add nodes - for node, row in nodes.iterrows(): - grp = floods.get_group(str(node)) - f, ax = plt.subplots(figsize=(4, 3)) - grp.set_index("date").value.plot(ylabel="flooding (l)", title=node, ax=ax) - img = BytesIO() - f.tight_layout() - f.savefig(img, format="png", dpi=94) - plt.close(f) - img.seek(0) - img_base64 = base64.b64encode(img.read()).decode() - img_html = f'' - if row.outfall == node: - color = "red" - else: - color = "black" - folium.CircleMarker( - [nodes.loc[node].geometry.y, nodes.loc[node].geometry.x], - color=color, - radius=3, - weight=0, - fill_color=color, - fill_opacity=1, - popup=folium.Popup(img_html), - ).add_to(m) - - return m - - -# Display the map -clickable_map(model_dir) +m # %% [markdown] # If we explore around, clicking on edges, we can see that flows are often diff --git a/pyproject.toml b/pyproject.toml index 4be8c39a..d402e848 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,11 +36,13 @@ dynamic = [ dependencies = [ "cdsapi", "cytoolz", + "folium", "geopandas>=1", "geopy", "joblib", "jsonschema", "loguru", + "matplotlib", "netcdf4", "networkx>=3", "numpy>=2", @@ -62,8 +64,6 @@ dependencies = [ "xarray", ] optional-dependencies.dev = [ - "folium", - "matplotlib", "mypy", "pip-tools", "pre-commit", @@ -74,8 +74,6 @@ optional-dependencies.dev = [ "ruff", ] optional-dependencies.doc = [ - "folium", - "matplotlib", "mkdocs", "mkdocs-include-markdown-plugin", "mkdocs-jupyter", diff --git a/src/swmmanywhere/utilities.py b/src/swmmanywhere/utilities.py index 770b1bf6..b61962dc 100644 --- a/src/swmmanywhere/utilities.py +++ b/src/swmmanywhere/utilities.py @@ -5,11 +5,17 @@ from __future__ import annotations +import base64 import functools +from io import BytesIO from pathlib import Path from typing import TYPE_CHECKING, Any +import folium +import geopandas as gpd +import pandas as pd import yaml +from matplotlib import pyplot as plt if TYPE_CHECKING: SafeDumper = yaml.SafeDumper @@ -49,3 +55,145 @@ def yaml_dump(o: Any, stream: Any = None, **kwargs: Any) -> str: sort_keys=False, **kwargs, ) + + +def plot_basic(model_dir: Path): + """Create a basic map with nodes and edges. + + Args: + model_dir (Path): The directory containing the model files. + + Returns: + folium.Map: The folium map. + """ + # Load and inspect results + nodes = gpd.read_file(model_dir / "nodes.geoparquet") + edges = gpd.read_file(model_dir / "edges.geoparquet") + + # Convert to EPSG 4326 for plotting + nodes = nodes.to_crs(4326) + edges = edges.to_crs(4326) + + # Identify outfalls + outfall = nodes.id == nodes.outfall + + # Plot on map + m = folium.Map( + location=[nodes.geometry.y.mean(), nodes.geometry.x.mean()], zoom_start=16 + ) + folium.GeoJson(edges, color="black", weight=1).add_to(m) + folium.GeoJson( + nodes.loc[~outfall], + marker=folium.CircleMarker( + radius=3, # Radius in metres + weight=0, # outline weight + fill_color="black", + fill_opacity=1, + ), + ).add_to(m) + + folium.GeoJson( + nodes.loc[outfall], + marker=folium.CircleMarker( + radius=3, # Radius in metres + weight=0, # outline weight + fill_color="red", + fill_opacity=1, + ), + ).add_to(m) + + # Display the map + return m + + +def plot_clickable(model_dir: Path): + """Create a clickable map with nodes, edges and results. + + Args: + model_dir (Path): The directory containing the model files. + + Returns: + folium.Map: The folium map. + """ + # Load and inspect results + nodes = gpd.read_file(model_dir / "nodes.geoparquet") + edges = gpd.read_file(model_dir / "edges.geoparquet") + df = pd.read_parquet(model_dir / "results.parquet") + df.id = df.id.astype(str) + floods = df.loc[df.variable == "flooding"].groupby("id") + flows = df.loc[df.variable == "flow"].groupby("id") + + # Convert to EPSG 4326 for plotting + nodes = nodes.to_crs(4326).set_index("id") + edges = edges.to_crs(4326).set_index("id") + + # Create map + m = folium.Map( + location=[nodes.geometry.y.mean(), nodes.geometry.x.mean()], zoom_start=16 + ) + + # Add edges + for edge, row in edges.iterrows(): + grp = flows.get_group(str(edge)) + f, ax = plt.subplots(figsize=(4, 3)) + grp.set_index("date").value.plot(ylabel="flow (l/s)", title=edge, ax=ax) + img = BytesIO() + f.tight_layout() + f.savefig(img, format="png", dpi=94) + plt.close(f) + img.seek(0) + img_base64 = base64.b64encode(img.read()).decode() + img_html = f'' + folium.PolyLine( + [[c[1], c[0]] for c in row.geometry.coords], + color="black", + weight=2, + popup=folium.Popup(img_html), + ).add_to(m) + + # Add nodes + for node, row in nodes.iterrows(): + grp = floods.get_group(str(node)) + f, ax = plt.subplots(figsize=(4, 3)) + grp.set_index("date").value.plot(ylabel="flooding (l)", title=node, ax=ax) + img = BytesIO() + f.tight_layout() + f.savefig(img, format="png", dpi=94) + plt.close(f) + img.seek(0) + img_base64 = base64.b64encode(img.read()).decode() + img_html = f'' + if row.outfall == node: + color = "red" + else: + color = "black" + folium.CircleMarker( + [nodes.loc[node].geometry.y, nodes.loc[node].geometry.x], + color=color, + radius=3, + weight=0, + fill_color=color, + fill_opacity=1, + popup=folium.Popup(img_html), + ).add_to(m) + + return m + + +def map(model_dir: Path): + """Create a map from a model directory. + + Args: + model_dir (Path): The directory containing the model files. + + Returns: + folium.Map: The folium map. + """ + if not (any(model_dir.glob("nodes.*")) and any(model_dir.glob("edges.*"))): + raise FileNotFoundError("No nodes or edges found in model directory.") + + if any(model_dir.glob("results.*")): + m = plot_clickable(model_dir) + else: + m = plot_basic(model_dir) + return m From 358112454e521719081349dbb6fac8d8372a230f Mon Sep 17 00:00:00 2001 From: barneydobson Date: Tue, 22 Oct 2024 16:58:50 +0100 Subject: [PATCH 30/36] update text --- docs/notebooks/extended_demo.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/notebooks/extended_demo.py b/docs/notebooks/extended_demo.py index d3f775e0..76c8df09 100644 --- a/docs/notebooks/extended_demo.py +++ b/docs/notebooks/extended_demo.py @@ -62,7 +62,8 @@ # around the synthesised `nodes` and `edges`. These are # created in the same directory as the `model_file`. Let's have a look at them. # Note that the `outfall` that each node drains to is specified in the `outfall` -# attribute, we will plot these in red and other nodes in black. +# attribute, we will plot these in red and other nodes in black with the built- +# in `swmmanywhere.utilities.map` function. # %% # Create a folium map and add the nodes and edges map(model_file.parent) @@ -171,13 +172,14 @@ # %% [markdown] -# Since folium is super clever, we can make these clickable on our map - and +# If `results` are present in the `model_dir`, `map` will make clickable +# elements to view plots, # now you can inspect your results in a much more elegant way than the SWMM GUI. # Just click a node or link to view the flooding or flow timeseries! # %% -m +map(model_dir) # %% [markdown] # If we explore around, clicking on edges, we can see that flows are often From 0f06f94b59efd2fea55e47fa218c07b99af5a4e1 Mon Sep 17 00:00:00 2001 From: barneydobson Date: Tue, 22 Oct 2024 17:10:40 +0100 Subject: [PATCH 31/36] improve doc --- src/swmmanywhere/utilities.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/swmmanywhere/utilities.py b/src/swmmanywhere/utilities.py index b61962dc..9cdc72f2 100644 --- a/src/swmmanywhere/utilities.py +++ b/src/swmmanywhere/utilities.py @@ -134,6 +134,7 @@ def plot_clickable(model_dir: Path): # Add edges for edge, row in edges.iterrows(): + # Create a plot for each edge grp = flows.get_group(str(edge)) f, ax = plt.subplots(figsize=(4, 3)) grp.set_index("date").value.plot(ylabel="flow (l/s)", title=edge, ax=ax) @@ -141,9 +142,13 @@ def plot_clickable(model_dir: Path): f.tight_layout() f.savefig(img, format="png", dpi=94) plt.close(f) + + # Convert plot to base64 img.seek(0) img_base64 = base64.b64encode(img.read()).decode() img_html = f'' + + # Add edge to map folium.PolyLine( [[c[1], c[0]] for c in row.geometry.coords], color="black", From 76a6b9824e532dcaf3bc9b81375b3614254b4dbe Mon Sep 17 00:00:00 2001 From: barneydobson Date: Tue, 22 Oct 2024 17:16:11 +0100 Subject: [PATCH 32/36] change map to plot_map --- docs/notebooks/extended_demo.py | 14 +++++++------- src/swmmanywhere/utilities.py | 2 +- tests/test_swmmanywhere.py | 7 +++++++ 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/docs/notebooks/extended_demo.py b/docs/notebooks/extended_demo.py index 76c8df09..f4b2c26a 100644 --- a/docs/notebooks/extended_demo.py +++ b/docs/notebooks/extended_demo.py @@ -32,7 +32,7 @@ from swmmanywhere.logging import set_verbose from swmmanywhere.swmmanywhere import swmmanywhere -from swmmanywhere.utilities import map +from swmmanywhere.utilities import plot_map # Create temporary directory temp_dir = tempfile.TemporaryDirectory() @@ -63,10 +63,10 @@ # created in the same directory as the `model_file`. Let's have a look at them. # Note that the `outfall` that each node drains to is specified in the `outfall` # attribute, we will plot these in red and other nodes in black with the built- -# in `swmmanywhere.utilities.map` function. +# in `swmmanywhere.utilities.plot_map` function. # %% # Create a folium map and add the nodes and edges -map(model_file.parent) +plot_map(model_file.parent) # %% [markdown] # OK, it's done something! Though perhaps we're not super satisfied with the output. @@ -105,7 +105,7 @@ }, } outputs = swmmanywhere(config) -map(outputs[0].parent) +plot_map(outputs[0].parent) # %% [markdown] # OK that clearly helped, although we have appear to have stranded pipes (e.g., along @@ -130,7 +130,7 @@ # Run again outputs = swmmanywhere(config) model_dir = outputs[0].parent -m = map(model_dir) +m = plot_map(model_dir) # %% [markdown] # That's a lot of information! However, the reason we are currently interested @@ -172,14 +172,14 @@ # %% [markdown] -# If `results` are present in the `model_dir`, `map` will make clickable +# If `results` are present in the `model_dir`, `plot_map` will make clickable # elements to view plots, # now you can inspect your results in a much more elegant way than the SWMM GUI. # Just click a node or link to view the flooding or flow timeseries! # %% -map(model_dir) +plot_map(model_dir) # %% [markdown] # If we explore around, clicking on edges, we can see that flows are often diff --git a/src/swmmanywhere/utilities.py b/src/swmmanywhere/utilities.py index 9cdc72f2..c98130fd 100644 --- a/src/swmmanywhere/utilities.py +++ b/src/swmmanywhere/utilities.py @@ -185,7 +185,7 @@ def plot_clickable(model_dir: Path): return m -def map(model_dir: Path): +def plot_map(model_dir: Path): """Create a map from a model directory. Args: diff --git a/tests/test_swmmanywhere.py b/tests/test_swmmanywhere.py index 34617aaa..afaaef09 100644 --- a/tests/test_swmmanywhere.py +++ b/tests/test_swmmanywhere.py @@ -13,6 +13,7 @@ from swmmanywhere import swmmanywhere from swmmanywhere.graph_utilities import graphfcns from swmmanywhere.metric_utilities import metrics +from swmmanywhere.utilities import plot_map, plot_basic def test_run(): @@ -94,6 +95,12 @@ def test_swmmanywhere(): assert (inp.parent / "results.parquet").exists() assert (config["real"]["inp"].parent / "real_results.parquet").exists() + # Check the map functions + plot_basic(inp.parent) + plot_map(inp.parent) + + + def test_load_config_file_validation(): """Test the file validation of the config.""" From 67e709a3674a25d8bb6d10c2509010cef3708b95 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2024 16:16:59 +0000 Subject: [PATCH 33/36] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_swmmanywhere.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_swmmanywhere.py b/tests/test_swmmanywhere.py index afaaef09..2e421b9b 100644 --- a/tests/test_swmmanywhere.py +++ b/tests/test_swmmanywhere.py @@ -13,7 +13,7 @@ from swmmanywhere import swmmanywhere from swmmanywhere.graph_utilities import graphfcns from swmmanywhere.metric_utilities import metrics -from swmmanywhere.utilities import plot_map, plot_basic +from swmmanywhere.utilities import plot_basic, plot_map def test_run(): @@ -100,8 +100,6 @@ def test_swmmanywhere(): plot_map(inp.parent) - - def test_load_config_file_validation(): """Test the file validation of the config.""" with tempfile.TemporaryDirectory() as temp_dir: From bc84017c04410fb93e9370ea2e83f791c8555fd6 Mon Sep 17 00:00:00 2001 From: barneydobson Date: Tue, 22 Oct 2024 19:28:03 +0100 Subject: [PATCH 34/36] couple typos --- docs/notebooks/extended_demo.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/notebooks/extended_demo.py b/docs/notebooks/extended_demo.py index f4b2c26a..681973c4 100644 --- a/docs/notebooks/extended_demo.py +++ b/docs/notebooks/extended_demo.py @@ -8,7 +8,7 @@ # # %% [markdown] # ## Introduction -# This script demonstrates a simple use case of `SWMManywhere`, building on the +# This script demonstrates a simple use case of `swmmanywhere`, building on the # [quickstart](https://imperialcollegelondon.github.io/SWMManywhere/quickstart/) # example, but including plotting and alterations. # @@ -80,9 +80,6 @@ # - We will also remove any types under the `omit_edges` entry, here you can specify # to not allow pipes to cross bridges, tunnels, motorways, etc., however, this is # such a small area we probably don't want to restrict things so much. -# - Due to the apparent steep slopes, it seems like other topological factors, such -# as pipe length, are less important. We can adjust the `weights` parameter to -# include only slope, whose parameter name is `chahinian_slope`. # - We have far too few outfalls, it seems implausible that so many riverside streets # would not have outfalls. Furthermore, there are points that are quite far from the # river that have been assigned as outfalls. We can reduce the `river_buffer_distance` @@ -109,7 +106,7 @@ # %% [markdown] # OK that clearly helped, although we have appear to have stranded pipes (e.g., along -# *Carrer de la Grella* in North West), presumably due to some mistake in the +# *Carrer dels Canals* in North West), presumably due to some mistake in the # OSM specifying that it # is connected via a pedestrian route. We won't remedy this in the tutorial, but you can # manually provide your From e62d49e4a411b09b3c5c68afd3beb853536fffc3 Mon Sep 17 00:00:00 2001 From: Dobson Date: Wed, 23 Oct 2024 10:36:40 +0100 Subject: [PATCH 35/36] better explanation --- docs/notebooks/extended_demo.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/notebooks/extended_demo.py b/docs/notebooks/extended_demo.py index 681973c4..b868d3eb 100644 --- a/docs/notebooks/extended_demo.py +++ b/docs/notebooks/extended_demo.py @@ -181,8 +181,10 @@ # %% [markdown] # If we explore around, clicking on edges, we can see that flows are often # looking sensible, though we can definitely some areas that have been hampered -# by our starting street graph (e.g., the Western portion of *Carrer del Sant Andreu* -# in North West). The first suggestion here would be to examine the starting graph, +# by our starting street graph (e.g., in the Western portion of *Carrer del Sant Andreu* +# in North West we can see negative flows meaning the direction is different from +# what the topology derivation assumed flow would be going in!). +# The first suggestion here would be to examine the starting graph, # however, if you want to # make more sophisticated customisations then your probably want to learn about # [graph functions](https://imperialcollegelondon.github.io/SWMManywhere/graphfcns_guide/). From 4700608239ff7dc8bfb4c1140edf50b00b30fc18 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 23 Oct 2024 09:37:06 +0000 Subject: [PATCH 36/36] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/notebooks/extended_demo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/notebooks/extended_demo.py b/docs/notebooks/extended_demo.py index b868d3eb..aab915d0 100644 --- a/docs/notebooks/extended_demo.py +++ b/docs/notebooks/extended_demo.py @@ -183,7 +183,7 @@ # looking sensible, though we can definitely some areas that have been hampered # by our starting street graph (e.g., in the Western portion of *Carrer del Sant Andreu* # in North West we can see negative flows meaning the direction is different from -# what the topology derivation assumed flow would be going in!). +# what the topology derivation assumed flow would be going in!). # The first suggestion here would be to examine the starting graph, # however, if you want to # make more sophisticated customisations then your probably want to learn about