Merge pull request #83 from ImperialCollegeLondon/10-configuration-file

Create demo_config.yml

dev-requirements.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.10
+# This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
 #    pip-compile --extra=dev --output-file=dev-requirements.txt
@@ -17,8 +17,10 @@ annotated-types==0.6.0
 attrs==23.2.0
     # via
     #   fiona
+    #   jsonschema
     #   pytest-mypy
     #   rasterio
+    #   referencing
 build==1.0.3
     # via pip-tools
 cdsapi==0.6.1
@@ -74,8 +76,6 @@ dill==0.3.7
     # via multiprocess
 distlib==0.3.8
     # via virtualenv
-exceptiongroup==1.2.0
-    # via pytest
 fastparquet==2023.10.1
     # via swmmanywhere (pyproject.toml)
 filelock==3.13.1
@@ -100,6 +100,10 @@ geopandas==0.14.2
     #   swmmanywhere (pyproject.toml)
 geopy==2.4.1
     # via swmmanywhere (pyproject.toml)
+gitdb==4.0.11
+    # via gitpython
+gitpython==3.1.42
+    # via swmmanywhere (pyproject.toml)
 identify==2.5.33
     # via pre-commit
 idna==3.6
@@ -110,6 +114,10 @@ iniconfig==2.0.0
     # via pytest
 joblib==1.3.2
     # via swmmanywhere (pyproject.toml)
+jsonschema==4.21.1
+    # via swmmanywhere (pyproject.toml)
+jsonschema-specifications==2023.12.1
+    # via jsonschema
 julian==0.14
     # via pyswmm
 kiwisolver==1.4.5
@@ -253,12 +261,20 @@ rasterio==1.3.9
     #   pysheds
     #   rioxarray
     #   swmmanywhere (pyproject.toml)
+referencing==0.33.0
+    # via
+    #   jsonschema
+    #   jsonschema-specifications
 requests==2.31.0
     # via
     #   cdsapi
     #   osmnx
 rioxarray==0.15.1
     # via swmmanywhere (pyproject.toml)
+rpds-py==0.18.0
+    # via
+    #   jsonschema
+    #   referencing
 ruff==0.1.11
     # via swmmanywhere (pyproject.toml)
 salib==1.4.7
@@ -281,20 +297,14 @@ six==1.16.0
     # via
     #   fiona
     #   python-dateutil
+smmap==5.0.1
+    # via gitdb
 snuggs==1.4.7
     # via rasterio
 swmm-toolkit==0.15.3
     # via pyswmm
 tifffile==2024.1.30
     # via scikit-image
-tomli==2.0.1
-    # via
-    #   build
-    #   coverage
-    #   mypy
-    #   pip-tools
-    #   pyproject-hooks
-    #   pytest
 toolz==0.12.1
     # via cytoolz
 tqdm==4.66.2

pyproject.toml

@@ -32,6 +32,7 @@ dependencies = [
   "geopy",
   "GitPython",
   "joblib",
+  "jsonschema",
   "loguru",
   "matplotlib",
   "netcdf4",
@@ -94,4 +95,7 @@ skip = "swmmanywhere/defs/iso_converter.yml,*.inp"
 ignore-words-list = "gage,gages"
 
 [tool.refurb]
-ignore = [184]
+ignore = [
+        184, # Because some frankly bizarre suggestions
+        109  # Because pyyaml doesn't support tuples
+        ]

requirements.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.10
+# This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
 #    pip-compile
@@ -17,7 +17,9 @@ annotated-types==0.6.0
 attrs==23.2.0
     # via
     #   fiona
+    #   jsonschema
     #   rasterio
+    #   referencing
 cdsapi==0.6.1
     # via swmmanywhere (pyproject.toml)
 certifi==2023.11.17
@@ -80,12 +82,20 @@ geopandas==0.14.2
     #   swmmanywhere (pyproject.toml)
 geopy==2.4.1
     # via swmmanywhere (pyproject.toml)
+gitdb==4.0.11
+    # via gitpython
+gitpython==3.1.42
+    # via swmmanywhere (pyproject.toml)
 idna==3.6
     # via requests
 imageio==2.33.1
     # via scikit-image
 joblib==1.3.2
     # via swmmanywhere (pyproject.toml)
+jsonschema==4.21.1
+    # via swmmanywhere (pyproject.toml)
+jsonschema-specifications==2023.12.1
+    # via jsonschema
 julian==0.14
     # via pyswmm
 kiwisolver==1.4.5
@@ -195,12 +205,20 @@ rasterio==1.3.9
     #   pysheds
     #   rioxarray
     #   swmmanywhere (pyproject.toml)
+referencing==0.33.0
+    # via
+    #   jsonschema
+    #   jsonschema-specifications
 requests==2.31.0
     # via
     #   cdsapi
     #   osmnx
 rioxarray==0.15.1
     # via swmmanywhere (pyproject.toml)
+rpds-py==0.18.0
+    # via
+    #   jsonschema
+    #   referencing
 salib==1.4.7
     # via swmmanywhere (pyproject.toml)
 scikit-image==0.22.0
@@ -221,6 +239,8 @@ six==1.16.0
     # via
     #   fiona
     #   python-dateutil
+smmap==5.0.1
+    # via gitdb
 snuggs==1.4.7
     # via rasterio
 swmm-toolkit==0.15.3

swmmanywhere/defs/schema.yml

@@ -0,0 +1,33 @@
+type: object
+properties:
+  base_dir: {type: string}
+  project: {type: string}
+  bbox: {type: array, items: {type: number}, minItems: 4, maxItems: 4}
+  api_keys: {type: string}
+  run_settings:
+    type: object
+    properties:
+      reporting_iters: {type: integer, minimum: 1}
+      duration: {type: number}
+      storevars:
+        type: array
+        items: 
+          type: string
+          enum: [flooding, flow, depth, runoff]
+  real:
+    type: ['object', 'null']
+    properties:
+      inp: {type: string}
+      graph: {type: string}
+      subcatchments: {type: string}
+      results: {type: ['string', 'null']}
+    required: [graph, subcatchments]
+    anyOf:
+      - required: [inp]
+      - required: [results]
+  starting_graph: {type: ['string', 'null']}
+  graphfcn_list: {type: array, items: {type: string}}
+  metric_list: {type: array, items: {type: string}}
+  address_overrides: {type: ['object', 'null']}
+  parameter_overrides: {type: ['object', 'null']}
+required: [base_dir, project, bbox, api_keys, graphfcn_list]

swmmanywhere/geospatial_utilities.py

@@ -258,7 +258,7 @@ def reproject_graph(G: nx.Graph,
                                         G_new.nodes[u]['y']],
                                        [G_new.nodes[v]['x'],
                                         G_new.nodes[v]['y']]])
-
+    G_new.graph['crs'] = target_crs
     return G_new
 
 
@@ -751,20 +751,23 @@ def edges_to_features(G: nx.Graph):
     return features
 
 def graph_to_geojson(graph: nx.Graph, 
-                     fid: Path, 
+                     fid_nodes: Path, 
+                     fid_edges: Path,
                      crs: str):
     """Write a graph to a GeoJSON file.
 
     Args:
         graph (nx.Graph): The input graph.
-        fid (Path): The filepath to save the GeoJSON file.
+        fid_nodes (Path): The filepath to save the nodes GeoJSON file.
+        fid_edges (Path): The filepath to save the edges GeoJSON file.
         crs (str): The CRS of the graph.
     """
     graph = graph.copy()
     nodes = nodes_to_features(graph)
     edges = edges_to_features(graph)
 
-    for iterable, label in zip([nodes, edges], ['nodes', 'edges']):
+    for iterable, fid in zip([nodes, edges], 
+                             [fid_nodes, fid_edges]):
         geojson = {
             'type': 'FeatureCollection',
             'features' : iterable,
@@ -775,7 +778,6 @@ def graph_to_geojson(graph: nx.Graph,
                 }
             }
             }
-        fid_ = fid.with_stem(fid.stem + f'_{label}').with_suffix('.geojson')
 
-        with fid_.open('w') as output_file:
+        with fid.open('w') as output_file:
             json.dump(geojson, output_file, indent=2)

swmmanywhere/graph_utilities.py

@@ -157,9 +157,32 @@ def get_osmid_id(data: dict) -> Hashable:
         id_ = id_[0]
     return id_
 
+def iterate_graphfcns(G: nx.Graph, 
+                      graphfcn_list: list[str], 
+                      params: dict,
+                      addresses: parameters.FilePaths) -> nx.Graph:
+    """Iterate a list of graph functions over a graph.
+
+    Args:
+        G (nx.Graph): The graph to iterate over.
+        graphfcn_list (list[str]): A list of graph functions to iterate.
+        params (dict): A dictionary of parameters to pass to the graph
+            functions.
+        addresses (parameters.FilePaths): A FilePaths parameter object
+
+    Returns:
+        nx.Graph: The graph after the graph functions have been applied.
+    """
+    not_exists = [g for g in graphfcn_list if g not in graphfcns]
+    if not_exists:
+        raise ValueError(f"Graphfcns are not registered:\n{', '.join(not_exists)}")
+    for function in graphfcn_list:
+        G = graphfcns[function](G, addresses = addresses, **params)
+        logger.info(f"graphfcn: {function} completed.")
+    return G
+
 @register_graphfcn
 class assign_id(BaseGraphFunction, 
-                required_edge_attributes = ['osmid'], 
                 adds_edge_attributes = ['id']
                 ):
     """assign_id class."""
@@ -180,8 +203,12 @@ def __call__(self,
         Returns:
             G (nx.Graph): The same graph with an ID assigned to each edge
         """
+        edge_ids: set[str] = set()
         for u, v, data in G.edges(data=True):
-            data['id'] = get_osmid_id(data)
+            data['id'] = f'{u}-{v}'
+            if data['id'] in edge_ids:
+                logger.warning(f"Duplicate edge ID: {data['id']}")
+            edge_ids.add(data['id'])
         return G
 
 @register_graphfcn
@@ -423,7 +450,7 @@ def __call__(self, G: nx.Graph,
             subs_gdf = go.derive_subcatchments(G,temp_fid)
 
         # Calculate runoff coefficient (RC)
-        if addresses.building.suffix == '.parquet':
+        if addresses.building.suffix in ('.geoparquet','.parquet'):
             buildings = gpd.read_parquet(addresses.building)
         else:
             buildings = gpd.read_file(addresses.building)
@@ -451,7 +478,7 @@ def __call__(self, G: nx.Graph,
 @register_graphfcn
 class set_elevation(BaseGraphFunction,
                     required_node_attributes = ['x', 'y'],
-                    adds_node_attributes = ['elevation']):
+                    adds_node_attributes = ['surface_elevation']):
     """set_elevation class."""
 
     def __call__(self, G: nx.Graph, 
@@ -477,12 +504,12 @@ def __call__(self, G: nx.Graph,
                                                     y, 
                                                     addresses.elevation)
         elevations_dict = {id_: elev for id_, elev in zip(G.nodes, elevations)}
-        nx.set_node_attributes(G, elevations_dict, 'elevation')
+        nx.set_node_attributes(G, elevations_dict, 'surface_elevation')
         return G
 
 @register_graphfcn
 class set_surface_slope(BaseGraphFunction,
-                        required_node_attributes = ['elevation'],
+                        required_node_attributes = ['surface_elevation'],
                         adds_edge_attributes = ['surface_slope']):
     """set_surface_slope class."""
 
@@ -502,7 +529,8 @@ def __call__(self, G: nx.Graph,
         """
         G = G.copy()
         # Compute the slope for each edge
-        slope_dict = {(u, v, k): (G.nodes[u]['elevation'] - G.nodes[v]['elevation']) 
+        slope_dict = {(u, v, k): (G.nodes[u]['surface_elevation'] - \
+                                  G.nodes[v]['surface_elevation']) 
                         / d['length'] for u, v, k, d in G.edges(data=True,
                                                                 keys=True)}
 
@@ -977,8 +1005,9 @@ def process_successors(G: nx.Graph,
 
 @register_graphfcn
 class pipe_by_pipe(BaseGraphFunction,
-                   required_edge_attributes = ['length', 'elevation'],
-                   required_node_attributes = ['contributing_area', 'elevation'],
+                   required_edge_attributes = ['length'],
+                   required_node_attributes = ['contributing_area', 
+                                               'surface_elevation'],
                    adds_edge_attributes = ['diameter'],
                    adds_node_attributes = ['chamber_floor_elevation']):
     """pipe_by_pipe class."""
@@ -1015,7 +1044,7 @@ def __call__(self,
             G (nx.Graph): A graph
         """
         G = G.copy()
-        surface_elevations = {n : d['elevation'] for n, d in G.nodes(data=True)}
+        surface_elevations = nx.get_node_attributes(G, 'surface_elevation')
         topological_order = list(nx.topological_sort(G))
         chamber_floor = {}
         edge_diams: dict[tuple[Hashable,Hashable,int],float] = {}