diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3942281f..228edfb1 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -4,6 +4,23 @@ jobs: test: uses: ./.github/workflows/ci.yml + publish-docs: + needs: publish-PyPI + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install dependencies + run: pip install -r doc-requirements.txt + + - name: Deploy Docs + run: mkdocs gh-deploy --force + # publish: # runs-on: ubuntu-latest # needs: test diff --git a/.gitignore b/.gitignore index 3a6c1aee..b17a5bb4 100644 --- a/.gitignore +++ b/.gitignore @@ -130,3 +130,6 @@ dmypy.json # Pysheds cache cache/ + +# Documentation generated models +swmmanywhere_models/ \ No newline at end of file diff --git a/doc-requirements.txt b/doc-requirements.txt new file mode 100644 index 00000000..3ca0b41b --- /dev/null +++ b/doc-requirements.txt @@ -0,0 +1,506 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --extra=doc --output-file=doc-requirements.txt +# +aenum==3.1.11 + # via + # pyswmm + # swmm-toolkit +affine==2.4.0 + # via + # pysheds + # rasterio +annotated-types==0.6.0 + # via pydantic +asttokens==2.4.1 + # via stack-data +attrs==23.2.0 + # via + # fiona + # jsonschema + # rasterio + # referencing +babel==2.14.0 + # via mkdocs-material +beautifulsoup4==4.12.3 + # via nbconvert +bleach==6.1.0 + # via nbconvert +cdsapi==0.6.1 + # via swmmanywhere (pyproject.toml) +certifi==2024.2.2 + # via + # fiona + # netcdf4 + # pyproj + # rasterio + # requests +cftime==1.6.3 + # via netcdf4 +charset-normalizer==3.3.2 + # via requests +click==8.1.7 + # via + # click-plugins + # cligj + # fiona + # mkdocs + # mkdocstrings + # rasterio +click-plugins==1.1.1 + # via + # fiona + # rasterio +cligj==0.7.2 + # via + # fiona + # rasterio +colorama==0.4.6 + # via + # click + # griffe + # ipython + # loguru + # mkdocs + # mkdocs-material + # tqdm +comm==0.2.2 + # via ipykernel +contourpy==1.2.0 + # via matplotlib +cramjam==2.8.3 + # via fastparquet +cycler==0.12.1 + # via matplotlib +cytoolz==0.12.3 + # via swmmanywhere (pyproject.toml) +debugpy==1.8.1 + # via ipykernel +decorator==5.1.1 + # via ipython +defusedxml==0.7.1 + # via nbconvert +dill==0.3.8 + # via multiprocess +executing==2.0.1 + # via stack-data +fastjsonschema==2.19.1 + # via nbformat +fastparquet==2024.2.0 + # via swmmanywhere (pyproject.toml) +fiona==1.9.6 + # via + # geopandas + # swmmanywhere (pyproject.toml) +fonttools==4.50.0 + # via matplotlib +fsspec==2024.3.1 + # via fastparquet +geographiclib==2.0 + # via geopy +geojson==3.1.0 + # via pysheds +geopandas==0.14.3 + # via + # osmnx + # swmmanywhere (pyproject.toml) +geopy==2.4.1 + # via swmmanywhere (pyproject.toml) +ghp-import==2.1.0 + # via mkdocs +gitdb==4.0.11 + # via gitpython +gitpython==3.1.42 + # via swmmanywhere (pyproject.toml) +griffe==0.42.1 + # via mkdocstrings-python +idna==3.6 + # via requests +imageio==2.34.0 + # via scikit-image +ipykernel==6.29.3 + # via mkdocs-jupyter +ipython==8.22.2 + # via ipykernel +jedi==0.19.1 + # via ipython +jinja2==3.1.3 + # via + # mkdocs + # mkdocs-material + # mkdocstrings + # nbconvert +joblib==1.3.2 + # via swmmanywhere (pyproject.toml) +jsonschema==4.21.1 + # via + # nbformat + # swmmanywhere (pyproject.toml) +jsonschema-specifications==2023.12.1 + # via jsonschema +julian==0.14 + # via pyswmm +jupyter-client==8.6.1 + # via + # ipykernel + # nbclient +jupyter-core==5.7.2 + # via + # ipykernel + # jupyter-client + # nbclient + # nbconvert + # nbformat +jupyterlab-pygments==0.3.0 + # via nbconvert +jupytext==1.16.1 + # via mkdocs-jupyter +kiwisolver==1.4.5 + # via matplotlib +latexcodec==3.0.0 + # via pybtex +lazy-loader==0.3 + # via scikit-image +llvmlite==0.42.0 + # via numba +loguru==0.7.2 + # via swmmanywhere (pyproject.toml) +markdown==3.5.2 + # via + # mkdocs + # mkdocs-autorefs + # mkdocs-material + # mkdocstrings + # mkdocstrings-python + # pymdown-extensions +markdown-it-py==3.0.0 + # via + # jupytext + # mdit-py-plugins +markupsafe==2.1.5 + # via + # jinja2 + # mkdocs + # mkdocs-autorefs + # mkdocstrings + # nbconvert +matplotlib==3.8.3 + # via + # salib + # swmmanywhere (pyproject.toml) +matplotlib-inline==0.1.6 + # via + # ipykernel + # ipython +mdit-py-plugins==0.4.0 + # via jupytext +mdurl==0.1.2 + # via markdown-it-py +mergedeep==1.3.4 + # via mkdocs +mistune==3.0.2 + # via nbconvert +mkdocs==1.5.3 + # via + # mkdocs-autorefs + # mkdocs-bibtex + # mkdocs-coverage + # mkdocs-gen-files + # mkdocs-jupyter + # mkdocs-material + # mkdocstrings + # swmmanywhere (pyproject.toml) +mkdocs-autorefs==1.0.1 + # via + # mkdocstrings + # swmmanywhere (pyproject.toml) +mkdocs-bibtex==2.14.1 + # via swmmanywhere (pyproject.toml) +mkdocs-coverage==1.0.0 + # via swmmanywhere (pyproject.toml) +mkdocs-gen-files==0.5.0 + # via swmmanywhere (pyproject.toml) +mkdocs-jupyter==0.24.6 + # via swmmanywhere (pyproject.toml) +mkdocs-material==9.5.14 + # via + # mkdocs-jupyter + # swmmanywhere (pyproject.toml) +mkdocs-material-extensions==1.3.1 + # via + # mkdocs-material + # swmmanywhere (pyproject.toml) +mkdocstrings[python]==0.24.1 + # via + # mkdocstrings-python + # swmmanywhere (pyproject.toml) +mkdocstrings-python==1.9.0 + # via mkdocstrings +multiprocess==0.70.16 + # via salib +nbclient==0.10.0 + # via nbconvert +nbconvert==7.16.2 + # via mkdocs-jupyter +nbformat==5.10.3 + # via + # jupytext + # nbclient + # nbconvert +nest-asyncio==1.6.0 + # via ipykernel +netcdf4==1.6.5 + # via swmmanywhere (pyproject.toml) +netcomp @ git+https://github.com/barneydobson/NetComp.git + # via swmmanywhere (pyproject.toml) +networkx==3.2.1 + # via + # netcomp + # osmnx + # scikit-image + # swmmanywhere (pyproject.toml) +numba==0.59.1 + # via pysheds +numpy==1.26.4 + # via + # cftime + # contourpy + # fastparquet + # imageio + # matplotlib + # netcdf4 + # netcomp + # numba + # osmnx + # pandas + # pyarrow + # pysheds + # rasterio + # rioxarray + # salib + # scikit-image + # scipy + # shapely + # snuggs + # swmmanywhere (pyproject.toml) + # tifffile + # xarray +osmnx==1.9.1 + # via swmmanywhere (pyproject.toml) +packaging==24.0 + # via + # fastparquet + # geopandas + # ipykernel + # jupytext + # matplotlib + # mkdocs + # nbconvert + # pyswmm + # rioxarray + # scikit-image + # xarray +paginate==0.5.6 + # via mkdocs-material +pandas==2.2.1 + # via + # fastparquet + # geopandas + # osmnx + # pysheds + # salib + # swmmanywhere (pyproject.toml) + # xarray +pandocfilters==1.5.1 + # via nbconvert +parso==0.8.3 + # via jedi +pathspec==0.12.1 + # via mkdocs +pillow==10.2.0 + # via + # imageio + # matplotlib + # scikit-image +platformdirs==4.2.0 + # via + # jupyter-core + # mkdocs + # mkdocstrings +prompt-toolkit==3.0.43 + # via ipython +psutil==5.9.8 + # via ipykernel +pure-eval==0.2.2 + # via stack-data +pyarrow==15.0.2 + # via swmmanywhere (pyproject.toml) +pybtex==0.24.0 + # via mkdocs-bibtex +pydantic==2.6.4 + # via swmmanywhere (pyproject.toml) +pydantic-core==2.16.3 + # via pydantic +pygments==2.17.2 + # via + # ipython + # mkdocs-jupyter + # mkdocs-material + # nbconvert +pymdown-extensions==10.7.1 + # via + # mkdocs-material + # mkdocstrings +pypandoc==1.13 + # via + # mkdocs-bibtex + # swmmanywhere (pyproject.toml) +pyparsing==3.1.2 + # via + # matplotlib + # snuggs +pyproj==3.6.1 + # via + # geopandas + # pysheds + # rioxarray +pysheds==0.3.5 + # via swmmanywhere (pyproject.toml) +pyswmm==2.0.0 + # via swmmanywhere (pyproject.toml) +python-dateutil==2.9.0.post0 + # via + # ghp-import + # jupyter-client + # matplotlib + # pandas +pytz==2024.1 + # via pandas +pywin32==306 + # via jupyter-core +pyyaml==6.0.1 + # via + # jupytext + # mkdocs + # pybtex + # pymdown-extensions + # pyyaml-env-tag + # swmmanywhere (pyproject.toml) +pyyaml-env-tag==0.1 + # via mkdocs +pyzmq==25.1.2 + # via + # ipykernel + # jupyter-client +rasterio==1.3.9 + # via + # pysheds + # rioxarray + # swmmanywhere (pyproject.toml) +referencing==0.34.0 + # via + # jsonschema + # jsonschema-specifications +regex==2023.12.25 + # via mkdocs-material +requests==2.31.0 + # via + # cdsapi + # mkdocs-bibtex + # mkdocs-material + # osmnx +rioxarray==0.15.1 + # via swmmanywhere (pyproject.toml) +rpds-py==0.18.0 + # via + # jsonschema + # referencing +salib==1.4.8 + # via swmmanywhere (pyproject.toml) +scikit-image==0.22.0 + # via pysheds +scipy==1.12.0 + # via + # netcomp + # pysheds + # salib + # scikit-image + # swmmanywhere (pyproject.toml) +shapely==2.0.3 + # via + # geopandas + # osmnx + # swmmanywhere (pyproject.toml) +six==1.16.0 + # via + # asttokens + # bleach + # fiona + # pybtex + # python-dateutil +smmap==5.0.1 + # via gitdb +snuggs==1.4.7 + # via rasterio +soupsieve==2.5 + # via beautifulsoup4 +stack-data==0.6.3 + # via ipython +swmm-toolkit==0.15.3 + # via pyswmm +tifffile==2024.2.12 + # via scikit-image +tinycss2==1.2.1 + # via nbconvert +toml==0.10.2 + # via jupytext +toolz==0.12.1 + # via cytoolz +tornado==6.4 + # via + # ipykernel + # jupyter-client +tqdm==4.66.2 + # via + # cdsapi + # swmmanywhere (pyproject.toml) +traitlets==5.14.2 + # via + # comm + # ipykernel + # ipython + # jupyter-client + # jupyter-core + # matplotlib-inline + # nbclient + # nbconvert + # nbformat +typing-extensions==4.10.0 + # via + # pydantic + # pydantic-core +tzdata==2024.1 + # via pandas +urllib3==2.2.1 + # via requests +validators==0.23.2 + # via mkdocs-bibtex +watchdog==4.0.0 + # via mkdocs +wcwidth==0.2.13 + # via prompt-toolkit +webencodings==0.5.1 + # via + # bleach + # tinycss2 +win32-setctime==1.1.0 + # via loguru +xarray==2024.2.0 + # via + # rioxarray + # swmmanywhere (pyproject.toml) + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/AUTHORS.md b/docs/AUTHORS.md similarity index 100% rename from AUTHORS.md rename to docs/AUTHORS.md diff --git a/CODE_OF_CONDUCT.md b/docs/CODE_OF_CONDUCT.md similarity index 100% rename from CODE_OF_CONDUCT.md rename to docs/CODE_OF_CONDUCT.md diff --git a/CONTRIBUTING.md b/docs/CONTRIBUTING.md similarity index 100% rename from CONTRIBUTING.md rename to docs/CONTRIBUTING.md diff --git a/swmmanywhere/HISTORY.md b/docs/HISTORY.md similarity index 100% rename from swmmanywhere/HISTORY.md rename to docs/HISTORY.md diff --git a/docs/images/andorra_swmm_screenshot.png b/docs/images/andorra_swmm_screenshot.png new file mode 100644 index 00000000..73eb367f Binary files /dev/null and b/docs/images/andorra_swmm_screenshot.png differ diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..d57e1942 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,20 @@ +# SWMManywhere + +This is the documentation for SWMManywhere. It is a software that lets you +derive a synthetic urban drainage network anywhere in the world. + +## Table of contents + +- [Home](index.md) +- [Quickstart](quickstart.md) +- [Contributing](CONTRIBUTING.md) +- [API reference](reference-overview.md) + - [SWMManywhere](reference.md) + - [Graph utilities](reference-graph-utilities.md) + - [Geospatial utilities](reference-geospatial-utilities.md) + - [Metric utilities](reference-metric-utilities.md) + - [Logging](reference-logging.md) + - [Parameters](reference-parameters.md) + - [Post processing](reference-post-processing.md) + - [Preprocessing](reference-preprocessing.md) + \ No newline at end of file diff --git a/docs/quickstart.md b/docs/quickstart.md new file mode 100644 index 00000000..326f178d --- /dev/null +++ b/docs/quickstart.md @@ -0,0 +1,58 @@ +# Quickstart + +SWMManywhere is a Python tool to synthesise Urban Drainage Models (UDM) anywhere in the world. +It handles everything from data acquisition to running the UDM in the [SWMM](https://www.epa.gov/sites/default/files/2019-02/documents/epaswmm5_1_manual_master_8-2-15.pdf) software. + +## Configuration + +SWMManywhere is primarily designed to be used via a Command Line Interface (CLI). +The user provides a `config` file address that specifies a variety of options to customise the synthesis process. +However, the minimum requirements for a user to provide are simply: + +- a base directory, +- a project name, +- a bounding box that specifies the latitude and longitude (EPSG:4326) of the bottom left and upper right corners of the region within which to create the UDM. + +We can define a simple configuration `.yml` file here: + +```yml +base_dir: /path/to/base/directory +project: my_first_swmm +bbox: [1.52740,42.50524,1.54273,42.51259] +``` + +## Run SWMManywhere + +The basic command is: + +```sh +python -m swmmanywhere --config_path=/path/to/file.yml +``` + +which will create a SWMM input file (`.inp`) at the file location: + +```text +//bbox_1/model_1/model_1.inp +``` + +## Use your model + +If you prefer GUIs then the easiest thing now is to download the [SWMM software](https://www.epa.gov/water-research/storm-water-management-model-swmm) and load your model in there. +The example above looks as follows: + +![SWMM Model](images/andorra_swmm_screenshot.png) + +From here you can run or edit your model. + +If you want to investigate your model in GIS, then the geospatial data that was formatted into the model file (`model_1.inp`) is also available at: + +```text +//bbox_1/model_1/nodes.geojson +//bbox_1/model_1/edges.geojson +//bbox_1/model_1/subcatchments.geojson +``` + +## Not happy with your model? + +Then it sounds like you want to explore the wide range of customisability that SWMManywhere offers! +See our notebooks to understand what is going on in greater detail and how to create better synthetic UDMs. diff --git a/docs/reference-geospatial-utilities.md b/docs/reference-geospatial-utilities.md new file mode 100644 index 00000000..d756135e --- /dev/null +++ b/docs/reference-geospatial-utilities.md @@ -0,0 +1,3 @@ +# Reference for SWMManywhere/geospatial_utilities.py + +::: swmmanywhere.geospatial_utilities diff --git a/docs/reference-graph-utilities.md b/docs/reference-graph-utilities.md new file mode 100644 index 00000000..2e9b697d --- /dev/null +++ b/docs/reference-graph-utilities.md @@ -0,0 +1,3 @@ +# Reference for SWMManywhere/graph_utilities.py + +::: swmmanywhere.graph_utilities diff --git a/docs/reference-logging.md b/docs/reference-logging.md new file mode 100644 index 00000000..03807aa1 --- /dev/null +++ b/docs/reference-logging.md @@ -0,0 +1,3 @@ +# Reference for SWMManywhere/logging.py + +::: swmmanywhere.logging diff --git a/docs/reference-metric-utilities.md b/docs/reference-metric-utilities.md new file mode 100644 index 00000000..4c6c0b35 --- /dev/null +++ b/docs/reference-metric-utilities.md @@ -0,0 +1,3 @@ +# Reference for SWMManywhere/metric_utilties.py + +::: swmmanywhere.metric_utilities diff --git a/docs/reference-overview.md b/docs/reference-overview.md new file mode 100644 index 00000000..7e5c8095 --- /dev/null +++ b/docs/reference-overview.md @@ -0,0 +1,12 @@ +# Reference + +Different API sections are documented separately. + +- [SWMManywhere](reference.md) +- [Graph utilities](reference-graph-utilities.md) +- [Geospatial utilities](reference-geospatial-utilities.md) +- [Metric utilities](reference-metric-utilities.md) +- [Logging](reference-logging.md) +- [Parameters](reference-parameters.md) +- [Post processing](reference-post-processing.md) +- [Preprocessing](reference-preprocessing.md) diff --git a/docs/reference-parameters.md b/docs/reference-parameters.md new file mode 100644 index 00000000..e21e0e1c --- /dev/null +++ b/docs/reference-parameters.md @@ -0,0 +1,3 @@ +# Reference for SWMManywhere/parameters.py + +::: swmmanywhere.parameters diff --git a/docs/reference-post-processing.md b/docs/reference-post-processing.md new file mode 100644 index 00000000..a07ec01e --- /dev/null +++ b/docs/reference-post-processing.md @@ -0,0 +1,3 @@ +# Reference for SWMManywhere/post_processing.py + +::: swmmanywhere.post_processing diff --git a/docs/reference-preprocessing.md b/docs/reference-preprocessing.md new file mode 100644 index 00000000..bf3a8186 --- /dev/null +++ b/docs/reference-preprocessing.md @@ -0,0 +1,3 @@ +# Reference for SWMManywhere/preprocessing.py + +::: swmmanywhere.preprocessing diff --git a/docs/reference.md b/docs/reference.md new file mode 100644 index 00000000..04c7a5bd --- /dev/null +++ b/docs/reference.md @@ -0,0 +1,3 @@ +# Reference for SWMManywhere/swmmanywhere.py + +::: swmmanywhere.swmmanywhere diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 00000000..6f034f92 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,43 @@ +site_name: SWMManywhere docs + +theme: + name: "material" + palette: + primary: 'cyan' + +extra_css: + - stylesheets/extra.css + +plugins: + - mkdocstrings + - mkdocs-jupyter: + execute: false + - search + - coverage: + page_name: coverage # default + html_report_dir: htmlcov # default + +markdown_extensions: + - footnotes + - pymdownx.arithmatex: + generic: true + +extra_javascript: + - javascripts/mathjax.js + - https://polyfill.io/v3/polyfill.min.js?features=es6 + - https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js + +nav: + - Home: index.md + - Quickstart: quickstart.md + - Contributing: CONTRIBUTING.md + - API reference: + - SWMManywhere: reference.md + - Graph utilities: reference-graph-utilities.md + - Geospatial utilities: reference-geospatial-utilities.md + - Metric utilities: reference-metric-utilities.md + - Logging: reference-logging.md + - Parameters: reference-parameters.md + - Post processing: reference-post-processing.md + - Preprocessing: reference-preprocessing.md + - Coverage report: coverage.md \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 3d93acec..b8328ec8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,18 @@ dev = [ "pytest-mypy", "ruff", ] +doc = [ + "mkdocs", + "mkdocs-autorefs", + "mkdocs-bibtex", + "mkdocs-coverage", + "mkdocs-gen-files", + "mkdocs-jupyter", + "mkdocs-material", + "mkdocs-material-extensions", + "mkdocstrings[python]", + "pypandoc", +] [tool.mypy] disallow_any_explicit = false @@ -79,7 +91,7 @@ module = "tests.*" disallow_untyped_defs = false [tool.pytest.ini_options] -addopts = "-v -p no:warnings --cov=swmmanywhere --cov-report=html --doctest-modules --ignore=swmmanywhere/__main__.py" +addopts = "-v -p no:warnings --cov=swmmanywhere --cov-report=html --doctest-modules --ignore=swmmanywhere/logging.py" [tool.ruff] select = ["D", "E", "F", "I"] # pydocstyle, pycodestyle, Pyflakes, isort diff --git a/tests/test_data/demo_config.yml b/swmmanywhere/defs/demo_config.yml similarity index 77% rename from tests/test_data/demo_config.yml rename to swmmanywhere/defs/demo_config.yml index 43aebd55..9ff3dcaf 100644 --- a/tests/test_data/demo_config.yml +++ b/swmmanywhere/defs/demo_config.yml @@ -42,6 +42,7 @@ metric_list: - outlet_relerror_length - outlet_relerror_npipes - outlet_relerror_nmanholes + - outlet_relerror_diameter - outlet_nse_flooding - outlet_kge_flooding - outlet_relerror_flooding @@ -51,48 +52,10 @@ metric_list: - subcatchment_nse_flooding - subcatchment_kge_flooding - subcatchment_relerror_flooding - - nc_deltacon0 - - nc_laplacian_dist - - nc_laplacian_norm_dist - - nc_adjacency_dist - - nc_vertex_edge_distance - - nc_resistance_distance - bias_flood_depth - kstest_edge_betweenness - kstest_betweenness - outlet_kstest_diameters -parameter_overrides: - hydraulic_design: - diameters: - - 0.15 - - 0.225 - - 0.3 - - 0.375 - - 0.45 - - 0.525 - - 0.6 - - 0.675 - - 0.75 - - 0.825 - - 0.9 - - 1.125 - - 1.35 - - 1.5 - - 1.95 - - 3.0 -address_overrides: null -parameters_to_sample: - - min_v: [0.5, 1.5] - - max_v - - max_fr - - precipitation - - outlet_length - - chahinian_slope_scaling - - length_scaling - - contributing_area_scaling - - chahinian_slope_exponent - - length_exponent - - contributing_area_exponent - - lane_width - - max_street_length -sample_magnitude: 9 \ No newline at end of file + - nc_deltacon0 + - nc_laplacian_dist + - nc_vertex_edge_distance \ No newline at end of file diff --git a/swmmanywhere/defs/schema.yml b/swmmanywhere/defs/schema.yml index abc7b6a0..87c8106a 100644 --- a/swmmanywhere/defs/schema.yml +++ b/swmmanywhere/defs/schema.yml @@ -30,4 +30,4 @@ properties: metric_list: {type: array, items: {type: string}} address_overrides: {type: ['object', 'null']} parameter_overrides: {type: ['object', 'null']} -required: [base_dir, project, bbox, graphfcn_list] \ No newline at end of file +required: [base_dir, project, bbox] \ No newline at end of file diff --git a/swmmanywhere/geospatial_utilities.py b/swmmanywhere/geospatial_utilities.py index 16743594..50a35677 100644 --- a/swmmanywhere/geospatial_utilities.py +++ b/swmmanywhere/geospatial_utilities.py @@ -31,7 +31,7 @@ from shapely.strtree import STRtree from tqdm.auto import tqdm -from swmmanywhere.logging import verbose +from swmmanywhere.logging import logger, verbose os.environ['NUMBA_NUM_THREADS'] = '1' import pyflwdir # noqa: E402 @@ -644,12 +644,20 @@ def delineate_catchment_pyflwdir(grid: pysheds.sgrid.sGrid, return gdf_bas def derive_subbasins_streamorder(fid: Path, - streamorder: int): - """Derive subbasins of a given stream order. + streamorder: int | None = None, + x: list[float] = [], + y: list[float] = []) -> gpd.GeoDataFrame: + """Derive subbasins. + + Use the pyflwdir snap function to find the most downstream points in each + subbasin. If streamorder is provided it will use that instead, although + defaulting to snap if there are no cells of the correct streamorder. Args: fid (Path): Filepath to the DEM. streamorder (int): The stream order to delineate subbasins for. + x (list): X coordinates. + y (list): Y coordinates. Returns: gpd.GeoDataFrame: A GeoDataFrame containing polygons. @@ -663,13 +671,24 @@ def derive_subbasins_streamorder(fid: Path, check_ftype = False, transform = grid.affine, ) + xy = [(x_,y_) for x_, y_ in zip(x,y) + if (x_ > grid.bbox[0]) and (x_ < grid.bbox[2]) + and (y_ > grid.bbox[1]) and (y_ < grid.bbox[3])] + + idxs, _ = flw.snap(xy=list(zip(*xy))) + subbasins = flw.basins(idxs=np.unique(idxs)) + + if streamorder is not None: + # Identify stream order + subbasins_, _ = flw.subbasins_streamorder(min_sto=streamorder) - # Identify stream order - subbasins, _ = flw.subbasins_streamorder(min_sto=streamorder) - if np.unique(subbasins.reshape(-1)).shape[0] == 1: - raise ValueError("""No subbasins found in derive_subbasins_streamorder. - Use a lower `subcatchment_derivation.subbasin_streamorder` and - probably check your DEM.""") + if np.unique(subbasins_).shape[0] == 1: + logger.warning("""No subbasins found in `derive_subbasins_streamorder`. + Instead subbasins have been selected based on the most downstream + points. But you should inspect `subbasins` and probably check your + DEM.""") + else: + subbasins = subbasins_ gdf_bas = vectorize(subbasins.astype(np.int32), 0, diff --git a/swmmanywhere/graph_utilities.py b/swmmanywhere/graph_utilities.py index d03780af..1c67ab93 100644 --- a/swmmanywhere/graph_utilities.py +++ b/swmmanywhere/graph_utilities.py @@ -274,10 +274,12 @@ def __call__(self, This function removes links that are not allowable for pipes. The non- allowable links are specified in the `omit_edges` attribute of the topology_derivation parameter. There two cases handled: + 1. The `highway` property of the edge. In `osmnx`, `highway` is a category that contains the road type, e.g., motorway, trunk, primary. If the edge contains a value in the `highway` property that is in `omit_edges`, the edge is removed. + 2. Any other properties of the edge that are in `omit_edges`. If the property is not null in the edge data, the edge is removed. e.g., if `bridge` is in `omit_edges` and the `bridge` entry of the edge @@ -639,7 +641,9 @@ def __call__(self, # Derive subbasins subbasins = go.derive_subbasins_streamorder(addresses.elevation, - subcatchment_derivation.subbasin_streamorder) + subcatchment_derivation.subbasin_streamorder, + x = list(nx.get_node_attributes(G, 'x').values()), + y = list(nx.get_node_attributes(G, 'y').values())) if verbose(): subbasins.to_file( diff --git a/swmmanywhere/logging.py b/swmmanywhere/logging.py index f3b332cf..b2ab217b 100644 --- a/swmmanywhere/logging.py +++ b/swmmanywhere/logging.py @@ -1,14 +1,16 @@ """Logging module for SWMManywhere. Example: +```python >>> import os >>> os.environ["SWMMANYWHERE_VERBOSE"] = "true" >>> # logging is now enabled in any swmmanywhere module >>> from swmmanywhere.logging import logger # You can now log yourself ->>> logger.info("This is an info message.") # Write to stdout # doctest: +SKIP +>>> logger.add("file.log") # Add a log file +>>> logger.info("This is an info message.") # Write to stdout and file.log This is an info message. ->>> logger.add("file.log") # Add a log file # doctest: +SKIP >>> os.environ["SWMMANYWHERE_VERBOSE"] = "false" # Disable logging +``` """ from __future__ import annotations diff --git a/swmmanywhere/metric_utilities.py b/swmmanywhere/metric_utilities.py index 272fd0d1..5d30b12f 100644 --- a/swmmanywhere/metric_utilities.py +++ b/swmmanywhere/metric_utilities.py @@ -215,13 +215,16 @@ def relerror(y: np.ndarray, Calculate the relative error: - .. math:: - - relerror = \\frac{{\mean(synthetic) - \mean(real)}}{{\mean(real)}} + $$ + relerror = \frac{mean(synthetic) - mean(real)} + {mean(real)} + $$ where: - - :math:`synthetic` is the synthetic data, - - :math:`real` is the real data. + + - \(synthetic\) is the synthetic data, + - \(real\) is the real data, + Args: y (np.ndarray): The real data. @@ -238,7 +241,21 @@ def relerror(y: np.ndarray, @register_coef def nse(y: np.ndarray, yhat: np.ndarray) -> float: - """Calculate Nash-Sutcliffe efficiency (NSE). + r"""Calculate Nash-Sutcliffe efficiency (NSE). + + Calculate the Nash-Sutcliffe efficiency (NSE): + + $$ + NSE = 1 - \frac{\sum_{i=1}^{n} (Q_{obs,i} - Q_{sim,i})^2} + {\sum_{i=1}^{n} (Q_{obs,i} - \overline{Q}_{obs})^2} + $$ + + where: + + - $Q_{obs,i}$ is the observed value at time $i$, + - $Q_{sim,i}$ is the simulated value at time $i$, + - $\overline{Q}_{obs}$ is the mean observed value over the simulation period, + - $n$ is the number of time steps in the simulation period. Args: y (np.array): Observed data array. @@ -253,8 +270,26 @@ def nse(y: np.ndarray, @register_coef def kge(y: np.ndarray,yhat: np.ndarray) -> float: - """Calculate the Kling-Gupta Efficiency (KGE) between simulated and observed data. + r"""Calculate the Kling-Gupta Efficiency (KGE) between simulated and observed data. + Calculate KGE with the 2009 formulation: + $$ + KGE = 1 - + \sqrt{ (r - 1)^2 + + (\frac{\sigma_{sim}}{\sigma_{obs}} - 1)^2 + + (\frac{\mu_{sim}}{\mu_{obs}} - 1)^2 + } + $$ + + where: + + - $r$ is the correlation coefficient between observed and simulated value, + - $\sigma_{sim}$ and $\sigma_{obs}$ are the standard deviations of the + simulated and observed value, respectively, + - $\mu_{sim}$ and $\mu_{obs}$ are the means of the simulated and observed + value, respectively. + + Args: y (np.array): Observed data array. yhat (np.array): Simulated data array. diff --git a/swmmanywhere/parameters.py b/swmmanywhere/parameters.py index 143b136d..965f228a 100644 --- a/swmmanywhere/parameters.py +++ b/swmmanywhere/parameters.py @@ -35,7 +35,7 @@ def get_full_parameters_flat(): class SubcatchmentDerivation(BaseModel): """Parameters for subcatchment derivation.""" - subbasin_streamorder: int = Field(default = 7, + subbasin_streamorder: int = Field(default = None, ge = 1, le = 20, unit = "-", diff --git a/swmmanywhere/swmmanywhere.py b/swmmanywhere/swmmanywhere.py index d688717b..bb64a665 100644 --- a/swmmanywhere/swmmanywhere.py +++ b/swmmanywhere/swmmanywhere.py @@ -18,6 +18,22 @@ from swmmanywhere.utilities import yaml_dump, yaml_load +def _check_defaults(config: dict) -> dict: + """Check the config for needed values and add them from defaults if missing. + + Args: + config (dict): The configuration. + + Returns: + dict: The configuration with defaults added. + """ + config_ = load_config(validation=False) + for key in ['run_settings', 'graphfcn_list', 'metric_list']: + if key not in config: + config[key] = config_[key] + + return config + def swmmanywhere(config: dict) -> tuple[Path, dict | None]: """Run SWMManywhere processes. @@ -50,6 +66,11 @@ def swmmanywhere(config: dict) -> tuple[Path, dict | None]: for key, val in config.get('address_overrides', {}).items(): logger.info(f"Setting {key} to {val}") setattr(addresses, key, val) + + # Check for defaults + config = _check_defaults(config) + if not addresses.precipitation.exists(): + addresses.precipitation = Path(__file__).parent / 'defs' / 'storm.dat' # Load the parameters and perform any manual overrides logger.info("Loading and setting parameters.") @@ -108,7 +129,7 @@ def swmmanywhere(config: dict) -> tuple[Path, dict | None]: # Write to .inp synthetic_write(addresses) - + # Run the model logger.info("Running the synthetic model.") synthetic_results = run(addresses.inp, @@ -119,11 +140,10 @@ def swmmanywhere(config: dict) -> tuple[Path, dict | None]: f'results.{addresses.extension}') # Get the real results - if config['real'].get('results',None): + if config.get('real', {}).get('results',None): logger.info("Loading real results.") - # TODO.. bit messy real_results = pd.read_parquet(config['real']['results']) - elif config['real']['inp']: + elif config.get('real', {}).get('inp',None): logger.info("Running the real model.") real_results = run(config['real']['inp'], **config['run_settings']) @@ -133,7 +153,7 @@ def swmmanywhere(config: dict) -> tuple[Path, dict | None]: else: logger.info("No real network provided, returning SWMM .inp file.") return addresses.inp, None - + # Iterate the metrics logger.info("Iterating metrics.") metrics = iterate_metrics(synthetic_results, @@ -261,7 +281,7 @@ def save_config(config: dict, config_path: Path): """ yaml_dump(config, config_path.open('w')) -def load_config(config_path: Path, +def load_config(config_path: Path = Path(__file__).parent / 'defs' / 'demo_config.yml', validation: bool = True, schema_fid: Path | None = None): """Load, validate, and convert Paths in a configuration file. diff --git a/tests/test_graph_utilities.py b/tests/test_graph_utilities.py index d4e58c10..021124a6 100644 --- a/tests/test_graph_utilities.py +++ b/tests/test_graph_utilities.py @@ -23,6 +23,7 @@ save_graph, ) from swmmanywhere.graph_utilities import graphfcns as gu +from swmmanywhere.logging import logger @pytest.fixture @@ -574,6 +575,21 @@ def test_clip_to_catchments(street_network): addresses.nodes = addresses.base_dir / 'nodes.geojson' addresses.elevation = Path(__file__).parent / 'test_data' / 'elevation.tif' + # Test default clipping + subcatchment_derivation = parameters.SubcatchmentDerivation() + G_ = gu.clip_to_catchments(G, + addresses=addresses, + subcatchment_derivation=subcatchment_derivation) + assert len(G_.edges) == 9 + + # Test default clipping streamorder + subcatchment_derivation = parameters.SubcatchmentDerivation() + subcatchment_derivation.subbasin_streamorder = 4 + G_ = gu.clip_to_catchments(G, + addresses=addresses, + subcatchment_derivation=subcatchment_derivation) + assert len(G_.edges) == 4 + # Test clipping subcatchment_derivation = parameters.SubcatchmentDerivation( subbasin_streamorder = 3, @@ -619,7 +635,12 @@ def test_clip_to_catchments(street_network): assert len(G_.edges) == 28 # Check streamorder adjustment - with pytest.raises(ValueError): + with tempfile.NamedTemporaryFile(suffix='.log', + mode = 'w+b', + delete=False) as temp_file: + fid = Path(temp_file.name) + os.environ['SWMMANYWHERE_VERBOSE'] = 'true' + logger.add(fid) subcatchment_derivation = parameters.SubcatchmentDerivation( subbasin_streamorder = 5, subbasin_membership = 0.9 @@ -627,7 +648,13 @@ def test_clip_to_catchments(street_network): G_ = gu.clip_to_catchments(G, addresses=addresses, subcatchment_derivation=subcatchment_derivation) - + ftext = str(temp_file.read()) + assert """No subbasins found""" in ftext + assert """WARNING""" in ftext + logger.remove() + os.environ['SWMMANYWHERE_VERBOSE'] = 'false' + assert (addresses.nodes.parent / 'subbasins.geojson').exists() + def test_filter_streets(): """Test the _filter_streets function.""" # Create a sample graph diff --git a/tests/test_swmmanywhere.py b/tests/test_swmmanywhere.py index fb8899e4..89e82a25 100644 --- a/tests/test_swmmanywhere.py +++ b/tests/test_swmmanywhere.py @@ -48,7 +48,7 @@ def test_swmmanywhere(): # Load the config test_data_dir = Path(__file__).parent / 'test_data' defs_dir = Path(__file__).parent.parent / 'swmmanywhere' / 'defs' - with (test_data_dir / 'demo_config.yml').open('r') as f: + with (defs_dir / 'demo_config.yml').open('r') as f: config = yaml.safe_load(f) # Set some test values @@ -103,7 +103,6 @@ def test_swmmanywhere(): def test_load_config_file_validation(): """Test the file validation of the config.""" with tempfile.TemporaryDirectory() as temp_dir: - test_data_dir = Path(__file__).parent / 'test_data' defs_dir = Path(__file__).parent.parent / 'swmmanywhere' / 'defs' base_dir = Path(temp_dir) @@ -112,7 +111,7 @@ def test_load_config_file_validation(): swmmanywhere.load_config(base_dir / 'test_config.yml') assert "test_config.yml" in str(exc_info.value) - with (test_data_dir / 'demo_config.yml').open('r') as f: + with (defs_dir / 'demo_config.yml').open('r') as f: config = yaml.safe_load(f) # Correct and avoid filevalidation errors @@ -130,11 +129,11 @@ def test_load_config_file_validation(): def test_load_config_schema_validation(): """Test the schema validation of the config.""" with tempfile.TemporaryDirectory() as temp_dir: - test_data_dir = Path(__file__).parent / 'test_data' + defs_dir = Path(__file__).parent.parent / 'swmmanywhere' / 'defs' base_dir = Path(temp_dir) # Load the config - with (test_data_dir / 'demo_config.yml').open('r') as f: + with (defs_dir / 'demo_config.yml').open('r') as f: config = yaml.safe_load(f) # Make an edit not to schema @@ -152,10 +151,9 @@ def test_save_config(): """Test the save_config function.""" with tempfile.TemporaryDirectory() as temp_dir: temp_dir = Path(temp_dir) - test_data_dir = Path(__file__).parent / 'test_data' defs_dir = Path(__file__).parent.parent / 'swmmanywhere' / 'defs' - with (test_data_dir / 'demo_config.yml').open('r') as f: + with (defs_dir / 'demo_config.yml').open('r') as f: config = yaml.safe_load(f) # Correct and avoid filevalidation errors @@ -168,3 +166,13 @@ def test_save_config(): # Reload to check OK config = swmmanywhere.load_config(temp_dir / 'test.yml') + +@pytest.mark.downloads +def test_minimal_req(): + """Test SWMManywhere with minimal info.""" + with tempfile.TemporaryDirectory() as temp_dir: + config = {'base_dir' : Path(temp_dir), + 'project' : 'my_test', + 'bbox' : [1.52740,42.50524,1.54273,42.51259]} + + swmmanywhere.swmmanywhere(config) \ No newline at end of file