From f157ecb35af5b2a61304e7159590f8818e2a01cb Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Tue, 17 Sep 2024 09:28:57 +0100 Subject: [PATCH 01/28] :green_heart: Update publish workflow to deploy in PyPI --- .github/workflows/publish.yml | 86 ++++++++++++++++++++++++++--------- 1 file changed, 64 insertions(+), 22 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 228edfb1..7e2cbc81 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -4,6 +4,70 @@ jobs: test: uses: ./.github/workflows/ci.yml + build-wheel: + needs: test + name: Build source distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Build sdist + run: | + python -m pip install --upgrade build + python -m build + + - uses: actions/upload-artifact@v3 + with: + path: dist/swmmanywhere* + + - uses: softprops/action-gh-release@v1 + with: + files: dist/swmmanywhere* + + publish-TestPyPI: + needs: build-wheel + name: Publish SWMManywhere to TestPyPI + runs-on: ubuntu-latest + permissions: + id-token: write + + steps: + - name: Download sdist artifact + uses: actions/download-artifact@v3 + with: + name: artifact + path: dist + + - name: Display structure of downloaded files + run: ls -R dist + + - name: Publish package distributions to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + skip-existing: true + + publish-PyPI: + needs: publish-TestPyPI + name: Publish SWMManywhere to PyPI + runs-on: ubuntu-latest + permissions: + id-token: write + + steps: + - name: Download sdist artifact + uses: actions/download-artifact@v3 + with: + name: artifact + path: dist + + - name: Display structure of downloaded files + run: ls -R dist + + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + publish-docs: needs: publish-PyPI runs-on: ubuntu-latest @@ -20,25 +84,3 @@ jobs: - name: Deploy Docs run: mkdocs gh-deploy --force - - # publish: - # runs-on: ubuntu-latest - # needs: test - # # The following steps to build a Docker image and publish to the GitHub container registry on release. Alternatively, can replace with other publishing steps (ie. publishing to PyPI, deploying documentation etc.) - # steps: - # - name: Login to GitHub Container Registry - # uses: docker/login-action@v3 - # with: - # registry: ghcr.io - # username: ${{ github.actor }} - # password: ${{ secrets.GITHUB_TOKEN }} - # - name: Get image metadata - # id: meta - # uses: docker/metadata-action@v5 - # with: - # images: ghcr.io/${{ github.repository }} - # - name: Build and push Docker image - # uses: docker/build-push-action@v5 - # with: - # push: true - # tags: ${{ steps.meta.outputs.tags }} From 913ad15f094c7b3760980cc090f93c8bf5878f25 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Tue, 17 Sep 2024 16:38:00 +0100 Subject: [PATCH 02/28] :wrench: Replace build system with hatchling --- pyproject.toml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 98a9d170..d8ed7911 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,7 @@ [build-system] -build-backend = "setuptools.build_meta" +build-backend = "hatchling.build" requires = [ - "setuptools", - "setuptools-scm", + "hatchling", ] [tool.setuptools.packages.find] From 71ca6e96df5e7bbc384234fc254d39e99c8aef6c Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Tue, 17 Sep 2024 16:39:09 +0100 Subject: [PATCH 03/28] :wrench: Add site URL to mkdocs --- mkdocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/mkdocs.yml b/mkdocs.yml index e4926910..c561e5fd 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -23,6 +23,7 @@ plugins: - include-markdown repo_url: https://github.com/ImperialCollegeLondon/SWMManywhere +site_url: https://imperialcollegelondon.github.io/SWMManywhere/ markdown_extensions: - footnotes From 8bc18a5f57224c80d331992fc9f4a4e4a4f42f76 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Tue, 17 Sep 2024 16:59:40 +0100 Subject: [PATCH 04/28] Update to use attest-build-provenance-github --- .github/workflows/publish.yml | 82 +++++++++++++++++------------------ 1 file changed, 40 insertions(+), 42 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 7e2cbc81..5d5d1933 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,5 +1,10 @@ on: [release] +permissions: + contents: write + attestations: write + id-token: write + jobs: test: uses: ./.github/workflows/ci.yml @@ -9,63 +14,56 @@ jobs: name: Build source distribution runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + with: + fetch-depth: 0 - - name: Build sdist - run: | - python -m pip install --upgrade build - python -m build - - - uses: actions/upload-artifact@v3 - with: - path: dist/swmmanywhere* - - - uses: softprops/action-gh-release@v1 - with: - files: dist/swmmanywhere* + - uses: hynek/build-and-inspect-python-package@v2 + with: + attest-build-provenance-github: true publish-TestPyPI: needs: build-wheel name: Publish SWMManywhere to TestPyPI runs-on: ubuntu-latest - permissions: - id-token: write steps: - - name: Download sdist artifact - uses: actions/download-artifact@v3 - with: - name: artifact - path: dist - - - name: Display structure of downloaded files - run: ls -R dist - - - name: Publish package distributions to TestPyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - repository-url: https://test.pypi.org/legacy/ - skip-existing: true + - name: Download packages built by build-and-inspect-python-package + uses: actions/download-artifact@v4 + with: + name: Packages + path: dist + + - name: Generate artifact attestation for sdist and wheel + uses: actions/attest-build-provenance@v1 + with: + subject-path: dist + + - name: Publish package distributions to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + skip-existing: true publish-PyPI: needs: publish-TestPyPI name: Publish SWMManywhere to PyPI runs-on: ubuntu-latest - permissions: - id-token: write steps: - - name: Download sdist artifact - uses: actions/download-artifact@v3 - with: - name: artifact - path: dist - - - name: Display structure of downloaded files - run: ls -R dist - - - name: Publish package distributions to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + - name: Download packages built by build-and-inspect-python-package + uses: actions/download-artifact@v4 + with: + name: Packages + path: dist + + - name: Generate artifact attestation for sdist and wheel + uses: actions/attest-build-provenance@v1 + with: + subject-path: dist + + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 publish-docs: From f3f1fcbcbb94f42125cfefe721e0944da1ae0560 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Tue, 17 Sep 2024 17:20:10 +0100 Subject: [PATCH 05/28] :wrench: Add urls --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index d8ed7911..9e25ead7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,6 +73,10 @@ doc = [ "mkdocs-material-extensions", "mkdocstrings[python]", ] +[project.urls] +Documentation = "https://imperialcollegelondon.github.io/SWMManywhere/" +Issues = "https://github.com/ImperialCollegeLondon/SWMManywhere/issues" +Source = "https://github.com/ImperialCollegeLondon/SWMManywhere" [tool.mypy] disallow_any_explicit = false From 917d41583e40c470c3834bc2ef860cd73c8f47ae Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Wed, 18 Sep 2024 15:34:46 +0100 Subject: [PATCH 06/28] :pushpin: Pin pywbt<=0.1.1 --- dev-requirements.txt | 90 +++++++++++++++------------------ doc-requirements.txt | 118 ++++++++++++++++++++----------------------- pyproject.toml | 6 ++- requirements.txt | 66 ++++++++++-------------- 4 files changed, 127 insertions(+), 153 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index c09b373c..83091e49 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile --extra=dev --output-file=dev-requirements.txt pyproject.toml +# pip-compile --extra=dev --output-file=dev-requirements.txt # aenum==3.1.11 # via @@ -14,7 +14,7 @@ affine==2.4.0 # rasterio annotated-types==0.7.0 # via pydantic -attrs==23.2.0 +attrs==24.2.0 # via # cads-api-client # fiona @@ -22,17 +22,16 @@ attrs==23.2.0 # pytest-mypy # rasterio # referencing -build==1.2.1 +build==1.2.2 # via pip-tools -cads-api-client==1.2.0 +cads-api-client==1.3.2 # via cdsapi -cdsapi==0.7.0 +cdsapi==0.7.3 # via swmmanywhere (pyproject.toml) -certifi==2024.7.4 +certifi==2024.8.30 # via # fiona # netcdf4 - # pyogrio # pyproj # rasterio # requests @@ -65,7 +64,7 @@ colorama==0.4.6 # loguru # pytest # tqdm -coverage[toml]==7.5.4 +coverage[toml]==7.6.1 # via pytest-cov cramjam==2.8.3 # via fastparquet @@ -75,17 +74,19 @@ distlib==0.3.8 # via virtualenv fastparquet==2024.5.0 # via swmmanywhere (pyproject.toml) -filelock==3.15.4 +filelock==3.16.1 # via # pytest-mypy # virtualenv -fiona==1.9.6 - # via swmmanywhere (pyproject.toml) -fsspec==2024.6.1 +fiona==1.10.1 + # via + # geopandas + # swmmanywhere (pyproject.toml) +fsspec==2024.9.0 # via fastparquet geographiclib==2.0 # via geopy -geopandas==1.0.1 +geopandas==0.14.4 # via # osmnx # swmmanywhere (pyproject.toml) @@ -95,9 +96,9 @@ gitdb==4.0.11 # via gitpython gitpython==3.1.43 # via swmmanywhere (pyproject.toml) -identify==2.6.0 +identify==2.6.1 # via pre-commit -idna==3.8 +idna==3.10 # via requests iniconfig==2.0.0 # via pytest @@ -117,19 +118,16 @@ loguru==0.7.2 # via swmmanywhere (pyproject.toml) multiurl==0.3.1 # via cads-api-client -mypy==1.10.1 +mypy==1.11.2 # via # pytest-mypy # swmmanywhere (pyproject.toml) mypy-extensions==1.0.0 # via mypy -netcdf4==1.7.1.post1 - # via swmmanywhere (pyproject.toml) -netcomp @ git+https://github.com/barneydobson/NetComp.git +netcdf4==1.7.1.post2 # via swmmanywhere (pyproject.toml) networkx==3.3 # via - # netcomp # osmnx # swmmanywhere (pyproject.toml) nodeenv==1.9.1 @@ -142,13 +140,11 @@ numpy==1.26.4 # fastparquet # geopandas # netcdf4 - # netcomp # numba # osmnx # pandas # pyarrow # pyflwdir - # pyogrio # rasterio # rioxarray # scipy @@ -156,7 +152,7 @@ numpy==1.26.4 # snuggs # swmmanywhere (pyproject.toml) # xarray -osmnx==1.9.3 +osmnx==1.9.4 # via swmmanywhere (pyproject.toml) packaging==24.1 # via @@ -164,7 +160,6 @@ packaging==24.1 # fastparquet # geopandas # planetary-computer - # pyogrio # pyswmm # pytest # rioxarray @@ -180,25 +175,23 @@ pip-tools==7.4.1 # via swmmanywhere (pyproject.toml) planetary-computer==1.0.0 # via swmmanywhere (pyproject.toml) -platformdirs==4.2.2 +platformdirs==4.3.6 # via virtualenv pluggy==1.5.0 # via pytest -pre-commit==3.7.1 +pre-commit==3.8.0 # via swmmanywhere (pyproject.toml) -pyarrow==16.1.0 +pyarrow==17.0.0 # via swmmanywhere (pyproject.toml) -pydantic==2.8.2 +pydantic==2.9.2 # via # planetary-computer # swmmanywhere (pyproject.toml) -pydantic-core==2.20.1 +pydantic-core==2.23.4 # via pydantic pyflwdir==0.5.8 # via swmmanywhere (pyproject.toml) -pyogrio==0.9.0 - # via geopandas -pyparsing==3.1.2 +pyparsing==3.1.4 # via snuggs pyproj==3.6.1 # via @@ -218,7 +211,7 @@ pystac-client==0.8.3 # swmmanywhere (pyproject.toml) pyswmm==2.0.1 # via swmmanywhere (pyproject.toml) -pytest==8.2.2 +pytest==8.3.3 # via # pytest-cov # pytest-mock @@ -238,18 +231,18 @@ python-dateutil==2.9.0.post0 # pystac-client python-dotenv==1.0.1 # via planetary-computer -pytz==2024.1 +pytz==2024.2 # via # multiurl # pandas # planetary-computer pywbt==0.1.1 # via swmmanywhere (pyproject.toml) -pyyaml==6.0.1 +pyyaml==6.0.2 # via # pre-commit # swmmanywhere (pyproject.toml) -rasterio==1.3.10 +rasterio==1.3.11 # via # rioxarray # swmmanywhere (pyproject.toml) @@ -267,26 +260,23 @@ requests==2.32.3 # pystac-client rioxarray==0.17.0 # via swmmanywhere (pyproject.toml) -rpds-py==0.19.1 +rpds-py==0.20.0 # via # jsonschema # referencing -ruff==0.5.5 +ruff==0.6.5 # via swmmanywhere (pyproject.toml) -scipy==1.14.0 +scipy==1.14.1 # via - # netcomp # pyflwdir # swmmanywhere (pyproject.toml) -shapely==2.0.5 +shapely==2.0.6 # via # geopandas # osmnx # swmmanywhere (pyproject.toml) six==1.16.0 - # via - # fiona - # python-dateutil + # via python-dateutil smmap==5.0.1 # via gitdb snuggs==1.4.7 @@ -295,7 +285,7 @@ swmm-toolkit==0.15.5 # via pyswmm toolz==0.12.1 # via cytoolz -tqdm==4.66.4 +tqdm==4.66.5 # via # cdsapi # multiurl @@ -308,15 +298,15 @@ typing-extensions==4.12.2 # pydantic-core tzdata==2024.1 # via pandas -urllib3==2.2.2 +urllib3==2.2.3 # via requests -virtualenv==20.26.3 +virtualenv==20.26.5 # via pre-commit -wheel==0.43.0 +wheel==0.44.0 # via pip-tools win32-setctime==1.1.0 # via loguru -xarray==2024.6.0 +xarray==2024.9.0 # via # rioxarray # swmmanywhere (pyproject.toml) diff --git a/doc-requirements.txt b/doc-requirements.txt index d4cc7566..324cb9ad 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile --extra=doc --output-file=doc-requirements.txt pyproject.toml +# pip-compile --extra=doc --output-file=doc-requirements.txt # aenum==3.1.11 # via @@ -16,30 +16,29 @@ annotated-types==0.7.0 # via pydantic asttokens==2.4.1 # via stack-data -attrs==23.2.0 +attrs==24.2.0 # via # cads-api-client # fiona # jsonschema # rasterio # referencing -babel==2.15.0 +babel==2.16.0 # via mkdocs-material beautifulsoup4==4.12.3 # via nbconvert bleach==6.1.0 # via nbconvert -bracex==2.4 +bracex==2.5 # via wcmatch -cads-api-client==1.2.0 +cads-api-client==1.3.2 # via cdsapi -cdsapi==0.7.0 +cdsapi==0.7.3 # via swmmanywhere (pyproject.toml) -certifi==2024.7.4 +certifi==2024.8.30 # via # fiona # netcdf4 - # pyogrio # pyproj # rasterio # requests @@ -85,19 +84,21 @@ decorator==5.1.1 # via ipython defusedxml==0.7.1 # via nbconvert -executing==2.0.1 +executing==2.1.0 # via stack-data fastjsonschema==2.20.0 # via nbformat fastparquet==2024.5.0 # via swmmanywhere (pyproject.toml) -fiona==1.9.6 - # via swmmanywhere (pyproject.toml) -fsspec==2024.6.1 +fiona==1.10.1 + # via + # geopandas + # swmmanywhere (pyproject.toml) +fsspec==2024.9.0 # via fastparquet geographiclib==2.0 # via geopy -geopandas==1.0.1 +geopandas==0.14.4 # via # osmnx # swmmanywhere (pyproject.toml) @@ -109,13 +110,13 @@ gitdb==4.0.11 # via gitpython gitpython==3.1.43 # via swmmanywhere (pyproject.toml) -griffe==0.47.0 +griffe==1.3.1 # via mkdocstrings-python -idna==3.8 +idna==3.10 # via requests ipykernel==6.29.5 # via mkdocs-jupyter -ipython==8.26.0 +ipython==8.27.0 # via ipykernel jedi==0.19.1 # via ipython @@ -136,7 +137,7 @@ jsonschema-specifications==2023.12.1 # via jsonschema julian==0.14 # via pyswmm -jupyter-client==8.6.2 +jupyter-client==8.6.3 # via # ipykernel # nbclient @@ -149,13 +150,13 @@ jupyter-core==5.7.2 # nbformat jupyterlab-pygments==0.3.0 # via nbconvert -jupytext==1.16.3 +jupytext==1.16.4 # via mkdocs-jupyter llvmlite==0.43.0 # via numba loguru==0.7.2 # via swmmanywhere (pyproject.toml) -markdown==3.6 +markdown==3.7 # via # mkdocs # mkdocs-autorefs @@ -177,7 +178,7 @@ matplotlib-inline==0.1.7 # via # ipykernel # ipython -mdit-py-plugins==0.4.1 +mdit-py-plugins==0.4.2 # via jupytext mdurl==0.1.2 # via markdown-it-py @@ -187,7 +188,7 @@ mergedeep==1.3.4 # mkdocs-get-deps mistune==3.0.2 # via nbconvert -mkdocs==1.6.0 +mkdocs==1.6.1 # via # mkdocs-autorefs # mkdocs-coverage @@ -196,17 +197,19 @@ mkdocs==1.6.0 # mkdocs-material # mkdocstrings # swmmanywhere (pyproject.toml) -mkdocs-autorefs==1.0.1 - # via mkdocstrings +mkdocs-autorefs==1.2.0 + # via + # mkdocstrings + # mkdocstrings-python mkdocs-coverage==1.1.0 # via swmmanywhere (pyproject.toml) mkdocs-get-deps==0.2.0 # via mkdocs -mkdocs-include-markdown-plugin==6.2.1 +mkdocs-include-markdown-plugin==6.2.2 # via swmmanywhere (pyproject.toml) -mkdocs-jupyter==0.24.8 +mkdocs-jupyter==0.25.0 # via swmmanywhere (pyproject.toml) -mkdocs-material==9.5.28 +mkdocs-material==9.5.35 # via # mkdocs-jupyter # swmmanywhere (pyproject.toml) @@ -214,11 +217,11 @@ mkdocs-material-extensions==1.3.1 # via # mkdocs-material # swmmanywhere (pyproject.toml) -mkdocstrings[python]==0.25.2 +mkdocstrings[python]==0.26.1 # via # mkdocstrings-python # swmmanywhere (pyproject.toml) -mkdocstrings-python==1.10.5 +mkdocstrings-python==1.11.1 # via mkdocstrings multiurl==0.3.1 # via cads-api-client @@ -233,13 +236,10 @@ nbformat==5.10.4 # nbconvert nest-asyncio==1.6.0 # via ipykernel -netcdf4==1.7.1.post1 - # via swmmanywhere (pyproject.toml) -netcomp @ git+https://github.com/barneydobson/NetComp.git +netcdf4==1.7.1.post2 # via swmmanywhere (pyproject.toml) networkx==3.3 # via - # netcomp # osmnx # swmmanywhere (pyproject.toml) numba==0.60.0 @@ -250,13 +250,11 @@ numpy==1.26.4 # fastparquet # geopandas # netcdf4 - # netcomp # numba # osmnx # pandas # pyarrow # pyflwdir - # pyogrio # rasterio # rioxarray # scipy @@ -264,7 +262,7 @@ numpy==1.26.4 # snuggs # swmmanywhere (pyproject.toml) # xarray -osmnx==1.9.3 +osmnx==1.9.4 # via swmmanywhere (pyproject.toml) packaging==24.1 # via @@ -275,7 +273,6 @@ packaging==24.1 # mkdocs # nbconvert # planetary-computer - # pyogrio # pyswmm # rioxarray # xarray @@ -296,7 +293,7 @@ pathspec==0.12.1 # via mkdocs planetary-computer==1.0.0 # via swmmanywhere (pyproject.toml) -platformdirs==4.2.2 +platformdirs==4.3.6 # via # jupyter-core # mkdocs-get-deps @@ -305,15 +302,15 @@ prompt-toolkit==3.0.47 # via ipython psutil==6.0.0 # via ipykernel -pure-eval==0.2.2 +pure-eval==0.2.3 # via stack-data -pyarrow==16.1.0 +pyarrow==17.0.0 # via swmmanywhere (pyproject.toml) -pydantic==2.8.2 +pydantic==2.9.2 # via # planetary-computer # swmmanywhere (pyproject.toml) -pydantic-core==2.20.1 +pydantic-core==2.23.4 # via pydantic pyflwdir==0.5.8 # via swmmanywhere (pyproject.toml) @@ -323,13 +320,11 @@ pygments==2.18.0 # mkdocs-jupyter # mkdocs-material # nbconvert -pymdown-extensions==10.8.1 +pymdown-extensions==10.9 # via # mkdocs-material # mkdocstrings -pyogrio==0.9.0 - # via geopandas -pyparsing==3.1.2 +pyparsing==3.1.4 # via snuggs pyproj==3.6.1 # via @@ -355,7 +350,7 @@ python-dateutil==2.9.0.post0 # pystac-client python-dotenv==1.0.1 # via planetary-computer -pytz==2024.1 +pytz==2024.2 # via # multiurl # pandas @@ -364,7 +359,7 @@ pywbt==0.1.1 # via swmmanywhere (pyproject.toml) pywin32==306 # via jupyter-core -pyyaml==6.0.1 +pyyaml==6.0.2 # via # jupytext # mkdocs @@ -374,11 +369,11 @@ pyyaml==6.0.1 # swmmanywhere (pyproject.toml) pyyaml-env-tag==0.1 # via mkdocs -pyzmq==26.0.3 +pyzmq==26.2.0 # via # ipykernel # jupyter-client -rasterio==1.3.10 +rasterio==1.3.11 # via # rioxarray # swmmanywhere (pyproject.toml) @@ -386,7 +381,7 @@ referencing==0.35.1 # via # jsonschema # jsonschema-specifications -regex==2024.5.15 +regex==2024.9.11 # via mkdocs-material requests==2.32.3 # via @@ -399,16 +394,15 @@ requests==2.32.3 # pystac-client rioxarray==0.17.0 # via swmmanywhere (pyproject.toml) -rpds-py==0.19.1 +rpds-py==0.20.0 # via # jsonschema # referencing -scipy==1.14.0 +scipy==1.14.1 # via - # netcomp # pyflwdir # swmmanywhere (pyproject.toml) -shapely==2.0.5 +shapely==2.0.6 # via # geopandas # osmnx @@ -417,13 +411,12 @@ six==1.16.0 # via # asttokens # bleach - # fiona # python-dateutil smmap==5.0.1 # via gitdb snuggs==1.4.7 # via rasterio -soupsieve==2.5 +soupsieve==2.6 # via beautifulsoup4 stack-data==0.6.3 # via ipython @@ -437,7 +430,7 @@ tornado==6.4.1 # via # ipykernel # jupyter-client -tqdm==4.66.4 +tqdm==4.66.5 # via # cdsapi # multiurl @@ -456,16 +449,15 @@ traitlets==5.14.3 typing-extensions==4.12.2 # via # cads-api-client - # ipython # pydantic # pydantic-core tzdata==2024.1 # via pandas -urllib3==2.2.2 +urllib3==2.2.3 # via requests -watchdog==4.0.1 +watchdog==5.0.2 # via mkdocs -wcmatch==8.5.2 +wcmatch==9.0 # via mkdocs-include-markdown-plugin wcwidth==0.2.13 # via prompt-toolkit @@ -475,7 +467,7 @@ webencodings==0.5.1 # tinycss2 win32-setctime==1.1.0 # via loguru -xarray==2024.6.0 +xarray==2024.9.0 # via # rioxarray # swmmanywhere (pyproject.toml) diff --git a/pyproject.toml b/pyproject.toml index 9e25ead7..b88eac3e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,6 @@ dependencies = [ "jsonschema", "loguru", "netcdf4", - "netcomp@ git+https://github.com/barneydobson/NetComp.git", "networkx>=3", "numpy", "osmnx", @@ -44,7 +43,7 @@ dependencies = [ "pyflwdir", "pystac_client", "pyswmm", - "pywbt", + "pywbt<=0.1.1", "PyYAML", "rasterio", "rioxarray", @@ -78,6 +77,9 @@ Documentation = "https://imperialcollegelondon.github.io/SWMManywhere/" Issues = "https://github.com/ImperialCollegeLondon/SWMManywhere/issues" Source = "https://github.com/ImperialCollegeLondon/SWMManywhere" +[tool.hatch.build.targets.sdist] +only-include = ["swmmanywhere", "netcomp"] + [tool.mypy] disallow_any_explicit = false disallow_any_generics = false diff --git a/requirements.txt b/requirements.txt index 5f23ec4f..9e8910cf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile @@ -14,22 +14,21 @@ affine==2.4.0 # rasterio annotated-types==0.7.0 # via pydantic -attrs==23.2.0 +attrs==24.2.0 # via # cads-api-client # fiona # jsonschema # rasterio # referencing -cads-api-client==1.2.0 +cads-api-client==1.3.2 # via cdsapi -cdsapi==0.7.0 +cdsapi==0.7.3 # via swmmanywhere (pyproject.toml) -certifi==2024.7.4 +certifi==2024.8.30 # via # fiona # netcdf4 - # pyogrio # pyproj # rasterio # requests @@ -63,13 +62,15 @@ cytoolz==0.12.3 # via swmmanywhere (pyproject.toml) fastparquet==2024.5.0 # via swmmanywhere (pyproject.toml) -fiona==1.9.6 - # via swmmanywhere (pyproject.toml) -fsspec==2024.6.1 +fiona==1.10.1 + # via + # geopandas + # swmmanywhere (pyproject.toml) +fsspec==2024.9.0 # via fastparquet geographiclib==2.0 # via geopy -geopandas==1.0.1 +geopandas==0.14.4 # via # osmnx # swmmanywhere (pyproject.toml) @@ -79,7 +80,7 @@ gitdb==4.0.11 # via gitpython gitpython==3.1.43 # via swmmanywhere (pyproject.toml) -idna==3.8 +idna==3.10 # via requests joblib==1.4.2 # via swmmanywhere (pyproject.toml) @@ -97,13 +98,10 @@ loguru==0.7.2 # via swmmanywhere (pyproject.toml) multiurl==0.3.1 # via cads-api-client -netcdf4==1.7.1.post1 - # via swmmanywhere (pyproject.toml) -netcomp @ git+https://github.com/barneydobson/NetComp.git +netcdf4==1.7.1.post2 # via swmmanywhere (pyproject.toml) networkx==3.3 # via - # netcomp # osmnx # swmmanywhere (pyproject.toml) numba==0.60.0 @@ -114,13 +112,11 @@ numpy==1.26.4 # fastparquet # geopandas # netcdf4 - # netcomp # numba # osmnx # pandas # pyarrow # pyflwdir - # pyogrio # rasterio # rioxarray # scipy @@ -128,14 +124,13 @@ numpy==1.26.4 # snuggs # swmmanywhere (pyproject.toml) # xarray -osmnx==1.9.3 +osmnx==1.9.4 # via swmmanywhere (pyproject.toml) packaging==24.1 # via # fastparquet # geopandas # planetary-computer - # pyogrio # pyswmm # rioxarray # xarray @@ -148,19 +143,17 @@ pandas==2.2.2 # xarray planetary-computer==1.0.0 # via swmmanywhere (pyproject.toml) -pyarrow==16.1.0 +pyarrow==17.0.0 # via swmmanywhere (pyproject.toml) -pydantic==2.8.2 +pydantic==2.9.2 # via # planetary-computer # swmmanywhere (pyproject.toml) -pydantic-core==2.20.1 +pydantic-core==2.23.4 # via pydantic pyflwdir==0.5.8 # via swmmanywhere (pyproject.toml) -pyogrio==0.9.0 - # via geopandas -pyparsing==3.1.2 +pyparsing==3.1.4 # via snuggs pyproj==3.6.1 # via @@ -184,16 +177,16 @@ python-dateutil==2.9.0.post0 # pystac-client python-dotenv==1.0.1 # via planetary-computer -pytz==2024.1 +pytz==2024.2 # via # multiurl # pandas # planetary-computer pywbt==0.1.1 # via swmmanywhere (pyproject.toml) -pyyaml==6.0.1 +pyyaml==6.0.2 # via swmmanywhere (pyproject.toml) -rasterio==1.3.10 +rasterio==1.3.11 # via # rioxarray # swmmanywhere (pyproject.toml) @@ -211,24 +204,21 @@ requests==2.32.3 # pystac-client rioxarray==0.17.0 # via swmmanywhere (pyproject.toml) -rpds-py==0.19.1 +rpds-py==0.20.0 # via # jsonschema # referencing -scipy==1.14.0 +scipy==1.14.1 # via - # netcomp # pyflwdir # swmmanywhere (pyproject.toml) -shapely==2.0.5 +shapely==2.0.6 # via # geopandas # osmnx # swmmanywhere (pyproject.toml) six==1.16.0 - # via - # fiona - # python-dateutil + # via python-dateutil smmap==5.0.1 # via gitdb snuggs==1.4.7 @@ -237,7 +227,7 @@ swmm-toolkit==0.15.5 # via pyswmm toolz==0.12.1 # via cytoolz -tqdm==4.66.4 +tqdm==4.66.5 # via # cdsapi # multiurl @@ -249,11 +239,11 @@ typing-extensions==4.12.2 # pydantic-core tzdata==2024.1 # via pandas -urllib3==2.2.2 +urllib3==2.2.3 # via requests win32-setctime==1.1.0 # via loguru -xarray==2024.6.0 +xarray==2024.9.0 # via # rioxarray # swmmanywhere (pyproject.toml) From c263cedf9fb3fdd6ba29a221f553be86675969fc Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Wed, 18 Sep 2024 15:43:45 +0100 Subject: [PATCH 07/28] :heavy_plus_sign: Remove netcomp as dependency and add it as internal package --- .pre-commit-config.yaml | 4 +- netcomp/LICENSE.txt | 21 +++ netcomp/README.md | 3 + netcomp/__init__.py | 25 +++ netcomp/distance/__init__.py | 10 ++ netcomp/distance/exact.py | 330 +++++++++++++++++++++++++++++++++++ netcomp/distance/features.py | 141 +++++++++++++++ netcomp/exception.py | 19 ++ netcomp/linalg/__init__.py | 14 ++ netcomp/linalg/eigenstuff.py | 131 ++++++++++++++ netcomp/linalg/fast_bp.py | 56 ++++++ netcomp/linalg/matrices.py | 94 ++++++++++ netcomp/linalg/resistance.py | 259 +++++++++++++++++++++++++++ 13 files changed, 1106 insertions(+), 1 deletion(-) create mode 100644 netcomp/LICENSE.txt create mode 100644 netcomp/README.md create mode 100644 netcomp/__init__.py create mode 100644 netcomp/distance/__init__.py create mode 100644 netcomp/distance/exact.py create mode 100644 netcomp/distance/features.py create mode 100644 netcomp/exception.py create mode 100644 netcomp/linalg/__init__.py create mode 100644 netcomp/linalg/eigenstuff.py create mode 100644 netcomp/linalg/fast_bp.py create mode 100644 netcomp/linalg/matrices.py create mode 100644 netcomp/linalg/resistance.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a9e24ae0..1d4b6f5c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,4 +42,6 @@ repos: - id: refurb name: Modernizing Python codebases using Refurb additional_dependencies: - - numpy \ No newline at end of file + - numpy + +exclude: netcomp \ No newline at end of file diff --git a/netcomp/LICENSE.txt b/netcomp/LICENSE.txt new file mode 100644 index 00000000..8aa26455 --- /dev/null +++ b/netcomp/LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) [year] [fullname] + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/netcomp/README.md b/netcomp/README.md new file mode 100644 index 00000000..6f2e6964 --- /dev/null +++ b/netcomp/README.md @@ -0,0 +1,3 @@ +See https://github.com/barneydobson/NetComp for more details on NetComp. + +NetComp is distributed under a MIT license \ No newline at end of file diff --git a/netcomp/__init__.py b/netcomp/__init__.py new file mode 100644 index 00000000..1fb4d162 --- /dev/null +++ b/netcomp/__init__.py @@ -0,0 +1,25 @@ +"""NetComp v0.2.2 +============== + + NetComp is a Python package for comparing networks using pairwise distances, + and for performing anomaly detection on a time series of networks. It is + built on top of the NetworkX package. For details on usage and installation, + see README.md. + +""" +from __future__ import annotations + +__author__ = "Peter Wills " +__version__ = "0.2.2" +__license__ = "MIT" + +import sys + +if sys.version_info[0] < 3: + m = "Python 3.x required (%d.%d detected)." + raise ImportError(m % sys.version_info[:2]) +del sys + + +from .distance import * +from .linalg import * diff --git a/netcomp/distance/__init__.py b/netcomp/distance/__init__.py new file mode 100644 index 00000000..5136643d --- /dev/null +++ b/netcomp/distance/__init__.py @@ -0,0 +1,10 @@ +"""********* +Distances +********* + +Calculation of distances between graphs. +""" +from __future__ import annotations + +from netcomp.distance.exact import * +from netcomp.distance.features import * diff --git a/netcomp/distance/exact.py b/netcomp/distance/exact.py new file mode 100644 index 00000000..4225fb03 --- /dev/null +++ b/netcomp/distance/exact.py @@ -0,0 +1,330 @@ +"""*************** +Exact Distances +*************** + +Calculation of exact distances between graphs. Generally slow (quadratic in +graph size). +""" +from __future__ import annotations + +import networkx as nx +import numpy as np +from numpy import linalg as la + +from netcomp.distance import aggregate_features, get_features +from netcomp.exception import InputError +from netcomp.linalg import ( + _eigs, + _flat, + _pad, + fast_bp, + laplacian_matrix, + normalized_laplacian_eig, + renormalized_res_mat, + resistance_matrix, +) + +###################### +## Helper Functions ## +###################### + + +def _canberra_dist(v1, v2): + """The canberra distance between two vectors. We need to carefully handle + the case in which both v1 and v2 are zero in a certain dimension. + """ + eps = 10 ** (-15) + v1, v2 = [_flat(v) for v in [v1, v2]] + d_can = 0 + for u, w in zip(v1, v2): + if np.abs(u) < eps and np.abs(w) < eps: + d_update = 1 + else: + d_update = np.abs(u - w) / (np.abs(u) + np.abs(w)) + d_can += d_update + return d_can + + +############################# +## Distance Between Graphs ## +############################# + + +def edit_distance(A1, A2): + """The edit distance between graphs, defined as the number of changes one + needs to make to put the edge lists in correspondence. + + Parameters + ---------- + A1, A2 : NumPy matrices + Adjacency matrices of graphs to be compared + + Returns: + ------- + dist : float + The edit distance between the two graphs + """ + dist = np.abs((A1 - A2)).sum() / 2 + return dist + + +def vertex_edge_overlap(A1, A2): + """Vertex-edge overlap. Basically a souped-up edit distance, but in similarity + form. The VEO similarity is defined as + + VEO(G1,G2) = (|V1&V2| + |E1&E2|) / (|V1|+|V2|+|E1|+|E2|) + + where |S| is the size of a set S and U&T is the union of U and T. + + Parameters + ---------- + A1, A2 : NumPy matrices + Adjacency matrices of graphs to be compared + + Returns: + ------- + sim : float + The similarity between the two graphs + + + References: + ---------- + + """ + try: + [G1, G2] = [nx.from_scipy_sparse_array(A) for A in [A1, A2]] + except AttributeError: + [G1, G2] = [nx.from_numpy_array(A) for A in [A1, A2]] + V1, V2 = [set(G.nodes()) for G in [G1, G2]] + E1, E2 = [set(G.edges()) for G in [G1, G2]] + V_overlap = len(V1 | V2) # set union + E_overlap = len(E1 | E2) + sim = (V_overlap + E_overlap) / (len(V1) + len(V2) + len(E1) + len(E2)) + return sim + + +def vertex_edge_distance(A1, A2): + """Vertex-edge overlap transformed into a distance via + + D = (1-VEO)/VEO + + which is the inversion of the common distance-to-similarity function + + sim = 1/(1+D). + + Parameters + ---------- + A1, A2 : NumPy matrices + Adjacency matrices of graphs to be compared + + Returns: + ------- + dist : float + The distance between the two graphs + """ + sim = vertex_edge_overlap(A1, A2) + dist = (1 - sim) / sim + return dist + + +def lambda_dist(A1, A2, k=None, p=2, kind="laplacian"): + """The lambda distance between graphs, which is defined as + + d(G1,G2) = norm(L_1 - L_2) + + where L_1 is a vector of the top k eigenvalues of the appropriate matrix + associated with G1, and L2 is defined similarly. + + Parameters + ---------- + A1, A2 : NumPy matrices + Adjacency matrices of graphs to be compared + + k : Integer + The number of eigenvalues to be compared + + p : non-zero Float + The p-norm is used to compare the resulting vector of eigenvalues. + + kind : String , in {'laplacian','laplacian_norm','adjacency'} + The matrix for which eigenvalues will be calculated. + + Returns: + ------- + dist : float + The distance between the two graphs + + Notes: + ----- + The norm can be any p-norm; by default we use p=2. If p<0 is used, the + result is not a mathematical norm, but may still be interesting and/or + useful. + + If k is provided, then we use the k SMALLEST eigenvalues for the Laplacian + distances, and we use the k LARGEST eigenvalues for the adjacency + distance. This is because the corresponding order flips, as L = D-A. + + References: + ---------- + + See Also: + -------- + netcomp.linalg._eigs + normalized_laplacian_eigs + + """ + # ensure valid k + n1, n2 = [A.shape[0] for A in [A1, A2]] + N = min(n1, n2) # minimum size between the two graphs + if k is None or k > N: + k = N + if kind == "laplacian": + # form matrices + L1, L2 = [laplacian_matrix(A) for A in [A1, A2]] + # get eigenvalues, ignore eigenvectors + evals1, evals2 = [_eigs(L)[0] for L in [L1, L2]] + elif kind == "laplacian_norm": + # use our function to graph evals of normalized laplacian + evals1, evals2 = [normalized_laplacian_eig(A)[0] for A in [A1, A2]] + elif kind == "adjacency": + evals1, evals2 = [_eigs(A)[0] for A in [A1, A2]] + # reverse, so that we are sorted from large to small, since we care + # about the k LARGEST eigenvalues for the adjacency distance + evals1, evals2 = [evals[::-1] for evals in [evals1, evals2]] + else: + raise InputError( + "Invalid type, choose from 'laplacian', " + "'laplacian_norm', and 'adjacency'." + ) + dist = la.norm(evals1[:k] - evals2[:k], ord=p) + return dist + + +def netsimile(A1, A2): + """NetSimile distance between two graphs. + + Parameters + ---------- + A1, A2 : SciPy sparse array + Adjacency matrices of the graphs in question. + + Returns: + ------- + d_can : Float + The distance between the two graphs. + + Notes: + ----- + NetSimile works on graphs without node correspondence. Graphs to not need to + be the same size. + + See Also: + -------- + + References: + ---------- + """ + feat_A1, feat_A2 = [get_features(A) for A in [A1, A2]] + agg_A1, agg_A2 = [aggregate_features(feat) for feat in [feat_A1, feat_A2]] + # calculate Canberra distance between two aggregate vectors + d_can = _canberra_dist(agg_A1, agg_A2) + return d_can + + +def resistance_distance( + A1, A2, p=2, renormalized=False, attributed=False, check_connected=True, beta=1 +): + """Compare two graphs using resistance distance (possibly renormalized). + + Parameters + ---------- + A1, A2 : NumPy Matrices + Adjacency matrices of graphs to be compared. + + p : float + Function returns the p-norm of the flattened matrices. + + renormalized : Boolean, optional (default = False) + If true, then renormalized resistance distance is computed. + + attributed : Boolean, optional (default=False) + If true, then the resistance distance PER NODE is returned. + + check_connected : Boolean, optional (default=True) + If false, then no check on connectivity is performed. See Notes of + resistance_matrix for more information. + + beta : float, optional (default=1) + A parameter used in the calculation of the renormalized resistance + matrix. If using regular resistance, this is irrelevant. + + Returns: + ------- + dist : float of numpy array + The RR distance between the two graphs. If attributed is True, then + vector distance per node is returned. + + Notes: + ----- + The distance is calculated by assuming the nodes are in correspondence, and + any nodes not present are treated as isolated by renormalized resistance. + + References: + ---------- + + See Also: + -------- + resistance_matrix + """ + # Calculate resistance matricies and compare + if renormalized: + # pad smaller adj. mat. so they're the same size + n1, n2 = [A.shape[0] for A in [A1, A2]] + N = max(n1, n2) + A1, A2 = [_pad(A, N) for A in [A1, A2]] + R1, R2 = [renormalized_res_mat(A, beta=beta) for A in [A1, A2]] + else: + R1, R2 = [ + resistance_matrix(A, check_connected=check_connected) for A in [A1, A2] + ] + try: + distance_vector = np.sum((R1 - R2) ** p, axis=1) + except ValueError: + raise InputError( + "Input matrices are different sizes. Please use " + "renormalized resistance distance." + ) + if attributed: + return distance_vector ** (1 / p) + else: + return np.sum(distance_vector) ** (1 / p) + + +def deltacon0(A1, A2, eps=None): + """DeltaCon0 distance between two graphs. The distance is the Frobenius norm + of the element-wise square root of the fast belief propogation matrix. + + Parameters + ---------- + A1, A2 : NumPy Matrices + Adjacency matrices of graphs to be compared. + + Returns: + ------- + dist : float + DeltaCon0 distance between graphs. + + References: + ---------- + + See Also: + -------- + fast_bp + """ + # pad smaller adj. mat. so they're the same size + n1, n2 = [A.shape[0] for A in [A1, A2]] + N = max(n1, n2) + A1, A2 = [_pad(A, N) for A in [A1, A2]] + S1, S2 = [fast_bp(A, eps=eps) for A in [A1, A2]] + dist = np.abs(np.sqrt(S1) - np.sqrt(S2)).sum() + return dist diff --git a/netcomp/distance/features.py b/netcomp/distance/features.py new file mode 100644 index 00000000..9f89a49b --- /dev/null +++ b/netcomp/distance/features.py @@ -0,0 +1,141 @@ +"""******** +Features +******** + +Calculation of features for NetSimile algorithm. +""" +from __future__ import annotations + +import networkx as nx +import numpy as np +from scipy import stats + +from netcomp.linalg import _eps + + +def get_features(A): + """Feature grabber for NetSimile algorithm. Features used are + + 1. Degree of node + 2. Clustering coefficient of node + 3. Average degree of node's neighbors + 4. Average clustering coefficient of node's neighbors + 5. Number of edges in node's egonet + 6. Number of neighbors of node's egonet + 7. Number of outgoing edges from node's egonet + + Parameters + --------- + A : NumPy matrix + Adjacency matrix of graph in question. Preferably a SciPy sparse matrix + for large graphs. + + Returns: + ------- + feature_mat : NumPy array + An n by 7 array of features, where n = A.shape[0] + + References: + ----- + [Berlingerio 2012] + + """ + try: + G = nx.from_scipy_sparse_array(A) + except AttributeError: + G = nx.from_numpy_array(A) + n = len(G) + # degrees, array so we can slice nice + d_vec = np.array(list(dict(G.degree()).values())) + # list of clustering coefficient + clust_vec = np.array(list(nx.clustering(G).values())) + neighbors = [G.neighbors(i) for i in range(n)] + # average degree of neighbors (0 if node is isolated) + neighbors = [list(G.neighbors(i)) for i in range(n)] + # average degree of neighbors (0 if node is isolated) + neighbor_deg = [ + d_vec[neighbors[i]].sum() / d_vec[i] if d_vec[i] > _eps else 0 for i in range(n) + ] + # avg. clustering coefficient of neighbors (0 if node is isolated) + neighbor_clust = [ + clust_vec[neighbors[i]].sum() / d_vec[i] if d_vec[i] > _eps else 0 + for i in range(n) + ] + egonets = [nx.ego_graph(G, i) for i in range(n)] + # number of edges in egonet + ego_size = [G.number_of_edges() for G in egonets] + # number of neighbors of egonet + ego_neighbors = [ + len( + set.union(*[set(neighbors[j]) for j in egonets[i].nodes()]) + - set(egonets[i].nodes()) + ) + for i in range(n) + ] + # number of edges outgoing from egonet + outgoing_edges = [ + len( + [ + edge + for edge in G.edges(egonets[i].nodes()) + if edge[1] not in egonets[i].nodes() + ] + ) + for i in range(n) + ] + # use mat.T so that each node is a row (standard format) + feature_mat = np.array( + [ + d_vec, + clust_vec, + neighbor_deg, + neighbor_clust, + ego_size, + ego_neighbors, + outgoing_edges, + ] + ).T + return feature_mat + + +def aggregate_features(feature_mat, row_var=False, as_matrix=False): + """Returns column-wise descriptive statistics of a feature matrix. + + Parameters + ---------- + feature_mat : NumPy array + Matrix on which statistics are to be calculated. Assumed to be formatted + so each row is an observation (a node, in the case of NetSimile). + + row_var : Boolean, optional (default=False) + If true, then each variable has it's own row, and statistics are + computed along rows rather than columns. + + as_matrix : Boolean, optional (default=False) + If true, then description is returned as matrix. Otherwise, it is + flattened into a vector. + + Returns: + ------- + description : NumPy array + Descriptive statistics of feature_mat + + Notes: + ----- + + References: + ---------- + """ + axis = int(row_var) # 0 if column-oriented, 1 if not + description = np.array( + [ + feature_mat.mean(axis=axis), + np.median(feature_mat, axis=axis), + np.std(feature_mat, axis=axis), + stats.skew(feature_mat, axis=axis), + stats.kurtosis(feature_mat, axis=axis), + ] + ) + if not as_matrix: + description = description.flatten() + return description diff --git a/netcomp/exception.py b/netcomp/exception.py new file mode 100644 index 00000000..53dc70ed --- /dev/null +++ b/netcomp/exception.py @@ -0,0 +1,19 @@ +"""********** +Exceptions +********** + +Custom exceptions for NetComp. +""" +from __future__ import annotations + + +class UndefinedException(Exception): + """Raised when matrix to be returned is undefined""" + + +class InputError(Exception): + """Raised when input to algorithm is invalid""" + + +class KathleenError(Exception): + """Raised when food is gross or it is too cold""" diff --git a/netcomp/linalg/__init__.py b/netcomp/linalg/__init__.py new file mode 100644 index 00000000..7e8fadbb --- /dev/null +++ b/netcomp/linalg/__init__.py @@ -0,0 +1,14 @@ +"""************** +Linear Algebra +************** + +Linear algebraic functions, calculations of important matrices. +""" +from __future__ import annotations + +from netcomp.linalg.eigenstuff import * +from netcomp.linalg.fast_bp import * +from netcomp.linalg.matrices import * + +# import helper functions for use in other places +from netcomp.linalg.resistance import * diff --git a/netcomp/linalg/eigenstuff.py b/netcomp/linalg/eigenstuff.py new file mode 100644 index 00000000..c55efb1c --- /dev/null +++ b/netcomp/linalg/eigenstuff.py @@ -0,0 +1,131 @@ +"""********** +Eigenstuff +********** + +Functions for calculating eigenstuff of graphs. +""" +from __future__ import annotations + +import numpy as np +from numpy import linalg as la +from scipy import sparse as sps +from scipy.sparse import issparse +from scipy.sparse import linalg as spla + +from netcomp.linalg.matrices import _eps, _flat + +###################### +## Helper Functions ## +###################### + + +def _eigs(M, which="SR", k=None): + """Helper function for getting eigenstuff. + + Parameters + ---------- + M : matrix, numpy or scipy sparse + The matrix for which we hope to get eigenstuff. + which : string in {'SR','LR'} + If 'SR', get eigenvalues with smallest real part. If 'LR', get largest. + k : int + Number of eigenvalues to return + + Returns: + ------- + evals, evecs : numpy arrays + Eigenvalues and eigenvectors of matrix M, sorted in ascending or + descending order, depending on 'which'. + + See Also: + -------- + numpy.linalg.eig + scipy.sparse.eigs + """ + n, _ = M.shape + if k is None: + k = n + if which not in ["LR", "SR"]: + raise ValueError("which must be either 'LR' or 'SR'.") + M = M.astype(float) + if issparse(M) and k < n - 1: + evals, evecs = spla.eigs(M, k=k, which=which) + else: + try: + M = M.todense() + except: + pass + evals, evecs = la.eig(M) + # sort dem eigenvalues + inds = np.argsort(evals) + if which == "LR": + inds = inds[::-1] + else: + pass + inds = inds[:k] + evals = evals[inds] + evecs = np.matrix(evecs[:, inds]) + return np.real(evals), np.real(evecs) + + +##################### +## Get Eigenstuff ## +##################### + + +def normalized_laplacian_eig(A, k=None): + """Return the eigenstuff of the normalized Laplacian matrix of graph + associated with adjacency matrix A. + + Calculates via eigenvalues if + + K = D^(-1/2) A D^(-1/2) + + where `A` is the adjacency matrix and `D` is the diagonal matrix of + node degrees. Since L = I - K, the eigenvalues and vectors of L can + be easily recovered. + + Parameters + ---------- + A : NumPy matrix + Adjacency matrix of a graph + + k : int, 0 < k < A.shape[0]-1 + The number of eigenvalues to grab. + + Returns: + ------- + lap_evals : NumPy array + Eigenvalues of L + + evecs : NumPy matrix + Columns are the eigenvectors of L + + Notes: + ----- + This way of calculating the eigenvalues of the normalized graph laplacian is + more numerically stable than simply forming the matrix L = I - K and doing + numpy.linalg.eig on the result. This is because the eigenvalues of L are + close to zero, whereas the eigenvalues of K are close to 1. + + References: + ---------- + + See Also: + -------- + nx.laplacian_matrix + nx.normalized_laplacian_matrix + """ + n, m = A.shape + ## + ## TODO: implement checks on the adjacency matrix + ## + degs = _flat(A.sum(axis=1)) + # the below will break if + inv_root_degs = [d ** (-1 / 2) if d > _eps else 0 for d in degs] + inv_rootD = sps.spdiags(inv_root_degs, [0], n, n, format="csr") + # build normalized diffusion matrix + K = inv_rootD * A * inv_rootD + evals, evecs = _eigs(K, k=k, which="LR") + lap_evals = 1 - evals + return np.real(lap_evals), np.real(evecs) diff --git a/netcomp/linalg/fast_bp.py b/netcomp/linalg/fast_bp.py new file mode 100644 index 00000000..0c8d8ccb --- /dev/null +++ b/netcomp/linalg/fast_bp.py @@ -0,0 +1,56 @@ +"""*********************** +Fast Belief Propagation +*********************** + +The fast approximation of the Belief Propogation matrix. +""" +from __future__ import annotations + +import numpy as np +from numpy import linalg as la +from scipy import sparse as sps + + +def fast_bp(A, eps=None): + """Return the fast belief propogation matrix of graph associated with A. + + Parameters + ---------- + A : NumPy matrix or Scipy sparse matrix + Adjacency matrix of a graph. If sparse, can be any format; CSC or CSR + recommended. + + eps : float, optional (default=None) + Small parameter used in calculation of matrix. If not provided, it is + set to 1/(1+d_max) where d_max is the maximum degree. + + Returns: + ------- + S : NumPy matrix or Scipy sparse matrix + The fast belief propogation matrix. If input is sparse, will be returned + as (sparse) CSC matrix. + + Notes: + ----- + + References: + ---------- + + """ + n, m = A.shape + ## + ## TODO: implement checks on the adjacency matrix + ## + degs = np.array(A.sum(axis=1)).flatten() + if eps is None: + eps = 1 / (1 + max(degs)) + I = sps.identity(n) + D = sps.dia_matrix((degs, [0]), shape=(n, n)) + # form inverse of S and invert (slow!) + Sinv = I + eps**2 * D - eps * A + try: + S = la.inv(Sinv) + except: + Sinv = sps.csc_matrix(Sinv) + S = sps.linalg.inv(Sinv) + return S diff --git a/netcomp/linalg/matrices.py b/netcomp/linalg/matrices.py new file mode 100644 index 00000000..e5a6207d --- /dev/null +++ b/netcomp/linalg/matrices.py @@ -0,0 +1,94 @@ +"""******** +Matrices +******** + +Matrices associated with graphs. Also contains linear algebraic helper functions. +""" +from __future__ import annotations + +import numpy as np +from scipy import sparse as sps +from scipy.sparse import issparse + +_eps = 10 ** (-10) # a small parameter + +###################### +## Helper Functions ## +###################### + + +def _flat(D): + """Flatten column or row matrices, as well as arrays.""" + if issparse(D): + raise ValueError("Cannot flatten sparse matrix.") + d_flat = np.array(D).flatten() + return d_flat + + +def _pad(A, N): + """Pad A so A.shape is (N,N)""" + n, _ = A.shape + if n >= N: + return A + else: + if issparse(A): + # thrown if we try to np.concatenate sparse matrices + side = sps.csr_matrix((n, N - n)) + bottom = sps.csr_matrix((N - n, N)) + A_pad = sps.hstack([A, side]) + A_pad = sps.vstack([A_pad, bottom]) + else: + side = np.zeros((n, N - n)) + bottom = np.zeros((N - n, N)) + A_pad = np.concatenate([A, side], axis=1) + A_pad = np.concatenate([A_pad, bottom]) + return A_pad + + +######################## +## Matrices of Graphs ## +######################## + + +def degree_matrix(A): + """Diagonal degree matrix of graph with adjacency matrix A + + Parameters + ---------- + A : matrix + Adjacency matrix + + Returns: + ------- + D : SciPy sparse matrix + Diagonal matrix of degrees. + """ + n, m = A.shape + degs = _flat(A.sum(axis=1)) + D = sps.spdiags(degs, [0], n, n, format="csr") + return D + + +def laplacian_matrix(A, normalized=False): + """Diagonal degree matrix of graph with adjacency matrix A + + Parameters + ---------- + A : matrix + Adjacency matrix + normalized : Bool, optional (default=False) + If true, then normalized laplacian is returned. + + Returns: + ------- + L : SciPy sparse matrix + Combinatorial laplacian matrix. + """ + n, m = A.shape + D = degree_matrix(A) + L = D - A + if normalized: + degs = _flat(A.sum(axis=1)) + rootD = sps.spdiags(np.power(degs, -1 / 2), [0], n, n, format="csr") + L = rootD * L * rootD + return L diff --git a/netcomp/linalg/resistance.py b/netcomp/linalg/resistance.py new file mode 100644 index 00000000..09b89487 --- /dev/null +++ b/netcomp/linalg/resistance.py @@ -0,0 +1,259 @@ +"""********** +Resistance +********** + +Resistance matrix. Renormalized version, as well as conductance and commute matrices. +""" +from __future__ import annotations + +import networkx as nx +import numpy as np +from numpy import linalg as la +from scipy import linalg as spla +from scipy.sparse import issparse + +from netcomp.exception import UndefinedException +from netcomp.linalg.matrices import laplacian_matrix + + +def resistance_matrix(A, check_connected=True): + """Return the resistance matrix of G. + + Parameters + ---------- + A : NumPy matrix or SciPy sparse matrix + Adjacency matrix of a graph. + + check_connected : Boolean, optional (default=True) + If false, then the resistance matrix will be computed even for + disconnected matrices. See Notes. + + Returns: + ------- + R : NumPy matrix + Matrix of pairwise resistances between nodes. + + Notes: + ----- + Uses formula for resistance matrix R in terms of Moore-Penrose of + pseudoinverse (non-normalized) graph Laplacian. See e.g. Theorem 2.1 in [1]. + + This formula can be computed even for disconnected graphs, although the + interpretation in this case is unclear. Thus, the usage of + check_connected=False is recommended only to reduce computation time in a + scenario in which the user is confident the graph in question is, in fact, + connected. + + Since we do not expect the pseudoinverse of the laplacian to be sparse, we + convert L to dense form before running np.linalg.pinv(). The returned + resistance matrix is dense. + + See Also: + -------- + nx.laplacian_matrix + + References: + ---------- + .. [1] W. Ellens, et al. (2011) + Effective graph resistance. + Linear Algebra and its Applications, 435 (2011) + + """ + n, m = A.shape + # check if graph is connected + if check_connected: + if issparse(A): + G = nx.from_scipy_sparse_array(A) + else: + G = nx.from_numpy_array(A) + if not nx.is_connected(G): + raise UndefinedException( + "Graph is not connected. " "Resistance matrix is undefined." + ) + L = laplacian_matrix(A) + try: + L = L.todense() + except: + pass + M = la.pinv(L) + # calculate R in terms of M + d = np.reshape(np.diag(M), (n, 1)) + ones = np.ones((n, 1)) + R = np.dot(d, ones.T) + np.dot(ones, d.T) - M - M.T + return R + + +def commute_matrix(A): + """Return the commute matrix of the graph associated with adj. matrix A. + + Parameters + ---------- + A : NumPy matrix or SciPy sparse matrix + Adjacency matrix of a graph. + + Returns: + ------- + C : NumPy matrix + Matrix of pairwise resistances between nodes. + + Notes: + ----- + Uses formula for commute time matrix in terms of resistance matrix, + + C = R*2*|E| + + where |E| is the number of edges in G. See e.g. Theorem 2.8 in [1]. + + See Also: + -------- + laplacian_matrix + resistance_matrix + + References: + ---------- + .. [1] W. Ellens, et al. (2011) + Effective graph resistance. + Linear Algebra and its Applications, 435 (2011) + + """ + R = resistance_matrix(A) + E = A.sum() / 2 # number of edges in graph + C = 2 * E * R + return C + + +def renormalized_res_mat(A, beta=1): + """Return the renormalized resistance matrix of graph associated with A. + + To renormalize a resistance R, we apply the function + + R' = R / (R + beta) + + In this way, the renormalized resistance of nodes in disconnected components + is 1. The parameter beta determines the penalty for disconnection. If we set + beta to be approximately the maximum resistance found in the network, then + the penalty for disconnection is at least 1/2. + + Parameters + ---------- + A : NumPy matrix or SciPy sparse matrix + Adjacency matrix of a graph. + + nodelist : list, optional + The rows and columns are ordered according to the nodes in nodelist. If + nodelist is None, then the ordering is produced by G.nodes(). + + weight : string or None, optional (default='weight') + The edge data key used to compute each value in the matrix. + If None, then each edge has weight 1. + + beta : float, optional + Scaling parameter in renormalization. Must be greater than or equal to + 1. Determines how heavily disconnection is penalized. + + Returns: + ------- + R : NumPy array + Matrix of pairwise renormalized resistances between nodes. + + Notes: + ----- + This function converts to a NetworkX graph, as it uses the algorithms + therein for identifying connected components. + + See Also: + -------- + resistance_matrix + + """ + if issparse(A): + G = nx.from_scipy_sparse_array(A) + else: + G = nx.from_numpy_array(A) + + if isinstance(G, nx.Graph): + cc = nx.connected_components + else: + cc = nx.weakly_connected_components + + n = len(G) + subgraphR = [] + for c in cc(G): + subgraph = G.subgraph(c).copy() + a_sub = nx.adjacency_matrix(subgraph) + r_sub = resistance_matrix(a_sub) + subgraphR.append(r_sub) + R = spla.block_diag(*subgraphR) + # now, resort R so that it matches the original node list + component_order = [] + for component in cc(G): + component_order += list(component) + component_order = list(np.argsort(component_order)) + R = R[component_order, :] + R = R[:, component_order] + renorm = np.vectorize(lambda r: r / (r + beta)) + R = renorm(R) + # set resistance for different components to 1 + R[R == 0] = 1 + R = R - np.eye(n) # don't want diagonal to be 1 + return R + + +def conductance_matrix(A): + """Return the conductance matrix of G. + + The conductance matrix of G is the element-wise inverse of the resistance + matrix. The diagonal is set to 0, although it is formally infinite. Nodes in + disconnected components have 0 conductance. + + Parameters + ---------- + G : graph + A NetworkX graph + + nodelist : list, optional + The rows and columns are ordered according to the nodes in nodelist. If + nodelist is None, then the ordering is produced by G.nodes(). + + weight : string or None, optional (default='weight') + The edge data key used to compute each value in the matrix. + If None, then each edge has weight 1. + + Returns: + ------- + C : NumPy array + Matrix of pairwise conductances between nodes. + + + See Also: + -------- + resistance_matrix + renormalized_res_mat + + """ + if issparse(A): + G = nx.from_scipy_sparse_array(A) + else: + G = nx.from_numpy_array(A) + if isinstance(G, nx.Graph): + cc = nx.connected_components + else: + cc = nx.weakly_connected_components + subgraphC = [] + for c in cc(G): + subgraph = G.subgraph(c).copy() + a_sub = nx.adjacency_matrix(subgraph) + r_sub = resistance_matrix(a_sub) + m = len(subgraph) + # add one to diagonal, invert, remove one from diagonal: + c_sub = 1 / (r_sub + np.eye(m)) - np.eye(m) + subgraphC.append(c_sub) + C = spla.block_diag(*subgraphC) + # resort C so that it matches the original node list + component_order = [] + for component in cc(G): + component_order += list(component) + component_order = list(np.argsort(component_order)) + C = C[component_order, :] + C = C[:, component_order] + return C From a956b60ffd1c1f1d1b2d8bed1d50f56c0a00da7b Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Wed, 18 Sep 2024 15:46:06 +0100 Subject: [PATCH 08/28] :heavy_minus_sign: Remove unnecessary dependencies --- dev-requirements.txt | 19 +------------------ doc-requirements.txt | 19 +------------------ pyproject.toml | 3 --- requirements.txt | 19 +------------------ 4 files changed, 3 insertions(+), 57 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 83091e49..c101accf 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -66,24 +66,16 @@ colorama==0.4.6 # tqdm coverage[toml]==7.6.1 # via pytest-cov -cramjam==2.8.3 - # via fastparquet cytoolz==0.12.3 # via swmmanywhere (pyproject.toml) distlib==0.3.8 # via virtualenv -fastparquet==2024.5.0 - # via swmmanywhere (pyproject.toml) filelock==3.16.1 # via # pytest-mypy # virtualenv fiona==1.10.1 - # via - # geopandas - # swmmanywhere (pyproject.toml) -fsspec==2024.9.0 - # via fastparquet + # via geopandas geographiclib==2.0 # via geopy geopandas==0.14.4 @@ -92,10 +84,6 @@ geopandas==0.14.4 # swmmanywhere (pyproject.toml) geopy==2.4.1 # via swmmanywhere (pyproject.toml) -gitdb==4.0.11 - # via gitpython -gitpython==3.1.43 - # via swmmanywhere (pyproject.toml) identify==2.6.1 # via pre-commit idna==3.10 @@ -137,7 +125,6 @@ numba==0.60.0 numpy==1.26.4 # via # cftime - # fastparquet # geopandas # netcdf4 # numba @@ -157,7 +144,6 @@ osmnx==1.9.4 packaging==24.1 # via # build - # fastparquet # geopandas # planetary-computer # pyswmm @@ -166,7 +152,6 @@ packaging==24.1 # xarray pandas==2.2.2 # via - # fastparquet # geopandas # osmnx # swmmanywhere (pyproject.toml) @@ -277,8 +262,6 @@ shapely==2.0.6 # swmmanywhere (pyproject.toml) six==1.16.0 # via python-dateutil -smmap==5.0.1 - # via gitdb snuggs==1.4.7 # via rasterio swmm-toolkit==0.15.5 diff --git a/doc-requirements.txt b/doc-requirements.txt index 324cb9ad..681d632d 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -74,8 +74,6 @@ colorama==0.4.6 # tqdm comm==0.2.2 # via ipykernel -cramjam==2.8.3 - # via fastparquet cytoolz==0.12.3 # via swmmanywhere (pyproject.toml) debugpy==1.8.5 @@ -88,14 +86,8 @@ executing==2.1.0 # via stack-data fastjsonschema==2.20.0 # via nbformat -fastparquet==2024.5.0 - # via swmmanywhere (pyproject.toml) fiona==1.10.1 - # via - # geopandas - # swmmanywhere (pyproject.toml) -fsspec==2024.9.0 - # via fastparquet + # via geopandas geographiclib==2.0 # via geopy geopandas==0.14.4 @@ -106,10 +98,6 @@ geopy==2.4.1 # via swmmanywhere (pyproject.toml) ghp-import==2.1.0 # via mkdocs -gitdb==4.0.11 - # via gitpython -gitpython==3.1.43 - # via swmmanywhere (pyproject.toml) griffe==1.3.1 # via mkdocstrings-python idna==3.10 @@ -247,7 +235,6 @@ numba==0.60.0 numpy==1.26.4 # via # cftime - # fastparquet # geopandas # netcdf4 # numba @@ -266,7 +253,6 @@ osmnx==1.9.4 # via swmmanywhere (pyproject.toml) packaging==24.1 # via - # fastparquet # geopandas # ipykernel # jupytext @@ -280,7 +266,6 @@ paginate==0.5.7 # via mkdocs-material pandas==2.2.2 # via - # fastparquet # geopandas # osmnx # swmmanywhere (pyproject.toml) @@ -412,8 +397,6 @@ six==1.16.0 # asttokens # bleach # python-dateutil -smmap==5.0.1 - # via gitdb snuggs==1.4.7 # via rasterio soupsieve==2.6 diff --git a/pyproject.toml b/pyproject.toml index b88eac3e..3672ccca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,11 +24,8 @@ classifiers = [ dependencies = [ "cdsapi", "cytoolz", - "fastparquet", - "fiona", "geopandas", "geopy", - "GitPython", "joblib", "jsonschema", "loguru", diff --git a/requirements.txt b/requirements.txt index 9e8910cf..8e8f393d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -56,18 +56,10 @@ colorama==0.4.6 # click # loguru # tqdm -cramjam==2.8.3 - # via fastparquet cytoolz==0.12.3 # via swmmanywhere (pyproject.toml) -fastparquet==2024.5.0 - # via swmmanywhere (pyproject.toml) fiona==1.10.1 - # via - # geopandas - # swmmanywhere (pyproject.toml) -fsspec==2024.9.0 - # via fastparquet + # via geopandas geographiclib==2.0 # via geopy geopandas==0.14.4 @@ -76,10 +68,6 @@ geopandas==0.14.4 # swmmanywhere (pyproject.toml) geopy==2.4.1 # via swmmanywhere (pyproject.toml) -gitdb==4.0.11 - # via gitpython -gitpython==3.1.43 - # via swmmanywhere (pyproject.toml) idna==3.10 # via requests joblib==1.4.2 @@ -109,7 +97,6 @@ numba==0.60.0 numpy==1.26.4 # via # cftime - # fastparquet # geopandas # netcdf4 # numba @@ -128,7 +115,6 @@ osmnx==1.9.4 # via swmmanywhere (pyproject.toml) packaging==24.1 # via - # fastparquet # geopandas # planetary-computer # pyswmm @@ -136,7 +122,6 @@ packaging==24.1 # xarray pandas==2.2.2 # via - # fastparquet # geopandas # osmnx # swmmanywhere (pyproject.toml) @@ -219,8 +204,6 @@ shapely==2.0.6 # swmmanywhere (pyproject.toml) six==1.16.0 # via python-dateutil -smmap==5.0.1 - # via gitdb snuggs==1.4.7 # via rasterio swmm-toolkit==0.15.5 From 5110c0f1afcfc16d3960ff0926fe0d75452ab1d0 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Wed, 18 Sep 2024 15:52:02 +0100 Subject: [PATCH 09/28] :construction_worker: Exclude etcomp from QA --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 86629b0d..4bab762b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,8 @@ jobs: steps: - uses: actions/checkout@v4 - uses: pre-commit/action@v3.0.1 + with: + exclude: 'netcomp/*' test: needs: qa From 24242299db19b9c38a1bfb629a987f96fc9698c5 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Thu, 19 Sep 2024 13:32:27 +0100 Subject: [PATCH 10/28] :rotating_light: Fix most linting issues. Others ignored. --- .github/workflows/ci.yml | 2 -- .pre-commit-config.yaml | 2 -- netcomp/__init__.py | 5 +++-- netcomp/distance/exact.py | 11 ++++++----- netcomp/distance/features.py | 2 +- netcomp/linalg/eigenstuff.py | 8 +++++--- netcomp/linalg/fast_bp.py | 13 +++++++------ netcomp/linalg/resistance.py | 7 ++++--- pyproject.toml | 7 ++++--- 9 files changed, 30 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4bab762b..86629b0d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,8 +8,6 @@ jobs: steps: - uses: actions/checkout@v4 - uses: pre-commit/action@v3.0.1 - with: - exclude: 'netcomp/*' test: needs: qa diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1d4b6f5c..7ca9211b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,5 +43,3 @@ repos: name: Modernizing Python codebases using Refurb additional_dependencies: - numpy - -exclude: netcomp \ No newline at end of file diff --git a/netcomp/__init__.py b/netcomp/__init__.py index 1fb4d162..2daef3df 100644 --- a/netcomp/__init__.py +++ b/netcomp/__init__.py @@ -7,6 +7,7 @@ see README.md. """ + from __future__ import annotations __author__ = "Peter Wills " @@ -21,5 +22,5 @@ del sys -from .distance import * -from .linalg import * +from .distance import * # noqa +from .linalg import * # noqa diff --git a/netcomp/distance/exact.py b/netcomp/distance/exact.py index 4225fb03..5e851180 100644 --- a/netcomp/distance/exact.py +++ b/netcomp/distance/exact.py @@ -5,6 +5,7 @@ Calculation of exact distances between graphs. Generally slow (quadratic in graph size). """ + from __future__ import annotations import networkx as nx @@ -13,7 +14,7 @@ from netcomp.distance import aggregate_features, get_features from netcomp.exception import InputError -from netcomp.linalg import ( +from netcomp.linalg import ( # type: ignore _eigs, _flat, _pad, @@ -276,7 +277,7 @@ def resistance_distance( -------- resistance_matrix """ - # Calculate resistance matricies and compare + # Calculate resistance matrices and compare if renormalized: # pad smaller adj. mat. so they're the same size n1, n2 = [A.shape[0] for A in [A1, A2]] @@ -296,13 +297,13 @@ def resistance_distance( ) if attributed: return distance_vector ** (1 / p) - else: - return np.sum(distance_vector) ** (1 / p) + + return np.sum(distance_vector) ** (1 / p) def deltacon0(A1, A2, eps=None): """DeltaCon0 distance between two graphs. The distance is the Frobenius norm - of the element-wise square root of the fast belief propogation matrix. + of the element-wise square root of the fast belief propagation matrix. Parameters ---------- diff --git a/netcomp/distance/features.py b/netcomp/distance/features.py index 9f89a49b..c237303d 100644 --- a/netcomp/distance/features.py +++ b/netcomp/distance/features.py @@ -10,7 +10,7 @@ import numpy as np from scipy import stats -from netcomp.linalg import _eps +from netcomp.linalg import _eps # type: ignore def get_features(A): diff --git a/netcomp/linalg/eigenstuff.py b/netcomp/linalg/eigenstuff.py index c55efb1c..8d4b3dd0 100644 --- a/netcomp/linalg/eigenstuff.py +++ b/netcomp/linalg/eigenstuff.py @@ -4,8 +4,11 @@ Functions for calculating eigenstuff of graphs. """ + from __future__ import annotations +from contextlib import suppress + import numpy as np from numpy import linalg as la from scipy import sparse as sps @@ -51,10 +54,9 @@ def _eigs(M, which="SR", k=None): if issparse(M) and k < n - 1: evals, evecs = spla.eigs(M, k=k, which=which) else: - try: + with suppress(Exception): M = M.todense() - except: - pass + evals, evecs = la.eig(M) # sort dem eigenvalues inds = np.argsort(evals) diff --git a/netcomp/linalg/fast_bp.py b/netcomp/linalg/fast_bp.py index 0c8d8ccb..b1d1e8b2 100644 --- a/netcomp/linalg/fast_bp.py +++ b/netcomp/linalg/fast_bp.py @@ -2,8 +2,9 @@ Fast Belief Propagation *********************** -The fast approximation of the Belief Propogation matrix. +The fast approximation of the Belief propagation matrix. """ + from __future__ import annotations import numpy as np @@ -12,7 +13,7 @@ def fast_bp(A, eps=None): - """Return the fast belief propogation matrix of graph associated with A. + """Return the fast belief propagation matrix of graph associated with A. Parameters ---------- @@ -27,7 +28,7 @@ def fast_bp(A, eps=None): Returns: ------- S : NumPy matrix or Scipy sparse matrix - The fast belief propogation matrix. If input is sparse, will be returned + The fast belief propagation matrix. If input is sparse, will be returned as (sparse) CSC matrix. Notes: @@ -44,13 +45,13 @@ def fast_bp(A, eps=None): degs = np.array(A.sum(axis=1)).flatten() if eps is None: eps = 1 / (1 + max(degs)) - I = sps.identity(n) + identity = sps.identity(n) D = sps.dia_matrix((degs, [0]), shape=(n, n)) # form inverse of S and invert (slow!) - Sinv = I + eps**2 * D - eps * A + Sinv = identity + eps**2 * D - eps * A try: S = la.inv(Sinv) - except: + except Exception: Sinv = sps.csc_matrix(Sinv) S = sps.linalg.inv(Sinv) return S diff --git a/netcomp/linalg/resistance.py b/netcomp/linalg/resistance.py index 09b89487..e58cd6b2 100644 --- a/netcomp/linalg/resistance.py +++ b/netcomp/linalg/resistance.py @@ -6,6 +6,8 @@ """ from __future__ import annotations +from contextlib import suppress + import networkx as nx import numpy as np from numpy import linalg as la @@ -71,10 +73,9 @@ def resistance_matrix(A, check_connected=True): "Graph is not connected. " "Resistance matrix is undefined." ) L = laplacian_matrix(A) - try: + with suppress(Exception): L = L.todense() - except: - pass + M = la.pinv(L) # calculate R in terms of M d = np.reshape(np.diag(M), (n, 1)) diff --git a/pyproject.toml b/pyproject.toml index 3672ccca..8c879fe2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,13 +92,14 @@ disallow_untyped_defs = false [tool.pytest.ini_options] addopts = "-v -p no:warnings --cov=swmmanywhere --cov-report=html --doctest-modules --ignore=swmmanywhere/logging.py" -[tool.ruff] +[tool.ruff.lint] select = ["D", "E", "F", "I"] # pydocstyle, pycodestyle, Pyflakes, isort -[tool.ruff.per-file-ignores] +[tool.ruff.lint.per-file-ignores] "tests/*" = ["D100", "D104"] +"netcomp/*" = ["D", "F"] # Ignore all checks for netcomp -[tool.ruff.pydocstyle] +[tool.ruff.lint.pydocstyle] convention = "google" [tool.ruff.lint.isort] From 580b4f8b8715187f6f52c16ab119a05db5996283 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Thu, 19 Sep 2024 13:34:20 +0100 Subject: [PATCH 11/28] :rotating_light: Fix markdownlint issues. --- netcomp/README.md | 6 ++++-- swmmanywhere/metric_utilities.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/netcomp/README.md b/netcomp/README.md index 6f2e6964..e4e18526 100644 --- a/netcomp/README.md +++ b/netcomp/README.md @@ -1,3 +1,5 @@ -See https://github.com/barneydobson/NetComp for more details on NetComp. +# NetComp -NetComp is distributed under a MIT license \ No newline at end of file +See [original repo](https://github.com/barneydobson/NetComp) for more details on NetComp. + +NetComp is distributed under a MIT license diff --git a/swmmanywhere/metric_utilities.py b/swmmanywhere/metric_utilities.py index 0e6d57ae..b469d625 100644 --- a/swmmanywhere/metric_utilities.py +++ b/swmmanywhere/metric_utilities.py @@ -12,13 +12,13 @@ import cytoolz.curried as tlz import geopandas as gpd import joblib -import netcomp import networkx as nx import numpy as np import pandas as pd import shapely from scipy import stats +import netcomp from swmmanywhere.logging import logger from swmmanywhere.parameters import MetricEvaluation From 45f9bd1804d8cf654d6ed74de2366bce0b0167fa Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Thu, 19 Sep 2024 14:00:44 +0100 Subject: [PATCH 12/28] :bug: Import things right (or at least, better) --- netcomp/__init__.py | 3 +-- netcomp/distance/__init__.py | 4 ++-- netcomp/distance/exact.py | 12 +++++------- netcomp/distance/features.py | 2 +- netcomp/linalg/__init__.py | 8 ++++---- netcomp/linalg/eigenstuff.py | 2 +- netcomp/linalg/resistance.py | 4 ++-- 7 files changed, 16 insertions(+), 19 deletions(-) diff --git a/netcomp/__init__.py b/netcomp/__init__.py index 2daef3df..700fa492 100644 --- a/netcomp/__init__.py +++ b/netcomp/__init__.py @@ -21,6 +21,5 @@ raise ImportError(m % sys.version_info[:2]) del sys - -from .distance import * # noqa from .linalg import * # noqa +from .distance import * # noqa diff --git a/netcomp/distance/__init__.py b/netcomp/distance/__init__.py index 5136643d..1ae92672 100644 --- a/netcomp/distance/__init__.py +++ b/netcomp/distance/__init__.py @@ -6,5 +6,5 @@ """ from __future__ import annotations -from netcomp.distance.exact import * -from netcomp.distance.features import * +from .exact import * +from .features import * diff --git a/netcomp/distance/exact.py b/netcomp/distance/exact.py index 5e851180..6ca7f969 100644 --- a/netcomp/distance/exact.py +++ b/netcomp/distance/exact.py @@ -12,18 +12,16 @@ import numpy as np from numpy import linalg as la -from netcomp.distance import aggregate_features, get_features -from netcomp.exception import InputError -from netcomp.linalg import ( # type: ignore - _eigs, - _flat, - _pad, +from ..exception import InputError +from ..linalg import ( # type: ignore fast_bp, - laplacian_matrix, normalized_laplacian_eig, renormalized_res_mat, resistance_matrix, ) +from ..linalg.eigenstuff import _eigs, _flat +from ..linalg.matrices import _pad, laplacian_matrix +from .features import aggregate_features, get_features ###################### ## Helper Functions ## diff --git a/netcomp/distance/features.py b/netcomp/distance/features.py index c237303d..57ba3d9c 100644 --- a/netcomp/distance/features.py +++ b/netcomp/distance/features.py @@ -10,7 +10,7 @@ import numpy as np from scipy import stats -from netcomp.linalg import _eps # type: ignore +from ..linalg.matrices import _eps # type: ignore def get_features(A): diff --git a/netcomp/linalg/__init__.py b/netcomp/linalg/__init__.py index 7e8fadbb..8cfe75da 100644 --- a/netcomp/linalg/__init__.py +++ b/netcomp/linalg/__init__.py @@ -6,9 +6,9 @@ """ from __future__ import annotations -from netcomp.linalg.eigenstuff import * -from netcomp.linalg.fast_bp import * -from netcomp.linalg.matrices import * +from .eigenstuff import * +from .fast_bp import * +from .matrices import * # import helper functions for use in other places -from netcomp.linalg.resistance import * +from .resistance import * diff --git a/netcomp/linalg/eigenstuff.py b/netcomp/linalg/eigenstuff.py index 8d4b3dd0..c6ddf57b 100644 --- a/netcomp/linalg/eigenstuff.py +++ b/netcomp/linalg/eigenstuff.py @@ -15,7 +15,7 @@ from scipy.sparse import issparse from scipy.sparse import linalg as spla -from netcomp.linalg.matrices import _eps, _flat +from .matrices import _eps, _flat ###################### ## Helper Functions ## diff --git a/netcomp/linalg/resistance.py b/netcomp/linalg/resistance.py index e58cd6b2..e24c2265 100644 --- a/netcomp/linalg/resistance.py +++ b/netcomp/linalg/resistance.py @@ -14,8 +14,8 @@ from scipy import linalg as spla from scipy.sparse import issparse -from netcomp.exception import UndefinedException -from netcomp.linalg.matrices import laplacian_matrix +from ..exception import UndefinedException +from .matrices import laplacian_matrix def resistance_matrix(A, check_connected=True): From 04016fde73d0ec268c61253d98e85810402c82dc Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Thu, 19 Sep 2024 16:49:18 +0100 Subject: [PATCH 13/28] Remove requirements files --- .github/workflows/ci.yml | 2 +- .github/workflows/publish.yml | 2 +- dev-requirements.txt | 299 ---------------------- doc-requirements.txt | 459 ---------------------------------- pyproject.toml | 4 +- requirements.txt | 235 ----------------- 6 files changed, 4 insertions(+), 997 deletions(-) delete mode 100644 dev-requirements.txt delete mode 100644 doc-requirements.txt delete mode 100644 requirements.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 86629b0d..9450eadb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,6 +23,6 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install dependencies - run: pip install -r dev-requirements.txt + run: pip install .[dev] - name: Run tests run: pytest \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5d5d1933..658f92e3 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -78,7 +78,7 @@ jobs: python-version: "3.11" - name: Install dependencies - run: pip install -r doc-requirements.txt + run: pip install .[doc] - name: Deploy Docs run: mkdocs gh-deploy --force diff --git a/dev-requirements.txt b/dev-requirements.txt deleted file mode 100644 index c101accf..00000000 --- a/dev-requirements.txt +++ /dev/null @@ -1,299 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.12 -# by the following command: -# -# pip-compile --extra=dev --output-file=dev-requirements.txt -# -aenum==3.1.11 - # via - # pyswmm - # swmm-toolkit -affine==2.4.0 - # via - # pyflwdir - # rasterio -annotated-types==0.7.0 - # via pydantic -attrs==24.2.0 - # via - # cads-api-client - # fiona - # jsonschema - # pytest-mypy - # rasterio - # referencing -build==1.2.2 - # via pip-tools -cads-api-client==1.3.2 - # via cdsapi -cdsapi==0.7.3 - # via swmmanywhere (pyproject.toml) -certifi==2024.8.30 - # via - # fiona - # netcdf4 - # pyproj - # rasterio - # requests -cfgv==3.4.0 - # via pre-commit -cftime==1.6.4 - # via netcdf4 -charset-normalizer==3.3.2 - # via requests -click==8.1.7 - # via - # click-plugins - # cligj - # fiona - # pip-tools - # planetary-computer - # rasterio -click-plugins==1.1.1 - # via - # fiona - # rasterio -cligj==0.7.2 - # via - # fiona - # rasterio -colorama==0.4.6 - # via - # build - # click - # loguru - # pytest - # tqdm -coverage[toml]==7.6.1 - # via pytest-cov -cytoolz==0.12.3 - # via swmmanywhere (pyproject.toml) -distlib==0.3.8 - # via virtualenv -filelock==3.16.1 - # via - # pytest-mypy - # virtualenv -fiona==1.10.1 - # via geopandas -geographiclib==2.0 - # via geopy -geopandas==0.14.4 - # via - # osmnx - # swmmanywhere (pyproject.toml) -geopy==2.4.1 - # via swmmanywhere (pyproject.toml) -identify==2.6.1 - # via pre-commit -idna==3.10 - # via requests -iniconfig==2.0.0 - # via pytest -joblib==1.4.2 - # via swmmanywhere (pyproject.toml) -jsonschema==4.23.0 - # via - # pystac - # swmmanywhere (pyproject.toml) -jsonschema-specifications==2023.12.1 - # via jsonschema -julian==0.14 - # via pyswmm -llvmlite==0.43.0 - # via numba -loguru==0.7.2 - # via swmmanywhere (pyproject.toml) -multiurl==0.3.1 - # via cads-api-client -mypy==1.11.2 - # via - # pytest-mypy - # swmmanywhere (pyproject.toml) -mypy-extensions==1.0.0 - # via mypy -netcdf4==1.7.1.post2 - # via swmmanywhere (pyproject.toml) -networkx==3.3 - # via - # osmnx - # swmmanywhere (pyproject.toml) -nodeenv==1.9.1 - # via pre-commit -numba==0.60.0 - # via pyflwdir -numpy==1.26.4 - # via - # cftime - # geopandas - # netcdf4 - # numba - # osmnx - # pandas - # pyarrow - # pyflwdir - # rasterio - # rioxarray - # scipy - # shapely - # snuggs - # swmmanywhere (pyproject.toml) - # xarray -osmnx==1.9.4 - # via swmmanywhere (pyproject.toml) -packaging==24.1 - # via - # build - # geopandas - # planetary-computer - # pyswmm - # pytest - # rioxarray - # xarray -pandas==2.2.2 - # via - # geopandas - # osmnx - # swmmanywhere (pyproject.toml) - # xarray -pip-tools==7.4.1 - # via swmmanywhere (pyproject.toml) -planetary-computer==1.0.0 - # via swmmanywhere (pyproject.toml) -platformdirs==4.3.6 - # via virtualenv -pluggy==1.5.0 - # via pytest -pre-commit==3.8.0 - # via swmmanywhere (pyproject.toml) -pyarrow==17.0.0 - # via swmmanywhere (pyproject.toml) -pydantic==2.9.2 - # via - # planetary-computer - # swmmanywhere (pyproject.toml) -pydantic-core==2.23.4 - # via pydantic -pyflwdir==0.5.8 - # via swmmanywhere (pyproject.toml) -pyparsing==3.1.4 - # via snuggs -pyproj==3.6.1 - # via - # geopandas - # rioxarray -pyproject-hooks==1.1.0 - # via - # build - # pip-tools -pystac[validation]==1.10.1 - # via - # planetary-computer - # pystac-client -pystac-client==0.8.3 - # via - # planetary-computer - # swmmanywhere (pyproject.toml) -pyswmm==2.0.1 - # via swmmanywhere (pyproject.toml) -pytest==8.3.3 - # via - # pytest-cov - # pytest-mock - # pytest-mypy - # swmmanywhere (pyproject.toml) -pytest-cov==5.0.0 - # via swmmanywhere (pyproject.toml) -pytest-mock==3.14.0 - # via swmmanywhere (pyproject.toml) -pytest-mypy==0.10.3 - # via swmmanywhere (pyproject.toml) -python-dateutil==2.9.0.post0 - # via - # multiurl - # pandas - # pystac - # pystac-client -python-dotenv==1.0.1 - # via planetary-computer -pytz==2024.2 - # via - # multiurl - # pandas - # planetary-computer -pywbt==0.1.1 - # via swmmanywhere (pyproject.toml) -pyyaml==6.0.2 - # via - # pre-commit - # swmmanywhere (pyproject.toml) -rasterio==1.3.11 - # via - # rioxarray - # swmmanywhere (pyproject.toml) -referencing==0.35.1 - # via - # jsonschema - # jsonschema-specifications -requests==2.32.3 - # via - # cads-api-client - # cdsapi - # multiurl - # osmnx - # planetary-computer - # pystac-client -rioxarray==0.17.0 - # via swmmanywhere (pyproject.toml) -rpds-py==0.20.0 - # via - # jsonschema - # referencing -ruff==0.6.5 - # via swmmanywhere (pyproject.toml) -scipy==1.14.1 - # via - # pyflwdir - # swmmanywhere (pyproject.toml) -shapely==2.0.6 - # via - # geopandas - # osmnx - # swmmanywhere (pyproject.toml) -six==1.16.0 - # via python-dateutil -snuggs==1.4.7 - # via rasterio -swmm-toolkit==0.15.5 - # via pyswmm -toolz==0.12.1 - # via cytoolz -tqdm==4.66.5 - # via - # cdsapi - # multiurl - # swmmanywhere (pyproject.toml) -typing-extensions==4.12.2 - # via - # cads-api-client - # mypy - # pydantic - # pydantic-core -tzdata==2024.1 - # via pandas -urllib3==2.2.3 - # via requests -virtualenv==20.26.5 - # via pre-commit -wheel==0.44.0 - # via pip-tools -win32-setctime==1.1.0 - # via loguru -xarray==2024.9.0 - # via - # rioxarray - # swmmanywhere (pyproject.toml) - -# The following packages are considered to be unsafe in a requirements file: -# pip -# setuptools diff --git a/doc-requirements.txt b/doc-requirements.txt deleted file mode 100644 index 681d632d..00000000 --- a/doc-requirements.txt +++ /dev/null @@ -1,459 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.12 -# by the following command: -# -# pip-compile --extra=doc --output-file=doc-requirements.txt -# -aenum==3.1.11 - # via - # pyswmm - # swmm-toolkit -affine==2.4.0 - # via - # pyflwdir - # rasterio -annotated-types==0.7.0 - # via pydantic -asttokens==2.4.1 - # via stack-data -attrs==24.2.0 - # via - # cads-api-client - # fiona - # jsonschema - # rasterio - # referencing -babel==2.16.0 - # via mkdocs-material -beautifulsoup4==4.12.3 - # via nbconvert -bleach==6.1.0 - # via nbconvert -bracex==2.5 - # via wcmatch -cads-api-client==1.3.2 - # via cdsapi -cdsapi==0.7.3 - # via swmmanywhere (pyproject.toml) -certifi==2024.8.30 - # via - # fiona - # netcdf4 - # pyproj - # rasterio - # requests -cftime==1.6.4 - # via netcdf4 -charset-normalizer==3.3.2 - # via requests -click==8.1.7 - # via - # click-plugins - # cligj - # fiona - # mkdocs - # mkdocstrings - # planetary-computer - # rasterio -click-plugins==1.1.1 - # via - # fiona - # rasterio -cligj==0.7.2 - # via - # fiona - # rasterio -colorama==0.4.6 - # via - # click - # griffe - # ipython - # loguru - # mkdocs - # mkdocs-material - # tqdm -comm==0.2.2 - # via ipykernel -cytoolz==0.12.3 - # via swmmanywhere (pyproject.toml) -debugpy==1.8.5 - # via ipykernel -decorator==5.1.1 - # via ipython -defusedxml==0.7.1 - # via nbconvert -executing==2.1.0 - # via stack-data -fastjsonschema==2.20.0 - # via nbformat -fiona==1.10.1 - # via geopandas -geographiclib==2.0 - # via geopy -geopandas==0.14.4 - # via - # osmnx - # swmmanywhere (pyproject.toml) -geopy==2.4.1 - # via swmmanywhere (pyproject.toml) -ghp-import==2.1.0 - # via mkdocs -griffe==1.3.1 - # via mkdocstrings-python -idna==3.10 - # via requests -ipykernel==6.29.5 - # via mkdocs-jupyter -ipython==8.27.0 - # via ipykernel -jedi==0.19.1 - # via ipython -jinja2==3.1.4 - # via - # mkdocs - # mkdocs-material - # mkdocstrings - # nbconvert -joblib==1.4.2 - # via swmmanywhere (pyproject.toml) -jsonschema==4.23.0 - # via - # nbformat - # pystac - # swmmanywhere (pyproject.toml) -jsonschema-specifications==2023.12.1 - # via jsonschema -julian==0.14 - # via pyswmm -jupyter-client==8.6.3 - # via - # ipykernel - # nbclient -jupyter-core==5.7.2 - # via - # ipykernel - # jupyter-client - # nbclient - # nbconvert - # nbformat -jupyterlab-pygments==0.3.0 - # via nbconvert -jupytext==1.16.4 - # via mkdocs-jupyter -llvmlite==0.43.0 - # via numba -loguru==0.7.2 - # via swmmanywhere (pyproject.toml) -markdown==3.7 - # via - # mkdocs - # mkdocs-autorefs - # mkdocs-material - # mkdocstrings - # pymdown-extensions -markdown-it-py==3.0.0 - # via - # jupytext - # mdit-py-plugins -markupsafe==2.1.5 - # via - # jinja2 - # mkdocs - # mkdocs-autorefs - # mkdocstrings - # nbconvert -matplotlib-inline==0.1.7 - # via - # ipykernel - # ipython -mdit-py-plugins==0.4.2 - # via jupytext -mdurl==0.1.2 - # via markdown-it-py -mergedeep==1.3.4 - # via - # mkdocs - # mkdocs-get-deps -mistune==3.0.2 - # via nbconvert -mkdocs==1.6.1 - # via - # mkdocs-autorefs - # mkdocs-coverage - # mkdocs-include-markdown-plugin - # mkdocs-jupyter - # mkdocs-material - # mkdocstrings - # swmmanywhere (pyproject.toml) -mkdocs-autorefs==1.2.0 - # via - # mkdocstrings - # mkdocstrings-python -mkdocs-coverage==1.1.0 - # via swmmanywhere (pyproject.toml) -mkdocs-get-deps==0.2.0 - # via mkdocs -mkdocs-include-markdown-plugin==6.2.2 - # via swmmanywhere (pyproject.toml) -mkdocs-jupyter==0.25.0 - # via swmmanywhere (pyproject.toml) -mkdocs-material==9.5.35 - # via - # mkdocs-jupyter - # swmmanywhere (pyproject.toml) -mkdocs-material-extensions==1.3.1 - # via - # mkdocs-material - # swmmanywhere (pyproject.toml) -mkdocstrings[python]==0.26.1 - # via - # mkdocstrings-python - # swmmanywhere (pyproject.toml) -mkdocstrings-python==1.11.1 - # via mkdocstrings -multiurl==0.3.1 - # via cads-api-client -nbclient==0.10.0 - # via nbconvert -nbconvert==7.16.4 - # via mkdocs-jupyter -nbformat==5.10.4 - # via - # jupytext - # nbclient - # nbconvert -nest-asyncio==1.6.0 - # via ipykernel -netcdf4==1.7.1.post2 - # via swmmanywhere (pyproject.toml) -networkx==3.3 - # via - # osmnx - # swmmanywhere (pyproject.toml) -numba==0.60.0 - # via pyflwdir -numpy==1.26.4 - # via - # cftime - # geopandas - # netcdf4 - # numba - # osmnx - # pandas - # pyarrow - # pyflwdir - # rasterio - # rioxarray - # scipy - # shapely - # snuggs - # swmmanywhere (pyproject.toml) - # xarray -osmnx==1.9.4 - # via swmmanywhere (pyproject.toml) -packaging==24.1 - # via - # geopandas - # ipykernel - # jupytext - # mkdocs - # nbconvert - # planetary-computer - # pyswmm - # rioxarray - # xarray -paginate==0.5.7 - # via mkdocs-material -pandas==2.2.2 - # via - # geopandas - # osmnx - # swmmanywhere (pyproject.toml) - # xarray -pandocfilters==1.5.1 - # via nbconvert -parso==0.8.4 - # via jedi -pathspec==0.12.1 - # via mkdocs -planetary-computer==1.0.0 - # via swmmanywhere (pyproject.toml) -platformdirs==4.3.6 - # via - # jupyter-core - # mkdocs-get-deps - # mkdocstrings -prompt-toolkit==3.0.47 - # via ipython -psutil==6.0.0 - # via ipykernel -pure-eval==0.2.3 - # via stack-data -pyarrow==17.0.0 - # via swmmanywhere (pyproject.toml) -pydantic==2.9.2 - # via - # planetary-computer - # swmmanywhere (pyproject.toml) -pydantic-core==2.23.4 - # via pydantic -pyflwdir==0.5.8 - # via swmmanywhere (pyproject.toml) -pygments==2.18.0 - # via - # ipython - # mkdocs-jupyter - # mkdocs-material - # nbconvert -pymdown-extensions==10.9 - # via - # mkdocs-material - # mkdocstrings -pyparsing==3.1.4 - # via snuggs -pyproj==3.6.1 - # via - # geopandas - # rioxarray -pystac[validation]==1.10.1 - # via - # planetary-computer - # pystac-client -pystac-client==0.8.3 - # via - # planetary-computer - # swmmanywhere (pyproject.toml) -pyswmm==2.0.1 - # via swmmanywhere (pyproject.toml) -python-dateutil==2.9.0.post0 - # via - # ghp-import - # jupyter-client - # multiurl - # pandas - # pystac - # pystac-client -python-dotenv==1.0.1 - # via planetary-computer -pytz==2024.2 - # via - # multiurl - # pandas - # planetary-computer -pywbt==0.1.1 - # via swmmanywhere (pyproject.toml) -pywin32==306 - # via jupyter-core -pyyaml==6.0.2 - # via - # jupytext - # mkdocs - # mkdocs-get-deps - # pymdown-extensions - # pyyaml-env-tag - # swmmanywhere (pyproject.toml) -pyyaml-env-tag==0.1 - # via mkdocs -pyzmq==26.2.0 - # via - # ipykernel - # jupyter-client -rasterio==1.3.11 - # via - # rioxarray - # swmmanywhere (pyproject.toml) -referencing==0.35.1 - # via - # jsonschema - # jsonschema-specifications -regex==2024.9.11 - # via mkdocs-material -requests==2.32.3 - # via - # cads-api-client - # cdsapi - # mkdocs-material - # multiurl - # osmnx - # planetary-computer - # pystac-client -rioxarray==0.17.0 - # via swmmanywhere (pyproject.toml) -rpds-py==0.20.0 - # via - # jsonschema - # referencing -scipy==1.14.1 - # via - # pyflwdir - # swmmanywhere (pyproject.toml) -shapely==2.0.6 - # via - # geopandas - # osmnx - # swmmanywhere (pyproject.toml) -six==1.16.0 - # via - # asttokens - # bleach - # python-dateutil -snuggs==1.4.7 - # via rasterio -soupsieve==2.6 - # via beautifulsoup4 -stack-data==0.6.3 - # via ipython -swmm-toolkit==0.15.5 - # via pyswmm -tinycss2==1.3.0 - # via nbconvert -toolz==0.12.1 - # via cytoolz -tornado==6.4.1 - # via - # ipykernel - # jupyter-client -tqdm==4.66.5 - # via - # cdsapi - # multiurl - # swmmanywhere (pyproject.toml) -traitlets==5.14.3 - # via - # comm - # ipykernel - # ipython - # jupyter-client - # jupyter-core - # matplotlib-inline - # nbclient - # nbconvert - # nbformat -typing-extensions==4.12.2 - # via - # cads-api-client - # pydantic - # pydantic-core -tzdata==2024.1 - # via pandas -urllib3==2.2.3 - # via requests -watchdog==5.0.2 - # via mkdocs -wcmatch==9.0 - # via mkdocs-include-markdown-plugin -wcwidth==0.2.13 - # via prompt-toolkit -webencodings==0.5.1 - # via - # bleach - # tinycss2 -win32-setctime==1.1.0 - # via loguru -xarray==2024.9.0 - # via - # rioxarray - # swmmanywhere (pyproject.toml) - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/pyproject.toml b/pyproject.toml index 8c879fe2..70943c31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ dependencies = [ "cdsapi", "cytoolz", - "geopandas", + "geopandas>=1", "geopy", "joblib", "jsonschema", @@ -32,7 +32,7 @@ dependencies = [ "netcdf4", "networkx>=3", "numpy", - "osmnx", + "osmnx>=1.9.3", "pandas", "planetary_computer", "pyarrow", diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 8e8f393d..00000000 --- a/requirements.txt +++ /dev/null @@ -1,235 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.12 -# by the following command: -# -# pip-compile -# -aenum==3.1.11 - # via - # pyswmm - # swmm-toolkit -affine==2.4.0 - # via - # pyflwdir - # rasterio -annotated-types==0.7.0 - # via pydantic -attrs==24.2.0 - # via - # cads-api-client - # fiona - # jsonschema - # rasterio - # referencing -cads-api-client==1.3.2 - # via cdsapi -cdsapi==0.7.3 - # via swmmanywhere (pyproject.toml) -certifi==2024.8.30 - # via - # fiona - # netcdf4 - # pyproj - # rasterio - # requests -cftime==1.6.4 - # via netcdf4 -charset-normalizer==3.3.2 - # via requests -click==8.1.7 - # via - # click-plugins - # cligj - # fiona - # planetary-computer - # rasterio -click-plugins==1.1.1 - # via - # fiona - # rasterio -cligj==0.7.2 - # via - # fiona - # rasterio -colorama==0.4.6 - # via - # click - # loguru - # tqdm -cytoolz==0.12.3 - # via swmmanywhere (pyproject.toml) -fiona==1.10.1 - # via geopandas -geographiclib==2.0 - # via geopy -geopandas==0.14.4 - # via - # osmnx - # swmmanywhere (pyproject.toml) -geopy==2.4.1 - # via swmmanywhere (pyproject.toml) -idna==3.10 - # via requests -joblib==1.4.2 - # via swmmanywhere (pyproject.toml) -jsonschema==4.23.0 - # via - # pystac - # swmmanywhere (pyproject.toml) -jsonschema-specifications==2023.12.1 - # via jsonschema -julian==0.14 - # via pyswmm -llvmlite==0.43.0 - # via numba -loguru==0.7.2 - # via swmmanywhere (pyproject.toml) -multiurl==0.3.1 - # via cads-api-client -netcdf4==1.7.1.post2 - # via swmmanywhere (pyproject.toml) -networkx==3.3 - # via - # osmnx - # swmmanywhere (pyproject.toml) -numba==0.60.0 - # via pyflwdir -numpy==1.26.4 - # via - # cftime - # geopandas - # netcdf4 - # numba - # osmnx - # pandas - # pyarrow - # pyflwdir - # rasterio - # rioxarray - # scipy - # shapely - # snuggs - # swmmanywhere (pyproject.toml) - # xarray -osmnx==1.9.4 - # via swmmanywhere (pyproject.toml) -packaging==24.1 - # via - # geopandas - # planetary-computer - # pyswmm - # rioxarray - # xarray -pandas==2.2.2 - # via - # geopandas - # osmnx - # swmmanywhere (pyproject.toml) - # xarray -planetary-computer==1.0.0 - # via swmmanywhere (pyproject.toml) -pyarrow==17.0.0 - # via swmmanywhere (pyproject.toml) -pydantic==2.9.2 - # via - # planetary-computer - # swmmanywhere (pyproject.toml) -pydantic-core==2.23.4 - # via pydantic -pyflwdir==0.5.8 - # via swmmanywhere (pyproject.toml) -pyparsing==3.1.4 - # via snuggs -pyproj==3.6.1 - # via - # geopandas - # rioxarray -pystac[validation]==1.10.1 - # via - # planetary-computer - # pystac-client -pystac-client==0.8.3 - # via - # planetary-computer - # swmmanywhere (pyproject.toml) -pyswmm==2.0.1 - # via swmmanywhere (pyproject.toml) -python-dateutil==2.9.0.post0 - # via - # multiurl - # pandas - # pystac - # pystac-client -python-dotenv==1.0.1 - # via planetary-computer -pytz==2024.2 - # via - # multiurl - # pandas - # planetary-computer -pywbt==0.1.1 - # via swmmanywhere (pyproject.toml) -pyyaml==6.0.2 - # via swmmanywhere (pyproject.toml) -rasterio==1.3.11 - # via - # rioxarray - # swmmanywhere (pyproject.toml) -referencing==0.35.1 - # via - # jsonschema - # jsonschema-specifications -requests==2.32.3 - # via - # cads-api-client - # cdsapi - # multiurl - # osmnx - # planetary-computer - # pystac-client -rioxarray==0.17.0 - # via swmmanywhere (pyproject.toml) -rpds-py==0.20.0 - # via - # jsonschema - # referencing -scipy==1.14.1 - # via - # pyflwdir - # swmmanywhere (pyproject.toml) -shapely==2.0.6 - # via - # geopandas - # osmnx - # swmmanywhere (pyproject.toml) -six==1.16.0 - # via python-dateutil -snuggs==1.4.7 - # via rasterio -swmm-toolkit==0.15.5 - # via pyswmm -toolz==0.12.1 - # via cytoolz -tqdm==4.66.5 - # via - # cdsapi - # multiurl - # swmmanywhere (pyproject.toml) -typing-extensions==4.12.2 - # via - # cads-api-client - # pydantic - # pydantic-core -tzdata==2024.1 - # via pandas -urllib3==2.2.3 - # via requests -win32-setctime==1.1.0 - # via loguru -xarray==2024.9.0 - # via - # rioxarray - # swmmanywhere (pyproject.toml) - -# The following packages are considered to be unsafe in a requirements file: -# setuptools From 799a1a9da314d23fd3baf6985cfd942adfe536d4 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Thu, 19 Sep 2024 16:51:03 +0100 Subject: [PATCH 14/28] Gitignore a file --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b17a5bb4..1342093d 100644 --- a/.gitignore +++ b/.gitignore @@ -132,4 +132,5 @@ dmypy.json cache/ # Documentation generated models -swmmanywhere_models/ \ No newline at end of file +swmmanywhere_models/ +whiteboxtools_binaries.zip \ No newline at end of file From f944d9113eaaea32d1cce9bb55370ac302ec2315 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Thu, 19 Sep 2024 16:52:15 +0100 Subject: [PATCH 15/28] Pin numpy>=2 --- pyproject.toml | 2 +- swmmanywhere/graphfcns/topology_graphfcns.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 70943c31..5cda4e10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ "loguru", "netcdf4", "networkx>=3", - "numpy", + "numpy>=2", "osmnx>=1.9.3", "pandas", "planetary_computer", diff --git a/swmmanywhere/graphfcns/topology_graphfcns.py b/swmmanywhere/graphfcns/topology_graphfcns.py index 292dbdfc..b3ab4c3d 100644 --- a/swmmanywhere/graphfcns/topology_graphfcns.py +++ b/swmmanywhere/graphfcns/topology_graphfcns.py @@ -209,7 +209,7 @@ def __call__( G (nx.Graph): A graph """ # Calculate bounds to normalise between - bounds: Dict[Any, List[float]] = defaultdict(lambda: [np.Inf, -np.Inf]) + bounds: Dict[Any, List[float]] = defaultdict(lambda: [np.inf, -np.inf]) for w in topology_derivation.weights: bounds[w][0] = min(nx.get_edge_attributes(G, w).values()) # lower bound From 1eb2656dc408565d3c32801ed60cf10107872884 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Thu, 19 Sep 2024 16:54:53 +0100 Subject: [PATCH 16/28] Fix deprecation warning --- swmmanywhere/post_processing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/swmmanywhere/post_processing.py b/swmmanywhere/post_processing.py index 8c95271b..2c78e1fb 100644 --- a/swmmanywhere/post_processing.py +++ b/swmmanywhere/post_processing.py @@ -433,7 +433,8 @@ def _fill_backslash_columns( # Extract SWMM order and default values columns = conversion_dict[key]["iwcolumns"] - shp = shp.fillna(0) + numeric_cols = shp.select_dtypes(include=[np.number]).columns + shp[numeric_cols] = shp[numeric_cols].fillna(0) # Find columns with a default specified cols_default = [c[1:] for c in columns if c.startswith("/")] From ad9c1304a4a0fb7dfd393bf2b1ed4e15d594c081 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Thu, 19 Sep 2024 16:58:35 +0100 Subject: [PATCH 17/28] Pin pywbt>=0.2.2 and fix code --- pyproject.toml | 2 +- swmmanywhere/geospatial_utilities.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5cda4e10..34004794 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ dependencies = [ "pyflwdir", "pystac_client", "pyswmm", - "pywbt<=0.1.1", + "pywbt>=0.2.2", "PyYAML", "rasterio", "rioxarray", diff --git a/swmmanywhere/geospatial_utilities.py b/swmmanywhere/geospatial_utilities.py index c990d970..c5c83ecd 100644 --- a/swmmanywhere/geospatial_utilities.py +++ b/swmmanywhere/geospatial_utilities.py @@ -642,10 +642,12 @@ def flwdir_whitebox(fid: Path) -> np.array: "D8Pointer": ["-i=dem_corr.tif", "-o=fdir.tif"], } whitebox_tools( + temp_path, wbt_args, - work_dir=temp_path, + save_dir=temp_path, verbose=verbose(), wbt_root=temp_path / "WBT", + zip_path=fid.parent / "whiteboxtools_binaries.zip", max_procs=1, ) From 51b574479e78111bcec5296c2dc5bffb0e374386 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Fri, 20 Sep 2024 06:26:40 +0100 Subject: [PATCH 18/28] Add more classifiers and hatch-vcs --- .gitignore | 5 ++++- pyproject.toml | 48 ++++++++++++++++++++++++++-------------- swmmanywhere/__init__.py | 7 +++++- 3 files changed, 41 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index 1342093d..f9cabf09 100644 --- a/.gitignore +++ b/.gitignore @@ -133,4 +133,7 @@ cache/ # Documentation generated models swmmanywhere_models/ -whiteboxtools_binaries.zip \ No newline at end of file +whiteboxtools_binaries.zip + +# Hatch +_version.py \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 34004794..20d6609d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,25 +1,33 @@ [build-system] build-backend = "hatchling.build" requires = [ + "hatch-vcs", "hatchling", ] -[tool.setuptools.packages.find] -exclude = ["htmlcov"] # Exclude the coverage report file from setuptools package finder - [project] name = "swmmanywhere" -version = "0.0.1" authors = [ { name = "Barnaby Dobson", email = "b.dobson@imperial.ac.uk" }, { name = "Imperial College London RSE Team", email = "ict-rse-team@imperial.ac.uk" } ] requires-python = ">=3.10" classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python", "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: GIS", + "Typing :: Typed", +] +dynamic = [ + "version", ] dependencies = [ "cdsapi", @@ -77,20 +85,11 @@ Source = "https://github.com/ImperialCollegeLondon/SWMManywhere" [tool.hatch.build.targets.sdist] only-include = ["swmmanywhere", "netcomp"] -[tool.mypy] -disallow_any_explicit = false -disallow_any_generics = false -warn_unreachable = true -warn_unused_ignores = false -disallow_untyped_defs = false -exclude = [".venv/"] +[tool.hatch.version] +source = "vcs" -[[tool.mypy.overrides]] -module = "tests.*" -disallow_untyped_defs = false - -[tool.pytest.ini_options] -addopts = "-v -p no:warnings --cov=swmmanywhere --cov-report=html --doctest-modules --ignore=swmmanywhere/logging.py" +[tool.hatch.build.hooks.vcs] +version-file = "_version.py" [tool.ruff.lint] select = ["D", "E", "F", "I"] # pydocstyle, pycodestyle, Pyflakes, isort @@ -109,6 +108,21 @@ required-imports = ["from __future__ import annotations"] skip = "swmmanywhere/defs/iso_converter.yml,*.inp" ignore-words-list = "gage,gages" +[tool.pytest.ini_options] +addopts = "-v -p no:warnings --cov=swmmanywhere --cov-report=html --doctest-modules --ignore=swmmanywhere/logging.py" + +[tool.mypy] +disallow_any_explicit = false +disallow_any_generics = false +warn_unreachable = true +warn_unused_ignores = false +disallow_untyped_defs = false +exclude = [".venv/"] + +[[tool.mypy.overrides]] +module = "tests.*" +disallow_untyped_defs = false + [tool.refurb] ignore = [ 184, # Because some frankly bizarre suggestions diff --git a/swmmanywhere/__init__.py b/swmmanywhere/__init__.py index 6da981e7..65cd1222 100644 --- a/swmmanywhere/__init__.py +++ b/swmmanywhere/__init__.py @@ -1,7 +1,12 @@ """The main module for MyProject.""" from __future__ import annotations -__version__ = "0.1.0" +from importlib.metadata import PackageNotFoundError, version + +try: + __version__ = version("swmmanywhere") +except PackageNotFoundError: + __version__ = "999" # Importing module to register the graphfcns and made them available from . import graphfcns # noqa: F401 From dcd42c36f14e3391350e00457bec01137fd41fa6 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Fri, 20 Sep 2024 06:30:50 +0100 Subject: [PATCH 19/28] Remove dummy test version --- tests/test_swmmanywhere.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/test_swmmanywhere.py b/tests/test_swmmanywhere.py index 4886beef..afa91d2a 100644 --- a/tests/test_swmmanywhere.py +++ b/tests/test_swmmanywhere.py @@ -9,16 +9,11 @@ import pytest import yaml -from swmmanywhere import __version__, swmmanywhere +from swmmanywhere import swmmanywhere from swmmanywhere.graph_utilities import graphfcns from swmmanywhere.metric_utilities import metrics -def test_version(): - """Check that the version is acceptable.""" - assert __version__ == "0.1.0" - - def test_run(): """Test the run function.""" demo_dir = Path(__file__).parent.parent / "swmmanywhere" / "defs" From 32a7cb6ef04eca89262fa46b50fb3d769800ccb6 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Fri, 20 Sep 2024 08:35:02 +0100 Subject: [PATCH 20/28] :construction_worker: Add inputs to workflow call --- .github/workflows/ci.yml | 32 ++++++++------------------ .github/workflows/ci_template.yml | 38 +++++++++++++++++++++++++++++++ .github/workflows/publish.yml | 5 +++- 3 files changed, 51 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/ci_template.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9450eadb..2bb92253 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,28 +1,14 @@ name: Test -on: [push,pull_request,workflow_call] +on: + push: + branches: + - main + pull_request: jobs: - qa: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: pre-commit/action@v3.0.1 - test: - needs: qa - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, windows-latest] - python-version: [ "3.10" ] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: pip install .[dev] - - name: Run tests - run: pytest \ No newline at end of file + uses: ./.github/workflows/ci_template.yml + with: + os: '["ubuntu-latest", "windows-latest", "macos-latest"]' + python-version: '["3.9", "3.10", "3.11", "3.12"]' \ No newline at end of file diff --git a/.github/workflows/ci_template.yml b/.github/workflows/ci_template.yml new file mode 100644 index 00000000..db9d3f70 --- /dev/null +++ b/.github/workflows/ci_template.yml @@ -0,0 +1,38 @@ +name: Test + +on: + workflow_call: + inputs: + os: + description: 'Operating system' + required: true + default: '["ubuntu-latest", "windows-latest"]' + python-version: + description: 'Python version' + required: true + default: '["3.10"]' + +jobs: + qa: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pre-commit/action@v3.0.1 + + test: + needs: qa + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ${{fromJson(inputs.os)}} + python-version: ${{fromJson(inputs.python-version)}} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: pip install .[dev] + - name: Run tests + run: pytest \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 658f92e3..2e3acf91 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -7,7 +7,10 @@ permissions: jobs: test: - uses: ./.github/workflows/ci.yml + uses: ./.github/workflows/ci_template.yml + with: + os: '["ubuntu-latest", "windows-latest", "macos-latest"]' + python-version: '["3.9", "3.10", "3.11", "3.12"]' build-wheel: needs: test From b8597492a0768b1bb1a2c10b18d67df676945bbb Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Fri, 20 Sep 2024 08:38:56 +0100 Subject: [PATCH 21/28] :construction_worker: Amend triggers --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2bb92253..e50bf999 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,8 +2,6 @@ name: Test on: push: - branches: - - main pull_request: jobs: From f3e9feb8a30160a3ed66599937f326715be126b9 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Fri, 20 Sep 2024 08:41:15 +0100 Subject: [PATCH 22/28] :construction_worker: Amend workflow file --- .github/workflows/ci.yml | 2 ++ .github/workflows/ci_template.yml | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e50bf999..2bb92253 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,6 +2,8 @@ name: Test on: push: + branches: + - main pull_request: jobs: diff --git a/.github/workflows/ci_template.yml b/.github/workflows/ci_template.yml index db9d3f70..839c0f0d 100644 --- a/.github/workflows/ci_template.yml +++ b/.github/workflows/ci_template.yml @@ -5,12 +5,12 @@ on: inputs: os: description: 'Operating system' - required: true default: '["ubuntu-latest", "windows-latest"]' + type: string python-version: description: 'Python version' - required: true default: '["3.10"]' + type: string jobs: qa: From 828d926728a2d1a0f19887d288f10a4aed48c201 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Fri, 20 Sep 2024 08:43:36 +0100 Subject: [PATCH 23/28] :construction_worker: Remove py3.9 from workflow --- .github/workflows/ci.yml | 5 +---- .github/workflows/publish.yml | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2bb92253..a8e57302 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,4 @@ on: jobs: test: - uses: ./.github/workflows/ci_template.yml - with: - os: '["ubuntu-latest", "windows-latest", "macos-latest"]' - python-version: '["3.9", "3.10", "3.11", "3.12"]' \ No newline at end of file + uses: ./.github/workflows/ci_template.yml \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2e3acf91..ec52321b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -10,7 +10,7 @@ jobs: uses: ./.github/workflows/ci_template.yml with: os: '["ubuntu-latest", "windows-latest", "macos-latest"]' - python-version: '["3.9", "3.10", "3.11", "3.12"]' + python-version: '["3.10", "3.11", "3.12"]' build-wheel: needs: test From 1b83f17ecfa5904123c26ba9fce7ebc281e12843 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Fri, 20 Sep 2024 08:44:44 +0100 Subject: [PATCH 24/28] :construction_worker: Rename workflows --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a8e57302..f98b771a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,3 @@ -name: Test - on: push: branches: From 9dbfaf80b2e9a3c86f38c8fa873ab0ea2e64e329 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Fri, 20 Sep 2024 09:24:03 +0100 Subject: [PATCH 25/28] :memo: Fix graphfncs_guide.md --- docs/graphfcns_guide.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/graphfcns_guide.md b/docs/graphfcns_guide.md index 61f214cb..1570344b 100644 --- a/docs/graphfcns_guide.md +++ b/docs/graphfcns_guide.md @@ -9,11 +9,11 @@ an updated graph. ## Using graph functions -Let's look at a [graph function](reference-graph-utilities.md#swmmanywhere.graph_utilities.to_undirected) +Let's look at a [graph function](reference-graph-utilities.md#swmmanywhere.graphfcns.network_cleaning_graphfcns.to_undirected) that is simply a wrapper for [`networkx.to_undirected`](https://networkx.org/documentation/stable/reference/classes/generated/networkx.DiGraph.to_undirected.html): -:::swmmanywhere.graph_utilities.to_undirected +:::swmmanywhere.graphfcns.network_cleaning_graphfcns.to_undirected handler: python options: members: no @@ -64,9 +64,9 @@ view the [`parameters`](reference-parameters.md) and [`FilePaths`](reference-filepaths.md) APIs. We can see an example of using a parameter category with this -[graph function](reference-graph-utilities.md#swmmanywhere.graph_utilities.remove_non_pipe_allowable_links): +[graph function](reference-graph-utilities.md#swmmanywhere.graphfcns.network_cleaning_graphfcns.remove_non_pipe_allowable_links): -:::swmmanywhere.graph_utilities.remove_non_pipe_allowable_links +:::swmmanywhere.graphfcns.network_cleaning_graphfcns.remove_non_pipe_allowable_links handler: python options: members: no @@ -138,9 +138,9 @@ are not provided, `iterate_graphfcns` uses the default values for all Furthermore, this `graphfcn_list` also provides opportunities for validation. For example, see the -[following graph function](reference-graph-utilities.md#swmmanywhere.graph_utilities.set_surface_slope): +[following graph function](reference-graph-utilities.md#swmmanywhere.graphfcns.topology_graphfcns.set_surface_slope): -:::swmmanywhere.graph_utilities.set_surface_slope +:::swmmanywhere.graphfcns.topology_graphfcns.set_surface_slope handler: python options: members: no @@ -168,7 +168,7 @@ can be used to specify what, if any, parameters are added to the graph by the graph function. Let us inspect the `set_elevation` graph function: -:::swmmanywhere.graph_utilities.set_elevation +:::swmmanywhere.graphfcns.topology_graphfcns.set_elevation handler: python options: members: no From 62e9d4adda764a90a39c8a4c83cc36e1b6d7db60 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Fri, 20 Sep 2024 09:27:43 +0100 Subject: [PATCH 26/28] :memo: Replace outlet with outfall in docs --- docs/metrics_guide.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/metrics_guide.md b/docs/metrics_guide.md index 7245a365..0c9c2305 100644 --- a/docs/metrics_guide.md +++ b/docs/metrics_guide.md @@ -39,15 +39,15 @@ registered metrics to be called from one place. ``` py >>> from swmmanywhere.metric_utilities import metrics >>> print(metrics.keys()) -dict_keys(['outlet_nse_flow', 'outlet_kge_flow', 'outlet_relerror_flow', -'outlet_relerror_length', 'outlet_relerror_npipes', 'outlet_relerror_nmanholes', -'outlet_relerror_diameter', 'outlet_nse_flooding', 'outlet_kge_flooding', -'outlet_relerror_flooding', 'grid_nse_flooding', 'grid_kge_flooding', +dict_keys(['outfall_nse_flow', 'outfall_kge_flow', 'outfall_relerror_flow', +'outfall_relerror_length', 'outfall_relerror_npipes', 'outfall_relerror_nmanholes', +'outfall_relerror_diameter', 'outfall_nse_flooding', 'outfall_kge_flooding', +'outfall_relerror_flooding', 'grid_nse_flooding', 'grid_kge_flooding', 'grid_relerror_flooding', 'subcatchment_nse_flooding', 'subcatchment_kge_flooding', 'subcatchment_relerror_flooding', 'nc_deltacon0', 'nc_laplacian_dist', 'nc_laplacian_norm_dist', 'nc_adjacency_dist', 'nc_vertex_edge_distance', 'nc_resistance_distance', 'bias_flood_depth', -'kstest_edge_betweenness', 'kstest_betweenness', 'outlet_kstest_diameters']) +'kstest_edge_betweenness', 'kstest_betweenness', 'outfall_kstest_diameters']) ``` We will later demonstrate how to [add a new metric](#add-a-new-metric) to the @@ -66,9 +66,9 @@ any `metric` has access to a range of arguments for calculation: - the [`MetricEvaluation`](reference-parameters.md#swmmanywhere.parameters.MetricEvaluation) parameters category. -For example, see the [following metric](reference-metric-utilities.md#swmmanywhere.metric_utilities.outlet_kstest_diameters) +For example, see the [following metric](reference-metric-utilities.md#swmmanywhere.metric_utilities.outfall_kstest_diameters) -:::swmmanywhere.metric_utilities.outlet_kstest_diameters +:::swmmanywhere.metric_utilities.outfall_kstest_diameters handler: python options: members: no @@ -184,7 +184,7 @@ us create a metric with `metric_factory`: ``` py >>> from swmmanywhere.metric_utilities import metric_factory ->>> metric_factory('outlet_nse_flow') +>>> metric_factory('outfall_nse_flow') .new_metric at 0x000001EECEA7C220> ``` @@ -243,7 +243,7 @@ in `metrics` we can also register that. ... metrics ... ) >>> import numpy as np ->>> metric_factory('outlet_rmse_flow') # Try creating the metric +>>> metric_factory('outfall_rmse_flow') # Try creating the metric Traceback (most recent call last): ... KeyError: 'rmse' @@ -253,9 +253,9 @@ KeyError: 'rmse' >>> print(coef_registry.keys()) dict_keys(['relerror', 'nse', 'kge', 'rmse']) ->>> metrics.register(metric_factory('outlet_rmse_flow')) # Create and register new metric +>>> metrics.register(metric_factory('outfall_rmse_flow')) # Create and register new metric .new_metric at 0x00000227D219E020> ->>> 'outlet_rmse_flow' in metrics # Check that the metric is available for use +>>> 'outfall_rmse_flow' in metrics # Check that the metric is available for use True ``` @@ -267,7 +267,7 @@ As with coefficients, these are stored in a registry. ``` py >>> from swmmanywhere.metric_utilities import scale_registry >>> print(scale_registry.keys()) -dict_keys(['subcatchment', 'grid', 'outlet']) +dict_keys(['subcatchment', 'grid', 'outfall']) ``` For example, `subcatchment` aligns `real` and `synthesised` subcatchments @@ -305,7 +305,7 @@ is a description of the designed UDM. ``` py >>> from swmmanywhere.metric_utilities import metric_factory ->>> metric_factory('outlet_nse_npipes') +>>> metric_factory('outfall_nse_npipes') Traceback (most recent call last): ... , in restriction_on_metric raise ValueError(f"Variable {variable} only valid with relerror metric") From 33203ddeda9454561d6a3e8832a4bdc8ba46c8cd Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Tue, 24 Sep 2024 15:25:43 +0100 Subject: [PATCH 27/28] :white_check_mark: Use importlib import mode in pytrst and not supress watnings. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 20d6609d..4c4735e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -109,7 +109,7 @@ skip = "swmmanywhere/defs/iso_converter.yml,*.inp" ignore-words-list = "gage,gages" [tool.pytest.ini_options] -addopts = "-v -p no:warnings --cov=swmmanywhere --cov-report=html --doctest-modules --ignore=swmmanywhere/logging.py" +addopts = "-v --import-mode=importlib --cov=swmmanywhere --cov-report=html --doctest-modules --ignore=swmmanywhere/logging.py" [tool.mypy] disallow_any_explicit = false From 3260a3d854d51ee4d7df8c04b1a41367fb7c8020 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Tue, 24 Sep 2024 15:47:41 +0100 Subject: [PATCH 28/28] Add download marker in pytest settings --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 4c4735e5..58115f86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -110,6 +110,9 @@ ignore-words-list = "gage,gages" [tool.pytest.ini_options] addopts = "-v --import-mode=importlib --cov=swmmanywhere --cov-report=html --doctest-modules --ignore=swmmanywhere/logging.py" +markers = [ + "downloads: mark a test as requiring downloads", +] [tool.mypy] disallow_any_explicit = false