From 34c65db037eab349e1a2a5c8b9d0d8772ff4ff45 Mon Sep 17 00:00:00 2001 From: John A Stevenson Date: Mon, 4 Mar 2024 18:59:55 +0000 Subject: [PATCH] Update FastAPI container and dependencies This commit also adds pip-tools for dependency management. The versions of FastAPI and related core libraries (starlette, pydantic) were set to match versions in the container. --- .github/workflows/lint_and_test.yml | 6 ++ Dockerfile | 2 +- README.md | 18 +++- requirements.in | 18 ++++ requirements.txt | 131 +++++++++++++++++++++++++--- requirements_dev.in | 9 ++ requirements_dev.txt | 114 ++++++++++++++++++++++-- 7 files changed, 272 insertions(+), 26 deletions(-) create mode 100644 requirements.in create mode 100644 requirements_dev.in diff --git a/.github/workflows/lint_and_test.yml b/.github/workflows/lint_and_test.yml index f26a4b6c..ed8d822f 100644 --- a/.github/workflows/lint_and_test.yml +++ b/.github/workflows/lint_and_test.yml @@ -23,6 +23,12 @@ jobs: python-version: 3.11 architecture: x64 + - name: Ensure Nexus mirror configuration is not committed + run: | + # Grep checks for Nexus mirror in the index-url parameter, + # the ! negates the grep exit code, which would be 0 if URL was present + ! grep -E '.*index-url.*nexus.*' requirements*.txt + - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/Dockerfile b/Dockerfile index b65585ca..af1ce78e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM tiangolo/uvicorn-gunicorn-fastapi:python3.11-slim-2023-02-20 +FROM tiangolo/uvicorn-gunicorn-fastapi:python3.11-slim-2024-03-04 ## Install python-ags4 COPY requirements.txt . diff --git a/README.md b/README.md index 6ed5afe8..8195e1c6 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,20 @@ pytest -vs test The test environment is configured so that adding `--pdb` to the test command will start an IPython debugger session in the event of test failure. +### Updating dependencies + +We are using [pip-tools](https://pip-tools.readthedocs.io/en/stable/) to create a pinned list of all dependencies from the ones that we need to specify. + +To refresh the dependency list, update `requirements.in` and `requirements-dev.in` then run the following: + +```bash +pip-compile -o requirements.txt requirements.in +pip-compile -o requirements_dev.txt requirements_dev.in +``` + +The updated requirements files must be edited to remove reference to the Nexus mirror before they can then be committed. + + ### GUI Customisation To ammend the GUI HTML we recommend running via `Docker` using your own `Dockerfile` like the below to `COPY` in your own templates. @@ -108,7 +122,7 @@ COPY content/templates /app/app/templates Containers for the application are hosted in the GitHub Container Registry -Every push to `Main` branch commits builds `pyagsapi:latest`. +Every push to `main` branch commits builds `pyagsapi:latest`. Push Tagged Releases with `^v?[0-9]+[.][0-9]+([.][0-9])?` (v* == v2.0) builds `pyagsapi:2.0` (the "v" gets dropped for the tag). @@ -116,7 +130,7 @@ You can also push release candidates using the format `/^v?[0-9]+[.][0-9]+([.][0 ## Example Files -Files in [https://github.com/BritishGeologicalSurvey/pyagsapi/tree/main/test/files/real](https://github.com/BritishGeologicalSurvey/pyagsapi/tree/main/test/files/real) are a random collection of real AGS files which have been submitted to the BGS and are available under OGL, we have included them here as example files for testing pyagsapi. +Files in [https://github.com/BritishGeologicalSurvey/pyagsapi/tree/main/test/files/real](https://github.com/BritishGeologicalSurvey/pyagsapi/tree/main/test/files/real) are a collection of real AGS files which have been submitted to the BGS and are available under OGL, we have included them here as example files for testing pyagsapi. ## Licence diff --git a/requirements.in b/requirements.in new file mode 100644 index 00000000..6bc1a6f2 --- /dev/null +++ b/requirements.in @@ -0,0 +1,18 @@ +Fiona +Jinja2 +Shapely +aiofiles +colorlog +geopandas +numpy +pyproj +python-ags4==0.4.1 +requests +shortuuid +# These libraries are already in FastAPI container but need updated +fastapi==0.88.0 +h11==0.14.0 +pydantic==1.10.14 +python-multipart==0.0.9 +starlette==0.22.0 +uvicorn==0.20.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 6b00ee99..565aae92 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,119 @@ -aiofiles==23.1.0 -numpy==1.24.2 +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --output-file=requirements.txt requirements.in +# + +aiofiles==23.2.1 + # via -r requirements.in +anyio==4.3.0 + # via starlette +attrs==23.2.0 + # via fiona +certifi==2024.2.2 + # via + # fiona + # pyproj + # requests +charset-normalizer==3.3.2 + # via requests +click==8.1.7 + # via + # click-plugins + # cligj + # fiona + # python-ags4 + # uvicorn +click-plugins==1.1.1 + # via fiona +cligj==0.7.2 + # via fiona +colorlog==6.8.2 + # via -r requirements.in +commonmark==0.9.1 + # via rich +defusedxml==0.7.1 + # via python-ags4 +et-xmlfile==1.1.0 + # via openpyxl +fastapi==0.88.0 + # via -r requirements.in +fiona==1.9.5 + # via + # -r requirements.in + # geopandas +geopandas==0.14.3 + # via -r requirements.in +h11==0.14.0 + # via + # -r requirements.in + # uvicorn +idna==3.6 + # via + # anyio + # requests +jinja2==3.1.3 + # via -r requirements.in +markupsafe==2.1.5 + # via jinja2 +numpy==1.26.4 + # via + # -r requirements.in + # pandas + # shapely +openpyxl==3.1.2 + # via python-ags4 +packaging==23.2 + # via geopandas +pandas==1.5.3 + # via + # geopandas + # python-ags4 +pydantic==1.10.14 + # via + # -r requirements.in + # fastapi +pygments==2.17.2 + # via rich +pyproj==3.6.1 + # via + # -r requirements.in + # geopandas python-ags4==0.4.1 -python-multipart==0.0.5 -colorlog==6.7.0 -shortuuid==1.0.11 -Jinja2==3.1.2 -Fiona==1.9.1 -Shapely==2.0.1 -pyproj==3.4.1 -geopandas==0.12.2 -requests==2.28.2 -# These libraries are already in FastAPI container but need updated -fastapi==0.92.0 + # via -r requirements.in +python-dateutil==2.9.0.post0 + # via pandas +python-multipart==0.0.9 + # via -r requirements.in +pytz==2024.1 + # via pandas +requests==2.31.0 + # via -r requirements.in +rich==12.6.0 + # via python-ags4 +shapely==2.0.3 + # via + # -r requirements.in + # geopandas +shortuuid==1.0.12 + # via -r requirements.in +six==1.16.0 + # via + # fiona + # python-dateutil +sniffio==1.3.1 + # via anyio +starlette==0.22.0 + # via + # -r requirements.in + # fastapi +typing-extensions==4.10.0 + # via pydantic +urllib3==2.2.1 + # via requests uvicorn==0.20.0 -h11==0.14.0 + # via -r requirements.in + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements_dev.in b/requirements_dev.in new file mode 100644 index 00000000..c9f00722 --- /dev/null +++ b/requirements_dev.in @@ -0,0 +1,9 @@ +ipython +ipdb +pytest +flake8 +requests +httpx +pytest-asyncio +requests-toolbelt +freezegun diff --git a/requirements_dev.txt b/requirements_dev.txt index d27f7670..5d4342c3 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,9 +1,105 @@ -ipython==8.10.0 -ipdb==0.13.11 -pytest==7.2.1 -flake8==6.0.0 -requests==2.28.2 -httpx==0.23.3 -pytest-asyncio==0.20.3 -requests-toolbelt==0.10.1 -freezegun==1.2.2 +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --output-file=requirements_dev.txt requirements_dev.in +# + +anyio==4.3.0 + # via httpx +asttokens==2.4.1 + # via stack-data +certifi==2024.2.2 + # via + # httpcore + # httpx + # requests +charset-normalizer==3.3.2 + # via requests +decorator==5.1.1 + # via + # ipdb + # ipython +executing==2.0.1 + # via stack-data +flake8==7.0.0 + # via -r requirements_dev.in +freezegun==1.4.0 + # via -r requirements_dev.in +h11==0.14.0 + # via httpcore +httpcore==1.0.4 + # via httpx +httpx==0.27.0 + # via -r requirements_dev.in +idna==3.6 + # via + # anyio + # httpx + # requests +iniconfig==2.0.0 + # via pytest +ipdb==0.13.13 + # via -r requirements_dev.in +ipython==8.22.2 + # via + # -r requirements_dev.in + # ipdb +jedi==0.19.1 + # via ipython +matplotlib-inline==0.1.6 + # via ipython +mccabe==0.7.0 + # via flake8 +packaging==23.2 + # via pytest +parso==0.8.3 + # via jedi +pexpect==4.9.0 + # via ipython +pluggy==1.4.0 + # via pytest +prompt-toolkit==3.0.43 + # via ipython +ptyprocess==0.7.0 + # via pexpect +pure-eval==0.2.2 + # via stack-data +pycodestyle==2.11.1 + # via flake8 +pyflakes==3.2.0 + # via flake8 +pygments==2.17.2 + # via ipython +pytest==8.1.0 + # via + # -r requirements_dev.in + # pytest-asyncio +pytest-asyncio==0.23.5 + # via -r requirements_dev.in +python-dateutil==2.9.0.post0 + # via freezegun +requests==2.31.0 + # via + # -r requirements_dev.in + # requests-toolbelt +requests-toolbelt==1.0.0 + # via -r requirements_dev.in +six==1.16.0 + # via + # asttokens + # python-dateutil +sniffio==1.3.1 + # via + # anyio + # httpx +stack-data==0.6.3 + # via ipython +traitlets==5.14.1 + # via + # ipython + # matplotlib-inline +urllib3==2.2.1 + # via requests +wcwidth==0.2.13 + # via prompt-toolkit