Switch to pipenv for dependency management

Makefile

@@ -80,46 +80,12 @@ mypy:
 ### REQUIREMENTS #############################################
 #
 
-GENERATED_REQUIREMENT_FILES=src/py/requirements.txt src/py/requirements-dev.txt
-SOURCE_REQUIREMENT_FILES=src/py/requirements.txt.in src/py/requirements-dev.txt.in
-
-# This rule pins the requirements with the minimal set of changes required to satisfy the
-# requirements.  This is typically run when a new requirement is added, and we want to
-# propagate the new requirement to the pin files.
-pin-requirements : $(GENERATED_REQUIREMENT_FILES)
-
-# This rule removes all existing pins and pins all requirements based on the latest set of libraries
-# that satisfy the requirements.  This is typically run periodically to make sure the pins don't
-# become too stale.
-pin-all-requirements:
-	@for target in $(GENERATED_REQUIREMENT_FILES); do \
-		echo -n '' >| $$target; \
-	done
-	@if [ $$(uname -s) == "Darwin" ]; then sleep 1; fi  # this is require because Darwin HFS+ only has second-resolution for timestamps.
-	@touch $(SOURCE_REQUIREMENT_FILES)
-	$(MAKE) $(GENERATED_REQUIREMENT_FILES)
-
-src/py/require%.txt : src/py/require%.txt.in src/py/requirements.txt.in
-	[ ! -e .$$(basename "$<")-env ] || exit 1
-	python -m venv .$$(basename "$<")-env
-	.$$(basename "$<")-env/bin/pip install -r $@
-	for src in $^; do \
-		.$$(basename "$<")-env/bin/pip install -r $$src; \
-	done
-	echo "# You should not edit this file directly.  Instead, you should edit one of the following files ($^) and run make $@" >| $@
-	.$$(basename "$<")-env/bin/pip freeze >> $@
-	rm -rf .$$(basename "$<")-env
-
-check-requirements:
-	if [[ $$(git status --porcelain $(GENERATED_REQUIREMENT_FILES)) ]]; then \
-	    echo "Modifications found in REQUIREMENTS files"; exit 2; \
-	fi
+update-deps :
+	pipenv update
+	pipenv lock -r >| src/py/requirements.txt
+	pipenv lock --dev -r >| src/py/requirements-dev.txt
 
-help-requirements:
-	$(call print_help, pin-requirements, pin requirements with minimal set of changes)
-	$(call print_help, pin-all-requirements, pin requirements with latest packages)
-	$(call print_help, check-requirements, fail if requirements files have been modified)
+.PHONY: update-deps
 
-.PHONY: pin-requirements pin-all-requirements check-requirements
 #
 ##############################################################

Pipfile

@@ -0,0 +1,29 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+Authlib = "*"
+Flask = "*"
+psycopg2-binary = "*"
+python-dotenv = "*"
+requests = "*"
+SQLAlchemy = "*"
+
+[dev-packages]
+alembic = "*"
+# version for black must be hardcoded, as pipenv does not like beta packages (by default).  as black does not have ANY
+# non-beta packages, pipenv panics and gives up.
+# TODO: (ttung) add a github workflow to check and fail if black releases a new version.
+black = "== 20.8b1"
+flake8 = "*"
+ipython = "*"
+isort = "*"
+mypy = "*"
+pytest = "*"
+pytest-xdist = "*"
+sqlalchemy-stubs = "*"
+
+[requires]
+python_version = "3.7"

README.md

@@ -3,14 +3,36 @@
 This repository contains the code for the gen-epi project.
 
 
-## Running the app in development mode: 
+## Setting up a development environment
+
+### Backend
+
+#### Installing the python dependencies
+
+1. Create a python virtual environment and activate it.  While other installation modes may be possible, this is the only tested development mode.
+```bash
+aspen% python3.7 -m venv .venv
+aspen% . .venv/bin/activate
+(.venv) aspen%
+```
+2. Install pipenv.
+```bash
+(.venv) aspen% pip install pipenv
+(.venv) aspen%
+```
+3. Install the packages required for development.
+```bash
+(.venv) aspen% pipenv install --dev
+```
+
+### Running the app in development mode:
 
 ```bash
-aspen% npm --prefix src/ts start
-aspen% export FLASK_APP=aspen.app
-aspen% export FLASK_ENV=development
-aspen% cd src/py
-aspen% flask run --host localhost --port 3000  # host and port needed for auth0
+(.venv) aspen% npm --prefix src/ts start
+(.venv) aspen% export FLASK_APP=aspen.app
+(.venv) aspen% export FLASK_ENV=development
+(.venv) aspen% cd src/py
+(.venv) aspen% flask run --host localhost --port 3000  # host and port needed for auth0
 ```
 
 ## Backend

docs/backend/README.md

@@ -69,3 +69,7 @@ In [3]:
 2. Run `ENV=dev alembic revision --autogenerate -m "SOME DESCRIPTIVE MESSAGE" --rev-id $(date +%Y%M%d_%H%M%S)`
 3. Verify that the schema migration generated in `database_migrations/versions/` is sane.
 4. Run `ENV=dev alembic upgrade head` to test the schema migration on your local database.
+
+## Updating python dependencies
+
+To update python dependencies, update the [`Pipfile`](../../Pipfile) and run `make update-deps`.  This will update [`Pipfile.lock`](../../Pipfile.lock) and [`requirements.txt`](../../src/py/requirements.txt) (used by setup.py), and [`requirements-dev.txt`](../../src/py/requirements-dev.txt) (used by tests).

src/py/requirements-dev.txt

@@ -1,61 +1,76 @@
-# You should not edit this file directly.  Instead, you should edit one of the following files (src/py/requirements-dev.txt.in src/py/requirements.txt.in) and run make src/py/requirements-dev.txt
-alembic==1.4.3
-apipkg==1.5
+#
+# These requirements were autogenerated by pipenv
+# To regenerate from the project's Pipfile, run:
+#
+#    pipenv lock --requirements --dev
+#
+
+# Note: in pipenv 2020.x, "--dev" changed to emit both default and development
+# requirements. To emit only development requirements, pass "--dev-only".
+
+-i https://pypi.org/simple
+alembic==1.5.4
+apipkg==1.5; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 appdirs==1.4.4
-appnope==0.1.2
-attrs==20.3.0
-Authlib==0.15.3
+appnope==0.1.2; sys_platform == 'darwin'
+attrs==20.3.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+authlib==0.15.3
 backcall==0.2.0
 black==20.8b1
+certifi==2020.12.5
 cffi==1.14.4
-click==7.1.2
-cryptography==3.3.1
+chardet==4.0.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+click==7.1.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+cryptography==3.3.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
 decorator==4.4.2
-execnet==1.7.1
+execnet==1.8.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 flake8==3.8.4
-Flask==1.1.2
-importlib-metadata==3.4.0
+flask==1.1.2
+idna==2.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+importlib-metadata==3.4.0; python_version < '3.8'
 iniconfig==1.1.1
-ipython==7.20.0
 ipython-genutils==0.2.0
+ipython==7.20.0
 isort==5.7.0
-itsdangerous==1.1.0
-jedi==0.18.0
-Jinja2==2.11.2
-Mako==1.1.4
-MarkupSafe==1.1.1
+itsdangerous==1.1.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+jedi==0.18.0; python_version >= '3.6'
+jinja2==2.11.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+mako==1.1.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+markupsafe==1.1.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 mccabe==0.6.1
-mypy==0.790
 mypy-extensions==0.4.3
-packaging==20.8
-parso==0.8.1
+mypy==0.800
+packaging==20.9; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+parso==0.8.1; python_version >= '3.6'
 pathspec==0.8.1
-pexpect==4.8.0
+pexpect==4.8.0; sys_platform != 'win32'
 pickleshare==0.7.5
-pluggy==0.13.1
-prompt-toolkit==3.0.14
+pluggy==0.13.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+prompt-toolkit==3.0.14; python_full_version >= '3.6.1'
 psycopg2-binary==2.8.6
 ptyprocess==0.7.0
-py==1.10.0
-pycodestyle==2.6.0
-pycparser==2.20
-pyflakes==2.2.0
-Pygments==2.7.4
-pyparsing==2.4.7
-pytest==6.2.1
-pytest-forked==1.3.0
+py==1.10.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+pycodestyle==2.6.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+pycparser==2.20; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+pyflakes==2.2.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+pygments==2.7.4; python_version >= '3.5'
+pyparsing==2.4.7; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
+pytest-forked==1.3.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 pytest-xdist==2.2.0
-python-dateutil==2.8.1
+pytest==6.2.2
+python-dateutil==2.8.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 python-dotenv==0.15.0
 python-editor==1.0.4
 regex==2020.11.13
-six==1.15.0
-SQLAlchemy==1.3.22
+requests==2.25.1
+six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 sqlalchemy-stubs==0.4
-toml==0.10.2
-traitlets==5.0.5
+sqlalchemy==1.3.23
+toml==0.10.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
+traitlets==5.0.5; python_version >= '3.7'
 typed-ast==1.4.2
-typing-extensions==3.7.4.3
+typing-extensions==3.7.4.3; python_version < '3.8'
+urllib3==1.26.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
 wcwidth==0.2.5
-Werkzeug==1.0.1
-zipp==3.4.0
+werkzeug==1.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+zipp==3.4.0; python_version >= '3.6'

src/py/requirements.txt

@@ -1,20 +1,27 @@
-# You should not edit this file directly.  Instead, you should edit one of the following files (src/py/requirements.txt.in) and run make src/py/requirements.txt
-Authlib==0.15.3
+#
+# These requirements were autogenerated by pipenv
+# To regenerate from the project's Pipfile, run:
+#
+#    pipenv lock --requirements
+#
+
+-i https://pypi.org/simple
+authlib==0.15.3
 certifi==2020.12.5
 cffi==1.14.4
-chardet==3.0.4
-click==7.1.2
-cryptography==3.3.1
-Flask==1.1.2
-idna==2.10
-itsdangerous==1.1.0
-Jinja2==2.11.2
-MarkupSafe==1.1.1
+chardet==4.0.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+click==7.1.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+cryptography==3.3.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
+flask==1.1.2
+idna==2.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+itsdangerous==1.1.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+jinja2==2.11.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+markupsafe==1.1.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 psycopg2-binary==2.8.6
-pycparser==2.20
+pycparser==2.20; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 python-dotenv==0.15.0
-requests==2.24.0
-six==1.15.0
-SQLAlchemy==1.3.22
-urllib3==1.25.11
-Werkzeug==1.0.1
+requests==2.25.1
+six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+sqlalchemy==1.3.23
+urllib3==1.26.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
+werkzeug==1.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'

src/py/setup.py

@@ -7,7 +7,11 @@
 # NOTE: this is not compatible with a sdist installation of aspen.
 requirements_file = Path(__file__).parent / "requirements.txt"
 with requirements_file.open("r") as fh:
-    requirement_lines = fh.readlines()
+    requirement_lines = [
+        line
+        for line in fh.readlines()
+        if not (line.startswith("-i") or line.startswith("./"))
+    ]
 
 setuptools.setup(
     name="aspen",