SQLAlchemy 2 update, and bump to 2.0.0

.github/scripts/build_docs.sh

@@ -4,30 +4,32 @@
 # Rebuild Sphinx docs from scratch and check generated files match those checked
 # in
 
-set -eux -o pipefail
+set -euo pipefail
+
+THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+PROJECT_ROOT_DIR=${THIS_DIR}/../..
+DOCS_DIR=${PROJECT_ROOT_DIR}/docs
+VENV_DIR=${HOME}/venv
+
+cd "${PROJECT_ROOT_DIR}"
 
 sudo apt-get install texlive-latex-extra dvipng
-python -m venv "${HOME}/venv"
-source "${HOME}/venv/bin/activate"
-python -VV
-python -m site
-python -m pip install -U pip
-echo installing pip packages
-python -m pip install -r docs/docs_requirements.txt
-python -m pip install -e .
+PYTHON=${VENV_DIR}/bin/python
+echo Installing pip packages for docs
+${PYTHON} -m pip install -r "${DOCS_DIR}/docs_requirements.txt"
 ########################################################################################
-cd "${GITHUB_WORKSPACE}/docs"
 echo Creating autodocs
-python ./create_all_autodocs.py --make --destroy_first
-cd "${GITHUB_WORKSPACE}"
+${PYTHON} "${DOCS_DIR}/create_all_autodocs.py" --make --destroy_first
 echo Checking if files generated by create_all_autodocs need to be checked in
 git diff
 git update-index --refresh
 git diff-index --quiet HEAD --
 test -z "$(git ls-files --exclude-standard --others)"
-cd docs
 echo Rebuilding docs
-python ./rebuild_docs.py --warnings_as_errors
+
+# Have to be in the virtualenv for sphinx-build to be picked up.
+source "${VENV_DIR}"/bin/activate
+python "${DOCS_DIR}/rebuild_docs.py" --warnings_as_errors
 echo Checking if files generated by rebuild_docs need to be checked in
 git diff
 git update-index --refresh

.github/scripts/create_virtualenv.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SYSTEM_PYTHON=python3
+
+if [ $# -eq 1 ]; then
+    # Script needs at least Python 3.10 for docs. Allow this to be specified
+    SYSTEM_PYTHON=$1
+fi
+
+VENV_DIR=${HOME}/venv
+
+${SYSTEM_PYTHON} -m venv "${VENV_DIR}"
+PYTHON=${VENV_DIR}/bin/python
+${PYTHON} -VV
+${PYTHON} -m site
+${PYTHON} -m pip install -U pip setuptools
+echo Dumping pre-installed packages
+${PYTHON} -m pip freeze

.github/scripts/install_base_python_packages.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+PROJECT_ROOT_DIR=${THIS_DIR}/../..
+VENV_DIR=${HOME}/venv
+
+PYTHON=${VENV_DIR}/bin/python
+echo Installing pip packages
+${PYTHON} -m pip install -e "${PROJECT_ROOT_DIR}"

.github/scripts/install_test_python_packages.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+VENV_DIR=${HOME}/venv
+
+PYTHON=${VENV_DIR}/bin/python
+${PYTHON} -m pip install "numpy<1.23"  # 1.23 incompatible with numba
+${PYTHON} -m pip install xlrd
+${PYTHON} -m pip install dogpile.cache==0.9.2  # Later versions incompatible
+${PYTHON} -m pip install pytest
+${PYTHON} -m pip install xhtml2pdf weasyprint pdfkit  # For PDF tests

.github/scripts/python_checks.sh

@@ -2,16 +2,19 @@
 
 # Run from .github/workflows/python_checks.yml
 
-set -eux -o pipefail
+set -euo pipefail
 
-python3 -m venv "${HOME}/venv"
-source "${HOME}/venv/bin/activate"
-python -m site
-python -m pip install -U pip setuptools
+THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+PROJECT_ROOT_DIR=${THIS_DIR}/../..
+VENV_DIR=${HOME}/venv
+PYTHON=${VENV_DIR}/bin/python
+PRECOMMIT=${VENV_DIR}/bin/pre-commit
 
-echo installing pip packages
-python -m pip install -e .
-python -m pip install pre-commit
+cd "${PROJECT_ROOT_DIR}"
 
-echo running pre-commit checks
-pre-commit run --all-files
+echo Installing pre-commit
+${PYTHON} -m pip install -e .
+${PYTHON} -m pip install pre-commit
+
+echo Running pre-commit checks
+${PRECOMMIT} run --all-files

.github/scripts/run_tests.sh

@@ -2,22 +2,16 @@
 
 # Run from .github/workflows/tests.yml
 
-set -eux -o pipefail
+set -euo pipefail
 
-sudo apt-get install wkhtmltopdf
+THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+PROJECT_ROOT_DIR=${THIS_DIR}/../..
+VENV_DIR=${HOME}/venv
+PYTEST=${VENV_DIR}/bin/pytest
 
-python -m venv "${HOME}/venv"
-source "${HOME}/venv/bin/activate"
-python -m site
-python -m pip install -U pip
-echo installing pip packages
+sudo apt-get install wkhtmltopdf
 
-python -m pip install "numpy<1.23"  # 1.23 incompatible with numba
-python -m pip install xlrd
-python -m pip install dogpile.cache==0.9.2  # Later versions incompatible
-python -m pip install pytest
-python -m pip install xhtml2pdf weasyprint pdfkit  # For PDF tests
-python -m pip install -e .
+cd "${PROJECT_ROOT_DIR}"
 
 # pytest --log-cli-level=INFO
-pytest
+${PYTEST}

.github/workflows/docs.yml

@@ -6,8 +6,27 @@ on: push
 
 jobs:
     build-docs:
-        runs-on: ubuntu-latest
+        strategy:
+            matrix:
+                include:
+                    - name: ubuntu-22.04
+                      os: ubuntu-22.04
+                      python-version: "3.10"
+        runs-on: ${{ matrix.os }}
         steps:
-            - uses: actions/checkout@v2
+            - uses: actions/checkout@v4
+            - uses: actions/setup-python@v5
+              with:
+                  python-version: ${{ matrix.python-version }}
+            - name: Create virtualenv
+              run: ${GITHUB_WORKSPACE}/.github/scripts/create_virtualenv.sh
+            - name: Install base Python packages
+              run: ${GITHUB_WORKSPACE}/.github/scripts/install_base_python_packages.sh
             - name: Build docs
               run: ${GITHUB_WORKSPACE}/.github/scripts/build_docs.sh
+            - name: Dump stuff on failure
+              if: failure()
+              run: |
+                  set -euxo pipefail
+                  ls -l ${HOME}/venv/bin
+                  ${HOME}/venv/bin/python -m pip freeze

.github/workflows/python_checks.yml

@@ -6,18 +6,24 @@ on:
     push:
         paths:
             - '**.py'
-            - .github/workflows/python-checks.yml
+            - .github/scripts/create_virtualenv.sh
+            - .github/scripts/install_base_python_packages.sh
             - .github/scripts/python_checks.sh
+            - .github/workflows/python-checks.yml
 jobs:
     python-checks:
         runs-on: ubuntu-latest
         strategy:
             matrix:
-                python-version: ["3.8", "3.9", "3.10"]
+                python-version: ["3.9", "3.10"]
         steps:
-            - uses: actions/checkout@v3
-            - uses: actions/setup-python@v4
+            - uses: actions/checkout@v4
+            - uses: actions/setup-python@v5
               with:
                   python-version: ${{ matrix.python-version }}
+            - name: Create virtualenv
+              run: ${GITHUB_WORKSPACE}/.github/scripts/create_virtualenv.sh
+            - name: Install base Python packages
+              run: ${GITHUB_WORKSPACE}/.github/scripts/install_base_python_packages.sh
             - name: Python checks
               run: ${GITHUB_WORKSPACE}/.github/scripts/python_checks.sh

.github/workflows/run_tests.yml

@@ -1,28 +1,35 @@
 ---
 # yamllint disable rule:line-length
-name: Tests
+name: Run tests
 # yamllint disable-line rule:truthy
 on:
     push:
         paths:
             - '**.py'
             - .github/scripts/change_apt_mirror.sh
-            - .github/workflows/run_tests.yml
+            - .github/scripts/create_virtualenv.sh
+            - .github/scripts/install_base_python_packages.sh
+            - .github/scripts/install_test_python_packages.sh
             - .github/scripts/run_tests.sh
+            - .github/workflows/run_tests.yml
 jobs:
-    pip-install-and-tests:
+    run-tests:
         runs-on: ubuntu-latest
         strategy:
             matrix:
-                python-version: ["3.8", "3.9", "3.10"]
+                python-version: ["3.9", "3.10"]
         steps:
-            - uses: actions/checkout@v3
-            - uses: actions/setup-python@v4
+            - uses: actions/checkout@v4
+            - uses: actions/setup-python@v5
               with:
                   python-version: ${{ matrix.python-version }}
             - name: Change apt mirror
-              run: |
-                  set -euxo pipefail
-                  ${GITHUB_WORKSPACE}/.github/scripts/change_apt_mirror.sh
+              run: ${GITHUB_WORKSPACE}/.github/scripts/change_apt_mirror.sh
+            - name: Create virtualenv
+              run: ${GITHUB_WORKSPACE}/.github/scripts/create_virtualenv.sh
+            - name: Install test Python packages
+              run: ${GITHUB_WORKSPACE}/.github/scripts/install_test_python_packages.sh
+            - name: Install base Python packages
+              run: ${GITHUB_WORKSPACE}/.github/scripts/install_base_python_packages.sh
             - name: Run tests
               run: ${GITHUB_WORKSPACE}/.github/scripts/run_tests.sh

.pre-commit-config.yaml

@@ -18,3 +18,11 @@ repos:
     rev: 5.0.4
     hooks:
     -   id: flake8
+-   repo: https://github.com/asottile/yesqa
+    rev: v1.5.0
+    hooks:
+    -   id: yesqa
+-   repo: https://github.com/pre-commit/pygrep-hooks
+    rev: v1.9.0
+    hooks:
+    -   id: python-check-blanket-noqa

cardinal_pythonlib/argparse_func.py

@@ -55,7 +55,7 @@ class ShowAllSubparserHelpAction(_HelpAction):
     shows help for all subparsers. As per
 
     https://stackoverflow.com/questions/20094215/argparse-subparser-monolithic-help-output
-    """  # noqa: E501
+    """
 
     def __call__(
         self,
@@ -147,7 +147,7 @@ def str2bool(v: str) -> bool:
             default=NICE,  # if the argument is entirely absent
             help="Activate nice mode.")
 
-    """  # noqa: E501
+    """
     lv = v.lower()
     if lv in ("yes", "true", "t", "y", "1"):
         return True

cardinal_pythonlib/athena_ohdsi.py

@@ -366,7 +366,7 @@ def get_athena_concepts(
         timeit.timeit(concept_testcode, number=1, globals=globals())
         # After speedup: 3.9 s for 1.1m rows.
 
-    """  # noqa
+    """  # noqa: E501
     assert bool(tsv_filename) != bool(
         cached_concepts
     ), "Specify either tsv_filename or cached_concepts"
@@ -461,7 +461,7 @@ def get_athena_concept_relationships(
     tsv_filename: str = "",
     cached_concept_relationships: Iterable[
         AthenaConceptRelationshipRow
-    ] = None,  # noqa
+    ] = None,
     concept_id_1_values: Collection[int] = None,
     concept_id_2_values: Collection[int] = None,
     relationship_id_values: Collection[str] = None,

cardinal_pythonlib/betweendict.py

@@ -72,7 +72,7 @@ class BetweenDict(dict):
       ... NB has initialization default argument bug
     - https://pypi.python.org/pypi/rangedict/0.1.5
     - https://stackoverflow.com/questions/30254739/is-there-a-library-implemented-rangedict-in-python
-    """  # noqa
+    """  # noqa: E501
     INVALID_MSG_TYPE = "Key must be an iterable with length 2"
     INVALID_MSG_VALUE = "First element of key must be less than second element"
 

cardinal_pythonlib/bulk_email/constants.py

@@ -31,7 +31,7 @@
 CONTENT_TYPE_MAX_LENGTH = 255
 # Can be quite long; see cardinal_pythonlib.httpconst.MimeType
 # 255 is the formal limit:
-# https://stackoverflow.com/questions/643690/maximum-mimetype-length-when-storing-type-in-db  # noqa
+# https://stackoverflow.com/questions/643690/maximum-mimetype-length-when-storing-type-in-db  # noqa: E501
 
 DEFAULT_TIME_BETWEEN_EMAILS_S = 0.5
 

cardinal_pythonlib/bulk_email/main.py

@@ -531,9 +531,9 @@ def main() -> None:
             f"via the environment variable {DB_URL_ENVVAR}"
         )
         sys.exit(EXIT_FAILURE)
-    engine = create_engine(db_url, echo=args.echo)
+    engine = create_engine(db_url, echo=args.echo, future=True)
     log.info(f"Using database: {get_safe_url_from_engine(engine)}")
-    session = Session(engine)
+    session = Session(engine, future=True)
 
     # -------------------------------------------------------------------------
     # Launch subcommand

cardinal_pythonlib/bulk_email/models.py

@@ -113,7 +113,7 @@ def make_table_args(*args, **kwargs) -> Tuple[Any]:
 # noinspection PyUnusedLocal
 @listens_for(Session, "before_flush")
 def before_flush(session: Session, flush_context, instances) -> None:
-    # https://docs.sqlalchemy.org/en/14/orm/events.html#sqlalchemy.orm.SessionEvents.before_flush  # noqa
+    # https://docs.sqlalchemy.org/en/14/orm/events.html#sqlalchemy.orm.SessionEvents.before_flush  # noqa: E501
     for instance in session.dirty:
         if not isinstance(instance, Config):
             continue
@@ -136,7 +136,7 @@ class Config(Base):
 
     - https://docs.sqlalchemy.org/en/14/_modules/examples/versioned_rows/versioned_rows.html
     - https://docs.sqlalchemy.org/en/14/orm/examples.html#module-examples.versioned_rows
-    """  # noqa
+    """  # noqa: E501
 
     __tablename__ = "config"
     __table_args__ = make_table_args(comment="Stores configuration records.")
@@ -148,9 +148,9 @@ class Config(Base):
     config_id = Column(
         Integer,
         # ... may not be supported by all databases; see
-        # https://docs.sqlalchemy.org/en/14/core/constraints.html#check-constraint  # noqa
+        # https://docs.sqlalchemy.org/en/14/core/constraints.html#check-constraint  # noqa: E501
         # ... though MySQL has recently added this:
-        # https://dev.mysql.com/doc/refman/8.0/en/create-table-check-constraints.html  # noqa
+        # https://dev.mysql.com/doc/refman/8.0/en/create-table-check-constraints.html  # noqa: E501
         primary_key=True,
         autoincrement=True,
         comment="Primary key.",
@@ -457,7 +457,7 @@ def _pending_job_query(session: Session) -> CountStarSpecializedQuery:
             .where(
                 and_(
                     SendAttempt.job_id == Job.job_id,
-                    SendAttempt.success == True,  # noqa
+                    SendAttempt.success == True,  # noqa: E712
                 )
             )
         )
@@ -474,7 +474,7 @@ def n_completed_jobs(session: Session) -> int:
         query = (
             CountStarSpecializedQuery([Job], session=session)
             .join(SendAttempt)
-            .filter(SendAttempt.success == True)  # noqa
+            .filter(SendAttempt.success == True)  # noqa: E712
         )
         return query.count_star()