From dfa69c6945405fa1db82e9ed4427643212e7581b Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Mon, 11 Sep 2023 14:22:28 +0100 Subject: [PATCH 1/3] Updated CI configuration Simplified publish workflow Make publish job dependent on tests Add back Python 3.7 Rename pytest46 to pytestoldest and add Python 3.12 testing Use latest developer version of Numpy and pytest for Python 3.12 Co-authored-by: P. L. Lim <2090236+pllim@users.noreply.github.com> --- .github/workflows/ci_workflows.yml | 66 +++++++++++------------------- .github/workflows/publish.yml | 48 ---------------------- tox.ini | 13 +++++- 3 files changed, 35 insertions(+), 92 deletions(-) delete mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/ci_workflows.yml b/.github/workflows/ci_workflows.yml index 286cdb6..2364eb4 100644 --- a/.github/workflows/ci_workflows.yml +++ b/.github/workflows/ci_workflows.yml @@ -9,45 +9,27 @@ on: jobs: tests: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - include: - - os: ubuntu-latest - python-version: 3.7 - toxenv: py37-test-pytest46 - - os: windows-latest - python-version: 3.7 - toxenv: py37-test-pytest50 - - os: macos-latest - python-version: 3.8 - toxenv: py38-test-pytest52 - - os: ubuntu-latest - python-version: 3.8 - toxenv: py38-test-pytest53 - - os: windows-latest - python-version: 3.9 - toxenv: py39-test-pytest60 - - os: macos-latest - python-version: 3.9 - toxenv: py39-test-pytest61 - - os: ubuntu-latest - python-version: '3.10' - toxenv: py310-test-pytest62 - - os: ubuntu-latest - python-version: '3.10' - toxenv: py310-test-pytestdev - - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install tox - run: python -m pip install tox - - name: Run tox - run: tox -v -e ${{ matrix.toxenv }} + uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1 + with: + envs: | + - linux: py37-test-pytestoldest + - macos: py37-test-pytest50 + - windows: py38-test-pytest52 + - linux: py38-test-pytest53 + - macos: py39-test-pytest60 + - windows: py39-test-pytest61 + - linux: py39-test-pytest62 + - macos: py310-test-pytest70 + - windows: py310-test-pytest71 + - linux: py310-test-pytest72 + - macos: py310-test-pytest73 + - windows: py311-test-pytest74 + - linux: py312-test-pytestdev + publish: + needs: tests + uses: OpenAstronomy/github-actions-workflows/.github/workflows/publish_pure_python.yml@v1 + with: + test_extras: test + test_command: pytest $GITHUB_WORKSPACE/tests; pytest --arraydiff $GITHUB_WORKSPACE/tests + secrets: + pypi_token: ${{ secrets.pypi_password }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index 61c38f1..0000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: Release - -on: - pull_request: - push: - tags: - - '*' - -jobs: - build-n-publish: - name: Build and publish Python 🐍 distributions 📦 to PyPI - runs-on: ubuntu-latest - if: ((github.event_name == 'push' && startsWith(github.ref, 'refs/tags')) || contains(github.event.pull_request.labels.*.name, 'Build wheels')) - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - - name: Install python-build and twine - run: python -m pip install pip build "twine>=3.3" -U - - - name: Build package - run: python -m build --sdist --wheel . - - - name: List result - run: ls -l dist - - - name: Check long_description - run: python -m twine check --strict dist/* - - # FIXME: pytest not found - #- name: Test package - # run: | - # cd .. - # python -m venv testenv - # testenv/bin/pip install pytest pytest-arraydiff/dist/*.whl - # testenv/bin/pytest pytest-arraydiff/tests --arraydiff - - - name: Publish distribution 📦 to PyPI - if: startsWith(github.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.pypi_password }} diff --git a/tox.ini b/tox.ini index c278ab9..95feef4 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = - py{37,38,39,310}-test{,-devdeps} + py{37,38,39,310,311}-test{,-devdeps} codestyle requires = setuptools >= 30.3.0 @@ -9,16 +9,25 @@ isolated_build = true [testenv] changedir = .tmp/{envname} +setenv = + # Until a released numpy version is 3.12 compatible, we install the dev wheel + py312: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/scientific-python-nightly-wheels/simple description = run tests deps = - pytest46: pytest==4.6.* + pytestoldest: pytest==4.6.0 pytest50: pytest==5.0.* pytest52: pytest==5.2.* pytest53: pytest==5.3.* pytest60: pytest==6.0.* pytest61: pytest==6.1.* pytest62: pytest==6.2.* + pytest70: pytest==7.0.* + pytest71: pytest==7.1.* + pytest72: pytest==7.2.* + pytest73: pytest==7.3.* + pytest74: pytest==7.4.* pytestdev: git+https://github.com/pytest-dev/pytest#egg=pytest + py312: numpy>=0.0.dev0 extras = test commands = From 86fd58c932fffe77be3ac356dea92ef2f0e84035 Mon Sep 17 00:00:00 2001 From: "P. L. Lim" <2090236+pllim@users.noreply.github.com> Date: Wed, 15 Nov 2023 17:10:11 -0500 Subject: [PATCH 2/3] Make sure test matrix is consistent across the board and add the codestyle check. Fix PEP 8 complaints. Auto-cancel duplicate Action jobs. --- .github/workflows/ci_workflows.yml | 7 ++++++- pytest_arraydiff/plugin.py | 7 ++----- setup.cfg | 2 ++ tox.ini | 12 +++++++----- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci_workflows.yml b/.github/workflows/ci_workflows.yml index 2364eb4..af72a01 100644 --- a/.github/workflows/ci_workflows.yml +++ b/.github/workflows/ci_workflows.yml @@ -7,11 +7,16 @@ on: # Run every Sunday at 06:53 UTC - cron: 53 6 * * 0 +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: tests: uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1 with: envs: | + - linux: codestyle - linux: py37-test-pytestoldest - macos: py37-test-pytest50 - windows: py38-test-pytest52 @@ -24,7 +29,7 @@ jobs: - linux: py310-test-pytest72 - macos: py310-test-pytest73 - windows: py311-test-pytest74 - - linux: py312-test-pytestdev + - linux: py312-test-devdeps publish: needs: tests uses: OpenAstronomy/github-actions-workflows/.github/workflows/publish_pure_python.yml@v1 diff --git a/pytest_arraydiff/plugin.py b/pytest_arraydiff/plugin.py index bb7c07e..e61809c 100755 --- a/pytest_arraydiff/plugin.py +++ b/pytest_arraydiff/plugin.py @@ -28,9 +28,6 @@ # # https://github.com/astrofrog/pytest-mpl - -from functools import wraps - import os import abc import shutil @@ -149,7 +146,7 @@ def read(filename): @staticmethod def write(filename, data, **kwargs): - import pandas as pd + import pandas as pd # noqa: F401 key = os.path.basename(filename).replace('.h5', '') return data.to_hdf(filename, key, **kwargs) @@ -201,7 +198,7 @@ def pytest_addoption(parser): def pytest_configure(config): config.getini('markers').append( 'array_compare: for functions using array comparison') - + if config.getoption("--arraydiff") or config.getoption("--arraydiff-generate-path") is not None: reference_dir = config.getoption("--arraydiff-reference-path") diff --git a/setup.cfg b/setup.cfg index 5bc3a65..68f7426 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,6 +16,8 @@ classifiers = Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 Programming Language :: Python :: Implementation :: CPython Topic :: Software Development :: Testing Topic :: Utilities diff --git a/tox.ini b/tox.ini index 95feef4..3481e2c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = - py{37,38,39,310,311}-test{,-devdeps} + py{37,38,39,310,311,312}-test{,-pytestoldest,-pytest50,-pytest52,-pytest53,-pytest60,-pytest61,-pytest62,-pytest70,-pytest71,-pytest72,-pytest73,-pytest74,-devdeps} codestyle requires = setuptools >= 30.3.0 @@ -10,8 +10,7 @@ isolated_build = true [testenv] changedir = .tmp/{envname} setenv = - # Until a released numpy version is 3.12 compatible, we install the dev wheel - py312: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/scientific-python-nightly-wheels/simple + devdeps: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/scientific-python-nightly-wheels/simple description = run tests deps = pytestoldest: pytest==4.6.0 @@ -26,17 +25,20 @@ deps = pytest72: pytest==7.2.* pytest73: pytest==7.3.* pytest74: pytest==7.4.* - pytestdev: git+https://github.com/pytest-dev/pytest#egg=pytest - py312: numpy>=0.0.dev0 + devdeps: git+https://github.com/pytest-dev/pytest#egg=pytest + devdeps: numpy>=0.0.dev0 extras = test commands = + # Force numpy-dev after something in the stack downgrades it + devdeps: python -m pip install --pre --upgrade --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy pip freeze pytest {toxinidir}/tests {posargs} pytest {toxinidir}/tests --arraydiff {posargs} [testenv:codestyle] skip_install = true +changedir = {toxinidir} description = check code style, e.g. with flake8 deps = flake8 commands = flake8 pytest_arraydiff --count From 754a8bd3a2cc45ec94951d9da04761fbd4555bb9 Mon Sep 17 00:00:00 2001 From: "P. L. Lim" <2090236+pllim@users.noreply.github.com> Date: Wed, 15 Nov 2023 17:26:39 -0500 Subject: [PATCH 3/3] Need astropy-dev and pandas-dev to go with numpy-dev Handle FutureWarning from pandas 3.0 Skip failing pandas test caused by numpy 2.0 Updated matrix to run a stable job in py312 --- .github/workflows/ci_workflows.yml | 8 ++++---- pytest_arraydiff/plugin.py | 2 +- tests/test_pytest_arraydiff.py | 4 ++++ tox.ini | 4 +++- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_workflows.yml b/.github/workflows/ci_workflows.yml index af72a01..179eb53 100644 --- a/.github/workflows/ci_workflows.yml +++ b/.github/workflows/ci_workflows.yml @@ -23,12 +23,12 @@ jobs: - linux: py38-test-pytest53 - macos: py39-test-pytest60 - windows: py39-test-pytest61 - - linux: py39-test-pytest62 + - linux: py310-test-pytest62 - macos: py310-test-pytest70 - windows: py310-test-pytest71 - - linux: py310-test-pytest72 - - macos: py310-test-pytest73 - - windows: py311-test-pytest74 + - linux: py311-test-pytest72 + - macos: py311-test-pytest73 + - windows: py312-test-pytest74 - linux: py312-test-devdeps publish: needs: tests diff --git a/pytest_arraydiff/plugin.py b/pytest_arraydiff/plugin.py index e61809c..f476e68 100755 --- a/pytest_arraydiff/plugin.py +++ b/pytest_arraydiff/plugin.py @@ -148,7 +148,7 @@ def read(filename): def write(filename, data, **kwargs): import pandas as pd # noqa: F401 key = os.path.basename(filename).replace('.h5', '') - return data.to_hdf(filename, key, **kwargs) + return data.to_hdf(filename, key=key, **kwargs) @classmethod def compare(cls, reference_file, test_file, atol=None, rtol=None): diff --git a/tests/test_pytest_arraydiff.py b/tests/test_pytest_arraydiff.py index e974706..5ad20dc 100644 --- a/tests/test_pytest_arraydiff.py +++ b/tests/test_pytest_arraydiff.py @@ -4,6 +4,9 @@ import pytest import numpy as np +from packaging.version import Version + +NUMPY_LT_2_0 = Version(np.__version__) < Version("2.0.dev") reference_dir = 'baseline' @@ -18,6 +21,7 @@ def test_succeeds_func_text(): return np.arange(3 * 5).reshape((3, 5)) +@pytest.mark.skipif(not NUMPY_LT_2_0, reason="AttributeError: `np.unicode_` was removed in the NumPy 2.0 release. Use `np.str_` instead.") @pytest.mark.array_compare(file_format='pd_hdf', reference_dir=reference_dir) def test_succeeds_func_pdhdf(): pd = pytest.importorskip('pandas') diff --git a/tox.ini b/tox.ini index 3481e2c..bde33c9 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ isolated_build = true [testenv] changedir = .tmp/{envname} setenv = - devdeps: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/scientific-python-nightly-wheels/simple + devdeps: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/astropy/simple https://pypi.anaconda.org/scientific-python-nightly-wheels/simple description = run tests deps = pytestoldest: pytest==4.6.0 @@ -27,6 +27,8 @@ deps = pytest74: pytest==7.4.* devdeps: git+https://github.com/pytest-dev/pytest#egg=pytest devdeps: numpy>=0.0.dev0 + devdeps: pandas>=0.0.dev0 + devdeps: astropy>=0.0.dev0 extras = test commands =