CI: merge conda and full tests (#1324)

* set testing matrix
* prune
* utils v0.4.1
* packaging
* doctest
* adjust
* Apply suggestions from code review

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

.github/workflows/ci-checks.yml

@@ -12,13 +12,13 @@ concurrency:
 
 jobs:
   check-code:
-    uses: Lightning-AI/utilities/.github/workflows/check-code.yml@v0.1.0
+    uses: Lightning-AI/utilities/.github/workflows/check-code.yml@v0.4.1
 
   check-schema:
-    uses: Lightning-AI/utilities/.github/workflows/check-schema.yml@v0.1.0
+    uses: Lightning-AI/utilities/.github/workflows/check-schema.yml@v0.4.1
 
   check-package:
-    uses: Lightning-AI/utilities/.github/workflows/check-package.yml@v0.1.0
+    uses: Lightning-AI/utilities/.github/workflows/check-package.yml@v0.4.1
     with:
       import-name: "torchmetrics"
       pypi-name: "torchmetrics"

.github/workflows/ci_test-full.yml

@@ -27,16 +27,23 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, macOS-11, windows-2022]
-        python-version: ['3.7', '3.8', '3.10']
-        requires: ['oldest', 'latest']
-        exclude:
-          - {python-version: '3.7', requires: 'latest'}
-          - {python-version: '3.10', requires: 'oldest'}
-          - {python-version: '3.10', os: 'windows'}  # todo: https://discuss.pytorch.org/t/numpy-is-not-available-error/146192
+        os: ['ubuntu-20.04']
+        python-version: ['3.8']
+        pytorch-version: ['1.9.1', '1.10.2', '1.11.0', '1.12.1']
         include:
-          - {os: 'ubuntu-22.04', python-version: '3.10'}
-          - {os: 'macOS-12', python-version: '3.10'}
+          - {os: 'ubuntu-22.04', python-version: '3.8', pytorch-version: '1.13.0'}
+          - {os: 'ubuntu-22.04', python-version: '3.10', pytorch-version: '1.13.0'}
+          - {os: 'macOS-11', python-version: '3.8', pytorch-version: '1.13.0'}
+          - {os: 'macOS-11', python-version: '3.9', pytorch-version: '1.13.0'}
+          - {os: 'windows-2022', python-version: '3.8', pytorch-version: '1.13.0'}
+          - {os: 'windows-2022', python-version: '3.9', pytorch-version: '1.13.0'}
+          # the oldest configurations
+          - {os: 'ubuntu-20.04', python-version: '3.7', pytorch-version: '1.8.1', requires: 'oldest'}
+          - {os: 'ubuntu-20.04', python-version: '3.8', pytorch-version: '1.8.1', requires: 'oldest'}
+          - {os: 'macOS-11', python-version: '3.7', pytorch-version: '1.8.1', requires: 'oldest'}
+          - {os: 'macOS-11', python-version: '3.8', pytorch-version: '1.8.1', requires: 'oldest'}
+          - {os: 'windows-2019', python-version: '3.7', pytorch-version: '1.8.1', requires: 'oldest'}
+          - {os: 'windows-2019', python-version: '3.8', pytorch-version: '1.8.1', requires: 'oldest'}
     env:
       PYTEST_ARTEFACT: test-results-${{ matrix.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}.xml
       PYTORCH_URL: https://download.pytorch.org/whl/cpu/torch_stable.html
@@ -56,19 +63,25 @@ jobs:
     - name: Setup macOS
       if: ${{ runner.os == 'macOS' }}
       run: |
-        python -c "print('UNITTEST_TIMEOUT=--timeout=120')" >> $GITHUB_ENV
+        echo 'UNITTEST_TIMEOUT=--timeout=120' >> $GITHUB_ENV
         brew install gcc libomp ffmpeg # https://github.com/pytorch/pytorch/issues/20030
     - name: Setup Linux
       if: ${{ runner.os == 'Linux' }}
       run: |
-        python -c "print('UNITTEST_TIMEOUT=--timeout=120')" >> $GITHUB_ENV
+        echo 'UNITTEST_TIMEOUT=--timeout=120' >> $GITHUB_ENV
         sudo apt update --fix-missing
         sudo apt install -y ffmpeg
     - name: Setup Windows
       if: ${{ runner.os == 'windows' }}
       run: |
         choco install ffmpeg
 
+    - name: Set PyTorch version
+      if: inputs.requires != 'oldest'
+      run: |
+        pip install packaging
+        python ./requirements/adjust-versions.py requirements.txt ${{ matrix.pytorch-version }}
+
     - uses: ./.github/actions/caching
       with:
         os: ${{ runner.os }}
@@ -78,9 +91,11 @@ jobs:
     - name: Install pkg
       run: |
         pip --version
-        pip install -e . -r requirements/test.txt --upgrade --find-links $PYTORCH_URL
+        pip install -e . -U --find-links $PYTORCH_URL
+        pip install "pytest" "pytest-doctestplus" -U
         pip list
 
+    # todo: copy this to install checks
     - name: DocTests
       working-directory: ./src
       # NOTE: run coverage on tests does not propagate failure status for Win, https://github.com/nedbat/coveragepy/issues/1003
@@ -97,8 +112,11 @@ jobs:
         pip install --requirement requirements/devel.txt -U --find-links $PYTORCH_URL
         pip list
 
-    - run: echo "::set-output name=TORCH::$(python -c 'import torch ; print(torch.__version__)' 2>&1)"
+    - name: Sanity check
       id: info
+      run: |
+        python -c "from torch import __version__ as ver; ver = ver.split('+')[0] ; assert ver == '${{ matrix.pytorch-version }}', ver"
+        python -c 'import torch ; print("TORCH=" + str(torch.__version__))' >> $GITHUB_OUTPUT
 
     - uses: ./.github/actions/unittesting
       if: ${{ needs.check-diff.outputs.test-dirs != '' }}

README.md

@@ -25,7 +25,6 @@ ______________________________________________________________________
 [![license](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/Lightning-AI/metrics/blob/master/LICENSE)
 
 [![CI testing - complete](https://github.com/Lightning-AI/metrics/actions/workflows/ci_test-full.yml/badge.svg?event=push)](https://github.com/Lightning-AI/metrics/actions/workflows/ci_test-full.yml)
-[![PyTorch & Conda](https://github.com/Lightning-AI/metrics/actions/workflows/ci_test-conda.yml/badge.svg?branch=master&event=push)](https://github.com/Lightning-AI/metrics/actions/workflows/ci_test-conda.yml)
 [![Build Status](https://dev.azure.com/Lightning-AI/Metrics/_apis/build/status/Lightning-AI.metrics?branchName=master)](https://dev.azure.com/Lightning-AI/Metrics/_build/latest?definitionId=3&branchName=master)
 [![codecov](https://codecov.io/gh/Lightning-AI/metrics/branch/master/graph/badge.svg?token=NER6LPI3HS)](https://codecov.io/gh/Lightning-AI/metrics)
 

requirements/adjust-versions.py

@@ -51,39 +51,36 @@ def find_latest(ver: str) -> Dict[str, str]:
     raise ValueError(f"Missing {ver} in {VERSIONS}")
 
 
-def main(path_req: str, torch_version: Optional[str] = None) -> None:
+def adjust(requires: str, torch_version: Optional[str] = None) -> str:
     if not torch_version:
         import torch
 
         torch_version = torch.__version__
     assert torch_version, f"invalid torch: {torch_version}"
-    latest = find_latest(torch_version)
-
-    if path_req == "conda":
-        # this is a special case when we need to get the remaining lib versions
-        # req = " ".join([f"{lib}={ver}" if ver else lib for lib, ver in latest.items() if lib != "torch"])
-        req = " ".join([f"{lib}={ver}" for lib, ver in latest.items() if lib != "torch" and ver])
-        print(req)
-        return
 
-    with open(path_req) as fp:
-        req = fp.readlines()
-    # remove comments
-    req = [r[: r.index("#")] if "#" in r else r for r in req]
-    req = [r.strip() for r in req]
+    # remove comments and strip whitespace
+    requires = re.sub(rf"\s*#.*{os.linesep}", os.linesep, requires).strip()
 
-    for lib, ver in latest.items():
-        for i, ln in enumerate(req):
-            m = re.search(r"(\w\d-_)*?[>=]{0,2}.*", ln)
-            if m and m.group() == lib:
-                req[i] = f"{lib}=={ver}" if ver else lib
+    latest = find_latest(torch_version)
+    for lib, version in latest.items():
+        replace = f"{lib}=={version}" if version else ""
+        requires = re.sub(rf"\b{lib}(?![-_\w]).*", replace, requires)
 
-    req = [r + os.linesep for r in req]
-    logging.info(req)  # on purpose - to debug
-    with open(path_req, "w") as fp:
-        fp.writelines(req)
+    return requires
 
 
 if __name__ == "__main__":
     logging.basicConfig(level=logging.INFO)
-    main(*sys.argv[1:])
+
+    if len(sys.argv) == 3:
+        requirements_path, torch_version = sys.argv[1:]
+    else:
+        requirements_path, torch_version = sys.argv[1], None
+    logging.info(f"requirements_path='{requirements_path}' with torch_version='{torch_version}'")
+
+    with open(requirements_path) as fp:
+        requirements = fp.read()
+    requirements = adjust(requirements, torch_version)
+    logging.info(requirements)  # on purpose - to debug
+    with open(requirements_path, "w") as fp:
+        fp.write(requirements)