From 0bf4dc0b806c6a7155fbe02ea98faaacf1e6bab1 Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Tue, 9 May 2023 11:35:38 +0200 Subject: [PATCH] replace local adjustment script with external (#1758) --- .azure/gpu-integrations.yml | 7 +- .azure/gpu-unittests.yml | 7 +- .github/actions/pull-caches/action.yml | 4 +- .github/workflows/ci-integrate.yml | 5 +- .github/workflows/ci-tests-full.yml | 5 +- Makefile | 1 - requirements/adjust-versions.py | 92 -------------------------- requirements/classification_test.txt | 1 + 8 files changed, 18 insertions(+), 104 deletions(-) delete mode 100644 requirements/adjust-versions.py diff --git a/.azure/gpu-integrations.yml b/.azure/gpu-integrations.yml index 4395b8a8758..ed7ef3648b7 100644 --- a/.azure/gpu-integrations.yml +++ b/.azure/gpu-integrations.yml @@ -64,9 +64,10 @@ jobs: - bash: | set -e - pip install -q packaging fire requests - python ./requirements/adjust-versions.py requirements.txt $(torch-ver) - python ./requirements/adjust-versions.py requirements/integrate.txt $(torch-ver) + pip install -q packaging fire requests wget + python -m wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py + python adjust-torch-versions.py requirements.txt $(torch-ver) + python adjust-torch-versions.py requirements/integrate.txt $(torch-ver) python .github/assistant.py set-oldest-versions --req_files='["requirements/integrate.txt"]' cat requirements/integrate.txt displayName: 'Adjust versions' diff --git a/.azure/gpu-unittests.yml b/.azure/gpu-unittests.yml index 37f208e6939..35047efbe4b 100644 --- a/.azure/gpu-unittests.yml +++ b/.azure/gpu-unittests.yml @@ -102,10 +102,11 @@ jobs: displayName: 'Image info & NVIDIA' - bash: | - pip install -q packaging - python ./requirements/adjust-versions.py requirements.txt $(torch-ver) + pip install -q packaging wget + python -m wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py + python adjust-torch-versions.py requirements.txt $(torch-ver) for fpath in `ls requirements/*.txt`; do - python ./requirements/adjust-versions.py $fpath $(torch-ver) + python adjust-torch-versions.py $fpath $(torch-ver) done displayName: 'Adjust versions' diff --git a/.github/actions/pull-caches/action.yml b/.github/actions/pull-caches/action.yml index b344acb9e5f..f46388c6e3d 100644 --- a/.github/actions/pull-caches/action.yml +++ b/.github/actions/pull-caches/action.yml @@ -25,7 +25,9 @@ runs: - name: Set PyTorch version if: inputs.pytorch-version != '' - run: python ./requirements/adjust-versions.py requirements.txt ${{ inputs.pytorch-version }} + run: | + curl https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py -o adjust-torch-versions.py + python adjust-torch-versions.py requirements.txt ${{ inputs.pytorch-version }} shell: bash - name: Set min. dependencies diff --git a/.github/workflows/ci-integrate.yml b/.github/workflows/ci-integrate.yml index 4f72707b435..ad6d1d4de07 100644 --- a/.github/workflows/ci-integrate.yml +++ b/.github/workflows/ci-integrate.yml @@ -64,10 +64,11 @@ jobs: - name: Install all dependencies run: | set -e + curl https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py -o adjust-torch-versions.py pip install -r requirements/test.txt -r requirements/integrate.txt \ --find-links $PYTORCH_URL -f $PYPI_CACHE --upgrade-strategy eager - python ./requirements/adjust-versions.py requirements.txt - python ./requirements/adjust-versions.py requirements/image.txt + python adjust-torch-versions.py requirements.txt + python adjust-torch-versions.py requirements/image.txt cat requirements.txt pip install -e . --find-links $PYTORCH_URL -f $PYPI_CACHE pip list diff --git a/.github/workflows/ci-tests-full.yml b/.github/workflows/ci-tests-full.yml index 326016ff4df..5b86c70ca88 100644 --- a/.github/workflows/ci-tests-full.yml +++ b/.github/workflows/ci-tests-full.yml @@ -119,10 +119,11 @@ jobs: - name: Install all dependencies run: | + curl https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py -o adjust-torch-versions.py pip install -q cython # needed for installing `pycocotools` in latest config - python ./requirements/adjust-versions.py requirements.txt + python adjust-torch-versions.py requirements.txt for fpath in `ls requirements/*.txt`; do - python ./requirements/adjust-versions.py $fpath + python adjust-torch-versions.py $fpath done pip install --requirement requirements/devel.txt -U \ --find-links $PYTORCH_URL -f $PYPI_CACHE diff --git a/Makefile b/Makefile index f0c753ab205..584b04e06e2 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,6 @@ docs: clean env: export FREEZE_REQUIREMENTS=1 pip install -e . -U - python ./requirements/adjust-versions.py requirements/image.txt pip install -r requirements/devel.txt data: diff --git a/requirements/adjust-versions.py b/requirements/adjust-versions.py deleted file mode 100644 index 1136b6b5d0b..00000000000 --- a/requirements/adjust-versions.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright The Lightning team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -import os -import re -import sys -from typing import Dict, Optional - -from packaging.version import Version - -VERSIONS = [ - {"torch": "2.0.0", "torchvision": "0.15.1", "torchtext": "0.15.1"}, # stable - {"torch": "1.14.0", "torchvision": "0.15.0", "torchtext": "0.15.0"}, # nightly - {"torch": "1.13.1", "torchvision": "0.14.1", "torchtext": "0.14.1"}, # stable - {"torch": "1.13.0", "torchvision": "0.14.0", "torchtext": "0.14.0"}, - {"torch": "1.12.1", "torchvision": "0.13.1", "torchtext": "0.13.1"}, - {"torch": "1.12.0", "torchvision": "0.13.0", "torchtext": "0.13.0"}, - {"torch": "1.11.0", "torchvision": "0.12.0", "torchtext": "0.12.0"}, - {"torch": "1.10.2", "torchvision": "0.11.3", "torchtext": "0.11.2"}, - {"torch": "1.10.1", "torchvision": "0.11.2", "torchtext": "0.11.1"}, - {"torch": "1.10.0", "torchvision": "0.11.1", "torchtext": "0.11.0"}, - {"torch": "1.9.1", "torchvision": "0.10.1", "torchtext": "0.10.1"}, - {"torch": "1.9.0", "torchvision": "0.10.0", "torchtext": "0.10.0"}, - {"torch": "1.8.2", "torchvision": "0.9.1", "torchtext": "0.9.1"}, - {"torch": "1.8.1", "torchvision": "0.9.1", "torchtext": "0.9.1"}, - {"torch": "1.8.0", "torchvision": "0.9.0", "torchtext": "0.9.0"}, -] -VERSIONS.sort(key=lambda v: Version(v["torch"]), reverse=True) - - -def find_latest(ver: str) -> Dict[str, str]: - """Find the latest version.""" - # drop all except semantic version - ver = re.search(r"([\.\d]+)", ver).groups()[0] - # in case there remaining dot at the end - e.g "1.9.0.dev20210504" - ver = ver[:-1] if ver[-1] == "." else ver - logging.info(f"finding ecosystem versions for: {ver}") - - # find first match - for option in VERSIONS: - if option["torch"].startswith(ver): - return option - - raise ValueError(f"Missing {ver} in {VERSIONS}") - - -def adjust(requires: str, torch_version: Optional[str] = None) -> str: - """Adjust the versions to be paired within pytorch ecosystem.""" - if not torch_version: - import torch - - torch_version = torch.__version__ - if not torch_version: - raise ValueError(f"invalid torch: {torch_version}") - - # remove comments and strip whitespace - requires = re.sub(rf"\s*#.*{os.linesep}", os.linesep, requires).strip() - - latest = find_latest(torch_version) - for lib, version in latest.items(): - replace = f"{lib}=={version}" if version else "" - requires = re.sub(rf"\b{lib}(?![-_\w]).*", replace, requires) - - return requires - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - - if len(sys.argv) == 3: - requirements_path, torch_version = sys.argv[1:] - else: - requirements_path, torch_version = sys.argv[1], None - logging.info(f"requirements_path='{requirements_path}' with arg torch_version='{torch_version}'") - - with open(requirements_path) as fp: - requirements = fp.read() - requirements = adjust(requirements, torch_version) - logging.info(requirements) # on purpose - to debug - with open(requirements_path, "w") as fp: - fp.write(requirements) diff --git a/requirements/classification_test.txt b/requirements/classification_test.txt index 5d0728493da..260395b37d2 100644 --- a/requirements/classification_test.txt +++ b/requirements/classification_test.txt @@ -3,4 +3,5 @@ pandas >=1.4.0, <=2.0.1 netcal >1.0.0, <=1.3.5 # calibration_error +numpy <1.24.0 # because of netcal compatibility with `np.float` fairlearn # group_fairness