From 05fe49213144235fc48ac63b5c5a79bec58dc032 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mochol=C3=AD?= Date: Fri, 11 Nov 2022 01:04:03 +0100 Subject: [PATCH 01/10] Refactor checkgroup to avoid duplicated checks --- .azure/gpu-benchmark.yml | 2 + .github/checkgroup.yml | 193 ++++++++++++++------------------------- 2 files changed, 72 insertions(+), 123 deletions(-) diff --git a/.azure/gpu-benchmark.yml b/.azure/gpu-benchmark.yml index 8fad0d69c15d1..ea1cc8afdbc8c 100644 --- a/.azure/gpu-benchmark.yml +++ b/.azure/gpu-benchmark.yml @@ -21,6 +21,8 @@ pr: paths: include: - ".azure/gpu-benchmark.yml" + - "tests/tests_pytorch/benchmarks/**" + - "requirements/pytorch/**" schedules: - cron: "0 0 * * *" # At the end of every day diff --git a/.github/checkgroup.yml b/.github/checkgroup.yml index 156d57e414edb..17f2e00edec44 100644 --- a/.github/checkgroup.yml +++ b/.github/checkgroup.yml @@ -6,15 +6,11 @@ subprojects: # SECTION: pytorch_lightning - - id: "pytorch_lightning" + - id: "pytorch_lightning: Tests workflow" paths: - # all examples don't need to be added because they aren't used in CI, but these are - - "examples/run_pl_examples.sh" - - "examples/pl_basics/backbone_image_classifier.py" - - "examples/pl_basics/autoencoder.py" - - "examples/pl_fault_tolerant/automatic.py" - - "examples/test_pl_examples.py" - - "examples/pl_integrations/dali_image_classifier.py" + - ".github/workflows/ci-pytorch-tests.yml" + - "requirements/lite/**" + - "src/lightning_lite/**" - "requirements/pytorch/**" - "src/pytorch_lightning/**" - "tests/tests_pytorch/**" @@ -39,40 +35,6 @@ subprojects: - "pl-cpu (macOS-11, lightning, 3.10, 1.12)" - "pl-cpu (ubuntu-20.04, lightning, 3.10, 1.12)" - "pl-cpu (windows-2022, lightning, 3.10, 1.12)" - - "pytorch-lightning (GPUs)" - - "pytorch-lightning (HPUs)" - - "pytorch-lightning (IPUs)" - - "pl-cpu (slow, macOS-11, pytorch, 3.7, 1.11)" - - "pl-cpu (slow, ubuntu-20.04, pytorch, 3.7, 1.11)" - - "pl-cpu (slow, windows-2022, pytorch, 3.7, 1.11)" - # TODO: since this job has intermittent availability, it cannot be required or it will block all PL PRs from forks - #- "test-on-tpus" - - - id: "pytorch_lightning: CPU workflow" - paths: - - ".github/workflows/ci-pytorch-tests.yml" - checks: - - "pl-cpu (macOS-11, pytorch, 3.8, 1.10)" - - "pl-cpu (macOS-11, pytorch, 3.9, 1.11)" - - "pl-cpu (macOS-11, pytorch, 3.10, 1.12)" - - "pl-cpu (macOS-11, pytorch, 3.10, 1.13, pre)" - - "pl-cpu (ubuntu-20.04, pytorch, 3.8, 1.10)" - - "pl-cpu (ubuntu-20.04, pytorch, 3.9, 1.11)" - - "pl-cpu (ubuntu-20.04, pytorch, 3.10, 1.11)" - - "pl-cpu (ubuntu-20.04, pytorch, 3.10, 1.12)" - - "pl-cpu (ubuntu-20.04, pytorch, 3.9, 1.13, pre)" - - "pl-cpu (windows-2022, pytorch, 3.9, 1.11)" - - "pl-cpu (windows-2022, pytorch, 3.10, 1.11)" - - "pl-cpu (windows-2022, pytorch, 3.10, 1.12)" - - "pl-cpu (windows-2022, pytorch, 3.8, 1.13, pre)" - - "pl-cpu (macOS-11, lightning, 3.10, 1.12)" - - "pl-cpu (ubuntu-20.04, lightning, 3.10, 1.12)" - - "pl-cpu (windows-2022, lightning, 3.10, 1.12)" - - - id: "pytorch_lightning: Slow workflow" - paths: - - ".github/workflows/ci-pytorch-tests-slow.yml" - checks: - "pl-cpu (slow, macOS-11, pytorch, 3.7, 1.11)" - "pl-cpu (slow, ubuntu-20.04, pytorch, 3.7, 1.11)" - "pl-cpu (slow, windows-2022, pytorch, 3.7, 1.11)" @@ -81,26 +43,71 @@ subprojects: paths: - ".azure/gpu-tests-pytorch.yml" - "tests/tests_pytorch/run_standalone_*.sh" + # only the azure GPU workflow runs the examples + # all examples don't need to be added because they aren't used in CI, but these are + - "examples/run_pl_examples.sh" + - "examples/pl_basics/backbone_image_classifier.py" + - "examples/pl_basics/autoencoder.py" + - "examples/pl_fault_tolerant/automatic.py" + - "examples/test_pl_examples.py" + - "examples/pl_integrations/dali_image_classifier.py" + - "requirements/lite/**" + - "src/lightning_lite/**" + - "requirements/pytorch/**" + - "src/pytorch_lightning/**" + - "tests/tests_pytorch/**" + - "setup.cfg" # includes pytest config + - ".actions/**" checks: - "pytorch-lightning (GPUs)" + - id: "pytorch_lightning: Benchmarks" + paths: + - ".azure/gpu-benchmark.yml" + - "tests/tests_pytorch/benchmarks/**" + - "requirements/pytorch/**" + checks: + - "pytorch-lightning.Benchmark" + - id: "pytorch_lightning: Azure HPU" paths: - ".azure/hpu-tests.yml" + - "requirements/lite/**" + - "src/lightning_lite/**" + - "requirements/pytorch/**" + - "src/pytorch_lightning/**" + - "tests/tests_pytorch/**" + - "setup.cfg" # includes pytest config + - ".actions/**" checks: - "pytorch-lightning (HPUs)" - id: "pytorch_lightning: Azure IPU" paths: - ".azure/ipu-tests.yml" + - "requirements/lite/**" + - "src/lightning_lite/**" + - "requirements/pytorch/**" + - "src/pytorch_lightning/**" + - "tests/tests_pytorch/**" + - "setup.cfg" # includes pytest config + - ".actions/**" checks: - "pytorch-lightning (IPUs)" - - id: "pytorch-lightning: TPU workflow" - paths: - - ".github/workflows/tpu-tests.yml" - checks: - - "test-on-tpus" + # TODO: since this job has intermittent availability, it cannot be required + #- id: "pytorch-lightning: TPU workflow" + # paths: + # - ".github/workflows/tpu-tests.yml" + # - "requirements/lite/**" + # - "src/lightning_lite/**" + # - "requirements/pytorch/**" + # - "src/pytorch_lightning/**" + # - "tests/tests_pytorch/**" + # - "setup.cfg" # includes pytest config + # - ".actions/**" + # checks: + # - "test-on-tpus" - id: "pytorch_lightning: Docs" paths: @@ -138,10 +145,11 @@ subprojects: # SECTION: lightning_lite - - id: "lightning_lite" + - id: "lightning_lite: CPU workflow" paths: - "requirements/lite/**" - "src/lightning_lite/**" + - "tests/tests_lite/**" - "setup.cfg" # includes pytest config - ".actions/**" checks: @@ -159,51 +167,6 @@ subprojects: - "lite-cpu (macOS-11, lightning, 3.8, 1.12)" - "lite-cpu (ubuntu-20.04, lightning, 3.8, 1.12)" - "lite-cpu (windows-2022, lightning, 3.8, 1.12)" - - "lightning-lite (GPUs)" - # Lite also requires PL checks as it depends on Lite - - "pl-cpu (macOS-11, pytorch, 3.8, 1.10)" - - "pl-cpu (macOS-11, pytorch, 3.9, 1.11)" - - "pl-cpu (macOS-11, pytorch, 3.10, 1.12)" - - "pl-cpu (macOS-11, pytorch, 3.10, 1.13, pre)" - - "pl-cpu (ubuntu-20.04, pytorch, 3.8, 1.10)" - - "pl-cpu (ubuntu-20.04, pytorch, 3.9, 1.11)" - - "pl-cpu (ubuntu-20.04, pytorch, 3.10, 1.11)" - - "pl-cpu (ubuntu-20.04, pytorch, 3.10, 1.12)" - - "pl-cpu (ubuntu-20.04, pytorch, 3.9, 1.13, pre)" - - "pl-cpu (windows-2022, pytorch, 3.9, 1.11)" - - "pl-cpu (windows-2022, pytorch, 3.10, 1.11)" - - "pl-cpu (windows-2022, pytorch, 3.10, 1.12)" - - "pl-cpu (windows-2022, pytorch, 3.8, 1.13, pre)" - - "pl-cpu (macOS-11, lightning, 3.10, 1.12)" - - "pl-cpu (ubuntu-20.04, lightning, 3.10, 1.12)" - - "pl-cpu (windows-2022, lightning, 3.10, 1.12)" - - "pytorch-lightning (GPUs)" - - "pytorch-lightning (HPUs)" - - "pytorch-lightning (IPUs)" - - "pl-cpu (slow, macOS-11, pytorch, 3.7, 1.11)" - - "pl-cpu (slow, ubuntu-20.04, pytorch, 3.7, 1.11)" - - "pl-cpu (slow, windows-2022, pytorch, 3.7, 1.11)" - #- "test-on-tpus" - - - id: "lightning_lite: Tests" - paths: - - "tests/tests_lite/**" - checks: - - "lite-cpu (macOS-11, lite, 3.9, 1.11)" - - "lite-cpu (macOS-11, lite, 3.8, 1.10)" - - "lite-cpu (macOS-11, lite, 3.10, 1.12)" - - "lite-cpu (macOS-11, lite, 3.10, 1.13, pre)" - - "lite-cpu (ubuntu-20.04, lite, 3.8, 1.11)" - - "lite-cpu (ubuntu-20.04, lite, 3.10, 1.12)" - - "lite-cpu (ubuntu-20.04, lite, 3.9, 1.13, pre)" - - "lite-cpu (windows-2022, lite, 3.9, 1.10)" - - "lite-cpu (windows-2022, lite, 3.10, 1.11)" - - "lite-cpu (windows-2022, lite, 3.10, 1.12)" - - "lite-cpu (windows-2022, lite, 3.8, 1.13, pre)" - - "lite-cpu (macOS-11, lightning, 3.8, 1.12)" - - "lite-cpu (ubuntu-20.04, lightning, 3.8, 1.12)" - - "lite-cpu (windows-2022, lightning, 3.8, 1.12)" - - "lightning-lite (GPUs)" - id: "lightning_lite: Azure GPU" paths: @@ -212,34 +175,25 @@ subprojects: - "examples/run_lite_examples.sh" - "tests/tests_lite/run_standalone_*.sh" - "tests/tests_pytorch/run_standalone_tests.sh" # used by Lite through a symlink + - "requirements/lite/**" + - "src/lightning_lite/**" + - "tests/tests_lite/**" + - "setup.cfg" # includes pytest config + - ".actions/**" checks: - "lightning-lite (GPUs)" # SECTION: lightning_app - - id: "lightning_app" + - id: "lightning_app: Tests workflow" paths: + - ".github/workflows/ci-app-tests.yml" - "requirements/app/**" - "src/lightning_app/**" - "tests/tests_app/**" - "examples/app_*/**" # some tests_app tests call examples files - "setup.py" - ".actions/**" - checks: - - "App.cloud-e2e" - - "app-pytest (macOS-11, app, 3.8, latest)" - - "app-pytest (macOS-11, app, 3.8, oldest)" - - "app-pytest (macOS-11, lightning, 3.9, latest)" - - "app-pytest (ubuntu-20.04, app, 3.8, latest)" - - "app-pytest (ubuntu-20.04, app, 3.8, oldest)" - - "app-pytest (ubuntu-20.04, lightning, 3.9, latest)" - - "app-pytest (windows-2022, app, 3.8, latest)" - - "app-pytest (windows-2022, app, 3.8, oldest)" - - "app-pytest (windows-2022, lightning, 3.8, latest)" - - - id: "lightning_app: Tests workflow" - paths: - - ".github/workflows/ci-app-tests.yml" checks: - "app-pytest (macOS-11, app, 3.8, latest)" - "app-pytest (macOS-11, app, 3.8, oldest)" @@ -253,6 +207,7 @@ subprojects: - id: "lightning_app: Examples" paths: + - ".github/workflows/ci-app-examples.yml" - "requirements/app/**" - "src/lightning_app/**" - "tests/tests_app_examples/**" @@ -270,23 +225,15 @@ subprojects: - "app-examples (windows-2022, app, 3.9, oldest)" - "app-examples (windows-2022, lightning, 3.9, latest)" - - id: "lightning_app: Examples workflow" - paths: - - ".github/workflows/ci-app-examples.yml" - checks: - - "app-examples (macOS-11, app, 3.9, latest)" - - "app-examples (macOS-11, app, 3.9, oldest)" - - "app-examples (macOS-11, lightning, 3.9, latest)" - - "app-examples (ubuntu-20.04, app, 3.9, latest)" - - "app-examples (ubuntu-20.04, app, 3.9, oldest)" - - "app-examples (ubuntu-20.04, lightning, 3.9, latest)" - - "app-examples (windows-2022, app, 3.9, latest)" - - "app-examples (windows-2022, app, 3.9, oldest)" - - "app-examples (windows-2022, lightning, 3.9, latest)" - - id: "lightning_app: Azure" paths: - ".azure/app-cloud-e2e.yml" + - "requirements/app/**" + - "src/lightning_app/**" + - "tests/tests_app/**" + - "examples/app_*/**" # some tests_app tests call examples files + - "setup.py" + - ".actions/**" checks: - "App.cloud-e2e" From fdaa0d3a25ce431301915c2e6886bfc9062abccb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mochol=C3=AD?= Date: Fri, 11 Nov 2022 01:37:34 +0100 Subject: [PATCH 02/10] Fixes --- .azure/app-cloud-e2e.yml | 4 ++- .azure/gpu-tests-lite.yml | 7 +++--- .azure/gpu-tests-pytorch.yml | 1 + .github/checkgroup.yml | 32 +++++++++++++----------- .github/workflows/ci-app-examples.yml | 1 - .github/workflows/ci-app-tests.yml | 1 - .github/workflows/ci-lite-tests.yml | 4 +-- .github/workflows/ci-pkg-install.yml | 1 - .github/workflows/ci-pytorch-dockers.yml | 7 ++---- .github/workflows/ci-pytorch-tests.yml | 1 - .github/workflows/docs-checks.yml | 1 - .github/workflows/docs-deploy.yml | 1 + .github/workflows/tpu-tests.yml | 4 ++- 13 files changed, 33 insertions(+), 32 deletions(-) diff --git a/.azure/app-cloud-e2e.yml b/.azure/app-cloud-e2e.yml index b8c8305715300..b99a9a4e03532 100644 --- a/.azure/app-cloud-e2e.yml +++ b/.azure/app-cloud-e2e.yml @@ -30,8 +30,10 @@ pr: - ".azure/app-cloud-e2e.yml" - "requirements/app/**" - "src/lightning_app/**" - - "examples/app_*" + - "tests/tests_app/**" + - "examples/app_*" # some tests_app tests call examples files - "tests/tests_app_examples/**" + - "setup.py" - ".actions/**" # variables are automatically exported as environment variables so this will override pip's default cache dir diff --git a/.azure/gpu-tests-lite.yml b/.azure/gpu-tests-lite.yml index 7125465f94f25..b1b69cb1ac138 100644 --- a/.azure/gpu-tests-lite.yml +++ b/.azure/gpu-tests-lite.yml @@ -20,14 +20,15 @@ pr: - "release/*" paths: include: + - ".azure/gpu-tests-lite.yml" - "examples/lite/**" - "examples/run_lite_examples.sh" - - ".azure/gpu-tests-lite.yml" + - "tests/tests_lite/run_standalone_*.sh" + - "tests/tests_pytorch/run_standalone_tests.sh" # used by Lite through a symlink - "requirements/lite/**" - "src/lightning_lite/**" - "tests/tests_lite/**" - - "tests/tests_pytorch/run_standalone_tests.sh" - - "tests/tests_lite/run_standalone_tests.sh" # a symlink to the one above + - "setup.cfg" # includes pytest config - ".actions/**" jobs: diff --git a/.azure/gpu-tests-pytorch.yml b/.azure/gpu-tests-pytorch.yml index 2c5500b26b67f..9ff3af76d4c91 100644 --- a/.azure/gpu-tests-pytorch.yml +++ b/.azure/gpu-tests-pytorch.yml @@ -21,6 +21,7 @@ pr: paths: include: - ".azure/gpu-tests-pytorch.yml" + - "tests/tests_pytorch/run_standalone_*.sh" - "examples/run_pl_examples.sh" - "examples/pl_basics/backbone_image_classifier.py" - "examples/pl_basics/autoencoder.py" diff --git a/.github/checkgroup.yml b/.github/checkgroup.yml index 17f2e00edec44..5509596ffb980 100644 --- a/.github/checkgroup.yml +++ b/.github/checkgroup.yml @@ -9,9 +9,9 @@ subprojects: - id: "pytorch_lightning: Tests workflow" paths: - ".github/workflows/ci-pytorch-tests.yml" - - "requirements/lite/**" + - "requirements/lite/*" - "src/lightning_lite/**" - - "requirements/pytorch/**" + - "requirements/pytorch/*" - "src/pytorch_lightning/**" - "tests/tests_pytorch/**" - "tests/legacy/back-compatible-versions.txt" @@ -51,12 +51,12 @@ subprojects: - "examples/pl_fault_tolerant/automatic.py" - "examples/test_pl_examples.py" - "examples/pl_integrations/dali_image_classifier.py" - - "requirements/lite/**" - - "src/lightning_lite/**" - "requirements/pytorch/**" - "src/pytorch_lightning/**" - "tests/tests_pytorch/**" - "setup.cfg" # includes pytest config + - "requirements/lite/**" + - "src/lightning_lite/**" - ".actions/**" checks: - "pytorch-lightning (GPUs)" @@ -72,6 +72,7 @@ subprojects: - id: "pytorch_lightning: Azure HPU" paths: - ".azure/hpu-tests.yml" + - "examples/pl_hpu/mnist_sample.py" - "requirements/lite/**" - "src/lightning_lite/**" - "requirements/pytorch/**" @@ -101,6 +102,7 @@ subprojects: # - ".github/workflows/tpu-tests.yml" # - "requirements/lite/**" # - "src/lightning_lite/**" + # - "tests/tests_lite/**" # - "requirements/pytorch/**" # - "src/pytorch_lightning/**" # - "tests/tests_pytorch/**" @@ -113,9 +115,11 @@ subprojects: paths: - "src/pytorch_lightning/**" - "docs/source-pytorch/**" - - ".github/workflows/docs-*.yml" + - ".github/workflows/docs-checks.yml" - "requirements/docs.txt" - "requirements/pytorch/**" + - "setup.py" + - ".actions/**" checks: - "make-doctest (pytorch)" - "make-html (pytorch)" @@ -124,11 +128,8 @@ subprojects: paths: - "dockers/**" - ".github/workflows/ci-pytorch-dockers.yml" - - "requirements.txt" - - "requirements/*.txt" - "requirements/pytorch/*" - "environment.yml" - - ".github/workflows/*docker*.yml" - "setup.py" - ".actions/**" checks: @@ -188,10 +189,10 @@ subprojects: - id: "lightning_app: Tests workflow" paths: - ".github/workflows/ci-app-tests.yml" - - "requirements/app/**" - "src/lightning_app/**" - "tests/tests_app/**" - - "examples/app_*/**" # some tests_app tests call examples files + - "examples/app_*" # some tests_app tests call examples files + - "requirements/app/*" - "setup.py" - ".actions/**" checks: @@ -208,10 +209,10 @@ subprojects: - id: "lightning_app: Examples" paths: - ".github/workflows/ci-app-examples.yml" - - "requirements/app/**" - "src/lightning_app/**" - "tests/tests_app_examples/**" - - "examples/app_*/**" + - "examples/app_*" + - "requirements/app/*" - "setup.py" - ".actions/**" checks: @@ -231,7 +232,8 @@ subprojects: - "requirements/app/**" - "src/lightning_app/**" - "tests/tests_app/**" - - "examples/app_*/**" # some tests_app tests call examples files + - "examples/app_*" # some tests_app tests call examples files + - "tests/tests_app_examples/**" - "setup.py" - ".actions/**" checks: @@ -241,9 +243,11 @@ subprojects: paths: - "src/lightning_app/**" - "docs/source-app/**" - - ".github/workflows/docs-*.yml" + - ".github/workflows/docs-checks.yml" - "requirements/docs.txt" - "requirements/app/**" + - "setup.py" + - ".actions/**" checks: - "make-doctest (app)" - "make-html (app)" diff --git a/.github/workflows/ci-app-examples.yml b/.github/workflows/ci-app-examples.yml index 457a01d643aab..f5b48f3563bb0 100644 --- a/.github/workflows/ci-app-examples.yml +++ b/.github/workflows/ci-app-examples.yml @@ -13,7 +13,6 @@ on: - "tests/tests_app_examples/**" - "examples/app_*" - "requirements/app/*" - - "!requirements/app/docs.txt" - "setup.py" - ".actions/**" diff --git a/.github/workflows/ci-app-tests.yml b/.github/workflows/ci-app-tests.yml index 4bcf1850eff72..76f865967519c 100644 --- a/.github/workflows/ci-app-tests.yml +++ b/.github/workflows/ci-app-tests.yml @@ -13,7 +13,6 @@ on: - "tests/tests_app/**" - "examples/app_*" # some tests_app tests call examples files - "requirements/app/*" - - "!requirements/app/docs.txt" - "setup.py" - ".actions/**" diff --git a/.github/workflows/ci-lite-tests.yml b/.github/workflows/ci-lite-tests.yml index 49a967526aa2e..4163658399969 100644 --- a/.github/workflows/ci-lite-tests.yml +++ b/.github/workflows/ci-lite-tests.yml @@ -8,12 +8,10 @@ on: branches: [master, "release/*"] types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped paths: - - "requirements/lite/*" - - "!requirements/lite/docs.txt" + - "requirements/lite/**" - "src/lightning_lite/**" - "tests/tests_lite/**" - "setup.cfg" # includes pytest config - - ".github/workflows/ci-lite-tests.yml" - ".actions/**" concurrency: diff --git a/.github/workflows/ci-pkg-install.yml b/.github/workflows/ci-pkg-install.yml index b1fad271779fc..63b8f4227e48e 100644 --- a/.github/workflows/ci-pkg-install.yml +++ b/.github/workflows/ci-pkg-install.yml @@ -13,7 +13,6 @@ on: - "setup.py" - "src/**" - "requirements/**" - - "!requirements/*/docs.txt" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} diff --git a/.github/workflows/ci-pytorch-dockers.yml b/.github/workflows/ci-pytorch-dockers.yml index 6ac2f5d280a42..99aed90649c64 100644 --- a/.github/workflows/ci-pytorch-dockers.yml +++ b/.github/workflows/ci-pytorch-dockers.yml @@ -8,12 +8,9 @@ on: types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped paths: - "dockers/**" - - "!dockers/README.md" - - "requirements.txt" - - "requirements/**" - - "!requirements/*/docs.txt" + - ".github/workflows/ci-pytorch-dockers.yml" + - "requirements/pytorch/*" - "environment.yml" - - ".github/workflows/*docker*.yml" - "setup.py" - ".actions/**" schedule: diff --git a/.github/workflows/ci-pytorch-tests.yml b/.github/workflows/ci-pytorch-tests.yml index 1b3aba144ad7e..81747db14b45a 100644 --- a/.github/workflows/ci-pytorch-tests.yml +++ b/.github/workflows/ci-pytorch-tests.yml @@ -17,7 +17,6 @@ on: - "requirements/lite/*" - "src/lightning_lite/**" - ".actions/**" - - "!requirements/*/docs.txt" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} diff --git a/.github/workflows/docs-checks.yml b/.github/workflows/docs-checks.yml index 4169bc33e4632..9ae096e75626c 100644 --- a/.github/workflows/docs-checks.yml +++ b/.github/workflows/docs-checks.yml @@ -14,7 +14,6 @@ on: - "docs/**" - "src/**" - "setup.py" - - "setup.cfg" - ".github/workflows/docs-checks.yml" concurrency: diff --git a/.github/workflows/docs-deploy.yml b/.github/workflows/docs-deploy.yml index 38c5db8a8946c..98942bde3287d 100644 --- a/.github/workflows/docs-deploy.yml +++ b/.github/workflows/docs-deploy.yml @@ -6,6 +6,7 @@ on: branches: ["master", "release/*"] paths: - ".github/workflows/docs-deploy.yml" + # FIXME: what is this job for? env: FREEZE_REQUIREMENTS: 1 diff --git a/.github/workflows/tpu-tests.yml b/.github/workflows/tpu-tests.yml index a7ffe3e10afe0..15a506e43d51d 100644 --- a/.github/workflows/tpu-tests.yml +++ b/.github/workflows/tpu-tests.yml @@ -8,8 +8,10 @@ on: types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped paths: - ".github/workflows/tpu-tests.yml" + - "requirements/lite/**" + - "src/lightning_lite/**" + - "tests/tests_lite/**" - "requirements/pytorch/*" - - "!requirements/pytorch/docs.txt" - "src/pytorch_lightning/**" - "tests/tests_pytorch/**" - "setup.cfg" # includes pytest config From 27fbc7d2ba9fd3104eaff276c56d7468711fce13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mochol=C3=AD?= Date: Fri, 11 Nov 2022 01:40:16 +0100 Subject: [PATCH 03/10] Fix --- .github/checkgroup.yml | 3 ++- .github/workflows/ci-lite-tests.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/checkgroup.yml b/.github/checkgroup.yml index 5509596ffb980..5743d33c97c98 100644 --- a/.github/checkgroup.yml +++ b/.github/checkgroup.yml @@ -148,10 +148,11 @@ subprojects: - id: "lightning_lite: CPU workflow" paths: - - "requirements/lite/**" + - "requirements/lite/*" - "src/lightning_lite/**" - "tests/tests_lite/**" - "setup.cfg" # includes pytest config + - ".github/workflows/ci-lite-tests.yml" - ".actions/**" checks: - "lite-cpu (macOS-11, lite, 3.9, 1.11)" diff --git a/.github/workflows/ci-lite-tests.yml b/.github/workflows/ci-lite-tests.yml index 4163658399969..31416d638100a 100644 --- a/.github/workflows/ci-lite-tests.yml +++ b/.github/workflows/ci-lite-tests.yml @@ -8,10 +8,11 @@ on: branches: [master, "release/*"] types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped paths: - - "requirements/lite/**" + - "requirements/lite/*" - "src/lightning_lite/**" - "tests/tests_lite/**" - "setup.cfg" # includes pytest config + - ".github/workflows/ci-lite-tests.yml" - ".actions/**" concurrency: From dbe283ff52dbb6704c14df24ac9228141e1924f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mochol=C3=AD?= Date: Fri, 11 Nov 2022 02:34:00 +0100 Subject: [PATCH 04/10] Update .github/workflows/ci-pytorch-dockers.yml Co-authored-by: Akihiro Nitta --- .github/workflows/ci-pytorch-dockers.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-pytorch-dockers.yml b/.github/workflows/ci-pytorch-dockers.yml index 99aed90649c64..582283705afd2 100644 --- a/.github/workflows/ci-pytorch-dockers.yml +++ b/.github/workflows/ci-pytorch-dockers.yml @@ -10,6 +10,7 @@ on: - "dockers/**" - ".github/workflows/ci-pytorch-dockers.yml" - "requirements/pytorch/*" + - "requirements/lite/*" - "environment.yml" - "setup.py" - ".actions/**" From 09c6d1055030c9181962ddd64d9333e475153324 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mochol=C3=AD?= Date: Fri, 11 Nov 2022 02:34:10 +0100 Subject: [PATCH 05/10] Akis suggestion --- .azure/gpu-tests-pytorch.yml | 1 - .github/checkgroup.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.azure/gpu-tests-pytorch.yml b/.azure/gpu-tests-pytorch.yml index 9ff3af76d4c91..2c5500b26b67f 100644 --- a/.azure/gpu-tests-pytorch.yml +++ b/.azure/gpu-tests-pytorch.yml @@ -21,7 +21,6 @@ pr: paths: include: - ".azure/gpu-tests-pytorch.yml" - - "tests/tests_pytorch/run_standalone_*.sh" - "examples/run_pl_examples.sh" - "examples/pl_basics/backbone_image_classifier.py" - "examples/pl_basics/autoencoder.py" diff --git a/.github/checkgroup.yml b/.github/checkgroup.yml index 5743d33c97c98..e05f7836f9fee 100644 --- a/.github/checkgroup.yml +++ b/.github/checkgroup.yml @@ -42,7 +42,6 @@ subprojects: - id: "pytorch_lightning: Azure GPU" paths: - ".azure/gpu-tests-pytorch.yml" - - "tests/tests_pytorch/run_standalone_*.sh" # only the azure GPU workflow runs the examples # all examples don't need to be added because they aren't used in CI, but these are - "examples/run_pl_examples.sh" From 7cfa4a4fae65d8f5700ad3508d57ee4648a8c247 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mochol=C3=AD?= Date: Fri, 11 Nov 2022 13:00:37 +0100 Subject: [PATCH 06/10] Add back setup.cfg for docs --- .github/checkgroup.yml | 2 ++ .github/workflows/docs-checks.yml | 1 + 2 files changed, 3 insertions(+) diff --git a/.github/checkgroup.yml b/.github/checkgroup.yml index e05f7836f9fee..8706d82cdadad 100644 --- a/.github/checkgroup.yml +++ b/.github/checkgroup.yml @@ -118,6 +118,7 @@ subprojects: - "requirements/docs.txt" - "requirements/pytorch/**" - "setup.py" + - "setup.cfg" # includes metadata used in the package creation - ".actions/**" checks: - "make-doctest (pytorch)" @@ -247,6 +248,7 @@ subprojects: - "requirements/docs.txt" - "requirements/app/**" - "setup.py" + - "setup.cfg" # includes metadata used in the package creation - ".actions/**" checks: - "make-doctest (app)" diff --git a/.github/workflows/docs-checks.yml b/.github/workflows/docs-checks.yml index 9ae096e75626c..c831a35f8f022 100644 --- a/.github/workflows/docs-checks.yml +++ b/.github/workflows/docs-checks.yml @@ -14,6 +14,7 @@ on: - "docs/**" - "src/**" - "setup.py" + - "setup.cfg" # includes metadata used in the package creation - ".github/workflows/docs-checks.yml" concurrency: From 21c94b2063647202458212e57483a4aef9dfa0c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mochol=C3=AD?= Date: Fri, 11 Nov 2022 13:02:20 +0100 Subject: [PATCH 07/10] Borda's suggestions --- .azure/app-cloud-e2e.yml | 2 +- .github/checkgroup.yml | 5 +++-- .github/workflows/ci-pytorch-dockers.yml | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.azure/app-cloud-e2e.yml b/.azure/app-cloud-e2e.yml index b99a9a4e03532..fb78404b9371e 100644 --- a/.azure/app-cloud-e2e.yml +++ b/.azure/app-cloud-e2e.yml @@ -31,7 +31,7 @@ pr: - "requirements/app/**" - "src/lightning_app/**" - "tests/tests_app/**" - - "examples/app_*" # some tests_app tests call examples files + - "examples/app_*/**" # some tests_app tests call examples files - "tests/tests_app_examples/**" - "setup.py" - ".actions/**" diff --git a/.github/checkgroup.yml b/.github/checkgroup.yml index 8706d82cdadad..32ca55500c7fe 100644 --- a/.github/checkgroup.yml +++ b/.github/checkgroup.yml @@ -128,7 +128,8 @@ subprojects: paths: - "dockers/**" - ".github/workflows/ci-pytorch-dockers.yml" - - "requirements/pytorch/*" + - "requirements/pytorch/**" + - "requirements/lite/**" - "environment.yml" - "setup.py" - ".actions/**" @@ -192,7 +193,7 @@ subprojects: - ".github/workflows/ci-app-tests.yml" - "src/lightning_app/**" - "tests/tests_app/**" - - "examples/app_*" # some tests_app tests call examples files + - "examples/app_*/**" # some tests_app tests call examples files - "requirements/app/*" - "setup.py" - ".actions/**" diff --git a/.github/workflows/ci-pytorch-dockers.yml b/.github/workflows/ci-pytorch-dockers.yml index 582283705afd2..ea41e76e7a88e 100644 --- a/.github/workflows/ci-pytorch-dockers.yml +++ b/.github/workflows/ci-pytorch-dockers.yml @@ -9,8 +9,8 @@ on: paths: - "dockers/**" - ".github/workflows/ci-pytorch-dockers.yml" - - "requirements/pytorch/*" - - "requirements/lite/*" + - "requirements/pytorch/**" + - "requirements/lite/**" - "environment.yml" - "setup.py" - ".actions/**" From 5317d671ca306a51a1e0c0a631d6dbaa60f49892 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mochol=C3=AD?= Date: Fri, 11 Nov 2022 13:03:56 +0100 Subject: [PATCH 08/10] TODO --- .github/workflows/docs-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs-deploy.yml b/.github/workflows/docs-deploy.yml index 98942bde3287d..acbb74290ac8e 100644 --- a/.github/workflows/docs-deploy.yml +++ b/.github/workflows/docs-deploy.yml @@ -6,7 +6,7 @@ on: branches: ["master", "release/*"] paths: - ".github/workflows/docs-deploy.yml" - # FIXME: what is this job for? + # TODO: this workflow is just for debugging. extend the paths that should trigger it env: FREEZE_REQUIREMENTS: 1 From 5e021a250d2dbb3c403b24ea46cbb7e025d8437d Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Fri, 11 Nov 2022 13:20:29 +0100 Subject: [PATCH 09/10] Apply suggestions from code review --- .github/checkgroup.yml | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/checkgroup.yml b/.github/checkgroup.yml index 32ca55500c7fe..4fe16c0abfbf2 100644 --- a/.github/checkgroup.yml +++ b/.github/checkgroup.yml @@ -50,13 +50,13 @@ subprojects: - "examples/pl_fault_tolerant/automatic.py" - "examples/test_pl_examples.py" - "examples/pl_integrations/dali_image_classifier.py" - - "requirements/pytorch/**" + - "requirements/pytorch/*" - "src/pytorch_lightning/**" - "tests/tests_pytorch/**" - "setup.cfg" # includes pytest config - - "requirements/lite/**" + - "requirements/lite/*" - "src/lightning_lite/**" - - ".actions/**" + - ".actions/*" checks: - "pytorch-lightning (GPUs)" @@ -64,7 +64,7 @@ subprojects: paths: - ".azure/gpu-benchmark.yml" - "tests/tests_pytorch/benchmarks/**" - - "requirements/pytorch/**" + - "requirements/pytorch/*" checks: - "pytorch-lightning.Benchmark" @@ -72,26 +72,26 @@ subprojects: paths: - ".azure/hpu-tests.yml" - "examples/pl_hpu/mnist_sample.py" - - "requirements/lite/**" + - "requirements/lite/*" - "src/lightning_lite/**" - - "requirements/pytorch/**" + - "requirements/pytorch/*" - "src/pytorch_lightning/**" - "tests/tests_pytorch/**" - "setup.cfg" # includes pytest config - - ".actions/**" + - ".actions/*" checks: - "pytorch-lightning (HPUs)" - id: "pytorch_lightning: Azure IPU" paths: - ".azure/ipu-tests.yml" - - "requirements/lite/**" + - "requirements/lite/*" - "src/lightning_lite/**" - - "requirements/pytorch/**" + - "requirements/pytorch/*" - "src/pytorch_lightning/**" - "tests/tests_pytorch/**" - "setup.cfg" # includes pytest config - - ".actions/**" + - ".actions/*" checks: - "pytorch-lightning (IPUs)" @@ -116,7 +116,7 @@ subprojects: - "docs/source-pytorch/**" - ".github/workflows/docs-checks.yml" - "requirements/docs.txt" - - "requirements/pytorch/**" + - "requirements/pytorch/*" - "setup.py" - "setup.cfg" # includes metadata used in the package creation - ".actions/**" @@ -128,8 +128,8 @@ subprojects: paths: - "dockers/**" - ".github/workflows/ci-pytorch-dockers.yml" - - "requirements/pytorch/**" - - "requirements/lite/**" + - "requirements/pytorch/*" + - "requirements/lite/*" - "environment.yml" - "setup.py" - ".actions/**" @@ -178,11 +178,11 @@ subprojects: - "examples/run_lite_examples.sh" - "tests/tests_lite/run_standalone_*.sh" - "tests/tests_pytorch/run_standalone_tests.sh" # used by Lite through a symlink - - "requirements/lite/**" + - "requirements/lite/*" - "src/lightning_lite/**" - "tests/tests_lite/**" - "setup.cfg" # includes pytest config - - ".actions/**" + - ".actions/*" checks: - "lightning-lite (GPUs)" @@ -231,13 +231,13 @@ subprojects: - id: "lightning_app: Azure" paths: - ".azure/app-cloud-e2e.yml" - - "requirements/app/**" + - "requirements/app/*" - "src/lightning_app/**" - "tests/tests_app/**" - - "examples/app_*" # some tests_app tests call examples files + - "examples/app_*/**" # some tests_app tests call examples files - "tests/tests_app_examples/**" - "setup.py" - - ".actions/**" + - ".actions/*" checks: - "App.cloud-e2e" @@ -250,7 +250,7 @@ subprojects: - "requirements/app/**" - "setup.py" - "setup.cfg" # includes metadata used in the package creation - - ".actions/**" + - ".actions/*" checks: - "make-doctest (app)" - "make-html (app)" From 2115f38c3a93cdfbc76e2e7e68eab7bfa68c7c32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mochol=C3=AD?= Date: Fri, 11 Nov 2022 13:36:34 +0100 Subject: [PATCH 10/10] Prefer ** over * in case directories are added one day --- .github/checkgroup.yml | 46 +++++++++++++------------- .github/workflows/ci-app-examples.yml | 2 +- .github/workflows/ci-app-tests.yml | 2 +- .github/workflows/ci-lite-tests.yml | 2 +- .github/workflows/ci-pytorch-tests.yml | 4 +-- .github/workflows/tpu-tests.yml | 2 +- 6 files changed, 29 insertions(+), 29 deletions(-) diff --git a/.github/checkgroup.yml b/.github/checkgroup.yml index 4fe16c0abfbf2..9c38a978de093 100644 --- a/.github/checkgroup.yml +++ b/.github/checkgroup.yml @@ -9,9 +9,9 @@ subprojects: - id: "pytorch_lightning: Tests workflow" paths: - ".github/workflows/ci-pytorch-tests.yml" - - "requirements/lite/*" + - "requirements/lite/**" - "src/lightning_lite/**" - - "requirements/pytorch/*" + - "requirements/pytorch/**" - "src/pytorch_lightning/**" - "tests/tests_pytorch/**" - "tests/legacy/back-compatible-versions.txt" @@ -50,13 +50,13 @@ subprojects: - "examples/pl_fault_tolerant/automatic.py" - "examples/test_pl_examples.py" - "examples/pl_integrations/dali_image_classifier.py" - - "requirements/pytorch/*" + - "requirements/pytorch/**" - "src/pytorch_lightning/**" - "tests/tests_pytorch/**" - "setup.cfg" # includes pytest config - - "requirements/lite/*" + - "requirements/lite/**" - "src/lightning_lite/**" - - ".actions/*" + - ".actions/**" checks: - "pytorch-lightning (GPUs)" @@ -64,7 +64,7 @@ subprojects: paths: - ".azure/gpu-benchmark.yml" - "tests/tests_pytorch/benchmarks/**" - - "requirements/pytorch/*" + - "requirements/pytorch/**" checks: - "pytorch-lightning.Benchmark" @@ -72,26 +72,26 @@ subprojects: paths: - ".azure/hpu-tests.yml" - "examples/pl_hpu/mnist_sample.py" - - "requirements/lite/*" + - "requirements/lite/**" - "src/lightning_lite/**" - - "requirements/pytorch/*" + - "requirements/pytorch/**" - "src/pytorch_lightning/**" - "tests/tests_pytorch/**" - "setup.cfg" # includes pytest config - - ".actions/*" + - ".actions/**" checks: - "pytorch-lightning (HPUs)" - id: "pytorch_lightning: Azure IPU" paths: - ".azure/ipu-tests.yml" - - "requirements/lite/*" + - "requirements/lite/**" - "src/lightning_lite/**" - - "requirements/pytorch/*" + - "requirements/pytorch/**" - "src/pytorch_lightning/**" - "tests/tests_pytorch/**" - "setup.cfg" # includes pytest config - - ".actions/*" + - ".actions/**" checks: - "pytorch-lightning (IPUs)" @@ -116,7 +116,7 @@ subprojects: - "docs/source-pytorch/**" - ".github/workflows/docs-checks.yml" - "requirements/docs.txt" - - "requirements/pytorch/*" + - "requirements/pytorch/**" - "setup.py" - "setup.cfg" # includes metadata used in the package creation - ".actions/**" @@ -128,8 +128,8 @@ subprojects: paths: - "dockers/**" - ".github/workflows/ci-pytorch-dockers.yml" - - "requirements/pytorch/*" - - "requirements/lite/*" + - "requirements/pytorch/**" + - "requirements/lite/**" - "environment.yml" - "setup.py" - ".actions/**" @@ -149,7 +149,7 @@ subprojects: - id: "lightning_lite: CPU workflow" paths: - - "requirements/lite/*" + - "requirements/lite/**" - "src/lightning_lite/**" - "tests/tests_lite/**" - "setup.cfg" # includes pytest config @@ -178,11 +178,11 @@ subprojects: - "examples/run_lite_examples.sh" - "tests/tests_lite/run_standalone_*.sh" - "tests/tests_pytorch/run_standalone_tests.sh" # used by Lite through a symlink - - "requirements/lite/*" + - "requirements/lite/**" - "src/lightning_lite/**" - "tests/tests_lite/**" - "setup.cfg" # includes pytest config - - ".actions/*" + - ".actions/**" checks: - "lightning-lite (GPUs)" @@ -194,7 +194,7 @@ subprojects: - "src/lightning_app/**" - "tests/tests_app/**" - "examples/app_*/**" # some tests_app tests call examples files - - "requirements/app/*" + - "requirements/app/**" - "setup.py" - ".actions/**" checks: @@ -214,7 +214,7 @@ subprojects: - "src/lightning_app/**" - "tests/tests_app_examples/**" - "examples/app_*" - - "requirements/app/*" + - "requirements/app/**" - "setup.py" - ".actions/**" checks: @@ -231,13 +231,13 @@ subprojects: - id: "lightning_app: Azure" paths: - ".azure/app-cloud-e2e.yml" - - "requirements/app/*" + - "requirements/app/**" - "src/lightning_app/**" - "tests/tests_app/**" - "examples/app_*/**" # some tests_app tests call examples files - "tests/tests_app_examples/**" - "setup.py" - - ".actions/*" + - ".actions/**" checks: - "App.cloud-e2e" @@ -250,7 +250,7 @@ subprojects: - "requirements/app/**" - "setup.py" - "setup.cfg" # includes metadata used in the package creation - - ".actions/*" + - ".actions/**" checks: - "make-doctest (app)" - "make-html (app)" diff --git a/.github/workflows/ci-app-examples.yml b/.github/workflows/ci-app-examples.yml index f5b48f3563bb0..76f7748e5a781 100644 --- a/.github/workflows/ci-app-examples.yml +++ b/.github/workflows/ci-app-examples.yml @@ -12,7 +12,7 @@ on: - "src/lightning_app/**" - "tests/tests_app_examples/**" - "examples/app_*" - - "requirements/app/*" + - "requirements/app/**" - "setup.py" - ".actions/**" diff --git a/.github/workflows/ci-app-tests.yml b/.github/workflows/ci-app-tests.yml index 76f865967519c..3975479ee4edc 100644 --- a/.github/workflows/ci-app-tests.yml +++ b/.github/workflows/ci-app-tests.yml @@ -12,7 +12,7 @@ on: - "src/lightning_app/**" - "tests/tests_app/**" - "examples/app_*" # some tests_app tests call examples files - - "requirements/app/*" + - "requirements/app/**" - "setup.py" - ".actions/**" diff --git a/.github/workflows/ci-lite-tests.yml b/.github/workflows/ci-lite-tests.yml index 31416d638100a..6da2ee4274b04 100644 --- a/.github/workflows/ci-lite-tests.yml +++ b/.github/workflows/ci-lite-tests.yml @@ -8,7 +8,7 @@ on: branches: [master, "release/*"] types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped paths: - - "requirements/lite/*" + - "requirements/lite/**" - "src/lightning_lite/**" - "tests/tests_lite/**" - "setup.cfg" # includes pytest config diff --git a/.github/workflows/ci-pytorch-tests.yml b/.github/workflows/ci-pytorch-tests.yml index 81747db14b45a..5bc4a48ef474b 100644 --- a/.github/workflows/ci-pytorch-tests.yml +++ b/.github/workflows/ci-pytorch-tests.yml @@ -8,13 +8,13 @@ on: branches: [master, "release/*"] types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped paths: - - "requirements/pytorch/*" + - "requirements/pytorch/**" - "src/pytorch_lightning/**" - "tests/tests_pytorch/**" - "tests/legacy/back-compatible-versions.txt" - "setup.cfg" # includes pytest config - ".github/workflows/ci-pytorch-tests.yml" - - "requirements/lite/*" + - "requirements/lite/**" - "src/lightning_lite/**" - ".actions/**" diff --git a/.github/workflows/tpu-tests.yml b/.github/workflows/tpu-tests.yml index 15a506e43d51d..b8f343ecff9d8 100644 --- a/.github/workflows/tpu-tests.yml +++ b/.github/workflows/tpu-tests.yml @@ -11,7 +11,7 @@ on: - "requirements/lite/**" - "src/lightning_lite/**" - "tests/tests_lite/**" - - "requirements/pytorch/*" + - "requirements/pytorch/**" - "src/pytorch_lightning/**" - "tests/tests_pytorch/**" - "setup.cfg" # includes pytest config