Merge branch 'master' into tests/grep-errors-standalone

Lightning-AI · Nov 4, 2022 · 4ecb32c · 4ecb32c
2 parents 4217679 + 921dc1c
commit 4ecb32c
Show file tree

Hide file tree

Showing 161 changed files with 2,690 additions and 1,385 deletions.
diff --git a/.azure/app-cloud-e2e.yml b/.azure/app-cloud-e2e.yml
@@ -75,7 +75,7 @@ jobs:
           name: "commands_and_api"
         # 'App: quick_start':  # TODO: Failed during installation
         #   name: "quick_start"
-    timeoutInMinutes: "30"
+    timeoutInMinutes: "20"
     cancelTimeoutInMinutes: "2"
     # values: https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#workspace
     workspace:

diff --git a/.github/checkgroup.yml b/.github/checkgroup.yml
@@ -50,7 +50,6 @@ subprojects:
       #- "pl-cpu (windows-2022, lightning, 3.7, 1.9, oldest)"
       - "make-doctest (pytorch)"
       - "make-html (pytorch)"
-      - "mypy"
       - "pytorch-lightning (GPUs)"
       - "pytorch-lightning (HPUs)"
       - "pytorch-lightning (IPUs)"
@@ -60,7 +59,7 @@ subprojects:
       # TODO: since this job has intermittent availability, it cannot be required or it will block all PL PRs from forks
       #- "test-on-tpus"
 
-  - id: "pytorch_lightning: CPU"
+  - id: "pytorch_lightning: CPU workflow"
     paths:
       - ".github/workflows/ci-pytorch-tests.yml"
     checks:
@@ -87,7 +86,7 @@ subprojects:
       - "pl-cpu (windows-2022, lightning, 3.10, 1.12)"
       #- "pl-cpu (windows-2022, lightning, 3.7, 1.9, oldest)"
 
-  - id: "pytorch_lightning: Slow"
+  - id: "pytorch_lightning: Slow workflow"
     paths:
       - ".github/workflows/ci-pytorch-tests-slow.yml"
     checks:
@@ -114,7 +113,7 @@ subprojects:
     checks:
       - "pytorch-lightning (IPUs)"
 
-  - id: "pytorch-lightning: TPU"
+  - id: "pytorch-lightning: TPU workflow"
     paths:
       - ".github/workflows/tpu-tests.yml"
     checks:
@@ -181,7 +180,6 @@ subprojects:
       - "lite-cpu (ubuntu-20.04, lightning, 3.8, 1.12)"
       - "lite-cpu (windows-2022, lightning, 3.8, 1.12)"
       - "lightning-lite (GPUs)"
-      - "mypy"
       # Lite also requires PL checks as it depends on Lite
       - "pl-cpu (macOS-11, pytorch, 3.8, 1.10)"
       - "pl-cpu (macOS-11, pytorch, 3.9, 1.11)"
@@ -254,8 +252,7 @@ subprojects:
       - "requirements/app/**"
       - "src/lightning_app/**"
       - "tests/tests_app/**"
-      - "examples/app_*"  # some tests_app tests call examples files
-      - ".github/workflows/ci-app-tests.yml"
+      - "examples/app_*/**"  # some tests_app tests call examples files
       - "setup.py"
       - ".actions/**"
     checks:
@@ -272,13 +269,26 @@ subprojects:
       - "app-pytest (windows-2022, app, 3.8, oldest)"
       - "app-pytest (windows-2022, lightning, 3.8, latest)"
 
+  - id: "lightning_app: Tests workflow"
+    paths:
+      - ".github/workflows/ci-app-tests.yml"
+    checks:
+      - "app-pytest (macOS-11, app, 3.8, latest)"
+      - "app-pytest (macOS-11, app, 3.8, oldest)"
+      - "app-pytest (macOS-11, lightning, 3.9, latest)"
+      - "app-pytest (ubuntu-20.04, app, 3.8, latest)"
+      - "app-pytest (ubuntu-20.04, app, 3.8, oldest)"
+      - "app-pytest (ubuntu-20.04, lightning, 3.9, latest)"
+      - "app-pytest (windows-2022, app, 3.8, latest)"
+      - "app-pytest (windows-2022, app, 3.8, oldest)"
+      - "app-pytest (windows-2022, lightning, 3.8, latest)"
+
   - id: "lightning_app: Examples"
     paths:
       - "requirements/app/**"
       - "src/lightning_app/**"
       - "tests/tests_app_examples/**"
-      - "examples/app_*"
-      - ".github/workflows/ci-app-examples.yml"
+      - "examples/app_*/**"
       - "setup.py"
       - ".actions/**"
     checks:
@@ -292,6 +302,20 @@ subprojects:
       - "app-examples (windows-2022, app, 3.9, oldest)"
       - "app-examples (windows-2022, lightning, 3.9, latest)"
 
+  - id: "lightning_app: Examples workflow"
+    paths:
+      - ".github/workflows/ci-app-examples.yml"
+    checks:
+      - "app-examples (macOS-11, app, 3.9, latest)"
+      - "app-examples (macOS-11, app, 3.9, oldest)"
+      - "app-examples (macOS-11, lightning, 3.9, latest)"
+      - "app-examples (ubuntu-20.04, app, 3.9, latest)"
+      - "app-examples (ubuntu-20.04, app, 3.9, oldest)"
+      - "app-examples (ubuntu-20.04, lightning, 3.9, latest)"
+      - "app-examples (windows-2022, app, 3.9, latest)"
+      - "app-examples (windows-2022, app, 3.9, oldest)"
+      - "app-examples (windows-2022, lightning, 3.9, latest)"
+
   - id: "lightning_app: Azure"
     paths:
       - ".azure/app-cloud-e2e.yml"
@@ -313,7 +337,10 @@ subprojects:
   - id: "mypy"
     paths:
       - ".github/workflows/code-checks.yml"
+      - "requirements/**"
+      - "src/**"
       - "pyproject.toml"  # includes mypy config
+      - ".actions/**"
     checks:
       - "mypy"
 

diff --git a/.github/workflows/ci-app-examples.yml b/.github/workflows/ci-app-examples.yml
@@ -12,7 +12,8 @@ on:
       - "src/lightning_app/**"
       - "tests/tests_app_examples/**"
       - "examples/app_*"
-      - "requirements/app/**"
+      - "requirements/app/*"
+      - "!requirements/app/docs.txt"
       - "setup.py"
       - ".actions/**"
 

diff --git a/.github/workflows/ci-app-tests.yml b/.github/workflows/ci-app-tests.yml
@@ -11,7 +11,8 @@ on:
       - "src/lightning_app/**"
       - "tests/tests_app/**"
       - "examples/app_*"  # some tests_app tests call examples files
-      - "requirements/app/**"
+      - "requirements/app/*"
+      - "!requirements/app/docs.txt"
       - "setup.py"
       - ".actions/**"
 

diff --git a/.github/workflows/ci-lite-tests.yml b/.github/workflows/ci-lite-tests.yml
@@ -8,7 +8,8 @@ on:
     branches: [master, "release/*"]
     types: [opened, reopened, ready_for_review, synchronize]  # add `ready_for_review` since draft is skipped
     paths:
-      - "requirements/lite/**"
+      - "requirements/lite/*"
+      - "!requirements/lite/docs.txt"
       - "src/lightning_lite/**"
       - "tests/tests_lite/**"
       - "setup.cfg"  # includes pytest config

diff --git a/.github/workflows/ci-pkg-install.yml b/.github/workflows/ci-pkg-install.yml
@@ -12,6 +12,7 @@ on:
       - "setup.py"
       - "src/**"
       - "requirements/**"
+      - "!requirements/*/docs.txt"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }}

diff --git a/.github/workflows/ci-pytorch-dockers.yml b/.github/workflows/ci-pytorch-dockers.yml
@@ -9,8 +9,8 @@ on:
       - "dockers/**"
       - "!dockers/README.md"
       - "requirements.txt"
-      - "requirements/*.txt"
-      - "requirements/pytorch/*"
+      - "requirements/**"
+      - "!requirements/*/docs.txt"
       - "environment.yml"
       - ".github/workflows/*docker*.yml"
       - "setup.py"

diff --git a/.github/workflows/ci-pytorch-tests-slow.yml b/.github/workflows/ci-pytorch-tests-slow.yml
@@ -9,13 +9,14 @@ on:
     types: [opened, reopened, ready_for_review, synchronize]  # add `ready_for_review` since draft is skipped
     paths:
       - ".github/workflows/ci-pytorch-tests-slow.yml"
-      - "requirements/pytorch/**"
+      - "requirements/pytorch/*"
       - "src/pytorch_lightning/**"
       - "tests/tests_pytorch/**"
       - "setup.cfg"  # includes pytest config
-      - "requirements/lite/**"
+      - "requirements/lite/*"
       - "src/lightning_lite/**"
       - ".actions/**"
+      - "!requirements/*/docs.txt"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }}

diff --git a/.github/workflows/ci-pytorch-tests.yml b/.github/workflows/ci-pytorch-tests.yml
@@ -8,15 +8,16 @@ on:
     branches: [master, "release/*"]
     types: [opened, reopened, ready_for_review, synchronize]  # add `ready_for_review` since draft is skipped
     paths:
-      - "requirements/pytorch/**"
+      - "requirements/pytorch/*"
       - "src/pytorch_lightning/**"
       - "tests/tests_pytorch/**"
       - "tests/legacy/back-compatible-versions.txt"
       - "setup.cfg"  # includes pytest config
       - ".github/workflows/ci-pytorch-tests.yml"
-      - "requirements/lite/**"
+      - "requirements/lite/*"
       - "src/lightning_lite/**"
       - ".actions/**"
+      - "!requirements/*/docs.txt"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }}

diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml
@@ -5,6 +5,12 @@ on:
     branches: [master, "release/*"]
   pull_request:
     branches: [master, "release/*"]
+    paths:
+      - ".github/workflows/code-checks.yml"
+      - "requirements/**"
+      - "src/**"
+      - "pyproject.toml"   # includes mypy config
+      - "actions/**"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }}

diff --git a/.github/workflows/docs-checks.yml b/.github/workflows/docs-checks.yml
@@ -4,7 +4,9 @@ name: Check Docs
 on:
   push:
     branches: ["master", "release/*"]
-  pull_request:
+  # use this event type to share secrets with forks.
+  # it's important that the PR head SHA is checked out to run the changes
+  pull_request_target:
     branches: ["master", "release/*"]
     paths:
       - ".actions/**"
@@ -30,6 +32,8 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
       - uses: actions/setup-python@v4
         with:
           python-version: "3.8"
@@ -56,6 +60,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: true
+          ref: ${{ github.event.pull_request.head.sha }}
       - uses: actions/download-artifact@v3
         with:
           name: ci-packages-${{ github.sha }}
@@ -64,6 +69,15 @@ jobs:
         with:
           python-version: "3.9"
 
+      - uses: aws-actions/configure-aws-credentials@v1
+        if: ${{ matrix.pkg-name == 'app' }}
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY_ID }}
+          aws-region: us-east-1
+      - run: aws s3 sync s3://sphinx-packages/ pypi/
+        if: ${{ matrix.pkg-name == 'app' }}
+
       # Note: This uses an internal pip API and may not always work
       # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow
       - name: Cache pip
@@ -82,9 +96,9 @@ jobs:
 
       - name: Install LAI package
         # This is needed as App docs is heavily using/referring to lightning package
-        if: ${{ matrix.pkg-name== 'app' }}
+        if: ${{ matrix.pkg-name == 'app' }}
         run: |
-          pip install -e . -U  --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --find-links pypi
+          pip install -e . -U  -f https://download.pytorch.org/whl/cpu/torch_stable.html -f pypi
           git checkout -- setup.py MANIFEST.in
 
       - name: Adjust docs refs
@@ -97,7 +111,8 @@ jobs:
         env:
           PACKAGE_NAME: ${{ matrix.pkg-name }}
         run: |
-          pip install -e .[extra,cloud,ui] -U -r requirements/${{ matrix.pkg-name }}/docs.txt --find-links pypi
+          pip install -e .[extra,cloud,ui] -U -r requirements/${{ matrix.pkg-name }}/docs.txt -f pypi
+          pip list
 
       - name: Test Documentation
         env:
@@ -120,6 +135,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: true
+          ref: ${{ github.event.pull_request.head.sha }}
       - uses: actions/download-artifact@v3
         with:
           name: ci-packages-${{ github.sha }}
@@ -128,6 +144,15 @@ jobs:
         with:
           python-version: "3.9"
 
+      - uses: aws-actions/configure-aws-credentials@v1
+        if: ${{ matrix.pkg-name != 'pytorch' }}
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY_ID }}
+          aws-region: us-east-1
+      - run: aws s3 sync s3://sphinx-packages/ pypi/
+        if: ${{ matrix.pkg-name != 'pytorch' }}
+
       # Note: This uses an internal pip API and may not always work
       # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow
       - name: Cache pip
@@ -145,9 +170,8 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y cmake pandoc texlive-latex-extra dvipng texlive-pictures
           pip --version
-          pip install -e . -r requirements/${{ matrix.pkg-name }}/docs.txt \
-            --find-links https://download.pytorch.org/whl/cpu/torch_stable.html \
-            --find-links pypi
+          pip install -e . -U -r requirements/${{ matrix.pkg-name }}/docs.txt \
+            -f https://download.pytorch.org/whl/cpu/torch_stable.html -f pypi
           pip list
         shell: bash
 

diff --git a/.github/workflows/docs-deploy.yml b/.github/workflows/docs-deploy.yml
@@ -28,6 +28,13 @@ jobs:
         with:
           python-version: 3.8
 
+      - uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY_ID }}
+          aws-region: us-east-1
+      - run: aws s3 sync s3://sphinx-packages/ pypi/
+
       # Note: This uses an internal pip API and may not always work
       # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow
       - name: Cache pip
@@ -44,7 +51,8 @@ jobs:
           sudo apt-get install -y cmake pandoc
           sudo apt-get install -y texlive-latex-extra dvipng texlive-pictures
           pip --version
-          pip install -e . --quiet -r requirements/app/docs.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
+          pip install -e . --quiet -r requirements/app/docs.txt \
+            -f https://download.pytorch.org/whl/cpu/torch_stable.html -f pypi
           pip list
 
       - name: Make Documentation
@@ -87,8 +95,3 @@ jobs:
         if: ${{ success() && github.ref == 'refs/heads/master' }}
         run: |-
           gsutil -m rsync -d -R docs/build/html/ gs://lightning-docs-latest
-
-      # fixme
-      - name: Upload docs/latest to GCS 🪣
-        run: |-
-          gsutil -m rsync -d -R docs/build/html/ gs://lightning-docs-latest