Update Python testing (#10269)

Lightning-AI · Nov 9, 2021 · 4f6dcb9 · 4f6dcb9
1 parent cd03e94
commit 4f6dcb9
Show file tree

Hide file tree

Showing 11 changed files with 31 additions and 28 deletions.
diff --git a/.azure-pipelines/gpu-benchmark.yml b/.azure-pipelines/gpu-benchmark.yml
@@ -28,8 +28,8 @@ jobs:
     cancelTimeoutInMinutes: "2"
     pool: gridai-spot-pool
     container:
-      # base ML image: mcr.microsoft.com/azureml/openmpi3.1.2-cuda10.2-cudnn8-ubuntu18.04
-      image: "pytorchlightning/pytorch_lightning:base-cuda-py3.8-torch1.8"
+      # should match the one in '.azure-pipelines/gpu-benchmark.yml'
+      image: "pytorchlightning/pytorch_lightning:base-cuda-py3.7-torch1.8"
       options: "--runtime=nvidia -e NVIDIA_VISIBLE_DEVICES=all --shm-size=32g"
     workspace:
       clean: all

diff --git a/.github/workflows/ci_dockers.yml b/.github/workflows/ci_dockers.yml
@@ -92,7 +92,7 @@ jobs:
       fail-fast: false
       matrix:
         # the config used in '.github/workflows/ci_test-conda.yml'
-        python_version: ["3.7"]
+        python_version: ["3.8"]
         pytorch_version: ["1.6", "1.7", "1.8", "1.9", "1.10"]
     steps:
       - name: Checkout
@@ -119,7 +119,7 @@ jobs:
       fail-fast: false
       matrix:
         # the config used in 'dockers/ipu-ci-runner/Dockerfile'
-        python_version: ["3.8"]  # latest
+        python_version: ["3.9"]  # latest
         # TODO: upgrade - PopTorch 2.2 uses torch 1.9, see:
         # https://docs.graphcore.ai/projects/poptorch-user-guide/en/latest/installation.html#version-compatibility
         pytorch_version: ["1.7"]

diff --git a/.github/workflows/ci_test-conda.yml b/.github/workflows/ci_test-conda.yml
@@ -14,7 +14,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7"]
+        python-version: ["3.8"]  # previous to last Python version as that one is already used in test-full
         pytorch-version: ["1.6", "1.7", "1.8", "1.9", "1.10"]
 
     # Timeout: https://stackoverflow.com/a/59076067/4521646

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
@@ -18,17 +18,21 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-18.04, windows-2019, macOS-10.15]
-        python-version: [3.6, 3.8, 3.9]
-        requires: ['minimal', 'latest']
-        release: ['stable']
-        exclude:
-          - python-version: 3.9
-            requires: 'minimal'
+        python-version: ["3.7", "3.9"]  # minimum, maximum
+        requires: ["oldest", "latest"]
+        release: ["stable"]
         include:
-          - os: ubuntu-20.04
-            python-version: 3.9
-            requires: 'latest'
-            release: 'pre'
+          # test 3.6 only on oldest until EOL: https://github.com/PyTorchLightning/pytorch-lightning/issues/9981
+          - {os: ubuntu-18.04, python-version: "3.6", requires: "oldest", release: "stable"}
+          - {os: windows-2019, python-version: "3.6", requires: "oldest", release: "stable"}
+          - {os: macOS-10.15, python-version: "3.6", requires: "oldest", release: "stable"}
+          # nightly: add when there's a release candidate
+          #- {os: ubuntu-20.04, python-version: "3.10", requires: "latest", release: "pre"}
+        exclude:
+          # PyTorch 1.6 is not available with Python 3.9: https://github.com/pytorch/pytorch/issues/46205
+          - {os: ubuntu-18.04, python-version: "3.9", requires: "oldest", release: "stable"}
+          - {os: windows-2019, python-version: "3.9", requires: "oldest", release: "stable"}
+          - {os: macOS-10.15, python-version: "3.9", requires: "oldest", release: "stable"}
 
     # Timeout: https://stackoverflow.com/a/59076067/4521646
     # TODO: the macOS is taking too long, probably caching did not work...
@@ -64,7 +68,7 @@ jobs:
         python .github/prune-packages.py requirements/extra.txt "horovod"
 
     - name: Set min. dependencies
-      if: matrix.requires == 'minimal'
+      if: matrix.requires == 'oldest'
       run: |
         python .github/set-min-requirements.py
 

diff --git a/.github/workflows/events-nightly.yml b/.github/workflows/events-nightly.yml
@@ -122,7 +122,7 @@ jobs:
       fail-fast: false
       matrix:
         # the config used in '.github/workflows/ci_test-conda.yml'
-        python_version: ["3.7"]
+        python_version: ["3.8"]
         pytorch_version: ["1.6", "1.7", "1.8", "1.9", "1.10"]
 
     steps:
@@ -163,7 +163,7 @@ jobs:
       matrix:
         # the config used in 'dockers/ipu-ci-runner/Dockerfile'
         include:
-          - python_version: "3.8"
+          - python_version: "3.9"
             pytorch_version: "1.7"
 
     steps:

diff --git a/dockers/base-cuda/Dockerfile b/dockers/base-cuda/Dockerfile
@@ -16,7 +16,7 @@ ARG CUDA_VERSION=10.2
 
 FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu18.04
 
-ARG PYTHON_VERSION=3.8
+ARG PYTHON_VERSION=3.9
 ARG PYTORCH_VERSION=1.6
 
 SHELL ["/bin/bash", "-c"]

diff --git a/dockers/base-ipu/Dockerfile b/dockers/base-ipu/Dockerfile
@@ -16,8 +16,7 @@ FROM ubuntu:20.04
 
 LABEL maintainer="PyTorchLightning <https://github.com/PyTorchLightning>"
 
-ARG PYTHON_VERSION=3.8
-ARG PYTORCH_VERSION=1.7
+ARG PYTHON_VERSION=3.9
 ARG CONDA_VERSION=4.9.2
 
 SHELL ["/bin/bash", "-c"]
@@ -41,7 +40,7 @@ RUN apt-get update -qq && \
     && \
 # Install conda and python.
 # NOTE new Conda does not forward the exit status... https://github.com/conda/conda/issues/8385
-    curl -o ~/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py38_${CONDA_VERSION}-Linux-x86_64.sh && \
+    curl -o ~/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py39_${CONDA_VERSION}-Linux-x86_64.sh && \
     chmod +x ~/miniconda.sh && \
     ~/miniconda.sh -b && \
     rm ~/miniconda.sh && \

diff --git a/dockers/base-xla/Dockerfile b/dockers/base-xla/Dockerfile
@@ -16,8 +16,8 @@ FROM google/cloud-sdk:slim
 
 LABEL maintainer="PyTorchLightning <https://github.com/PyTorchLightning>"
 
-# CALL: docker image build -t pytorch-lightning:XLA-extras-py3.6 -f dockers/base-xla/Dockerfile . --build-arg PYTHON_VERSION=3.6
-ARG PYTHON_VERSION=3.7
+# CALL: docker image build -t pytorch-lightning:XLA-extras-py3.6 -f dockers/base-xla/Dockerfile . --build-arg PYTHON_VERSION=3.8
+ARG PYTHON_VERSION=3.9
 ARG CONDA_VERSION=4.9.2
 ARG XLA_VERSION=1.6
 
@@ -42,7 +42,7 @@ RUN apt-get update -qq && \
     && \
     # Install conda and python.
     # NOTE new Conda does not forward the exit status... https://github.com/conda/conda/issues/8385
-    curl -o ~/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py38_${CONDA_VERSION}-Linux-x86_64.sh && \
+    curl -o ~/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py39_${CONDA_VERSION}-Linux-x86_64.sh && \
     chmod +x ~/miniconda.sh && \
     ~/miniconda.sh -b && \
     rm ~/miniconda.sh && \

diff --git a/dockers/ipu-ci-runner/Dockerfile b/dockers/ipu-ci-runner/Dockerfile
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARG PYTHON_VERSION=3.8
+ARG PYTHON_VERSION=3.9
 ARG PYTORCH_VERSION=1.7
 
 FROM pytorchlightning/pytorch_lightning:base-ipu-py${PYTHON_VERSION}-torch${PYTORCH_VERSION}

diff --git a/dockers/release/Dockerfile b/dockers/release/Dockerfile
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARG PYTHON_VERSION=3.7
+ARG PYTHON_VERSION=3.9
 ARG PYTORCH_VERSION=1.6
 
 FROM pytorchlightning/pytorch_lightning:base-cuda-py${PYTHON_VERSION}-torch${PYTORCH_VERSION}

diff --git a/dockers/tpu-tests/Dockerfile b/dockers/tpu-tests/Dockerfile
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARG PYTHON_VERSION=3.7
+ARG PYTHON_VERSION=3.9
 ARG PYTORCH_VERSION=1.6
 
 FROM pytorchlightning/pytorch_lightning:base-xla-py${PYTHON_VERSION}-torch${PYTORCH_VERSION}