From 21713e6de3f65e7972e91c25337767f0d4643d08 Mon Sep 17 00:00:00 2001 From: Akihiro Nitta Date: Tue, 31 May 2022 11:47:12 +0900 Subject: [PATCH 1/4] Add pull-legacy-checkpoints action --- .github/actions/pull-legacy-checkpoints/action.yml | 8 ++++++++ .github/scripts/pull_legacy_checkpoints.sh | 9 +++++++++ 2 files changed, 17 insertions(+) create mode 100644 .github/actions/pull-legacy-checkpoints/action.yml create mode 100644 .github/scripts/pull_legacy_checkpoints.sh diff --git a/.github/actions/pull-legacy-checkpoints/action.yml b/.github/actions/pull-legacy-checkpoints/action.yml new file mode 100644 index 0000000000000..42d86dfd4c9db --- /dev/null +++ b/.github/actions/pull-legacy-checkpoints/action.yml @@ -0,0 +1,8 @@ +name: 'Pull Legacy Checkpoints' +description: 'Pull legacy checkpoints to test checkpointing for backward compatibility.' + +runs: + using: 'composite' + steps: + - name: Download and unarchive checkpoints + run: bash .github/scripts/pull_legacy_checkpoints.sh diff --git a/.github/scripts/pull_legacy_checkpoints.sh b/.github/scripts/pull_legacy_checkpoints.sh new file mode 100644 index 0000000000000..8b3f791297b66 --- /dev/null +++ b/.github/scripts/pull_legacy_checkpoints.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Run this script from the project root. +URL="https://pl-public-data.s3.amazonaws.com/legacy/checkpoints.zip" +mkdir -p legacy +# wget is simpler but does not work on Windows +python -c "from urllib.request import urlretrieve; urlretrieve('$URL', 'legacy/checkpoints.zip')" +ls -l legacy/ +unzip -o legacy/checkpoints.zip -d legacy/ +ls -l legacy/checkpoints/ From 4e7d5ef02bca19a527baa4f5ab4a41d63d5e4e17 Mon Sep 17 00:00:00 2001 From: Akihiro Nitta Date: Tue, 31 May 2022 12:18:24 +0900 Subject: [PATCH 2/4] Replace pulls with the new action and script --- .azure-pipelines/gpu-tests.yml | 5 +---- .github/workflows/ci_test-conda.yml | 9 ++------- .github/workflows/ci_test-full.yml | 10 ++-------- dockers/tpu-tests/Dockerfile | 5 +---- legacy/README.md | 3 +-- tests/README.md | 3 +-- 6 files changed, 8 insertions(+), 27 deletions(-) diff --git a/.azure-pipelines/gpu-tests.yml b/.azure-pipelines/gpu-tests.yml index c0074adf4e81c..266363fbe8107 100644 --- a/.azure-pipelines/gpu-tests.yml +++ b/.azure-pipelines/gpu-tests.yml @@ -69,10 +69,7 @@ jobs: python requirements/check-avail-extras.py displayName: 'Env details' - - bash: | - wget https://pl-public-data.s3.amazonaws.com/legacy/checkpoints.zip -P legacy/ - unzip -o legacy/checkpoints.zip -d legacy/ - ls -l legacy/checkpoints/ + - bash: bash ./.github/scripts/pull_legacy_checkpoints.sh displayName: 'Get legacy checkpoints' - bash: | diff --git a/.github/workflows/ci_test-conda.yml b/.github/workflows/ci_test-conda.yml index c907e13b4ac23..2df0fedd752ec 100644 --- a/.github/workflows/ci_test-conda.yml +++ b/.github/workflows/ci_test-conda.yml @@ -55,13 +55,8 @@ jobs: # sanity check python requirements/check-avail-extras.py - - name: Pull checkpoints from S3 - working-directory: ./legacy - run: | - # enter legacy and update checkpoints from S3 - curl https://pl-public-data.s3.amazonaws.com/legacy/checkpoints.zip --output checkpoints.zip - unzip -o checkpoints.zip - ls -l checkpoints/ + - name: Pull legacy checkpoints + uses: ./.github/actions/pull-legacy-checkpoints.yml - name: Tests run: | diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index f260c67069912..e8e05f60f8df0 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -76,14 +76,8 @@ jobs: restore-keys: | ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-py${{ matrix.python-version }}-${{ matrix.release }}-${{ matrix.requires }}- - - name: Pull checkpoints from S3 - working-directory: ./legacy - run: | - # wget is simpler but does not work on Windows - python -c "from urllib.request import urlretrieve ; urlretrieve('https://pl-public-data.s3.amazonaws.com/legacy/checkpoints.zip', 'checkpoints.zip')" - ls -l . - unzip -o checkpoints.zip - ls -l checkpoints/ + - name: Pull legacy checkpoints + uses: ./.github/actions/pull-legacy-checkpoints.yml - name: Install dependencies run: | diff --git a/dockers/tpu-tests/Dockerfile b/dockers/tpu-tests/Dockerfile index 602ef1684b859..47a2dc4127a03 100644 --- a/dockers/tpu-tests/Dockerfile +++ b/dockers/tpu-tests/Dockerfile @@ -22,10 +22,7 @@ LABEL maintainer="PyTorchLightning " COPY ./ ./pytorch-lightning/ # Pull the legacy checkpoints -RUN cd pytorch-lightning && \ - wget https://pl-public-data.s3.amazonaws.com/legacy/checkpoints.zip -P legacy/ && \ - unzip -o legacy/checkpoints.zip -d legacy/ && \ - ls -l legacy/checkpoints/ +RUN cd pytorch-lightning && bash ./.github/scripts/pull_legacy_checkpoints.sh RUN \ pip install -q fire && \ diff --git a/legacy/README.md b/legacy/README.md index efbd18f7eede6..13f7b69a13de6 100644 --- a/legacy/README.md +++ b/legacy/README.md @@ -7,8 +7,7 @@ At this moment we focus on ability running old checkpoints, so the flow here is If you want to pull all saved version-checkpoints for local testing/development, call ```bash -wget https://pl-public-data.s3.amazonaws.com/legacy/checkpoints.zip -unzip -o checkpoints.zip +bash .github/scripts/pull_legacy_checkpoints.sh ``` To back populate collection with past version you can use following bash: diff --git a/tests/README.md b/tests/README.md index 665d0c9421584..ac9454f212e5a 100644 --- a/tests/README.md +++ b/tests/README.md @@ -22,8 +22,7 @@ pre-commit install Additionally, for testing backward compatibility with older versions of PyTorch Lightning, you also need to download all saved version-checkpoints from the public AWS storage. Run the following script to get all saved version-checkpoints: ```bash -wget https://pl-public-data.s3.amazonaws.com/legacy/checkpoints.zip -P legacy/ -unzip -o legacy/checkpoints.zip -d legacy/ +bash .github/scripts/pull_legacy_checkpoints.sh ``` Note: These checkpoints are generated to set baselines for maintaining backward compatibility with legacy versions of PyTorch Lightning. Details of checkpoints for back-compatibility can be found [here](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/legacy/README.md). From 6622cea315bd7a89642dffa76f4925bc7b8087f3 Mon Sep 17 00:00:00 2001 From: Akihiro Nitta Date: Tue, 31 May 2022 12:31:27 +0900 Subject: [PATCH 3/4] Simplify --- .github/actions/pull-legacy-checkpoints/action.yml | 8 -------- .github/workflows/ci_test-conda.yml | 2 +- .github/workflows/ci_test-full.yml | 2 +- 3 files changed, 2 insertions(+), 10 deletions(-) delete mode 100644 .github/actions/pull-legacy-checkpoints/action.yml diff --git a/.github/actions/pull-legacy-checkpoints/action.yml b/.github/actions/pull-legacy-checkpoints/action.yml deleted file mode 100644 index 42d86dfd4c9db..0000000000000 --- a/.github/actions/pull-legacy-checkpoints/action.yml +++ /dev/null @@ -1,8 +0,0 @@ -name: 'Pull Legacy Checkpoints' -description: 'Pull legacy checkpoints to test checkpointing for backward compatibility.' - -runs: - using: 'composite' - steps: - - name: Download and unarchive checkpoints - run: bash .github/scripts/pull_legacy_checkpoints.sh diff --git a/.github/workflows/ci_test-conda.yml b/.github/workflows/ci_test-conda.yml index 2df0fedd752ec..6565f512a3267 100644 --- a/.github/workflows/ci_test-conda.yml +++ b/.github/workflows/ci_test-conda.yml @@ -56,7 +56,7 @@ jobs: python requirements/check-avail-extras.py - name: Pull legacy checkpoints - uses: ./.github/actions/pull-legacy-checkpoints.yml + run: bash .github/scripts/pull_legacy_checkpoints.sh - name: Tests run: | diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index e8e05f60f8df0..826f3d6e0c087 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -77,7 +77,7 @@ jobs: ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-py${{ matrix.python-version }}-${{ matrix.release }}-${{ matrix.requires }}- - name: Pull legacy checkpoints - uses: ./.github/actions/pull-legacy-checkpoints.yml + run: bash .github/scripts/pull_legacy_checkpoints.sh - name: Install dependencies run: | From ecaf9d238ac41b2d9a6d23f0a2ff8701ed93f756 Mon Sep 17 00:00:00 2001 From: Akihiro Nitta Date: Wed, 1 Jun 2022 08:39:10 +0900 Subject: [PATCH 4/4] mv .github/scripts/ to .actions --- {.github/scripts => .actions}/pull_legacy_checkpoints.sh | 0 .azure-pipelines/gpu-tests.yml | 2 +- .github/workflows/ci_test-conda.yml | 2 +- .github/workflows/ci_test-full.yml | 2 +- dockers/tpu-tests/Dockerfile | 2 +- legacy/README.md | 2 +- tests/README.md | 2 +- 7 files changed, 6 insertions(+), 6 deletions(-) rename {.github/scripts => .actions}/pull_legacy_checkpoints.sh (100%) diff --git a/.github/scripts/pull_legacy_checkpoints.sh b/.actions/pull_legacy_checkpoints.sh similarity index 100% rename from .github/scripts/pull_legacy_checkpoints.sh rename to .actions/pull_legacy_checkpoints.sh diff --git a/.azure-pipelines/gpu-tests.yml b/.azure-pipelines/gpu-tests.yml index 266363fbe8107..157bb1d535f9a 100644 --- a/.azure-pipelines/gpu-tests.yml +++ b/.azure-pipelines/gpu-tests.yml @@ -69,7 +69,7 @@ jobs: python requirements/check-avail-extras.py displayName: 'Env details' - - bash: bash ./.github/scripts/pull_legacy_checkpoints.sh + - bash: bash .actions/pull_legacy_checkpoints.sh displayName: 'Get legacy checkpoints' - bash: | diff --git a/.github/workflows/ci_test-conda.yml b/.github/workflows/ci_test-conda.yml index 6565f512a3267..7e6b8842eab5d 100644 --- a/.github/workflows/ci_test-conda.yml +++ b/.github/workflows/ci_test-conda.yml @@ -56,7 +56,7 @@ jobs: python requirements/check-avail-extras.py - name: Pull legacy checkpoints - run: bash .github/scripts/pull_legacy_checkpoints.sh + run: bash .actions/pull_legacy_checkpoints.sh - name: Tests run: | diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index 826f3d6e0c087..b1a1ac4c5d4a3 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -77,7 +77,7 @@ jobs: ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-py${{ matrix.python-version }}-${{ matrix.release }}-${{ matrix.requires }}- - name: Pull legacy checkpoints - run: bash .github/scripts/pull_legacy_checkpoints.sh + run: bash .actions/pull_legacy_checkpoints.sh - name: Install dependencies run: | diff --git a/dockers/tpu-tests/Dockerfile b/dockers/tpu-tests/Dockerfile index 47a2dc4127a03..d4c58c665e7a5 100644 --- a/dockers/tpu-tests/Dockerfile +++ b/dockers/tpu-tests/Dockerfile @@ -22,7 +22,7 @@ LABEL maintainer="PyTorchLightning " COPY ./ ./pytorch-lightning/ # Pull the legacy checkpoints -RUN cd pytorch-lightning && bash ./.github/scripts/pull_legacy_checkpoints.sh +RUN cd pytorch-lightning && bash .actions/pull_legacy_checkpoints.sh RUN \ pip install -q fire && \ diff --git a/legacy/README.md b/legacy/README.md index 13f7b69a13de6..68eb718a98b07 100644 --- a/legacy/README.md +++ b/legacy/README.md @@ -7,7 +7,7 @@ At this moment we focus on ability running old checkpoints, so the flow here is If you want to pull all saved version-checkpoints for local testing/development, call ```bash -bash .github/scripts/pull_legacy_checkpoints.sh +bash .actions/pull_legacy_checkpoints.sh ``` To back populate collection with past version you can use following bash: diff --git a/tests/README.md b/tests/README.md index ac9454f212e5a..2951896748daf 100644 --- a/tests/README.md +++ b/tests/README.md @@ -22,7 +22,7 @@ pre-commit install Additionally, for testing backward compatibility with older versions of PyTorch Lightning, you also need to download all saved version-checkpoints from the public AWS storage. Run the following script to get all saved version-checkpoints: ```bash -bash .github/scripts/pull_legacy_checkpoints.sh +bash .actions/pull_legacy_checkpoints.sh ``` Note: These checkpoints are generated to set baselines for maintaining backward compatibility with legacy versions of PyTorch Lightning. Details of checkpoints for back-compatibility can be found [here](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/legacy/README.md).