diff --git a/.github/actions/pkg-publish/action.yml b/.github/actions/pkg-publish/action.yml index c9b202273f68c..b9362784ae6c5 100644 --- a/.github/actions/pkg-publish/action.yml +++ b/.github/actions/pkg-publish/action.yml @@ -5,9 +5,14 @@ inputs: pkg-pattern: description: what file pattern is searched in folder, so for example `*_app*` required: true + pypi-test-token: + description: login token for PyPI + default: '' + required: false pypi-token: description: login token for PyPI - required: true + default: '' + required: false runs: using: "composite" @@ -20,16 +25,19 @@ runs: shell: bash # We do this, since failures on test.pypi aren't that bad - #- name: Publish to Test PyPI - # uses: pypa/gh-action-pypi-publish@v1.5.1 - # with: - # user: __token__ - # password: ${{ secrets.test_pypi_token_lai }} - # repository_url: https://test.pypi.org/legacy/ - # verbose: true + - name: Publish to Test PyPI + uses: pypa/gh-action-pypi-publish@v1.5.1 + if: inputs.pypi-test-token != '' + with: + user: __token__ + password: ${{ secrets.test_pypi_token_lai }} + repository_url: https://test.pypi.org/legacy/ + packages_dir: pypi/ + verbose: true - name: Publish distribution 📦 to PyPI uses: pypa/gh-action-pypi-publish@v1.5.1 + if: inputs.pypi-token != '' with: user: __token__ password: ${{ inputs.pypi-token }} diff --git a/.github/workflows/legacy-checkpoints.yml b/.github/workflows/legacy-checkpoints.yml index 9fa98cc5b1049..5e08455d22c8f 100644 --- a/.github/workflows/legacy-checkpoints.yml +++ b/.github/workflows/legacy-checkpoints.yml @@ -41,11 +41,15 @@ on: AWS_SECRET_KEY_ID: required: true +defaults: + run: + shell: bash + jobs: create-legacy-ckpts: runs-on: ubuntu-20.04 outputs: - pl-version: ${{ steps.decide-pl-version.outputs.pl-version }} + pl-version: ${{ steps.decide-version.outputs.pl-version }} defaults: run: working-directory: tests/legacy @@ -69,16 +73,24 @@ jobs: env: PACKAGE_NAME: pytorch run: | - pip install -e . -f https://download.pytorch.org/whl/cpu/torch_stable.html + pip install . -f https://download.pytorch.org/whl/cpu/torch_stable.html pip list if: inputs.pl_version == '' + - name: Install PL version + run: | + pip install "pytorch-lightning==${{ inputs.pl_version }}" \ + -f https://download.pytorch.org/whl/cpu/torch_stable.html \ + --extra-index-url https://test.pypi.org/simple/ + pip list + if: inputs.pl_version != '' + - name: Pull legacy checkpoints working-directory: ./ run: bash .actions/pull_legacy_checkpoints.sh - name: Decide PL version to create a PR with - id: decide-pl-version + id: decide-version run: | python -c "import pytorch_lightning as pl; print(f'pl-version={pl.__version__}')" >> $GITHUB_OUTPUT || echo pl-version='' >> $GITHUB_OUTPUT diff --git a/.github/workflows/release-pypi.yml b/.github/workflows/release-pypi.yml index a3d7b85ab9935..ba7f18341ac01 100644 --- a/.github/workflows/release-pypi.yml +++ b/.github/workflows/release-pypi.yml @@ -7,6 +7,10 @@ on: release: types: [published] +defaults: + run: + shell: bash + jobs: init: runs-on: ubuntu-20.04 @@ -24,8 +28,7 @@ jobs: runs-on: ubuntu-20.04 strategy: fail-fast: true - # run sequential - max-parallel: 1 + max-parallel: 1 # run sequential to prevent download/upload collisions matrix: pkg-name: ["lightning", "app", "lite", "pytorch"] steps: @@ -70,7 +73,69 @@ jobs: files: 'dist/*' repo-token: ${{ secrets.GITHUB_TOKEN }} - publish-packages: + release-version: + runs-on: ubuntu-20.04 + outputs: + tag: ${{ steps.lai-package.outputs.version }} + steps: + - uses: actions/checkout@v3 + - name: install Package + env: + PACKAGE_NAME: "lightning" + run: pip install . -f https://download.pytorch.org/whl/cpu/torch_stable.html + - name: package Version + id: lai-package + run: python -c "import lightning as L; print(f'version={L.__version__}')" >> $GITHUB_OUTPUT + + signaling: + runs-on: ubuntu-20.04 + needs: [release-version, pre-publish-packages] + env: + TAG: ${{ needs.release-version.outputs.tag }} + steps: + - uses: actions/setup-python@v4 + with: + python-version: 3.8 + - uses: actions/checkout@v3 + with: + repository: gridai/base-images + token: ${{ secrets.PAT_GHOST }} + ref: main + - uses: fregante/setup-git-token@v1 + with: + token: ${{ secrets.PAT_GHOST }} + name: PL Ghost + email: pl-github@grid.ai + - name: Update lightning version + run: | + import json, os + with open("versions.json") as fo: + vers = json.load(fo) + vers["lightning_version"] = os.getenv('TAG') + with open("versions.json", "w") as fw: + json.dump(vers, fw) + shell: python + - name: GIT Commit + run: | + git add versions.json + git commit -m "bumping lightning version -> ${TAG}" + cat versions.json + - name: GIT Push + run: | + git status + git push + + waiting: + # TODO: replace with back signal from build images/ loop checking for a specific branch? + runs-on: ubuntu-20.04 + needs: signaling + steps: + - name: Delay releasing + uses: juliangruber/sleep-action@v1 + with: + time: 30m + + pre-publish-packages: runs-on: ubuntu-20.04 needs: build-packages if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' @@ -82,38 +147,59 @@ jobs: path: dist - run: ls -lh dist/ - run: mkdir pypi/ - - - name: Delay releasing - uses: juliangruber/sleep-action@v1 + - uses: ./.github/actions/pkg-publish with: - time: 10m + pkg-pattern: "*app*" + pypi-test-token: ${{ secrets.PYPI_TEST_TOKEN_APP }} + - uses: ./.github/actions/pkg-publish + with: + pkg-pattern: "*lite*" + pypi-test-token: ${{ secrets.PYPI_TEST_TOKEN_LITE }} + - uses: ./.github/actions/pkg-publish + with: + pkg-pattern: "*pytorch*" + pypi-test-token: ${{ secrets.PYPI_TEST_TOKEN_PYTORCH }} + - uses: ./.github/actions/pkg-publish + with: + pkg-pattern: "*" + pypi-test-token: ${{ secrets.PYPI_TEST_TOKEN_LAI }} + publish-packages: + runs-on: ubuntu-20.04 + needs: waiting + if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' + steps: + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 + with: + name: dist-packages-${{ github.sha }} + path: dist + - run: ls -lh dist/ + - run: mkdir pypi/ - uses: ./.github/actions/pkg-publish with: pkg-pattern: "*app*" pypi-token: ${{ secrets.PYPI_TOKEN_APP }} - - uses: ./.github/actions/pkg-publish with: pkg-pattern: "*lite*" pypi-token: ${{ secrets.PYPI_TOKEN_LITE }} - - uses: ./.github/actions/pkg-publish with: pkg-pattern: "*pytorch*" pypi-token: ${{ secrets.PYPI_TOKEN_PYTORCH }} - - uses: ./.github/actions/pkg-publish with: pkg-pattern: "*" pypi-token: ${{ secrets.PYPI_TOKEN_LAI }} legacy-checkpoints: - needs: publish-packages + needs: [release-version, pre-publish-packages] uses: ./.github/workflows/legacy-checkpoints.yml with: push_to_s3: true create_pr: true + pl_version: ${{ needs.release-version.outputs.tag }} secrets: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_KEY_ID: ${{ secrets.AWS_SECRET_KEY_ID }}