ci: Don't install things to home directory, add workflow to test rele…

…ase pipelines (#706) Summary: Noticed we were getting extra conda packges from the home directory on the M1 runners. Making this PR to reduce the amount that we're using the home directory. Will probably make it unwritable in the future so this is a pre-emptive step to that. Also adds a workflow to test the release pipelines without actually releasing a new binary. Signed-off-by: Eli Uriegas <[email protected]> Pull Request resolved: #706 Reviewed By: ejguan Differential Revision: D38406845 Pulled By: seemethere fbshipit-source-id: 1d312c3e1db34497c546a9b842ac7ba01d2a7ee3
pytorch · Aug 5, 2022 · fd648a4 · fd648a4
1 parent 9375899
commit fd648a4
Show file tree

Hide file tree

Showing 2 changed files with 56 additions and 28 deletions.
diff --git a/.github/workflows/_build_test_upload.yml b/.github/workflows/_build_test_upload.yml
@@ -12,6 +12,10 @@ on:
       pytorch_version:
         required: true
         type: string
+      do-upload:
+        required: false
+        default: true
+        type: boolean
     secrets:
       PYTORCH_BINARY_AWS_ACCESS_KEY_ID:
         required: true
@@ -28,24 +32,18 @@ on:
 
 jobs:
   get_release_type:
-    if: inputs.branch != ''
     runs-on: ubuntu-latest
     outputs:
       type: ${{ steps.get_release_type.outputs.type }}
     steps:
       - name: Get Release Type
         run: |
-          if [[ ${{ inputs.branch }} == v* ]] && [[ ${{ inputs.pre_dev_release }} == false ]]; then
+          if [[ "${{ inputs.branch }}" == v* ]] && [[ ${{ inputs.pre_dev_release }} == false ]]; then
             RELEASE_TYPE=official
-          elif [[ ${{ inputs.branch }} == release/* ]] && [[ ${{ inputs.pre_dev_release }} == true ]]; then
+          elif [[ "${{ inputs.branch }}" == release/* ]] && [[ ${{ inputs.pre_dev_release }} == true ]]; then
             RELEASE_TYPE=test
-          elif [[ ${{ inputs.branch }} == main ]] && [[ ${{ inputs.pre_dev_release }} == true ]]; then
-            RELEASE_TYPE=nightly
           else
-            echo "Invalid combination of inputs!"
-            echo "  branch: ${{ inputs.branch }}"
-            echo "  pre_dev_release: ${{ inputs.pre_dev_release }}"
-            exit 1
+            RELEASE_TYPE=nightly
           fi
           echo "Release Type: $RELEASE_TYPE"
           echo "::set-output name=type::$RELEASE_TYPE"
@@ -83,19 +81,23 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Add temp runner environment variables
+        shell: bash -l {0}
+        run: |
+          echo "MINICONDA_INSTALL_PATH_MACOS=${RUNNER_TEMP}/miniconda" >> "${GITHUB_ENV}"
       - name: Install Conda on MacOS
         if: ${{ startsWith( matrix.os, 'macos' ) }}
         shell: bash -l {0}
         run: |
-          mkdir -p ~/miniconda3
+          mkdir -p "${MINICONDA_INSTALL_PATH_MACOS}"
           if ${{ startsWith( matrix.os, 'macos-m1' ) }}; then
-            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh -o ~/miniconda3/miniconda.sh
+            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh -o "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh"
           else
-            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o ~/miniconda3/miniconda.sh
+            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh"
           fi
-          bash ~/miniconda3/miniconda.sh -b -u -p ~/miniconda3
-          rm -rf ~/miniconda3/miniconda.sh
-          echo "$(dirname ~)/$(basename ~)/miniconda3/bin" >> $GITHUB_PATH
+          bash "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh" -b -u -p "${MINICONDA_INSTALL_PATH_MACOS}"
+          rm -rf "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh"
+          echo "${MINICONDA_INSTALL_PATH_MACOS}/bin" >> $GITHUB_PATH
       - name: Setup Python ${{ matrix.python-version }} on MacOS
         if: ${{ startsWith( matrix.os, 'macos' ) }}
         shell: bash -l {0}
@@ -211,7 +213,7 @@ jobs:
           path: dist/torchdata*.whl
 
   wheel_upload:
-    if: always() && inputs.branch != ''
+    if: always() && inputs.branch != '' && inputs.do-upload == true
     needs: [get_release_type, wheel_build_test]
     runs-on: ubuntu-latest
     outputs:
@@ -289,31 +291,36 @@ jobs:
         with:
           ref: ${{ inputs.branch }}
           submodules: recursive
+      - name: Add temp runner environment variables
+        shell: bash -l {0}
+        run: |
+          echo "MINICONDA_INSTALL_PATH_MACOS=${RUNNER_TEMP}/miniconda" >> "${GITHUB_ENV}"
+          echo "CONDA_ENV_PATH=${RUNNER_TEMP}/conda_build_env" >> "${GITHUB_ENV}"
       - name: Create Conda Env
         if: ${{ ! startsWith( matrix.os, 'macos' ) }}
         uses: conda-incubator/setup-miniconda@v2
         with:
           python-version: ${{ matrix.python-version }}
-          activate-environment: conda_build_env
+          activate-environment: ${{ env.CONDA_ENV_PATH }}
       - name: Install Conda on MacOS
         if: ${{ startsWith( matrix.os, 'macos' ) }}
         shell: bash -l {0}
         run: |
-          mkdir -p ~/miniconda3
+          mkdir -p "${MINICONDA_INSTALL_PATH_MACOS}"
           if ${{ startsWith( matrix.os, 'macos-m1' ) }}; then
-            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh -o ~/miniconda3/miniconda.sh
+            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh -o "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh"
           else
-            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o ~/miniconda3/miniconda.sh
+            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh"
           fi
-          bash ~/miniconda3/miniconda.sh -b -u -p ~/miniconda3
-          rm -rf ~/miniconda3/miniconda.sh
-          echo "$(dirname ~)/$(basename ~)/miniconda3/bin" >> $GITHUB_PATH
+          bash "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh" -b -u -p "${MINICONDA_INSTALL_PATH_MACOS}"
+          rm -rf "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh"
+          echo "${MINICONDA_INSTALL_PATH_MACOS}/bin" >> $GITHUB_PATH
       - name: Create Conda Env on MacOS
         if: ${{ startsWith( matrix.os, 'macos' ) }}
         shell: bash -l {0}
         run: |
           conda init bash
-          conda create -y --name conda_build_env python=${{ matrix.python-version }}
+          conda create -y -p "${CONDA_ENV_PATH}" python=${{ matrix.python-version }}
       - name: Setup additional system libraries
         if: startsWith( matrix.os, 'ubuntu' )
         run: |
@@ -338,14 +345,14 @@ jobs:
           BUILD_S3: ${{ steps.build_s3.outputs.value }}
         run: |
           set -ex
-          conda activate conda_build_env
+          conda activate "${CONDA_ENV_PATH}"
           conda install -yq conda-build -c conda-forge
           packaging/build_conda.sh
           conda index ./conda-bld
       - name: Install TorchData Conda Package
         shell: bash -l {0}
         run: |
-          conda activate conda_build_env
+          conda activate "${CONDA_ENV_PATH}"
           if [[ ${{ needs.get_release_type.outputs.type }} == 'official' ]]; then
             CONDA_CHANNEL=pytorch
           else
@@ -355,7 +362,7 @@ jobs:
       - name: Run DataPipes Tests with pytest
         shell: bash -l {0}
         run: |
-          conda activate conda_build_env
+          conda activate "${CONDA_ENV_PATH}"
           pip3 install -r test/requirements.txt
           pytest --no-header -v test --ignore=test/test_period.py --ignore=test/test_text_examples.py --ignore=test/test_audio_examples.py
       - name: Upload Conda Package to Github
@@ -366,7 +373,7 @@ jobs:
           path: conda-bld/*/torchdata-*.tar.bz2
 
   conda_upload:
-    if: always() && inputs.branch != ''
+    if: always() && inputs.branch != '' && inputs.do-upload == true
     needs: [get_release_type, conda_build_test]
     runs-on: ubuntu-latest
     container: continuumio/miniconda3

diff --git a/.github/workflows/pull_release.yml b/.github/workflows/pull_release.yml
@@ -0,0 +1,21 @@
+name: Test Release Pipelines
+
+on:
+  pull_request:
+    paths:
+      - .github/workflows/pull_release.yml
+      - .github/workflows/_build_test_upload.yml
+
+jobs:
+  build_test_upload:
+    if: github.repository == 'pytorch/data'
+    uses: ./.github/workflows/_build_test_upload.yml
+    with:
+      branch: ""
+      pre_dev_release: true
+      pytorch_version: ""
+      do-upload: false
+    secrets:
+      PYTORCH_BINARY_AWS_ACCESS_KEY_ID: ""
+      PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY: ""
+      CONDA_TEST_PYTORCHBOT_TOKEN: ""