From 35f01462ec4ce0043a085c617ffc977d047156f6 Mon Sep 17 00:00:00 2001 From: atalman Date: Mon, 9 Jan 2023 10:15:57 -0800 Subject: [PATCH] Use MATRIX_* variables instead of redeefining new var each time --- .github/scripts/validate_binaries.sh | 31 +++++++------------ .github/workflows/validate-domain-library.yml | 27 ++-------------- .github/workflows/validate-linux-binaries.yml | 12 ++----- .../validate-macos-arm64-binaries.yml | 10 ++---- .github/workflows/validate-macos-binaries.yml | 11 ++----- .../workflows/validate-nightly-binaries.yml | 1 + .../workflows/validate-windows-binaries.yml | 12 ++----- check_binary.sh | 13 ++++++++ test/smoke_test/smoke_test.py | 6 ++-- 9 files changed, 41 insertions(+), 82 deletions(-) diff --git a/.github/scripts/validate_binaries.sh b/.github/scripts/validate_binaries.sh index c4a450698..20b9aaa43 100755 --- a/.github/scripts/validate_binaries.sh +++ b/.github/scripts/validate_binaries.sh @@ -1,38 +1,31 @@ -#!/usr/bin/env bash -set -ex - -if [[ ${TARGET_OS} == 'windows' ]]; then - source /c/Jenkins/Miniconda3/etc/profile.d/conda.sh -else - eval "$(conda shell.bash hook)" -fi - -if [[ ${PACKAGE_TYPE} == "libtorch" ]]; then - curl ${INSTALLATION} -o libtorch.zip +if [[ ${MATRIX_PACKAGE_TYPE} == "libtorch" ]]; then + curl ${MATRIX_INSTALLATION} -o libtorch.zip unzip libtorch.zip else - if [ $DESIRED_PYTHON == '3.11' ]; then + if [ $MATRIX_PYTHON_VERSION == '3.11' ]; then export CPYTHON_VERSIONS=3.11.0 sudo yum -y install openssl-devel libssl-dev bzip2-devel libffi-devel sudo yum -y groupinstall "Development Tools" export PYTHON_PATH="/opt/_internal/cpython-3.11.0/bin" export PIP_PATH="${PYTHON_PATH}/pip" - export PIP_INSTALLATION="${INSTALLATION/pip3/"$PIP_PATH"}" + export PIP_INSTALLATION="${MATRIX_INSTALLATION/pip3/"$PIP_PATH"}" export WITH_OPENSSL="/opt/openssl" ./common/install_cpython.sh eval ${PYTHON_PATH}/python --version eval ${PIP_INSTALLATION} eval ${PYTHON_PATH}/python ./test/smoke_test/smoke_test.py --package torchonly else - conda create -y -n ${ENV_NAME} python=${DESIRED_PYTHON} numpy pillow + conda create -y -n ${ENV_NAME} python=${MATRIX_PYTHON_VERSION} numpy pillow conda activate ${ENV_NAME} - export CONDA_LIBRARY_PATH="$(dirname $(which python))/../lib" - export LD_LIBRARY_PATH=$CONDA_LIBRARY_PATH:$LD_LIBRARY_PATH - INSTALLATION=${INSTALLATION/"conda install"/"conda install -y"} + INSTALLATION=${MATRIX_INSTALLATION/"conda install"/"conda install -y"} eval $INSTALLATION - python ./test/smoke_test/smoke_test.py - if [[ ${TARGET_OS} != 'macos' && ${TARGET_OS} != 'windows' ]]; then + + if [[ ${TARGET_OS} == 'linux' ]]; then + export CONDA_LIBRARY_PATH="$(dirname $(which python))/../lib" + export LD_LIBRARY_PATH=$CONDA_LIBRARY_PATH:$LD_LIBRARY_PATH ${PWD}/check_binary.sh fi + + python ./test/smoke_test/smoke_test.py fi fi diff --git a/.github/workflows/validate-domain-library.yml b/.github/workflows/validate-domain-library.yml index 9b1536d98..85eec729a 100644 --- a/.github/workflows/validate-domain-library.yml +++ b/.github/workflows/validate-domain-library.yml @@ -89,13 +89,7 @@ jobs: script: | set -ex export ENV_NAME="conda-env-${{ github.run_id }}" - export DESIRED_PYTHON="${{ matrix.python_version }}" - export PACKAGE_TYPE="${{ matrix.package_type }}" - export CHANNEL="${{ matrix.channel }}" export SMOKE_TEST="${{ inputs.smoke_test }}" - export GPU_ARCH_VER="${{ matrix.gpu_arch_version }}" - export GPU_ARCH_TYPE="${{ matrix.gpu_arch_type }}" - export DESIRED_CUDA="${{ matrix.desired_cuda }}" eval $SMOKE_TEST validate-windows: if: (inputs.os == 'windows' || inputs.os == 'all') @@ -113,13 +107,7 @@ jobs: script: | set -ex export ENV_NAME="conda-env-${{ github.run_id }}" - export DESIRED_PYTHON="${{ matrix.python_version }}" - export PACKAGE_TYPE="${{ matrix.package_type }}" - export CHANNEL="${{ matrix.channel }}" export SMOKE_TEST="${{ inputs.smoke_test }}" - export GPU_ARCH_VER="${{ matrix.gpu_arch_version }}" - export GPU_ARCH_TYPE="${{ matrix.gpu_arch_type }}" - export DESIRED_CUDA="${{ matrix.desired_cuda }}" export TARGET_OS="windows" eval $SMOKE_TEST validate-macos: @@ -138,13 +126,8 @@ jobs: script: | set -ex export ENV_NAME="conda-env-${{ github.run_id }}" - export DESIRED_PYTHON="${{ matrix.python_version }}" - export PACKAGE_TYPE="${{ matrix.package_type }}" - export CHANNEL="${{ matrix.channel }}" + export TARGET_OS="macos" export SMOKE_TEST="${{ inputs.smoke_test }}" - export GPU_ARCH_VER="${{ matrix.gpu_arch_version }}" - export GPU_ARCH_TYPE="${{ matrix.gpu_arch_type }}" - export DESIRED_CUDA="${{ matrix.desired_cuda }}" eval $SMOKE_TEST validate-macos-arm64: if: (inputs.os == 'macos-arm64' || inputs.os == 'all') @@ -162,11 +145,5 @@ jobs: script: | set -ex export ENV_NAME="conda-env-${{ github.run_id }}" - export DESIRED_PYTHON="${{ matrix.python_version }}" - export PACKAGE_TYPE="${{ matrix.package_type }}" - export CHANNEL="${{ matrix.channel }}" - export SMOKE_TEST="${{ inputs.smoke_test }}" - export GPU_ARCH_VER="${{ matrix.gpu_arch_version }}" - export GPU_ARCH_TYPE="${{ matrix.gpu_arch_type }}" - export DESIRED_CUDA="${{ matrix.desired_cuda }}" + export TARGET_OS="macos-arm64" eval $SMOKE_TEST diff --git a/.github/workflows/validate-linux-binaries.yml b/.github/workflows/validate-linux-binaries.yml index 004a7f5d2..857fb0a73 100644 --- a/.github/workflows/validate-linux-binaries.yml +++ b/.github/workflows/validate-linux-binaries.yml @@ -50,16 +50,10 @@ jobs: repository: "pytorch/builder" ref: ${{ inputs.ref || github.ref }} job-name: ${{ matrix.build_name }} + binary-matrix: ${{ toJSON(matrix) }} script: | set -ex export ENV_NAME="conda-env-${{ github.run_id }}" - export GPU_ARCH_VER="${{ matrix.gpu_arch_version }}" - export GPU_ARCH_TYPE="${{ matrix.gpu_arch_type }}" - export INSTALLATION="${{ matrix.installation }}" - export CUDA_VER="${{ matrix.desired_cuda }}" - export DESIRED_PYTHON="${{ matrix.python_version }}" - export DESIRED_CUDA="${{ matrix.desired_cuda }}" - export DESIRED_DEVTOOLSET="${{ matrix.devtoolset }}" - export PACKAGE_TYPE="${{ matrix.package_type }}" export TARGET_OS="linux" - ./.github/scripts/validate_binaries.sh + eval "$(conda shell.bash hook)" + source ./.github/scripts/validate_binaries.sh diff --git a/.github/workflows/validate-macos-arm64-binaries.yml b/.github/workflows/validate-macos-arm64-binaries.yml index aa9831a93..f321022d4 100644 --- a/.github/workflows/validate-macos-arm64-binaries.yml +++ b/.github/workflows/validate-macos-arm64-binaries.yml @@ -48,15 +48,9 @@ jobs: repository: "pytorch/builder" ref: ${{ inputs.ref || github.ref }} job-name: ${{ matrix.build_name }} + binary-matrix: ${{ toJSON(matrix) }} script: | set -ex export ENV_NAME="conda-env-${{ github.run_id }}" - export GPU_ARCH_VER="${{ matrix.gpu_arch_version }}" - export GPU_ARCH_TYPE="${{ matrix.gpu_arch_type }}" - export INSTALLATION="${{ matrix.installation }}" - export CUDA_VER="${{ matrix.desired_cuda }}" - export DESIRED_PYTHON="${{ matrix.python_version }}" - export DESIRED_CUDA="${{ matrix.desired_cuda }}" - export PACKAGE_TYPE="${{ matrix.package_type }}" export TARGET_OS="macos-arm64" - ./.github/scripts/validate_binaries.sh + source ./.github/scripts/validate_binaries.sh diff --git a/.github/workflows/validate-macos-binaries.yml b/.github/workflows/validate-macos-binaries.yml index da4d38795..0e3f38ff8 100644 --- a/.github/workflows/validate-macos-binaries.yml +++ b/.github/workflows/validate-macos-binaries.yml @@ -48,16 +48,9 @@ jobs: repository: "pytorch/builder" ref: ${{ inputs.ref || github.ref }} job-name: ${{ matrix.build_name }} + binary-matrix: ${{ toJSON(matrix) }} script: | set -ex export ENV_NAME="conda-env-${{ github.run_id }}" - export GPU_ARCH_VER="${{ matrix.gpu_arch_version }}" - export GPU_ARCH_TYPE="${{ matrix.gpu_arch_type }}" - export INSTALLATION="${{ matrix.installation }}" - export CUDA_VER="${{ matrix.desired_cuda }}" - export DESIRED_PYTHON="${{ matrix.python_version }}" - export DESIRED_CUDA="${{ matrix.desired_cuda }}" - export DESIRED_DEVTOOLSET="${{ matrix.devtoolset }}" - export PACKAGE_TYPE="${{ matrix.package_type }}" export TARGET_OS="macos" - ./.github/scripts/validate_binaries.sh + source ./.github/scripts/validate_binaries.sh diff --git a/.github/workflows/validate-nightly-binaries.yml b/.github/workflows/validate-nightly-binaries.yml index 8862009d7..32c6781ba 100644 --- a/.github/workflows/validate-nightly-binaries.yml +++ b/.github/workflows/validate-nightly-binaries.yml @@ -24,6 +24,7 @@ on: - .github/workflows/validate-windows-binaries.yml - .github/workflows/validate-macos-binaries.yml - .github/workflows/validate-macos-arm64-binaries.yml + - .github/scripts/validate_binaries.sh - test/smoke_test/* jobs: nightly: diff --git a/.github/workflows/validate-windows-binaries.yml b/.github/workflows/validate-windows-binaries.yml index d01ebbbff..497f9849d 100644 --- a/.github/workflows/validate-windows-binaries.yml +++ b/.github/workflows/validate-windows-binaries.yml @@ -49,17 +49,11 @@ jobs: repository: "pytorch/builder" ref: ${{ inputs.ref || github.ref }} job-name: ${{ matrix.build_name }} + binary-matrix: ${{ toJSON(matrix) }} timeout: 60 script: | set -ex export ENV_NAME="conda-env-${{ github.run_id }}" - export GPU_ARCH_VER="${{ matrix.gpu_arch_version }}" - export GPU_ARCH_TYPE="${{ matrix.gpu_arch_type }}" - export INSTALLATION="${{ matrix.installation }}" - export CUDA_VER="${{ matrix.desired_cuda }}" - export DESIRED_PYTHON="${{ matrix.python_version }}" - export DESIRED_CUDA="${{ matrix.desired_cuda }}" - export DESIRED_DEVTOOLSET="${{ matrix.devtoolset }}" - export PACKAGE_TYPE="${{ matrix.package_type }}" export TARGET_OS="windows" - ./.github/scripts/validate_binaries.sh + source /c/Jenkins/Miniconda3/etc/profile.d/conda.sh + source ./.github/scripts/validate_binaries.sh diff --git a/check_binary.sh b/check_binary.sh index 35a47766b..51b15dc23 100755 --- a/check_binary.sh +++ b/check_binary.sh @@ -22,6 +22,19 @@ set -eux -o pipefail # libtorch package. +if [[ -z ${DESIRED_PYTHON:-} ]]; then + export DESIRED_PYTHON=${MATRIX_PYTHON_VERSION:-} +fi +if [[ -z ${DESIRED_CUDA:-} ]]; then + export DESIRED_CUDA=${MATRIX_DESIRED_CUDA:-} +fi +if [[ -z ${DESIRED_DEVTOOLSET:-} ]]; then + export DESIRED_DEVTOOLSET=${MATRIX_DESIRED_DEVTOOLSET:-} +fi +if [[ -z ${PACKAGE_TYPE:-} ]]; then + export PACKAGE_TYPE=${MATRIX_PACKAGE_TYPE:-} +fi + # The install root depends on both the package type and the os # All MacOS packages use conda, even for the wheel packages. if [[ "$PACKAGE_TYPE" == libtorch ]]; then diff --git a/test/smoke_test/smoke_test.py b/test/smoke_test/smoke_test.py index e72d3c670..a20a94b76 100644 --- a/test/smoke_test/smoke_test.py +++ b/test/smoke_test/smoke_test.py @@ -8,10 +8,10 @@ import importlib import subprocess -gpu_arch_ver = os.getenv("GPU_ARCH_VER") -gpu_arch_type = os.getenv("GPU_ARCH_TYPE") +gpu_arch_ver = os.getenv("MATRIX_GPU_ARCH_VERSION") +gpu_arch_type = os.getenv("MATRIX_GPU_ARCH_TYPE") # use installation env variable to tell if it is nightly channel -installation_str = os.getenv("INSTALLATION") +installation_str = os.getenv("MATRIX_INSTALLATION") is_cuda_system = gpu_arch_type == "cuda" SCRIPT_DIR = Path(__file__).parent NIGHTLY_ALLOWED_DELTA = 3