Add test to validate torchscript backward compatibility

.circleci/config.yml

@@ -605,6 +605,42 @@ jobs:
           name: Run style check
           command: .circleci/unittest/linux/scripts/run_style_checks.sh
 
+  torchscript_bc_test:
+    docker:
+      - image: "pytorch/torchaudio_unittest_base:manylinux"
+    resource_class: medium
+    steps:
+      - checkout
+      - generate_cache_key
+      - restore_cache:
+
+          keys:
+            - torchscript-bc-test-release-envs-v1-{{ arch }}-{{ checksum ".cachekey" }}
+
+      - run:
+          name: Generate Objects
+          command: |
+             .circleci/torchscript_bc_test/setup_release_envs.sh
+             .circleci/torchscript_bc_test/generate_objects.sh
+      - save_cache:
+
+          key: torchscript-bc-test-release-envs-v1-{{ arch }}-{{ checksum ".cachekey" }}
+
+          paths:
+            - conda
+            - envs
+      - store_artifacts:
+          path: test/torchscript_bc_test/assets
+      - persist_to_workspace:
+          root: .
+          paths:
+            - test/torchscript_bc_test/assets
+      - run:
+          name: Run BC check test
+          command: |
+            .circleci/torchscript_bc_test/setup_master_envs.sh
+            .circleci/torchscript_bc_test/validate_objects.sh
+
 workflows:
   build:
     jobs:
@@ -691,6 +727,7 @@ workflows:
           python_version: '3.8'
   unittest:
     jobs:
+      - torchscript_bc_test
       - download_third_parties_nix:
           name: download_third_parties_nix
       - unittest_linux_cpu:

.circleci/config.yml.in

@@ -605,13 +605,50 @@ jobs:
           name: Run style check
           command: .circleci/unittest/linux/scripts/run_style_checks.sh
 
+  torchscript_bc_test:
+    docker:
+      - image: "pytorch/torchaudio_unittest_base:manylinux"
+    resource_class: medium
+    steps:
+      - checkout
+      - generate_cache_key
+      - restore_cache:
+          {% raw %}
+          keys:
+            - torchscript-bc-test-release-envs-v1-{{ arch }}-{{ checksum ".cachekey" }}
+          {% endraw %}
+      - run:
+          name: Generate Objects
+          command: |
+             .circleci/torchscript_bc_test/setup_release_envs.sh
+             .circleci/torchscript_bc_test/generate_objects.sh
+      - save_cache:
+          {% raw %}
+          key: torchscript-bc-test-release-envs-v1-{{ arch }}-{{ checksum ".cachekey" }}
+          {% endraw %}
+          paths:
+            - conda
+            - envs
+      - store_artifacts:
+          path: test/torchscript_bc_test/assets
+      - persist_to_workspace:
+          root: .
+          paths:
+            - test/torchscript_bc_test/assets
+      - run:
+          name: Run BC check test
+          command: |
+            .circleci/torchscript_bc_test/setup_master_envs.sh
+            .circleci/torchscript_bc_test/validate_objects.sh
+
 workflows:
   build:
     jobs:
       - circleci_consistency
       {{ build_workflows() }}
   unittest:
     jobs:
+      - torchscript_bc_test
       {{ unittest_workflows() }}
   nightly:
     jobs:

.circleci/torchscript_bc_test/common.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+
+declare -a TORCHAUDIO_VERSIONS=("0.6.0")
+declare -a PYTHON_VERSIONS=("3.6" "3.7" "3.8")
+
+export TORCHAUDIO_VERSIONS
+export PYTHON_VERSIONS
+
+export KALDI_ROOT="${KALDI_ROOT:-$HOME}"  # Just to disable warning emitted from kaldi_io
+
+_this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+_root_dir="$(git rev-parse --show-toplevel)"
+_conda_dir="${_root_dir}/conda"
+case "$(uname -s)" in
+    Darwin*) _os="MacOSX";;
+    *) _os="Linux"
+esac
+
+install_conda() {
+    if [ ! -d "${_conda_dir}" ]; then
+        printf "* Installing conda\n"
+        wget -nv -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${_os}-x86_64.sh"
+        bash ./miniconda.sh -b -f -p "${_conda_dir}"
+        rm miniconda.sh
+    fi
+}
+
+init_conda() {
+    eval "$("${_conda_dir}/bin/conda" shell.bash hook)"
+}
+
+get_name() {
+    echo "${1}-py${2}"
+}
+
+get_env_dir() {
+    echo "${_root_dir}/envs/$(get_name "$1" "$2")"
+}
+
+create_env() {
+    env_dir="$(get_env_dir "$1" "$2")"
+    if [ ! -d "${env_dir}" ]; then
+        printf "* Creating environment torchaudio: %s, Python: %s\n" "$1" "$2"
+        conda create -q --prefix "${env_dir}" -y python="$2"
+    fi
+}
+
+activate_env() {
+    printf "* Activating environment torchaudio: %s, Python: %s\n" "$1" "$2"
+    conda activate "$(get_env_dir "$1" "$2")"
+}
+
+install_release() {
+    printf "* Installing torchaudio: %s\n" "$1"
+    conda install -y -q torchaudio="$1" packaging -c pytorch
+    # packaging is required in test to validate the torchaudio version for dump
+}
+
+install_build_dependencies() {
+    printf "* Installing torchaudio dependencies except PyTorch - (Python: %s)\n" "$1"
+    conda env update -q --file "${_this_dir}/environment.yml" --prune
+    if [ "${_os}" == Linux ]; then
+        pip install clang-format
+    fi
+}
+
+build_master() {
+    printf "* Installing PyTorch (py%s)\n" "$1"
+    conda install -y -q pytorch "cpuonly" -c pytorch-nightly
+    printf "* Installing torchaudio\n"
+    cd "${_root_dir}" || exit 1
+    BUILD_SOX=1 python setup.py clean install
+}

.circleci/torchscript_bc_test/environment.yml

@@ -0,0 +1,17 @@
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - flake8
+  - numpy
+  - pytest
+  - pytest-cov
+  - codecov
+  - librosa
+  - llvmlite==0.31  # See https://github.com/pytorch/audio/pull/766
+  - pip
+  - pip:
+    - kaldi-io
+    - scipy
+    - parameterized
+    - numba==0.48  # See https://github.com/librosa/librosa/issues/1160

.circleci/torchscript_bc_test/generate_objects.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+set -e
+
+this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+pushd "${this_dir}"
+. "common.sh"
+popd
+
+init_conda
+
+# Move to test directory so that the checked out torchaudio source
+# will not shadow the conda-installed version of torchaudio
+cd test
+
+for torchaudio in "${TORCHAUDIO_VERSIONS[@]}" ; do
+    for python in "${PYTHON_VERSIONS[@]}" ; do
+        activate_env "${torchaudio}" "${python}"
+        python -m torch.utils.collect_env
+        printf "***********************************************************\n"
+        printf "* Generating\n"
+        printf "  Objects: Python: %s, torchaudio: %s\n" "${python}" "${torchaudio}"
+        printf "***********************************************************\n"
+        ./torchscript_bc_test/main.py --mode generate --version "${torchaudio}"
+    done
+done

.circleci/torchscript_bc_test/setup_master_envs.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -e
+
+cd "$( dirname "${BASH_SOURCE[0]}" )"
+. "common.sh"
+
+install_conda
+init_conda
+
+# Install torchaudio environments
+for python in "${PYTHON_VERSIONS[@]}" ; do
+    create_env master "${python}"
+    activate_env master "${python}"
+    install_build_dependencies "${python}"
+    build_master "${python}"
+done

.circleci/torchscript_bc_test/setup_release_envs.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -e
+
+cd "$( dirname "${BASH_SOURCE[0]}" )"
+. "common.sh"
+
+install_conda
+init_conda
+
+# Install torchaudio environments
+for torchaudio in "${TORCHAUDIO_VERSIONS[@]}" ; do
+    for python in "${PYTHON_VERSIONS[@]}" ; do
+        create_env "${torchaudio}" "${python}"
+        activate_env "${torchaudio}" "${python}"
+        install_release "${torchaudio}"
+    done
+done

.circleci/torchscript_bc_test/validate_objects.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+set -e
+
+this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+pushd "${this_dir}"
+. "common.sh"
+popd
+
+init_conda
+
+# Move to test directory so that the checked out torchaudio source
+# will not shadow the conda-installed version of torchaudio
+cd test
+
+# Validate torchscript objects for each
+for runtime_python in "${PYTHON_VERSIONS[@]}" ; do
+    activate_env master "${runtime_python}"
+    python -m torch.utils.collect_env
+    for object_torchaudio in "${TORCHAUDIO_VERSIONS[@]}" ; do
+        for object_python in "${PYTHON_VERSIONS[@]}" ; do
+            printf "***********************************************************\n"
+            printf "* Validating\n"
+            printf "  Runtime: Python: %s, torchaudio: master (%s)\n" "${runtime_python}" "$(python -c 'import torchaudio;print(torchaudio.__version__)')"
+            printf "  Objects: Python: %s, torchaudio: %s\n" "${object_python}" "${object_torchaudio}"
+            printf "***********************************************************\n"
+            ./torchscript_bc_test/main.py --mode validate --version "${object_torchaudio}"
+        done
+    done
+done