diff --git a/.github/scripts/unittest-linux/install.sh b/.github/scripts/unittest-linux/install.sh
index 885296219cd..70a8793c224 100755
--- a/.github/scripts/unittest-linux/install.sh
+++ b/.github/scripts/unittest-linux/install.sh
@@ -75,7 +75,7 @@ fi
 # Note: installing librosa via pip fail because it will try to compile numba.
 (
     set -x
-    conda install -y -c conda-forge ${NUMBA_DEV_CHANNEL} 'librosa==0.10.0' parameterized 'requests>=2.20' 'ffmpeg>=5,<7'
+    conda install -y -c conda-forge ${NUMBA_DEV_CHANNEL} sox libvorbis 'librosa==0.10.0' parameterized 'requests>=2.20' 'ffmpeg>=5,<7'
     pip install kaldi-io SoundFile coverage pytest pytest-cov 'scipy==1.7.3' expecttest unidecode inflect Pillow sentencepiece pytorch-lightning 'protobuf<4.21.0' demucs tinytag pyroomacoustics flashlight-text git+https://github.com/kpu/kenlm
 )
 # Install fairseq
diff --git a/.github/scripts/unittest-windows/install.sh b/.github/scripts/unittest-windows/install.sh
index 676de7e830a..f1b8c529a75 100644
--- a/.github/scripts/unittest-windows/install.sh
+++ b/.github/scripts/unittest-windows/install.sh
@@ -67,7 +67,7 @@ case "$(python --version)" in
 esac
 # Note: installing librosa via pip fail because it will try to compile numba.
 (
-    conda install -y -c conda-forge ${NUMBA_DEV_CHANNEL} 'librosa==0.10.0' parameterized 'requests>=2.20'
+    conda install -y -c conda-forge ${NUMBA_DEV_CHANNEL} sox libvorbis 'librosa==0.10.0' parameterized 'requests>=2.20'
     # Need to disable shell check since this'll fail out if SENTENCEPIECE_DEPENDENCY is empty
     # shellcheck disable=SC2086
     pip install \
diff --git a/.github/workflows/unittest-linux-cpu.yml b/.github/workflows/unittest-linux-cpu.yml
index 50207c0131b..b96dd200cb4 100644
--- a/.github/workflows/unittest-linux-cpu.yml
+++ b/.github/workflows/unittest-linux-cpu.yml
@@ -50,6 +50,8 @@ jobs:
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_AUDIO_OUT_DEVICE=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MACOS=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_TEMPORARY_DISABLED=true
+        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_DECODER=true
+        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_ENCODER=true
         echo '::endgroup::'
 
         set -euxo pipefail
diff --git a/.github/workflows/unittest-linux-gpu.yml b/.github/workflows/unittest-linux-gpu.yml
index 7e8b27cf4d0..cbb8b15299f 100644
--- a/.github/workflows/unittest-linux-gpu.yml
+++ b/.github/workflows/unittest-linux-gpu.yml
@@ -59,7 +59,7 @@ jobs:
         python3 -m pip install -v -e . --no-use-pep517
 
         # Install test tools
-        conda install -y --quiet -c conda-forge -c numba/label/dev 'librosa==0.10.0' parameterized 'requests>=2.20' 'ffmpeg>=5,<7'
+        conda install -y --quiet -c conda-forge -c numba/label/dev sox libvorbis 'librosa==0.10.0' parameterized 'requests>=2.20' 'ffmpeg>=5,<7'
         python3 -m pip install --quiet kaldi-io SoundFile coverage pytest pytest-cov 'scipy==1.7.3' transformers expecttest unidecode inflect Pillow sentencepiece pytorch-lightning 'protobuf<4.21.0' demucs tinytag flashlight-text git+https://github.com/kpu/kenlm/
         python3 -m pip install --quiet git+https://github.com/pytorch/fairseq.git@e47a4c8
 
@@ -74,6 +74,8 @@ jobs:
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_HW_ACCEL=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_ON_PYTHON_310=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_TEMPORARY_DISABLED=true
+        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_DECODER=true
+        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_ENCODER=true
 
         declare -a args=(
             '-v'
diff --git a/.github/workflows/unittest-macos-cpu.yml b/.github/workflows/unittest-macos-cpu.yml
index 4e3cc598799..274d5eb648e 100644
--- a/.github/workflows/unittest-macos-cpu.yml
+++ b/.github/workflows/unittest-macos-cpu.yml
@@ -49,6 +49,8 @@ jobs:
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_sentencepiece=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_AUDIO_OUT_DEVICE=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_TEMPORARY_DISABLED=true
+        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_DECODER=true
+        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_ENCODER=true
         echo '::endgroup::'
 
         set -euxo pipefail
diff --git a/.github/workflows/unittest-windows-cpu.yml b/.github/workflows/unittest-windows-cpu.yml
index e2eb73674d7..a3ed8e34827 100644
--- a/.github/workflows/unittest-windows-cpu.yml
+++ b/.github/workflows/unittest-windows-cpu.yml
@@ -45,6 +45,7 @@ jobs:
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_sentencepiece=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_AUDIO_OUT_DEVICE=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MACOS=true
+        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_TEMPORARY_DISABLED=true
 
         .github/scripts/unittest-windows/setup_env.sh
         .github/scripts/unittest-windows/install.sh