diff --git a/.github/scripts/validate_binaries.sh b/.github/scripts/validate_binaries.sh
index f05d9b20a..f11cd43ab 100755
--- a/.github/scripts/validate_binaries.sh
+++ b/.github/scripts/validate_binaries.sh
@@ -3,18 +3,13 @@ if [[ ${MATRIX_PACKAGE_TYPE} == "libtorch" ]]; then
     unzip libtorch.zip
 else
     #special case for Python 3.11
-    if [ $MATRIX_PYTHON_VERSION == '3.11' ]; then
-        export CPYTHON_VERSIONS=3.11.0
-        sudo yum -y install openssl-devel libssl-dev bzip2-devel libffi-devel
-        sudo yum -y groupinstall "Development Tools"
-        export PYTHON_PATH="/opt/_internal/cpython-3.11.0/bin"
-        export PIP_PATH="${PYTHON_PATH}/pip"
-        export PIP_INSTALLATION="${MATRIX_INSTALLATION/pip3/"$PIP_PATH"}"
-        export WITH_OPENSSL="/opt/openssl"
-        ./common/install_cpython.sh
-        eval ${PYTHON_PATH}/python --version
-        eval ${PIP_INSTALLATION}
-        eval ${PYTHON_PATH}/python ./test/smoke_test/smoke_test.py --package torchonly
+    if [[ ${MATRIX_PYTHON_VERSION} == '3.11' ]]; then
+        conda create -y -n ${ENV_NAME} python=${MATRIX_PYTHON_VERSION}
+        conda activate ${ENV_NAME}
+        eval $MATRIX_INSTALLATION
+        python ./test/smoke_test/smoke_test.py --package torchonly
+        conda deactivate
+        conda env remove -n ${ENV_NAME}
     else
 
         # Special case Pypi installation package, only applicable to linux nightly CUDA 11.7 builds, wheel package