From 27c47026e4108f8e2f02892ab5062a42f28a3160 Mon Sep 17 00:00:00 2001
From: Andrey Talman <atalman@fb.com>
Date: Thu, 28 Dec 2023 17:56:03 +0000
Subject: [PATCH] Add test ops validation for validation workflows (#1650)

* Add test ops validation

* include workflows
---
 .github/scripts/validate_test_ops.sh          | 16 ++++++++++++++++
 .github/workflows/validate-binaries.yml       | 11 +++++++++++
 .github/workflows/validate-linux-binaries.yml | 15 +++++++++++++++
 3 files changed, 42 insertions(+)
 create mode 100644 .github/scripts/validate_test_ops.sh

diff --git a/.github/scripts/validate_test_ops.sh b/.github/scripts/validate_test_ops.sh
new file mode 100644
index 000000000..7c80b601e
--- /dev/null
+++ b/.github/scripts/validate_test_ops.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -eux -o pipefail
+
+retry () {
+    $*  || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
+}
+
+# Clone the Pytorch branch
+retry git clone --depth 1 https://github.com/pytorch/pytorch.git
+retry git submodule update --init --recursive
+pushd pytorch
+
+# Run test_ops validation
+export CUDA_LAUNCH_BLOCKING=1
+python3 test/test_ops.py
diff --git a/.github/workflows/validate-binaries.yml b/.github/workflows/validate-binaries.yml
index 8dbe5f27c..558be8e56 100644
--- a/.github/workflows/validate-binaries.yml
+++ b/.github/workflows/validate-binaries.yml
@@ -27,6 +27,11 @@ on:
         default: false
         required: false
         type: boolean
+      include-test-ops:
+        description: 'Include Test Ops tests (only Linux)'
+        default: false
+        required: false
+        type: boolean
   workflow_dispatch:
     inputs:
       os:
@@ -65,6 +70,11 @@ on:
         default: ""
         required: false
         type: string
+      include-test-ops:
+        description: 'Include Test Ops tests (only Linux)'
+        default: false
+        required: false
+        type: boolean
 
 
 jobs:
@@ -94,6 +104,7 @@ jobs:
       torchonly: ${{ inputs.torchonly }}
       version: ${{ inputs.version }}
       release-matrix: ${{ needs.generate-release-matrix.outputs.matrix }}
+      include-test-ops: ${{ inputs.include-test-ops }}
 
   linux-aarch64:
     if:  inputs.os == 'linux-aarch64' || inputs.os == 'all'
diff --git a/.github/workflows/validate-linux-binaries.yml b/.github/workflows/validate-linux-binaries.yml
index d1c6c29bd..3f652eff8 100644
--- a/.github/workflows/validate-linux-binaries.yml
+++ b/.github/workflows/validate-linux-binaries.yml
@@ -27,6 +27,11 @@ on:
         default: ""
         required: false
         type: string
+      include-test-ops:
+        description: 'Include Test Ops tests (only Linux)'
+        default: false
+        required: false
+        type: boolean
   workflow_dispatch:
     inputs:
       channel:
@@ -58,6 +63,11 @@ on:
         default: ""
         required: false
         type: string
+      include-test-ops:
+        description: 'Include Test Ops tests (only Linux)'
+        default: false
+        required: false
+        type: boolean
 
 jobs:
   generate-linux-matrix:
@@ -84,6 +94,7 @@ jobs:
         set -ex
         export ENV_NAME="conda-env-${{ github.run_id }}"
         export TORCH_ONLY=${{ inputs.torchonly }}
+        export INCLUDE_TEST_OPS=${{ inputs.include-test-ops }}
         export RELEASE_VERSION=${{ inputs.version }}
         export TARGET_OS="linux"
         eval "$(conda shell.bash hook)"
@@ -97,3 +108,7 @@ jobs:
 
         # Standart case: Validate binaries
         source ./.github/scripts/validate_binaries.sh
+
+        if [[ ${INCLUDE_TEST_OPS} == 'true' ]]; then
+          source ./.github/scripts/validate_test_ops.sh
+        fi