forward unittest workloads to AWS

openvinotoolkit · Feb 6, 2024 · cf35662 · cf35662
1 parent 7c3617e
commit cf35662
Show file tree

Hide file tree

Showing 3 changed files with 147 additions and 38 deletions.
diff --git a/.github/workflows/pre_merge.yml b/.github/workflows/pre_merge.yml
@@ -34,7 +34,7 @@ jobs:
       - name: Code quality checks
         run: tox -vv -e pre-commit-all-py310-pt1
   Unit-Test:
-    runs-on: [self-hosted, linux, x64, dev]
+    runs-on: [otx-gpu-t4-1]
     needs: Code-Quality-Checks
     timeout-minutes: 120
     strategy:
@@ -52,41 +52,73 @@ jobs:
     concurrency:
       group: ${{ github.workflow }}-Unit-${{ github.event.pull_request.number || github.ref }}-${{ matrix.tox-env }}
       cancel-in-progress: true
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-      - name: Install Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install dependencies
-        run: python -m pip install -r requirements/dev.txt
-      - name: Run unit test
-        run: tox -vv -e unittest-all-${{ matrix.tox-env }}-pt1
-      - name: Upload coverage artifact
-        if: ${{ matrix.python-version == '3.8' }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: coverage
-          path: .tox/coverage.xml
-      - name: Upload coverage reports to Codecov
-        run: |
-          # If the workflow is triggered from PR then it gets the commit id from the PR.
-          # else it uses the commit id of the latest commit. This is because the commit
-          # of the checked-out branch/commit does not exist in the tree as it is grafted.
-          # Also note: GitHub does not pass secrets to pipelines triggered from a fork.
-          # This means that upload will fail for PRs from forks.
-          if [ -n "${{ github.event.pull_request.head.sha }}" ]
-          then
-            COMMIT_ID=${{ github.event.pull_request.head.sha }}
-          else
-            COMMIT_ID=${{ github.sha }}
-          fi
-          # current version of codecov-action does not support uploading reports through the proxy
-          # so we use the latest version of codecov uploader binary
-          curl -Os https://uploader.codecov.io/latest/linux/codecov
-          chmod +x codecov
-          ./codecov -t ${{ secrets.CODECOV_TOKEN }} --sha $COMMIT_ID -U $HTTP_PROXY -f .tox/coverage.xml -F ${{ matrix.tox-env }}
+    uses: ./.github/workflows/run_tests_in_tox.yml
+    with:
+      python-version: ${{ matrix.python-version }}
+      toxenv-pyver: ${{ matrix.tox-env }}
+      toxenv-task: all
+      tests_dir: tests/unit
+      timeout-minutes: 120
+      upload-artifact: true
+      artifact-prefix: "unit-test-results"
+      runs-on: "['otx-gpu-t4-1']"
+  # Unit-Test:
+  #   runs-on: [otx-gpu-t4-1]
+  #   container:
+  #     image: 219678651685.dkr.ecr.eu-central-1.amazonaws.com/ote-ci:pr-194-1334bef936df3b06d28bc3d1bd90e7248359b99e
+  #     options: "--runtime=nvidia --env-file=/home/runner/.nvidia.env --shm-size=24g"
+  #   needs: Code-Quality-Checks
+  #   timeout-minutes: 120
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       include:
+  #         - python-version: "3.8"
+  #           tox-env: "py38"
+  #         - python-version: "3.9"
+  #           tox-env: "py39"
+  #         - python-version: "3.10"
+  #           tox-env: "py310"
+  #   name: Unit-Test-with-Python${{ matrix.python-version }}
+  #   # This is what will cancel the job concurrency
+  #   concurrency:
+  #     group: ${{ github.workflow }}-Unit-${{ github.event.pull_request.number || github.ref }}-${{ matrix.tox-env }}
+  #     cancel-in-progress: true
+  #   steps:
+  #     - name: Checkout repository
+  #       uses: actions/checkout@v3
+  #     - name: Install Python
+  #       uses: actions/setup-python@v4
+  #       with:
+  #         python-version: ${{ matrix.python-version }}
+  #     - name: Install dependencies
+  #       run: python -m pip install -r requirements/dev.txt
+  #     - name: Run unit test
+  #       run: tox -vv -e unittest-all-${{ matrix.tox-env }}-pt1
+  #     - name: Upload coverage artifact
+  #       if: ${{ matrix.python-version == '3.8' }}
+  #       uses: actions/upload-artifact@v3
+  #       with:
+  #         name: coverage
+  #         path: .tox/coverage.xml
+  #     - name: Upload coverage reports to Codecov
+  #       run: |
+  #         # If the workflow is triggered from PR then it gets the commit id from the PR.
+  #         # else it uses the commit id of the latest commit. This is because the commit
+  #         # of the checked-out branch/commit does not exist in the tree as it is grafted.
+  #         # Also note: GitHub does not pass secrets to pipelines triggered from a fork.
+  #         # This means that upload will fail for PRs from forks.
+  #         if [ -n "${{ github.event.pull_request.head.sha }}" ]
+  #         then
+  #           COMMIT_ID=${{ github.event.pull_request.head.sha }}
+  #         else
+  #           COMMIT_ID=${{ github.sha }}
+  #         fi
+  #         # current version of codecov-action does not support uploading reports through the proxy
+  #         # so we use the latest version of codecov uploader binary
+  #         curl -Os https://uploader.codecov.io/latest/linux/codecov
+  #         chmod +x codecov
+  #         ./codecov -t ${{ secrets.CODECOV_TOKEN }} --sha $COMMIT_ID -U $HTTP_PROXY -f .tox/coverage.xml -F ${{ matrix.tox-env }}
   Integration-Test:
     needs: Unit-Test
     strategy:
@@ -114,7 +146,7 @@ jobs:
     concurrency:
       group: ${{ github.workflow }}-Integration-${{ github.event.pull_request.number || github.ref }}-${{ matrix.task }}
       cancel-in-progress: true
-    uses: ./.github/workflows/run_tests_in_tox.yml
+    uses: ./.github/workflows/run_tests_in_tox_ori.yml
     with:
       python-version: "3.10"
       toxenv-pyver: "py310"

diff --git a/.github/workflows/run_tests_in_tox.yml b/.github/workflows/run_tests_in_tox.yml
@@ -33,15 +33,21 @@ on:
       toxenv-ptver:
         type: string
         default: "pt1"
+      container-options:
+        type: string
+        default: "--runtime=nvidia --env-file=/home/runner/.nvidia.env --shm-size=24g"
 
 # Declare default permissions as read only.
 permissions: read-all
 
 jobs:
-  run_tests_in_tox:
+  run_tests_on_custom:
     # tricky workaround to pass list from the string input type
     # https://github.com/orgs/community/discussions/11692
     runs-on: ${{ fromJson(inputs.runs-on) }}
+    container:
+      image: 219678651685.dkr.ecr.eu-central-1.amazonaws.com/ote-ci:pr-194-1334bef936df3b06d28bc3d1bd90e7248359b99e
+      option: ${{ inputs.container-options }}
     timeout-minutes: ${{ inputs.timeout-minutes }}
     steps:
       - name: Checkout repository

diff --git a/.github/workflows/run_tests_in_tox_ori.yml b/.github/workflows/run_tests_in_tox_ori.yml
@@ -0,0 +1,71 @@
+on:
+  workflow_call:
+    inputs:
+      python-version:
+        type: string
+        default: "3.10"
+      toxenv-pyver:
+        description: "[py38, py39, py310]"
+        type: string
+        default: "py310"
+      toxenv-task:
+        description: "[all, act, ano, cls, det, seg, iseg]"
+        type: string
+        default: "all"
+      tests-dir:
+        type: string
+        default: ""
+      timeout-minutes:
+        type: number
+        default: 720
+      upload-artifact:
+        type: boolean
+        default: false
+      runs-on:
+        type: string
+        default: "['self-hosted', 'Linux', 'X64', 'dev']"
+      task:
+        type: string
+        default: "undefined"
+      artifact-prefix:
+        type: string
+        default: "test-results"
+      toxenv-ptver:
+        type: string
+        default: "pt1"
+
+# Declare default permissions as read only.
+permissions: read-all
+
+jobs:
+  run_tests_in_tox:
+    # tricky workaround to pass list from the string input type
+    # https://github.com/orgs/community/discussions/11692
+    runs-on: ${{ fromJson(inputs.runs-on) }}
+    timeout-minutes: ${{ inputs.timeout-minutes }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ inputs.python-version }}
+      - name: Install dependencies
+        run: python -m pip install -r requirements/dev.txt
+      - name: Run Tests
+        env:
+          MLFLOW_TRACKING_SERVER_URI: ${{ vars.MLFLOW_TRACKING_SERVER_URI }}
+          BENCHMARK_RESULTS_CLEAR: ${{ vars.BENCHMARK_RESULTS_CLEAR }}
+          GH_CTX_REF_NAME: ${{ github.ref_name }}
+          GH_CTX_SHA: ${{ github.sha  }}
+        run: tox -vv -e tests-${{ inputs.toxenv-task }}-${{ inputs.toxenv-pyver }}-${{ inputs.toxenv-ptver }} -- ${{ inputs.tests-dir }}
+      - name: Upload test results
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ inputs.artifact-prefix }}-${{ inputs.toxenv-task }}-${{ inputs.toxenv-pyver }}-${{ inputs.toxenv-ptver }}
+          path: |
+            .tox/tests-${{ inputs.toxenv-task }}-${{ inputs.toxenv-pyver }}-${{ inputs.toxenv-ptver }}.csv
+            .tox/tests-reg_${{ inputs.task }}*.csv
+            .tox/perf-*.csv
+        # Use always() to always run this step to publish test results when there are test failures
+        if: ${{ inputs.upload-artifact && always() }}