From cf35662bf396dc8d58888407414f93e0b3840027 Mon Sep 17 00:00:00 2001 From: Yunchu Lee Date: Tue, 6 Feb 2024 11:14:02 +0900 Subject: [PATCH] forward unittest workloads to AWS --- .github/workflows/pre_merge.yml | 106 ++++++++++++++------- .github/workflows/run_tests_in_tox.yml | 8 +- .github/workflows/run_tests_in_tox_ori.yml | 71 ++++++++++++++ 3 files changed, 147 insertions(+), 38 deletions(-) create mode 100644 .github/workflows/run_tests_in_tox_ori.yml diff --git a/.github/workflows/pre_merge.yml b/.github/workflows/pre_merge.yml index b71d7f66ba8..8e1f4ecfbc7 100644 --- a/.github/workflows/pre_merge.yml +++ b/.github/workflows/pre_merge.yml @@ -34,7 +34,7 @@ jobs: - name: Code quality checks run: tox -vv -e pre-commit-all-py310-pt1 Unit-Test: - runs-on: [self-hosted, linux, x64, dev] + runs-on: [otx-gpu-t4-1] needs: Code-Quality-Checks timeout-minutes: 120 strategy: @@ -52,41 +52,73 @@ jobs: concurrency: group: ${{ github.workflow }}-Unit-${{ github.event.pull_request.number || github.ref }}-${{ matrix.tox-env }} cancel-in-progress: true - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - name: Install Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: python -m pip install -r requirements/dev.txt - - name: Run unit test - run: tox -vv -e unittest-all-${{ matrix.tox-env }}-pt1 - - name: Upload coverage artifact - if: ${{ matrix.python-version == '3.8' }} - uses: actions/upload-artifact@v3 - with: - name: coverage - path: .tox/coverage.xml - - name: Upload coverage reports to Codecov - run: | - # If the workflow is triggered from PR then it gets the commit id from the PR. - # else it uses the commit id of the latest commit. This is because the commit - # of the checked-out branch/commit does not exist in the tree as it is grafted. - # Also note: GitHub does not pass secrets to pipelines triggered from a fork. - # This means that upload will fail for PRs from forks. - if [ -n "${{ github.event.pull_request.head.sha }}" ] - then - COMMIT_ID=${{ github.event.pull_request.head.sha }} - else - COMMIT_ID=${{ github.sha }} - fi - # current version of codecov-action does not support uploading reports through the proxy - # so we use the latest version of codecov uploader binary - curl -Os https://uploader.codecov.io/latest/linux/codecov - chmod +x codecov - ./codecov -t ${{ secrets.CODECOV_TOKEN }} --sha $COMMIT_ID -U $HTTP_PROXY -f .tox/coverage.xml -F ${{ matrix.tox-env }} + uses: ./.github/workflows/run_tests_in_tox.yml + with: + python-version: ${{ matrix.python-version }} + toxenv-pyver: ${{ matrix.tox-env }} + toxenv-task: all + tests_dir: tests/unit + timeout-minutes: 120 + upload-artifact: true + artifact-prefix: "unit-test-results" + runs-on: "['otx-gpu-t4-1']" + # Unit-Test: + # runs-on: [otx-gpu-t4-1] + # container: + # image: 219678651685.dkr.ecr.eu-central-1.amazonaws.com/ote-ci:pr-194-1334bef936df3b06d28bc3d1bd90e7248359b99e + # options: "--runtime=nvidia --env-file=/home/runner/.nvidia.env --shm-size=24g" + # needs: Code-Quality-Checks + # timeout-minutes: 120 + # strategy: + # fail-fast: false + # matrix: + # include: + # - python-version: "3.8" + # tox-env: "py38" + # - python-version: "3.9" + # tox-env: "py39" + # - python-version: "3.10" + # tox-env: "py310" + # name: Unit-Test-with-Python${{ matrix.python-version }} + # # This is what will cancel the job concurrency + # concurrency: + # group: ${{ github.workflow }}-Unit-${{ github.event.pull_request.number || github.ref }}-${{ matrix.tox-env }} + # cancel-in-progress: true + # steps: + # - name: Checkout repository + # uses: actions/checkout@v3 + # - name: Install Python + # uses: actions/setup-python@v4 + # with: + # python-version: ${{ matrix.python-version }} + # - name: Install dependencies + # run: python -m pip install -r requirements/dev.txt + # - name: Run unit test + # run: tox -vv -e unittest-all-${{ matrix.tox-env }}-pt1 + # - name: Upload coverage artifact + # if: ${{ matrix.python-version == '3.8' }} + # uses: actions/upload-artifact@v3 + # with: + # name: coverage + # path: .tox/coverage.xml + # - name: Upload coverage reports to Codecov + # run: | + # # If the workflow is triggered from PR then it gets the commit id from the PR. + # # else it uses the commit id of the latest commit. This is because the commit + # # of the checked-out branch/commit does not exist in the tree as it is grafted. + # # Also note: GitHub does not pass secrets to pipelines triggered from a fork. + # # This means that upload will fail for PRs from forks. + # if [ -n "${{ github.event.pull_request.head.sha }}" ] + # then + # COMMIT_ID=${{ github.event.pull_request.head.sha }} + # else + # COMMIT_ID=${{ github.sha }} + # fi + # # current version of codecov-action does not support uploading reports through the proxy + # # so we use the latest version of codecov uploader binary + # curl -Os https://uploader.codecov.io/latest/linux/codecov + # chmod +x codecov + # ./codecov -t ${{ secrets.CODECOV_TOKEN }} --sha $COMMIT_ID -U $HTTP_PROXY -f .tox/coverage.xml -F ${{ matrix.tox-env }} Integration-Test: needs: Unit-Test strategy: @@ -114,7 +146,7 @@ jobs: concurrency: group: ${{ github.workflow }}-Integration-${{ github.event.pull_request.number || github.ref }}-${{ matrix.task }} cancel-in-progress: true - uses: ./.github/workflows/run_tests_in_tox.yml + uses: ./.github/workflows/run_tests_in_tox_ori.yml with: python-version: "3.10" toxenv-pyver: "py310" diff --git a/.github/workflows/run_tests_in_tox.yml b/.github/workflows/run_tests_in_tox.yml index d2591cbcb50..897f4beddc9 100644 --- a/.github/workflows/run_tests_in_tox.yml +++ b/.github/workflows/run_tests_in_tox.yml @@ -33,15 +33,21 @@ on: toxenv-ptver: type: string default: "pt1" + container-options: + type: string + default: "--runtime=nvidia --env-file=/home/runner/.nvidia.env --shm-size=24g" # Declare default permissions as read only. permissions: read-all jobs: - run_tests_in_tox: + run_tests_on_custom: # tricky workaround to pass list from the string input type # https://github.com/orgs/community/discussions/11692 runs-on: ${{ fromJson(inputs.runs-on) }} + container: + image: 219678651685.dkr.ecr.eu-central-1.amazonaws.com/ote-ci:pr-194-1334bef936df3b06d28bc3d1bd90e7248359b99e + option: ${{ inputs.container-options }} timeout-minutes: ${{ inputs.timeout-minutes }} steps: - name: Checkout repository diff --git a/.github/workflows/run_tests_in_tox_ori.yml b/.github/workflows/run_tests_in_tox_ori.yml new file mode 100644 index 00000000000..d2591cbcb50 --- /dev/null +++ b/.github/workflows/run_tests_in_tox_ori.yml @@ -0,0 +1,71 @@ +on: + workflow_call: + inputs: + python-version: + type: string + default: "3.10" + toxenv-pyver: + description: "[py38, py39, py310]" + type: string + default: "py310" + toxenv-task: + description: "[all, act, ano, cls, det, seg, iseg]" + type: string + default: "all" + tests-dir: + type: string + default: "" + timeout-minutes: + type: number + default: 720 + upload-artifact: + type: boolean + default: false + runs-on: + type: string + default: "['self-hosted', 'Linux', 'X64', 'dev']" + task: + type: string + default: "undefined" + artifact-prefix: + type: string + default: "test-results" + toxenv-ptver: + type: string + default: "pt1" + +# Declare default permissions as read only. +permissions: read-all + +jobs: + run_tests_in_tox: + # tricky workaround to pass list from the string input type + # https://github.com/orgs/community/discussions/11692 + runs-on: ${{ fromJson(inputs.runs-on) }} + timeout-minutes: ${{ inputs.timeout-minutes }} + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python-version }} + - name: Install dependencies + run: python -m pip install -r requirements/dev.txt + - name: Run Tests + env: + MLFLOW_TRACKING_SERVER_URI: ${{ vars.MLFLOW_TRACKING_SERVER_URI }} + BENCHMARK_RESULTS_CLEAR: ${{ vars.BENCHMARK_RESULTS_CLEAR }} + GH_CTX_REF_NAME: ${{ github.ref_name }} + GH_CTX_SHA: ${{ github.sha }} + run: tox -vv -e tests-${{ inputs.toxenv-task }}-${{ inputs.toxenv-pyver }}-${{ inputs.toxenv-ptver }} -- ${{ inputs.tests-dir }} + - name: Upload test results + uses: actions/upload-artifact@v3 + with: + name: ${{ inputs.artifact-prefix }}-${{ inputs.toxenv-task }}-${{ inputs.toxenv-pyver }}-${{ inputs.toxenv-ptver }} + path: | + .tox/tests-${{ inputs.toxenv-task }}-${{ inputs.toxenv-pyver }}-${{ inputs.toxenv-ptver }}.csv + .tox/tests-reg_${{ inputs.task }}*.csv + .tox/perf-*.csv + # Use always() to always run this step to publish test results when there are test failures + if: ${{ inputs.upload-artifact && always() }}