From 25b7043d3863fcdaf7cf8a52f6e192f2dfe0a658 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Oct 2024 11:48:34 +0530 Subject: [PATCH 01/12] Create test-submission-generation-non-cm-based-benchmarks.yml --- ...ion-generation-non-cm-based-benchmarks.yml | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .github/workflows/test-submission-generation-non-cm-based-benchmarks.yml diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml new file mode 100644 index 0000000000..0e0e401f3d --- /dev/null +++ b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml @@ -0,0 +1,38 @@ +# This workflow will test the submission generation capability of CM for non-CM based benchmarks + +name: Submission generation tests for non-CM based MLPerf Inference Benchmarks + +on: + pull_request_target: + branches: [ "main", "dev", "mlperf-inference" ] + paths: + - '.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml' + - '**' # kept on for all the path instead of submission generation CM script so that this could help in trapping any bugs in any recent submission checker modification also + - '!**.md' +jobs: + Case-3: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + python-version: [ "3.12" ] + exclude: + - os: macos-latest + - os: windows-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python3 -m pip install cmind + cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} + - name: Pull repo where test cases are uploaded + run: | + cm pull repo anandhu-eng@externalSubmissionPOC + - name: Submission generation - datacenter open + run: | + cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repo/anandhu-eng@externalSubmissionPOC/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=closed --category=datacenter --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet From b11a57b52bf7f30b81bc4b0d698d3e2c133848ab Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Oct 2024 11:51:53 +0530 Subject: [PATCH 02/12] Update run conditions for testing --- .../test-submission-generation-non-cm-based-benchmarks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml index 0e0e401f3d..eb8837be4a 100644 --- a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml +++ b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml @@ -7,8 +7,8 @@ on: branches: [ "main", "dev", "mlperf-inference" ] paths: - '.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml' - - '**' # kept on for all the path instead of submission generation CM script so that this could help in trapping any bugs in any recent submission checker modification also - - '!**.md' + #- '**' # kept on for all the path instead of submission generation CM script so that this could help in trapping any bugs in any recent submission checker modification also + #- '!**.md' jobs: Case-3: runs-on: ${{ matrix.os }} From 0b9532f892bee2cbb26e985a5444e7616f06e030 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Oct 2024 14:03:54 +0530 Subject: [PATCH 03/12] test commit --- .../test-submission-generation-non-cm-based-benchmarks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml index eb8837be4a..0e0e401f3d 100644 --- a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml +++ b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml @@ -7,8 +7,8 @@ on: branches: [ "main", "dev", "mlperf-inference" ] paths: - '.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml' - #- '**' # kept on for all the path instead of submission generation CM script so that this could help in trapping any bugs in any recent submission checker modification also - #- '!**.md' + - '**' # kept on for all the path instead of submission generation CM script so that this could help in trapping any bugs in any recent submission checker modification also + - '!**.md' jobs: Case-3: runs-on: ${{ matrix.os }} From cb09d504c7b4d64a463491722ed510309f1062bf Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Oct 2024 14:23:59 +0530 Subject: [PATCH 04/12] Update test-submission-generation-non-cm-based-benchmarks.yml --- .../test-submission-generation-non-cm-based-benchmarks.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml index 0e0e401f3d..ad1d662378 100644 --- a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml +++ b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml @@ -1,14 +1,14 @@ # This workflow will test the submission generation capability of CM for non-CM based benchmarks -name: Submission generation tests for non-CM based MLPerf Inference Benchmarks +name: CM based Submission Generation on: pull_request_target: branches: [ "main", "dev", "mlperf-inference" ] paths: - '.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml' - - '**' # kept on for all the path instead of submission generation CM script so that this could help in trapping any bugs in any recent submission checker modification also - - '!**.md' + # - '**' # kept on for all the path instead of submission generation CM script so that this could help in trapping any bugs in any recent submission checker modification also + # - '!**.md' jobs: Case-3: runs-on: ${{ matrix.os }} From 265224afa0ec2a05e0a6a4ea4edc8606126a83d3 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Oct 2024 15:00:12 +0530 Subject: [PATCH 05/12] Update test-submission-generation-non-cm-based-benchmarks.yml --- .../test-submission-generation-non-cm-based-benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml index ad1d662378..dd5332d97c 100644 --- a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml +++ b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml @@ -3,7 +3,7 @@ name: CM based Submission Generation on: - pull_request_target: + pull_request: branches: [ "main", "dev", "mlperf-inference" ] paths: - '.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml' From 3fcfca22d6294e3823f17516e5b45f9422da73c3 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Oct 2024 15:03:27 +0530 Subject: [PATCH 06/12] update results dir --- .../test-submission-generation-non-cm-based-benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml index dd5332d97c..e78d3c3c58 100644 --- a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml +++ b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml @@ -35,4 +35,4 @@ jobs: cm pull repo anandhu-eng@externalSubmissionPOC - name: Submission generation - datacenter open run: | - cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repo/anandhu-eng@externalSubmissionPOC/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=closed --category=datacenter --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet + cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repoS/anandhu-eng@externalSubmissionPOC/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=closed --category=datacenter --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet From 5354309ea3338f491d60d31493096fa1699eabdd Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Oct 2024 15:04:29 +0530 Subject: [PATCH 07/12] Update test-submission-generation-non-cm-based-benchmarks.yml --- .../test-submission-generation-non-cm-based-benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml index e78d3c3c58..9d0d1e9bd4 100644 --- a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml +++ b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml @@ -35,4 +35,4 @@ jobs: cm pull repo anandhu-eng@externalSubmissionPOC - name: Submission generation - datacenter open run: | - cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repoS/anandhu-eng@externalSubmissionPOC/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=closed --category=datacenter --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet + cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repos/anandhu-eng@externalSubmissionPOC/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=closed --category=datacenter --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet From c89dfa1d3896e7a6c8ab34ef1739ef6df20d3b74 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Oct 2024 15:11:35 +0530 Subject: [PATCH 08/12] division and category made matrix --- ...test-submission-generation-non-cm-based-benchmarks.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml index 9d0d1e9bd4..8e7e7f6668 100644 --- a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml +++ b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml @@ -17,9 +17,13 @@ jobs: matrix: os: [ubuntu-latest, windows-latest, macos-latest] python-version: [ "3.12" ] + division: ["closed", "open"] + category: ["datacenter", "edge"] exclude: - os: macos-latest - os: windows-latest + - division: "open" + - division: "edge" steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} @@ -33,6 +37,6 @@ jobs: - name: Pull repo where test cases are uploaded run: | cm pull repo anandhu-eng@externalSubmissionPOC - - name: Submission generation - datacenter open + - name: Submission generation - ${{ matrix.category }} ${{ matrix.division }} run: | - cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repos/anandhu-eng@externalSubmissionPOC/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=closed --category=datacenter --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet + cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repos/anandhu-eng@externalSubmissionPOC/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --category=${{ matrix.category }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet From 02675cb6f1399f1097abea22d0b3d6bf3a396684 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Oct 2024 15:15:42 +0530 Subject: [PATCH 09/12] fix typo --- .../test-submission-generation-non-cm-based-benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml index 8e7e7f6668..2ae4e84250 100644 --- a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml +++ b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml @@ -23,7 +23,7 @@ jobs: - os: macos-latest - os: windows-latest - division: "open" - - division: "edge" + - category: "edge" steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} From 75f8ea13f3783629b8cac1e9300b66cbe780a2a6 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Oct 2024 15:28:00 +0530 Subject: [PATCH 10/12] added redhat os to test coverage --- .../test-submission-generation-non-cm-based-benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml index 2ae4e84250..5b6b453477 100644 --- a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml +++ b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest, windows-latest, macos-latest, rhel-latest] python-version: [ "3.12" ] division: ["closed", "open"] category: ["datacenter", "edge"] From 2ccf3737b339a1acb03b2a69676576ce859130d0 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Oct 2024 16:32:42 +0530 Subject: [PATCH 11/12] removed rhel latest --- ...ed-benchmarks.yml => test-cm-based-ubmission-generation.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{test-submission-generation-non-cm-based-benchmarks.yml => test-cm-based-ubmission-generation.yml} (96%) diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-cm-based-ubmission-generation.yml similarity index 96% rename from .github/workflows/test-submission-generation-non-cm-based-benchmarks.yml rename to .github/workflows/test-cm-based-ubmission-generation.yml index 5b6b453477..2ae4e84250 100644 --- a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml +++ b/.github/workflows/test-cm-based-ubmission-generation.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest, rhel-latest] + os: [ubuntu-latest, windows-latest, macos-latest] python-version: [ "3.12" ] division: ["closed", "open"] category: ["datacenter", "edge"] From c12942fd0a8a9dfa6e50cfd5fcc565d10e21705a Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Oct 2024 17:20:45 +0530 Subject: [PATCH 12/12] Updated the source test repo --- .github/workflows/test-cm-based-ubmission-generation.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-cm-based-ubmission-generation.yml b/.github/workflows/test-cm-based-ubmission-generation.yml index 2ae4e84250..4fc6c0bc45 100644 --- a/.github/workflows/test-cm-based-ubmission-generation.yml +++ b/.github/workflows/test-cm-based-ubmission-generation.yml @@ -1,4 +1,4 @@ -# This workflow will test the submission generation capability of CM for non-CM based benchmarks +# This workflow will test the submission generation capability of CM f name: CM based Submission Generation @@ -36,7 +36,7 @@ jobs: cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - name: Pull repo where test cases are uploaded run: | - cm pull repo anandhu-eng@externalSubmissionPOC - - name: Submission generation - ${{ matrix.category }} ${{ matrix.division }} + cm pull repo anandhu-eng@inference --checkout=submission-generation-tests + - name: Submission generation(model_mapping.json not present but model name is matching with the official one in submission checker) - ${{ matrix.category }} ${{ matrix.division }} run: | - cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repos/anandhu-eng@externalSubmissionPOC/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --category=${{ matrix.category }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet + cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repos/anandhu-eng@inference/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --category=${{ matrix.category }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet