From eaa793892ecceb969d2cd264114342396b622e2b Mon Sep 17 00:00:00 2001
From: Arjun Suresh <arjunsuresh1987@gmail.com>
Date: Wed, 30 Oct 2024 01:45:54 +0530
Subject: [PATCH 01/32] Fix docker container reuse during docker_detached mode

---
 script/run-mlperf-inference-app/customize.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/script/run-mlperf-inference-app/customize.py b/script/run-mlperf-inference-app/customize.py
index 629bc3a254..f0e1c02f5f 100644
--- a/script/run-mlperf-inference-app/customize.py
+++ b/script/run-mlperf-inference-app/customize.py
@@ -202,7 +202,7 @@ def preprocess(i):
                 docker_extra_input[k] = inp[k]
         inp = {}
         if str(docker_dt).lower() in ["yes", "true", "1"]:
-            env['CM_DOCKER_REUSE_EXISTING_CONTAINER'] = 'yes'
+            env['CM_DOCKER_REUSE_EXISTING_CONTAINER'] = 'no' # turning it off for the first run and after that we turn it on
 
         if env.get('CM_DOCKER_IMAGE_NAME', '') != '':
             docker_extra_input['docker_image_name'] = env['CM_DOCKER_IMAGE_NAME']
@@ -256,6 +256,7 @@ def preprocess(i):
                     print(f"\nStop Running loadgen scenario: {scenario} and mode: {mode}")
                     return {'return': 0} # We run commands interactively inside the docker container
                 else:
+                    env['CM_DOCKER_REUSE_EXISTING_CONTAINER'] = 'yes'
                     container_id = env_copy['CM_DOCKER_CONTAINER_ID']
                     env['CM_DOCKER_CONTAINER_ID'] = container_id
             if state.get('docker', {}):

From 018d09c32f2456bebac968a03975337def717dc6 Mon Sep 17 00:00:00 2001
From: Arjun Suresh <arjunsuresh1987@gmail.com>
Date: Tue, 29 Oct 2024 20:17:23 +0000
Subject: [PATCH 02/32] Update test-nvidia-mlperf-implementation.yml

---
 .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml
index e51634c468..f9fc94d7be 100644
--- a/.github/workflows/test-nvidia-mlperf-implementation.yml
+++ b/.github/workflows/test-nvidia-mlperf-implementation.yml
@@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations
 
 on:
   schedule:
-    - cron: "04 18 * * *" #to be adjusted
+    - cron: "23 20 * * *" #to be adjusted
 
 jobs:
   build_nvidia:

From 52dae77db81de4f60a972311d223e952ce23fb9a Mon Sep 17 00:00:00 2001
From: Arjun Suresh <arjunsuresh1987@gmail.com>
Date: Wed, 30 Oct 2024 02:35:35 +0530
Subject: [PATCH 03/32] Fix docker container reuse during docker_detached mode

---
 script/run-docker-container/customize.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/script/run-docker-container/customize.py b/script/run-docker-container/customize.py
index 20f8138bad..35d3fdf1e4 100644
--- a/script/run-docker-container/customize.py
+++ b/script/run-docker-container/customize.py
@@ -62,6 +62,7 @@ def preprocess(i):
     if len(output_split) > 1 and str(env.get('CM_DOCKER_REUSE_EXISTING_CONTAINER', '')).lower() in [ "1", "true", "yes" ]: #container exists
         out = output_split[1].split(" ")
         existing_container_id = out[0]
+        print(f"Reusing existing container {existing_container_id}")
         env['CM_DOCKER_CONTAINER_ID'] = existing_container_id
 
     else:

From 0a7423b8b651228c6cf952e3957ed6a8df68ae8b Mon Sep 17 00:00:00 2001
From: Arjun Suresh <arjunsuresh1987@gmail.com>
Date: Wed, 30 Oct 2024 02:48:20 +0530
Subject: [PATCH 04/32] By default use existing docker image

---
 automation/script/module_misc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py
index 92c8a5f717..c4e508c352 100644
--- a/automation/script/module_misc.py
+++ b/automation/script/module_misc.py
@@ -1692,7 +1692,7 @@ def docker(i):
     env=i.get('env', {})
 
     noregenerate_docker_file = i.get('docker_noregenerate', False)
-    norecreate_docker_image = i.get('docker_norecreate', False)
+    norecreate_docker_image = i.get('docker_norecreate', True)
 
     if i.get('docker_skip_build', False):
         noregenerate_docker_file = True

From fecec8587e4d61692d03354e4e095ddf31165c4e Mon Sep 17 00:00:00 2001
From: Arjun Suresh <arjunsuresh1987@gmail.com>
Date: Tue, 29 Oct 2024 21:19:39 +0000
Subject: [PATCH 05/32] Update test-nvidia-mlperf-implementation.yml

---
 .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml
index f9fc94d7be..b1cfb11d0a 100644
--- a/.github/workflows/test-nvidia-mlperf-implementation.yml
+++ b/.github/workflows/test-nvidia-mlperf-implementation.yml
@@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations
 
 on:
   schedule:
-    - cron: "23 20 * * *" #to be adjusted
+    - cron: "25 21 * * *" #to be adjusted
 
 jobs:
   build_nvidia:

From 25b7043d3863fcdaf7cf8a52f6e192f2dfe0a658 Mon Sep 17 00:00:00 2001
From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com>
Date: Wed, 30 Oct 2024 11:48:34 +0530
Subject: [PATCH 06/32] Create
 test-submission-generation-non-cm-based-benchmarks.yml

---
 ...ion-generation-non-cm-based-benchmarks.yml | 38 +++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 .github/workflows/test-submission-generation-non-cm-based-benchmarks.yml

diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml
new file mode 100644
index 0000000000..0e0e401f3d
--- /dev/null
+++ b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml
@@ -0,0 +1,38 @@
+# This workflow will test the submission generation capability of CM for non-CM based benchmarks
+
+name: Submission generation tests for non-CM based MLPerf Inference Benchmarks
+
+on:
+  pull_request_target:
+    branches: [ "main", "dev", "mlperf-inference" ]
+    paths:
+      - '.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml'
+      - '**'  # kept on for all the path instead of submission generation CM script so that this could help in trapping any bugs in any recent submission checker modification also
+      - '!**.md'
+jobs:
+  Case-3:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: [ "3.12" ]
+        exclude:
+          - os: macos-latest
+          - os: windows-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python3 -m pip install cmind
+        cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
+    - name: Pull repo where test cases are uploaded
+      run: |
+        cm pull repo anandhu-eng@externalSubmissionPOC
+    - name: Submission generation - datacenter open
+      run: |
+        cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repo/anandhu-eng@externalSubmissionPOC/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=closed --category=datacenter --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet

From b11a57b52bf7f30b81bc4b0d698d3e2c133848ab Mon Sep 17 00:00:00 2001
From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com>
Date: Wed, 30 Oct 2024 11:51:53 +0530
Subject: [PATCH 07/32] Update run conditions for testing

---
 .../test-submission-generation-non-cm-based-benchmarks.yml    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml
index 0e0e401f3d..eb8837be4a 100644
--- a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml
+++ b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml
@@ -7,8 +7,8 @@ on:
     branches: [ "main", "dev", "mlperf-inference" ]
     paths:
       - '.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml'
-      - '**'  # kept on for all the path instead of submission generation CM script so that this could help in trapping any bugs in any recent submission checker modification also
-      - '!**.md'
+      #- '**'  # kept on for all the path instead of submission generation CM script so that this could help in trapping any bugs in any recent submission checker modification also
+      #- '!**.md'
 jobs:
   Case-3:
     runs-on: ${{ matrix.os }}

From 0b9532f892bee2cbb26e985a5444e7616f06e030 Mon Sep 17 00:00:00 2001
From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com>
Date: Wed, 30 Oct 2024 14:03:54 +0530
Subject: [PATCH 08/32] test commit

---
 .../test-submission-generation-non-cm-based-benchmarks.yml    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml
index eb8837be4a..0e0e401f3d 100644
--- a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml
+++ b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml
@@ -7,8 +7,8 @@ on:
     branches: [ "main", "dev", "mlperf-inference" ]
     paths:
       - '.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml'
-      #- '**'  # kept on for all the path instead of submission generation CM script so that this could help in trapping any bugs in any recent submission checker modification also
-      #- '!**.md'
+      - '**'  # kept on for all the path instead of submission generation CM script so that this could help in trapping any bugs in any recent submission checker modification also
+      - '!**.md'
 jobs:
   Case-3:
     runs-on: ${{ matrix.os }}

From cb09d504c7b4d64a463491722ed510309f1062bf Mon Sep 17 00:00:00 2001
From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com>
Date: Wed, 30 Oct 2024 14:23:59 +0530
Subject: [PATCH 09/32] Update
 test-submission-generation-non-cm-based-benchmarks.yml

---
 .../test-submission-generation-non-cm-based-benchmarks.yml  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml
index 0e0e401f3d..ad1d662378 100644
--- a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml
+++ b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml
@@ -1,14 +1,14 @@
 # This workflow will test the submission generation capability of CM for non-CM based benchmarks
 
-name: Submission generation tests for non-CM based MLPerf Inference Benchmarks
+name: CM based Submission Generation
 
 on:
   pull_request_target:
     branches: [ "main", "dev", "mlperf-inference" ]
     paths:
       - '.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml'
-      - '**'  # kept on for all the path instead of submission generation CM script so that this could help in trapping any bugs in any recent submission checker modification also
-      - '!**.md'
+      # - '**'  # kept on for all the path instead of submission generation CM script so that this could help in trapping any bugs in any recent submission checker modification also
+      # - '!**.md'
 jobs:
   Case-3:
     runs-on: ${{ matrix.os }}

From 265224afa0ec2a05e0a6a4ea4edc8606126a83d3 Mon Sep 17 00:00:00 2001
From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com>
Date: Wed, 30 Oct 2024 15:00:12 +0530
Subject: [PATCH 10/32] Update
 test-submission-generation-non-cm-based-benchmarks.yml

---
 .../test-submission-generation-non-cm-based-benchmarks.yml      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml
index ad1d662378..dd5332d97c 100644
--- a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml
+++ b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml
@@ -3,7 +3,7 @@
 name: CM based Submission Generation
 
 on:
-  pull_request_target:
+  pull_request:
     branches: [ "main", "dev", "mlperf-inference" ]
     paths:
       - '.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml'

From 3fcfca22d6294e3823f17516e5b45f9422da73c3 Mon Sep 17 00:00:00 2001
From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com>
Date: Wed, 30 Oct 2024 15:03:27 +0530
Subject: [PATCH 11/32] update results dir

---
 .../test-submission-generation-non-cm-based-benchmarks.yml      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml
index dd5332d97c..e78d3c3c58 100644
--- a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml
+++ b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml
@@ -35,4 +35,4 @@ jobs:
         cm pull repo anandhu-eng@externalSubmissionPOC
     - name: Submission generation - datacenter open
       run: |
-        cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repo/anandhu-eng@externalSubmissionPOC/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=closed --category=datacenter --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet
+        cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repoS/anandhu-eng@externalSubmissionPOC/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=closed --category=datacenter --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet

From 5354309ea3338f491d60d31493096fa1699eabdd Mon Sep 17 00:00:00 2001
From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com>
Date: Wed, 30 Oct 2024 15:04:29 +0530
Subject: [PATCH 12/32] Update
 test-submission-generation-non-cm-based-benchmarks.yml

---
 .../test-submission-generation-non-cm-based-benchmarks.yml      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml
index e78d3c3c58..9d0d1e9bd4 100644
--- a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml
+++ b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml
@@ -35,4 +35,4 @@ jobs:
         cm pull repo anandhu-eng@externalSubmissionPOC
     - name: Submission generation - datacenter open
       run: |
-        cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repoS/anandhu-eng@externalSubmissionPOC/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=closed --category=datacenter --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet
+        cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repos/anandhu-eng@externalSubmissionPOC/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=closed --category=datacenter --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet

From c89dfa1d3896e7a6c8ab34ef1739ef6df20d3b74 Mon Sep 17 00:00:00 2001
From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com>
Date: Wed, 30 Oct 2024 15:11:35 +0530
Subject: [PATCH 13/32] division and category made matrix

---
 ...test-submission-generation-non-cm-based-benchmarks.yml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml
index 9d0d1e9bd4..8e7e7f6668 100644
--- a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml
+++ b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml
@@ -17,9 +17,13 @@ jobs:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
         python-version: [ "3.12" ]
+        division: ["closed", "open"]
+        category: ["datacenter", "edge"]
         exclude:
           - os: macos-latest
           - os: windows-latest
+          - division: "open"
+          - division: "edge"
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
@@ -33,6 +37,6 @@ jobs:
     - name: Pull repo where test cases are uploaded
       run: |
         cm pull repo anandhu-eng@externalSubmissionPOC
-    - name: Submission generation - datacenter open
+    - name: Submission generation - ${{ matrix.category }} ${{ matrix.division }}
       run: |
-        cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repos/anandhu-eng@externalSubmissionPOC/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=closed --category=datacenter --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet
+        cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repos/anandhu-eng@externalSubmissionPOC/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --category=${{ matrix.category }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet

From 02675cb6f1399f1097abea22d0b3d6bf3a396684 Mon Sep 17 00:00:00 2001
From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com>
Date: Wed, 30 Oct 2024 15:15:42 +0530
Subject: [PATCH 14/32] fix typo

---
 .../test-submission-generation-non-cm-based-benchmarks.yml      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml
index 8e7e7f6668..2ae4e84250 100644
--- a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml
+++ b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml
@@ -23,7 +23,7 @@ jobs:
           - os: macos-latest
           - os: windows-latest
           - division: "open"
-          - division: "edge"
+          - category: "edge"
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}

From 75f8ea13f3783629b8cac1e9300b66cbe780a2a6 Mon Sep 17 00:00:00 2001
From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com>
Date: Wed, 30 Oct 2024 15:28:00 +0530
Subject: [PATCH 15/32] added redhat os to test coverage

---
 .../test-submission-generation-non-cm-based-benchmarks.yml      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml
index 2ae4e84250..5b6b453477 100644
--- a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml
+++ b/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml
@@ -15,7 +15,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [ubuntu-latest, windows-latest, macos-latest, rhel-latest]
         python-version: [ "3.12" ]
         division: ["closed", "open"]
         category: ["datacenter", "edge"]

From 4b5b6c978c3fa0861273df19528d44a426564182 Mon Sep 17 00:00:00 2001
From: Arjun Suresh <arjunsuresh1987@gmail.com>
Date: Wed, 30 Oct 2024 10:24:46 +0000
Subject: [PATCH 16/32] Don't fail on dmidecode command in get-platform-details

---
 script/get-platform-details/run.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/script/get-platform-details/run.sh b/script/get-platform-details/run.sh
index d3fc7e0d37..acac8e0b23 100644
--- a/script/get-platform-details/run.sh
+++ b/script/get-platform-details/run.sh
@@ -120,7 +120,7 @@ echo "------------------------------------------------------------" >> $OUTPUT_F
 echo "21. dmidecode" >> $OUTPUT_FILE
 if [[ ${CM_SUDO_USER} == "yes" ]]; then
     eval "${CM_SUDO} dmidecode" >> $OUTPUT_FILE
-    test $? -eq 0 || exit $?
+    test $? -eq 0 || echo "FAILED: dmidecode" >> $OUTPUT_FILE
 else
     echo "Requires SUDO permission" >> $OUTPUT_FILE
 fi
@@ -129,7 +129,7 @@ echo "------------------------------------------------------------" >> $OUTPUT_F
 echo "22. BIOS" >> $OUTPUT_FILE
 if [[ ${CM_SUDO_USER} == "yes" ]]; then
     eval "${CM_SUDO} dmidecode -t bios" >> $OUTPUT_FILE
-    test $? -eq 0 || exit $?
+    test $? -eq 0 || echo "FAILED: dmidecode -t bios" >> $OUTPUT_FILE
 else
     echo "Requires SUDO permission" >> $OUTPUT_FILE
 fi

From a5eff8e6f4e0f1d0c59f4118f3a7c62b3f5a6b30 Mon Sep 17 00:00:00 2001
From: Arjun Suresh <arjunsuresh1987@gmail.com>
Date: Wed, 30 Oct 2024 16:13:02 +0530
Subject: [PATCH 17/32] Added fail safe option for get-generic-sys-util

---
 script/get-generic-sys-util/run.sh   | 23 ++++++++++++++++++-----
 script/get-platform-details/_cm.json |  5 ++++-
 script/get-platform-details/run.sh   |  2 +-
 3 files changed, 23 insertions(+), 7 deletions(-)

diff --git a/script/get-generic-sys-util/run.sh b/script/get-generic-sys-util/run.sh
index 27c2f62867..5074e756b8 100644
--- a/script/get-generic-sys-util/run.sh
+++ b/script/get-generic-sys-util/run.sh
@@ -1,6 +1,19 @@
-#!/bin/bash
+# Safe execution of a command stored in a variable
+cmd="${CM_SYS_UTIL_INSTALL_CMD}"
+echo "$cmd"
 
-cmd=${CM_SYS_UTIL_INSTALL_CMD}
-echo $cmd
-eval $cmd
-test $? -eq 0 || exit $?
+# Execute the command and capture the exit status directly
+if ! eval "$cmd"; then
+    echo "Command failed with status $?"
+    if [[ "${CM_TMP_FAIL_SAFE}" == 'yes' ]]; then
+        # Exit safely if fail-safe is enabled
+        echo "Fail-safe is enabled, exiting with status 0"
+        exit 0
+    else
+        # Otherwise exit with the actual error status
+        exit $?
+    fi
+else
+    #echo "Command succeeded"
+    exit 0
+fi
diff --git a/script/get-platform-details/_cm.json b/script/get-platform-details/_cm.json
index 11c57baa18..b02576ad9e 100644
--- a/script/get-platform-details/_cm.json
+++ b/script/get-platform-details/_cm.json
@@ -66,7 +66,10 @@
           "linux"
         ]
       },
-      "tags": "get,sys-util,generic,_linux-tools"
+      "tags": "get,sys-util,generic,_linux-tools",
+      "env": {
+        "CM_TMP_FAIL_SAFE": "yes"
+      }
     }
   ],
   "tags": [
diff --git a/script/get-platform-details/run.sh b/script/get-platform-details/run.sh
index acac8e0b23..ba2194e669 100644
--- a/script/get-platform-details/run.sh
+++ b/script/get-platform-details/run.sh
@@ -79,7 +79,7 @@ echo "------------------------------------------------------------" >> $OUTPUT_F
 
 echo "14. cpupower frequency-info" >> $OUTPUT_FILE
 eval "cpupower frequency-info" >> $OUTPUT_FILE
-test $? -eq 0 || exit $?
+test $? -eq 0 || echo "FAILED: cpupower frequency-info" >> $OUTPUT_FILE
 echo "------------------------------------------------------------" >> $OUTPUT_FILE
 
 echo "15. sysctl" >> $OUTPUT_FILE

From 2ccf3737b339a1acb03b2a69676576ce859130d0 Mon Sep 17 00:00:00 2001
From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com>
Date: Wed, 30 Oct 2024 16:32:42 +0530
Subject: [PATCH 18/32] removed rhel latest

---
 ...ed-benchmarks.yml => test-cm-based-ubmission-generation.yml} | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename .github/workflows/{test-submission-generation-non-cm-based-benchmarks.yml => test-cm-based-ubmission-generation.yml} (96%)

diff --git a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml b/.github/workflows/test-cm-based-ubmission-generation.yml
similarity index 96%
rename from .github/workflows/test-submission-generation-non-cm-based-benchmarks.yml
rename to .github/workflows/test-cm-based-ubmission-generation.yml
index 5b6b453477..2ae4e84250 100644
--- a/.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml
+++ b/.github/workflows/test-cm-based-ubmission-generation.yml
@@ -15,7 +15,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest, rhel-latest]
+        os: [ubuntu-latest, windows-latest, macos-latest]
         python-version: [ "3.12" ]
         division: ["closed", "open"]
         category: ["datacenter", "edge"]

From f14ae8e20e90ff6230d76dbdd1d7172342dd793b Mon Sep 17 00:00:00 2001
From: Arjun Suresh <arjunsuresh1987@gmail.com>
Date: Wed, 30 Oct 2024 16:40:59 +0530
Subject: [PATCH 19/32] Added test for testing all-variations (WIP)

---
 automation/script/module.py          | 19 ++++++++++++++++++-
 script/get-generic-sys-util/_cm.json |  3 +++
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/automation/script/module.py b/automation/script/module.py
index 183f7c48c0..d5388846f5 100644
--- a/automation/script/module.py
+++ b/automation/script/module.py
@@ -2428,7 +2428,24 @@ def test(self, i):
 
             if console:
                 logging.info(path)
-                logging.info('  Test: TBD')
+                test_config = meta.get('tests', '')
+                if test_config:
+                    log.info(test_config)
+                    test_all_variations = test_config.get('test-all-variations', False)
+                    if test_all_variations:
+                        variations = meta.get("variations")
+                        individual_variations = [ v for v in variations if variations[v].get('group', '') == '' and str(variations[v].get('exclude-in-test', '')).lower() not in [ "1", "true", "yes" ] ]
+                        tags_string = ",".join(meta.get("tags"))
+                        for variation in individual_variations:
+                            run_tags = f"{tags_string},_{variation}"
+                            r = self.cmind.access({'action':'run',
+                                'automation':'script',
+                                'tags': run_tags,
+                                'quiet': i.get('quiet') })
+                            if r['return'] > 0:
+                                return r
+
+                logging.info('  Test: WIP')
 
 
         return {'return':0, 'list': lst}
diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json
index a1f9d09918..e0db7c4944 100644
--- a/script/get-generic-sys-util/_cm.json
+++ b/script/get-generic-sys-util/_cm.json
@@ -721,5 +721,8 @@
         }
       }
     }
+  },
+  "tests": {
+    "test-all-variations": "yes"
   }
 }

From 5a4771b5e77f23ed0cbd2fb6d1fce9c09f7521ed Mon Sep 17 00:00:00 2001
From: Arjun Suresh <arjunsuresh1987@gmail.com>
Date: Wed, 30 Oct 2024 11:13:12 +0000
Subject: [PATCH 20/32] Update test-nvidia-mlperf-implementation.yml

---
 .github/workflows/test-nvidia-mlperf-implementation.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml
index b1cfb11d0a..0691a58604 100644
--- a/.github/workflows/test-nvidia-mlperf-implementation.yml
+++ b/.github/workflows/test-nvidia-mlperf-implementation.yml
@@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations
 
 on:
   schedule:
-    - cron: "25 21 * * *" #to be adjusted
+    - cron: "19 11 * * *" #to be adjusted
 
 jobs:
   build_nvidia:

From 0aba91ee687e76997c6abb1fcc5a1e01f0068254 Mon Sep 17 00:00:00 2001
From: Arjun Suresh <arjunsuresh1987@gmail.com>
Date: Wed, 30 Oct 2024 17:16:35 +0530
Subject: [PATCH 21/32] Added github action for individual CM script tests

---
 .../workflows/run-individual-script-tests.yml | 31 +++++++++++++++++++
 tests/script/process_tests.py                 | 24 ++++++++++++++
 2 files changed, 55 insertions(+)
 create mode 100644 .github/workflows/run-individual-script-tests.yml
 create mode 100644 tests/script/process_tests.py

diff --git a/.github/workflows/run-individual-script-tests.yml b/.github/workflows/run-individual-script-tests.yml
new file mode 100644
index 0000000000..8a4add3442
--- /dev/null
+++ b/.github/workflows/run-individual-script-tests.yml
@@ -0,0 +1,31 @@
+# This workflow will add/update the README.md files for any updated CM scripts
+name: Readme update for CM scripts
+
+on:
+  pull_request:
+    branches: [ "main", "mlperf-inference", "dev" ]
+    paths:
+      - 'script/**_cm.json'
+      - 'script/**_cm.yml'
+
+jobs:
+  run-script-tests:
+    runs-on: ubuntu-latest
+    steps:
+    - name: 'Checkout'
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 2
+    - name: Get changed files
+      id: getfile
+      run: |
+        echo "files=$(git diff --name-only ${{ github.event.before }} | xargs)" >> $GITHUB_OUTPUT
+    - name: RUN Script Tests
+      run: |
+        echo ${{ steps.getfile.outputs.files }}
+        for file in ${{ steps.getfile.outputs.files }}; do
+          echo $file
+        done
+        python3 -m pip install cmind
+        cm pull repo --url=https://github.com/${{ github.repository }} --checkout=${{ github.ref_name }}
+        python3 tests/script/process_tests.py ${{ steps.getfile.outputs.files }}
diff --git a/tests/script/process_tests.py b/tests/script/process_tests.py
new file mode 100644
index 0000000000..386110edf4
--- /dev/null
+++ b/tests/script/process_tests.py
@@ -0,0 +1,24 @@
+import sys
+import os
+import cmind as cm
+import check as checks
+import json
+import yaml
+
+files=sys.argv[1:]
+
+for file in files:
+    if not os.path.isfile(file):
+        continue
+    if not file.endswith("_cm.json") and not file.endswith("_cm.yaml"):
+        continue
+    script_path = os.path.dirname(file)
+    f = open(file)
+    if file.endswith(".json"):
+        data = json.load(f)
+    elif file.endswith(".yaml"):
+        data = yaml.safe_load(f)
+    uid = data['uid']
+
+    r = cm.access({'action':'test', 'automation':'script', 'artifact': uid, 'quiet': 'yes'})
+    checks.check_return(r)

From c12942fd0a8a9dfa6e50cfd5fcc565d10e21705a Mon Sep 17 00:00:00 2001
From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com>
Date: Wed, 30 Oct 2024 17:20:45 +0530
Subject: [PATCH 22/32] Updated the source test repo

---
 .github/workflows/test-cm-based-ubmission-generation.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test-cm-based-ubmission-generation.yml b/.github/workflows/test-cm-based-ubmission-generation.yml
index 2ae4e84250..4fc6c0bc45 100644
--- a/.github/workflows/test-cm-based-ubmission-generation.yml
+++ b/.github/workflows/test-cm-based-ubmission-generation.yml
@@ -1,4 +1,4 @@
-# This workflow will test the submission generation capability of CM for non-CM based benchmarks
+# This workflow will test the submission generation capability of CM f
 
 name: CM based Submission Generation
 
@@ -36,7 +36,7 @@ jobs:
         cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
     - name: Pull repo where test cases are uploaded
       run: |
-        cm pull repo anandhu-eng@externalSubmissionPOC
-    - name: Submission generation - ${{ matrix.category }} ${{ matrix.division }}
+        cm pull repo anandhu-eng@inference --checkout=submission-generation-tests
+    - name: Submission generation(model_mapping.json not present but model name is matching with the official one in submission checker) - ${{ matrix.category }} ${{ matrix.division }}
       run: |
-        cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repos/anandhu-eng@externalSubmissionPOC/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --category=${{ matrix.category }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet
+        cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repos/anandhu-eng@inference/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --category=${{ matrix.category }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet

From 00bfb46e3028c8e4e1cd30b30bd2cb5abd7dc942 Mon Sep 17 00:00:00 2001
From: Arjun Suresh <arjunsuresh1987@gmail.com>
Date: Wed, 30 Oct 2024 17:25:33 +0530
Subject: [PATCH 23/32] Fix test name

---
 .github/workflows/run-individual-script-tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-individual-script-tests.yml b/.github/workflows/run-individual-script-tests.yml
index 8a4add3442..97d40d4154 100644
--- a/.github/workflows/run-individual-script-tests.yml
+++ b/.github/workflows/run-individual-script-tests.yml
@@ -1,5 +1,5 @@
-# This workflow will add/update the README.md files for any updated CM scripts
-name: Readme update for CM scripts
+# This workflow will run configured tests for any updated CM scripts
+name: Individual CM script Tests
 
 on:
   pull_request:

From f1001126b2cfe73b19f7b6b6ccc0ab1dd6160d71 Mon Sep 17 00:00:00 2001
From: Arjun Suresh <arjunsuresh1987@gmail.com>
Date: Wed, 30 Oct 2024 17:29:38 +0530
Subject: [PATCH 24/32] Fix gh action for individual CM sript tests

---
 .github/workflows/run-individual-script-tests.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/run-individual-script-tests.yml b/.github/workflows/run-individual-script-tests.yml
index 97d40d4154..1ecd058af8 100644
--- a/.github/workflows/run-individual-script-tests.yml
+++ b/.github/workflows/run-individual-script-tests.yml
@@ -19,7 +19,9 @@ jobs:
     - name: Get changed files
       id: getfile
       run: |
-        echo "files=$(git diff --name-only ${{ github.event.before }} | xargs)" >> $GITHUB_OUTPUT
+        git remote add upstream ${{ github.event.pull_request.base.repo.clone_url }}
+        git fetch upstream
+        echo "files=$(git diff \"upstream\" \"${{ github.event.pull_request.base.ref }}\" --name-only | xargs)" >> $GITHUB_OUTPUT
     - name: RUN Script Tests
       run: |
         echo ${{ steps.getfile.outputs.files }}

From e064bb9d256550c3859b7b86d4487064edcf7410 Mon Sep 17 00:00:00 2001
From: Arjun Suresh <arjunsuresh1987@gmail.com>
Date: Wed, 30 Oct 2024 17:36:10 +0530
Subject: [PATCH 25/32] Fix gh action for individual CM sript tests

---
 .github/workflows/run-individual-script-tests.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/run-individual-script-tests.yml b/.github/workflows/run-individual-script-tests.yml
index 1ecd058af8..d8abb96d5c 100644
--- a/.github/workflows/run-individual-script-tests.yml
+++ b/.github/workflows/run-individual-script-tests.yml
@@ -13,15 +13,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: 'Checkout'
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
-        fetch-depth: 2
+        fetch-depth: 0
     - name: Get changed files
       id: getfile
       run: |
         git remote add upstream ${{ github.event.pull_request.base.repo.clone_url }}
         git fetch upstream
-        echo "files=$(git diff \"upstream\" \"${{ github.event.pull_request.base.ref }}\" --name-only | xargs)" >> $GITHUB_OUTPUT
+        echo "files=$(git diff upstream ${{ github.event.pull_request.base.ref }} --name-only | xargs)" >> $GITHUB_OUTPUT
     - name: RUN Script Tests
       run: |
         echo ${{ steps.getfile.outputs.files }}
@@ -29,5 +29,5 @@ jobs:
           echo $file
         done
         python3 -m pip install cmind
-        cm pull repo --url=https://github.com/${{ github.repository }} --checkout=${{ github.ref_name }}
+        cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
         python3 tests/script/process_tests.py ${{ steps.getfile.outputs.files }}

From e279a2faa59bea89ac1d553ab6f300af34aca911 Mon Sep 17 00:00:00 2001
From: Arjun Suresh <arjunsuresh1987@gmail.com>
Date: Wed, 30 Oct 2024 17:39:06 +0530
Subject: [PATCH 26/32] Fix gh action for individual CM sript tests

---
 .github/workflows/run-individual-script-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-individual-script-tests.yml b/.github/workflows/run-individual-script-tests.yml
index d8abb96d5c..d06696f746 100644
--- a/.github/workflows/run-individual-script-tests.yml
+++ b/.github/workflows/run-individual-script-tests.yml
@@ -21,7 +21,7 @@ jobs:
       run: |
         git remote add upstream ${{ github.event.pull_request.base.repo.clone_url }}
         git fetch upstream
-        echo "files=$(git diff upstream ${{ github.event.pull_request.base.ref }} --name-only | xargs)" >> $GITHUB_OUTPUT
+        echo "files=$(git diff upstream/${{ github.event.pull_request.base.ref }} --name-only | xargs)" >> $GITHUB_OUTPUT
     - name: RUN Script Tests
       run: |
         echo ${{ steps.getfile.outputs.files }}

From 566e103446d50c653d4c8fa399a53d80aadb56c6 Mon Sep 17 00:00:00 2001
From: Arjun Suresh <arjunsuresh1987@gmail.com>
Date: Wed, 30 Oct 2024 17:42:37 +0530
Subject: [PATCH 27/32] Fix gh action for individual CM sript tests

---
 .github/workflows/run-individual-script-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-individual-script-tests.yml b/.github/workflows/run-individual-script-tests.yml
index d06696f746..04a709c974 100644
--- a/.github/workflows/run-individual-script-tests.yml
+++ b/.github/workflows/run-individual-script-tests.yml
@@ -15,7 +15,7 @@ jobs:
     - name: 'Checkout'
       uses: actions/checkout@v4
       with:
-        fetch-depth: 0
+        fetch-depth: 2
     - name: Get changed files
       id: getfile
       run: |

From 0694dce0a02b0c0c19668f953272eb2bebfd3b5e Mon Sep 17 00:00:00 2001
From: Arjun Suresh <arjunsuresh1987@gmail.com>
Date: Wed, 30 Oct 2024 17:49:57 +0530
Subject: [PATCH 28/32] Fix gh action for individual CM sript tests

---
 automation/script/module.py   | 3 +--
 tests/script/process_tests.py | 3 ++-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/automation/script/module.py b/automation/script/module.py
index d5388846f5..7be9aa8554 100644
--- a/automation/script/module.py
+++ b/automation/script/module.py
@@ -2425,12 +2425,11 @@ def test(self, i):
 
             alias = meta.get('alias','')
             uid = meta.get('uid','')
-
             if console:
                 logging.info(path)
                 test_config = meta.get('tests', '')
                 if test_config:
-                    log.info(test_config)
+                    logging.info(test_config)
                     test_all_variations = test_config.get('test-all-variations', False)
                     if test_all_variations:
                         variations = meta.get("variations")
diff --git a/tests/script/process_tests.py b/tests/script/process_tests.py
index 386110edf4..558c47a2ee 100644
--- a/tests/script/process_tests.py
+++ b/tests/script/process_tests.py
@@ -8,6 +8,7 @@
 files=sys.argv[1:]
 
 for file in files:
+    print(file)
     if not os.path.isfile(file):
         continue
     if not file.endswith("_cm.json") and not file.endswith("_cm.yaml"):
@@ -20,5 +21,5 @@
         data = yaml.safe_load(f)
     uid = data['uid']
 
-    r = cm.access({'action':'test', 'automation':'script', 'artifact': uid, 'quiet': 'yes'})
+    r = cm.access({'action':'test', 'automation':'script', 'artifact': uid, 'quiet': 'yes', 'out': 'con'})
     checks.check_return(r)

From cafec0bd36567983d55b471c7e7066530df50464 Mon Sep 17 00:00:00 2001
From: Arjun Suresh <arjunsuresh1987@gmail.com>
Date: Wed, 30 Oct 2024 18:16:08 +0530
Subject: [PATCH 29/32] Enable docker run for individual CM script tests

---
 automation/script/module.py              | 32 +++++++++++++++++++-----
 script/get-generic-sys-util/_cm.json     |  6 ++++-
 script/run-docker-container/customize.py |  2 +-
 tests/script/process_tests.py            |  2 +-
 4 files changed, 33 insertions(+), 9 deletions(-)

diff --git a/automation/script/module.py b/automation/script/module.py
index 7be9aa8554..ce509b75cc 100644
--- a/automation/script/module.py
+++ b/automation/script/module.py
@@ -2431,18 +2431,38 @@ def test(self, i):
                 if test_config:
                     logging.info(test_config)
                     test_all_variations = test_config.get('test-all-variations', False)
+                    use_docker = test_config.get('use_docker', False)
                     if test_all_variations:
                         variations = meta.get("variations")
                         individual_variations = [ v for v in variations if variations[v].get('group', '') == '' and str(variations[v].get('exclude-in-test', '')).lower() not in [ "1", "true", "yes" ] ]
                         tags_string = ",".join(meta.get("tags"))
                         for variation in individual_variations:
                             run_tags = f"{tags_string},_{variation}"
-                            r = self.cmind.access({'action':'run',
-                                'automation':'script',
-                                'tags': run_tags,
-                                'quiet': i.get('quiet') })
-                            if r['return'] > 0:
-                                return r
+                            if use_docker:
+                                docker_images = test_config.get('docker_images', [ "ubuntu-22.04" ])
+                                for docker_image in docker_images:
+                                    ii = {'action':'docker',
+                                            'automation':'script',
+                                            'tags': run_tags,
+                                            'quiet': i.get('quiet'),
+                                            'docker_image': docker_image,
+                                            'docker_image_name': alias
+                                        }
+                                    if i.get('docker_cm_repo', '') != '':
+                                        ii['docker_cm_repo'] = i['docker_cm_repo']
+                                    if i.get('docker_cm_repo_branch', '') != '':
+                                        ii['docker_cm_repo_branch'] = i['docker_cm_repo_branch']
+
+                                    r = self.cmind.access(ii)
+                                    if r['return'] > 0:
+                                        return r
+                            else:
+                                r = self.cmind.access({'action':'run',
+                                    'automation':'script',
+                                    'tags': run_tags,
+                                    'quiet': i.get('quiet') })
+                                if r['return'] > 0:
+                                    return r
 
                 logging.info('  Test: WIP')
 
diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json
index e0db7c4944..08df073dfa 100644
--- a/script/get-generic-sys-util/_cm.json
+++ b/script/get-generic-sys-util/_cm.json
@@ -723,6 +723,10 @@
     }
   },
   "tests": {
-    "test-all-variations": "yes"
+    "test-all-variations": "yes",
+    "use_docker": "yes",
+    "docker_images": [
+      "ubuntu-22.04"
+    ]
   }
 }
diff --git a/script/run-docker-container/customize.py b/script/run-docker-container/customize.py
index 35d3fdf1e4..db7ef766ab 100644
--- a/script/run-docker-container/customize.py
+++ b/script/run-docker-container/customize.py
@@ -303,7 +303,7 @@ def update_docker_info(env):
     if env.get('CM_DOCKER_IMAGE_NAME', '') != '':
         docker_image_name = env['CM_DOCKER_IMAGE_NAME']
     else:
-        docker_image_name = 'cm-script-'+env['CM_DOCKER_RUN_SCRIPT_TAGS'].replace(',', '-').replace('_','-')
+        docker_image_name = 'cm-script-'+env['CM_DOCKER_RUN_SCRIPT_TAGS'].replace(',', '-').replace('_','-').replace('+','plus')
         env['CM_DOCKER_IMAGE_NAME'] = docker_image_name
 
     docker_image_tag_extra = env.get('CM_DOCKER_IMAGE_TAG_EXTRA', '-latest')
diff --git a/tests/script/process_tests.py b/tests/script/process_tests.py
index 558c47a2ee..59c3b67694 100644
--- a/tests/script/process_tests.py
+++ b/tests/script/process_tests.py
@@ -9,7 +9,7 @@
 
 for file in files:
     print(file)
-    if not os.path.isfile(file):
+    if not os.path.isfile(file) or not "script" in file:
         continue
     if not file.endswith("_cm.json") and not file.endswith("_cm.yaml"):
         continue

From dd40f0a61224881934cd88989cc1719f7dec0496 Mon Sep 17 00:00:00 2001
From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com>
Date: Wed, 30 Oct 2024 18:21:37 +0530
Subject: [PATCH 30/32] capture framework version from cm_sut_info.json

---
 script/generate-mlperf-inference-submission/customize.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py
index 34d6c97b95..032a68618d 100644
--- a/script/generate-mlperf-inference-submission/customize.py
+++ b/script/generate-mlperf-inference-submission/customize.py
@@ -209,6 +209,7 @@ def generate_submission(i):
             implementation = sut_info["implementation"]
             device = sut_info["device"]
             framework = sut_info["framework"].replace(" ","_")
+            framework_version = sut_info["framework_version"]
             run_config = sut_info["run_config"]
             new_res = f"{system}-{implementation}-{device}-{framework}-{run_config}"
         else:
@@ -234,7 +235,7 @@ def generate_submission(i):
                 system_meta_default['framework'] = framework + " " + framework_version
             else:
                 print(parts)
-                return {'return': 1, 'error': f"The required details for generating the inference submission:\n1.system_name\n2.implementation\n3.framework\n4.run_config\nInclude a cm-sut-info.json file with the above content in {result_path}"}
+                return {'return': 1, 'error': f"The required details for generating the inference submission:\n1.hardware_name\n2.implementation\n3.Device\n4.framework\n5.framework_version\n6.run_config\nInclude a cm-sut-info.json or sut-info.json file with the above content in {result_path}"}
             
         platform_prefix = inp.get('platform_prefix', '')
         if platform_prefix:

From 9e3392bee828e1fc7803fb81be12f2aae6860a82 Mon Sep 17 00:00:00 2001
From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com>
Date: Wed, 30 Oct 2024 19:12:47 +0530
Subject: [PATCH 31/32] Enable default values to required env keys

---
 script/generate-mlperf-inference-submission/customize.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py
index 032a68618d..f82893e39a 100644
--- a/script/generate-mlperf-inference-submission/customize.py
+++ b/script/generate-mlperf-inference-submission/customize.py
@@ -15,7 +15,7 @@ def fill_from_json(file_path, keys, sut_info):
     with open(file_path, 'r') as f:
         data = json.load(f)
         for key in keys:
-            if key in data and sut_info[key] is None:
+            if key in data and (sut_info[key] is None or sut_info[key] == "default"):
                 sut_info[key] = data[key]
             elif key in data and sut_info[key] != data[key]:
                 return -1 # error saying there is a mismatch in the value of a key
@@ -149,7 +149,8 @@ def generate_submission(i):
             "implementation": None,
             "device": None,
             "framework": None,
-            "run_config": None
+            "framework_version": "default",
+            "run_config": "default"
         } # variable to store the system meta 
 
         model_mapping_combined = {} # to store all the model mapping related to an SUT

From 7f58d670f7f6afae953d56b4649ff88afcc3663d Mon Sep 17 00:00:00 2001
From: Arjun Suresh <arjunsuresh1987@gmail.com>
Date: Wed, 30 Oct 2024 19:17:51 +0530
Subject: [PATCH 32/32] Fix huggingface downloader - fixes #437

---
 ...lperf-loadgen-onnx-huggingface-bert-fp32-squad.yml |  7 +++----
 script/get-ml-model-huggingface-zoo/download_model.py | 11 ++++++-----
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/test-mlperf-loadgen-onnx-huggingface-bert-fp32-squad.yml b/.github/workflows/test-mlperf-loadgen-onnx-huggingface-bert-fp32-squad.yml
index ed73c7ad03..597121fb0d 100644
--- a/.github/workflows/test-mlperf-loadgen-onnx-huggingface-bert-fp32-squad.yml
+++ b/.github/workflows/test-mlperf-loadgen-onnx-huggingface-bert-fp32-squad.yml
@@ -5,7 +5,7 @@ name: MLPerf loadgen with HuggingFace bert onnx fp32 squad model
 
 on:
   pull_request:
-    branches: [ "main", "dev" ]
+    branches: [ "main", "dev", "mlperf-inference" ]
     paths:
       - '.github/workflows/test-mlperf-loadgen-onnx-huggingface-bert-fp32-squad.yml'
       - '**'
@@ -18,7 +18,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ "3.12" ]
+        python-version: [ "3.10", "3.12" ]
 
     steps:
     - uses: actions/checkout@v3
@@ -30,7 +30,6 @@ jobs:
       run: |
         python3 -m pip install cmind
         cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
-        cm run script --quiet --tags=get,sys-utils-cm
     - name: Test MLPerf loadgen with HuggingFace bert onnx fp32 squad model
       run: |
-        cmr "python app loadgen-generic _onnxruntime _custom _huggingface _model-stub.ctuning/mlperf-inference-bert-onnx-fp32-squad-v1.1" --adr.hf-downloader.model_filename=model.onnx --quiet
+        cm run script --tags=python,app,loadgen-generic,_onnxruntime,_custom,_huggingface,_model-stub.ctuning/mlperf-inference-bert-onnx-fp32-squad-v1.1 --quiet
diff --git a/script/get-ml-model-huggingface-zoo/download_model.py b/script/get-ml-model-huggingface-zoo/download_model.py
index 4e6e9c86e8..87f9b25aeb 100644
--- a/script/get-ml-model-huggingface-zoo/download_model.py
+++ b/script/get-ml-model-huggingface-zoo/download_model.py
@@ -28,8 +28,7 @@
 
         model_filenames = model_filename.split(',') if ',' in model_filename else [model_filename]
 
-        # First must be model
-        base_model_filename = model_filenames[0]
+        base_model_filepath = None
 
         files = []
         if full_subfolder!='':
@@ -93,15 +92,17 @@ def list_hf_files(path):
             xrevision = None if revision == '' else revision
             xsubfolder = None if subfolder == '' else subfolder
 
-            hf_hub_download(repo_id=model_stub,
+            downloaded_path = hf_hub_download(repo_id=model_stub,
                        subfolder=xsubfolder,
                        filename=model_filename,
-                       force_filename=model_filename,
                        revision=xrevision,
                        cache_dir=os.getcwd())
+            print(downloaded_path)
+            if not base_model_filepath:
+                base_model_filepath = downloaded_path 
         
 
         print ('')
         
         with open('tmp-run-env.out', 'w') as f:
-            f.write(f"CM_ML_MODEL_FILE_WITH_PATH={os.path.join(os.getcwd(),base_model_filename)}")
+            f.write(f"CM_ML_MODEL_FILE_WITH_PATH={base_model_filepath}")