Merge remote-tracking branch 'upstream/main'

.github/CODEOWNERS

@@ -2,4 +2,8 @@
 # Unless a later match takes precedence,they will be requested for review when someone opens a pull request.
 * @mlcommons/wg-ck 
 
-/CODEOWNERS @mlcommons/staff
+/.github/CODEOWNERS @mlcommons/systems
+
+/.github/workflows/cla.yml @mlcommons/systems
+
+/LICENSE.md @mlcommons/systems

.github/workflows/test-cm-based-ubmission-generation.yml → .github/workflows/test-cm-based-submission-generation.yml

@@ -6,11 +6,11 @@ on:
   pull_request:
     branches: [ "main", "dev", "mlperf-inference" ]
     paths:
-      - '.github/workflows/test-submission-generation-non-cm-based-benchmarks.yml'
+      - '.github/workflows/test-cm-based-submission-generation.yml'
       # - '**'  # kept on for all the path instead of submission generation CM script so that this could help in trapping any bugs in any recent submission checker modification also
       # - '!**.md'
 jobs:
-  Case-3:
+  submission_generation:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -19,6 +19,7 @@ jobs:
         python-version: [ "3.12" ]
         division: ["closed", "open"]
         category: ["datacenter", "edge"]
+        case: ["case-3", "case-7"]
         exclude:
           - os: macos-latest
           - os: windows-latest
@@ -36,7 +37,18 @@ jobs:
         cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
     - name: Pull repo where test cases are uploaded
       run: |
-        cm pull repo anandhu-eng@inference --checkout=submission-generation-tests
-    - name: Submission generation(model_mapping.json not present but model name is matching with the official one in submission checker) - ${{ matrix.category }} ${{ matrix.division }}
+        git clone -b submission-generation-tests https://github.com/anandhu-eng/inference.git submission_generation_tests
+    - name: Run Submission Generation - ${{ matrix.case }} ${{ matrix.category }} ${{ matrix.division }}
       run: |
-        cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=$HOME/CM/repos/anandhu-eng@inference/case-3/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --category=${{ matrix.category }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet
+        if [ "${{ matrix.case }}" == "case-3" ]; then
+          #results_dir="submission_generation_tests/case-3/"
+          description="Submission generation (model_mapping.json not present but model name matches with official one)"
+        elif [ "${{ matrix.case }}" == "case-7" ]; then
+          #results_dir="submission_generation_tests/case-7/"
+          description="Submission generation (sut_info.json incomplete, SUT folder name in required format)"
+        fi
+        # Dynamically set the log group to simulate a dynamic step name
+        echo "::group::$description"
+        cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --results_dir=submission_generation_tests/${{ matrix.case }}/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --category=${{ matrix.category }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet
+        echo "::endgroup::"
+  

VERSION

@@ -1 +1 @@
-0.3.23
+0.3.24

automation/script/module.py

@@ -3984,7 +3984,11 @@ def parse_version(self, i):
             print(f"Regex Pattern: {match_text}")
             print(f"Matched String: {string}")
             print(f"Match Groups: {match.groups()}")
-        version = r['match'].group(group_number)
+
+        if r['match'].lastindex and r['match'].lastindex >= group_number:
+            version = r['match'].group(group_number)
+        else:
+            return {'return':1, 'error': 'Invalid version detection group number. Version was not detected. Last index of match = {}. Given group number = {}'.format(r['match'].lastindex, group_number)}
 
         which_env[env_key] = version
         which_env['CM_DETECTED_VERSION'] = version # to be recorded in the cache meta

script/get-generic-sys-util/_cm.json

@@ -554,9 +554,11 @@
     "md5sha1sum": {
       "env": {
         "CM_SYS_UTIL_NAME": "md5sha1sum",
-        "CM_SYS_UTIL_VERSION_CMD": "md5sum --version",
-        "CM_SYS_UTIL_VERSION_RE": "\\b\\d+\\.\\d+(?:\\.\\d+)?\\b"
+        "CM_SYS_UTIL_VERSION_CMD": "md5sum --version | grep sha1sum",
+        "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b",
+        "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0
       },
+      "new_env_keys": ["CM_MD5SHA1SUM_VERSION"],
       "state": {
         "md5sha1sum": {
           "apt": "",

script/get-generic-sys-util/customize.py

@@ -17,7 +17,8 @@ def preprocess(i):
     if env.get('CM_GENERIC_SYS_UTIL_RUN_MODE', '') == "detect":
         if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '':
             r = automation.run_native_script({'run_script_input':i['run_script_input'], 'env':env, 'script_name':'detect'})
-            if r['return'] > 0: #detection failed, do install via prehook_deps
+            if r['return'] != 0: #detection failed, do install via prehook_deps
+                print("detection failed, going for installation")
                 env['CM_GENERIC_SYS_UTIL_INSTALL_NEEDED'] = "yes"
                 return {'return': 0}
             else: #detection is successful, no need to install
@@ -55,6 +56,7 @@ def preprocess(i):
     if not package_name:
         if str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_MISSING_PACKAGE', '')).lower() in [ "1", "true", "yes" ]:
             print(f"WARNING: No package name specified for {pm} and util name {util}. Ignoring it...")
+            env['CM_TMP_GENERIC_SYS_UTIL_PACKAGE_INSTALL_IGNORED'] = 'yes'
             return {'return': 0}
         else:
             return {'return': 1, 'error': f'No package name specified for {pm} and util name {util}'}
@@ -106,20 +108,17 @@ def detect_version(i):
     env = i['env']
     version_env_key = f"CM_{env['CM_SYS_UTIL_NAME'].upper()}_VERSION"
     version_check_re = env.get('CM_SYS_UTIL_VERSION_RE', '')
+    group_number = env.get('CM_TMP_VERSION_DETECT_GROUP_NUMBER', 1)
 
     # Confirm that the regex pattern and file are present
     if version_check_re == '' or not os.path.exists("tmp-ver.out"):
         version = "undetected"
     else:
-        r = i['automation'].parse_version({
-            'match_text': version_check_re,
-            'group_number': 1,             
-            'env_key': version_env_key,
-            'which_env': env, #'debug': 1
-        })
-
-        if r['return'] > 0:
-            return r
+        r = i['automation'].parse_version({'match_text': version_check_re,
+                                       'group_number': group_number,
+                                       'env_key': version_env_key,
+                                       'which_env': env})
+        if r['return'] >0: return r
 
         version = r['version']
         print(i['recursion_spaces'] + '    Detected version: {}'.format(version))
@@ -132,11 +131,11 @@ def postprocess(i):
 
     version_env_key = f"CM_{env['CM_SYS_UTIL_NAME'].upper()}_VERSION"
 
-    if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' and (env['CM_GENERIC_SYS_UTIL_RUN_MODE'] == "install" or env.get(version_env_key, '') == '') :
+    if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' and (env['CM_GENERIC_SYS_UTIL_RUN_MODE'] == "install" or env.get(version_env_key, '') == '') and str(env.get('CM_TMP_GENERIC_SYS_UTIL_PACKAGE_INSTALL_IGNORED', '')).lower() not in ["yes", "1", "true"]:
         automation = i['automation']
         r = automation.run_native_script({'run_script_input':i['run_script_input'], 'env':env, 'script_name':'detect'})
-        if r['return'] > 0:
-            return r
+        if r['return'] > 0 and str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE', '')).lower() not in [ "1", "yes", "true" ] and str(env.get('CM_TMP_FAIL_SAFE', '')).lower() not in [ "1", "yes", "true" ]:
+            return {'return': 1, 'error': 'Version detection failed after installation. Please check the provided version command or use env.CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE=yes to ignore the error.'}
 
         r = detect_version(i)
 

script/get-generic-sys-util/detect.sh

@@ -3,12 +3,12 @@
 if [[ -n "${CM_SYS_UTIL_VERSION_CMD}" ]]; then
   if [[ "${CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM}" == "yes" ]]; then
     # Redirect both stdout and stderr to tmp-ver.out
-    ${CM_SYS_UTIL_VERSION_CMD}  > tmp-ver.out 2>&1
-  else		
-    ${CM_SYS_UTIL_VERSION_CMD}  > tmp-ver.out
+    cmd="${CM_SYS_UTIL_VERSION_CMD}  > tmp-ver.out 2>&1"
+  else
+    cmd="${CM_SYS_UTIL_VERSION_CMD}  > tmp-ver.out"
   fi
-  echo ${CM_SYS_UTIL_VERSION_CMD}
-  echo $PWD
+  echo $cmd
+  eval $cmd
+  test $? -eq 0 || exit $?
 fi
 
-test $? -eq 0 || exit $?

setup.py

@@ -139,7 +139,7 @@ def read_file(file_name, default=""):
     return default
 
 long_description = read_file("README.md", "No description available.")
-version_ = read_file("VERSION", "0.3.0")
+version_ = read_file("VERSION", "0.3.1")
 
 setup(
     name='cm4mlops',