Merge pull request #460 from GATEOverflow/mlperf-inference

Improvements to version-detect in get-generic-sys-util
mlcommons · Nov 1, 2024 · 1038a21 · 1038a21
2 parents ba9b9be + cc82180
commit 1038a21
Show file tree

Hide file tree

Showing 4 changed files with 26 additions and 9 deletions.
diff --git a/automation/script/module.py b/automation/script/module.py
@@ -3980,7 +3980,10 @@ def parse_version(self, i):
 
         string = r['string']
 
-        version = r['match'].group(group_number)
+        if r['match'].lastindex and r['match'].lastindex >= group_number:
+            version = r['match'].group(group_number)
+        else:
+            return {'return':1, 'error': 'Invalid version detection group number. Version was not detected. Last index of match = {}. Given group number = {}'.format(r['match'].lastindex, group_number)}
 
         which_env[env_key] = version
         which_env['CM_DETECTED_VERSION'] = version # to be recorded in the cache meta

diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json
@@ -457,8 +457,12 @@
     },
     "md5sha1sum": {
       "env": {
-        "CM_SYS_UTIL_NAME": "md5sha1sum"
+        "CM_SYS_UTIL_NAME": "md5sha1sum",
+        "CM_SYS_UTIL_VERSION_CMD": "md5sum --version | grep sha1sum",
+        "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b",
+        "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0
       },
+      "new_env_keys": ["CM_MD5SHA1SUM_VERSION"],
       "state": {
         "md5sha1sum": {
           "apt": "",

diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py
@@ -17,7 +17,8 @@ def preprocess(i):
     if env.get('CM_GENERIC_SYS_UTIL_RUN_MODE', '') == "detect":
         if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '':
             r = automation.run_native_script({'run_script_input':i['run_script_input'], 'env':env, 'script_name':'detect'})
-            if r['return'] > 0: #detection failed, do install via prehook_deps
+            if r['return'] != 0: #detection failed, do install via prehook_deps
+                print("detection failed, going for installation")
                 env['CM_GENERIC_SYS_UTIL_INSTALL_NEEDED'] = "yes"
                 return {'return': 0}
             else: #detection is successful, no need to install
@@ -55,6 +56,7 @@ def preprocess(i):
     if not package_name:
         if str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_MISSING_PACKAGE', '')).lower() in [ "1", "true", "yes" ]:
             print(f"WARNING: No package name specified for {pm} and util name {util}. Ignoring it...")
+            env['CM_TMP_GENERIC_SYS_UTIL_PACKAGE_INSTALL_IGNORED'] = 'yes'
             return {'return': 0}
         else:
             return {'return': 1, 'error': f'No package name specified for {pm} and util name {util}'}
@@ -104,13 +106,14 @@ def detect_version(i):
     env = i['env']
     version_env_key = f"CM_{env['CM_SYS_UTIL_NAME'].upper()}_VERSION"
     version_check_re = env.get('CM_SYS_UTIL_VERSION_RE', '')
+    group_number = env.get('CM_TMP_VERSION_DETECT_GROUP_NUMBER', 1)
 
     if version_check_re == '' or not os.path.exists("tmp-ver.out"):
         version = "undetected"
 
     else:
         r = i['automation'].parse_version({'match_text': version_check_re,
-                                       'group_number': 1,
+                                       'group_number': group_number,
                                        'env_key': version_env_key,
                                        'which_env': env})
         if r['return'] >0: return r
@@ -125,11 +128,11 @@ def postprocess(i):
 
     version_env_key = f"CM_{env['CM_SYS_UTIL_NAME'].upper()}_VERSION"
 
-    if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' and (env['CM_GENERIC_SYS_UTIL_RUN_MODE'] == "install" or env.get(version_env_key, '') == '') :
+    if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' and (env['CM_GENERIC_SYS_UTIL_RUN_MODE'] == "install" or env.get(version_env_key, '') == '') and str(env.get('CM_TMP_GENERIC_SYS_UTIL_PACKAGE_INSTALL_IGNORED', '')).lower() not in ["yes", "1", "true"]:
         automation = i['automation']
         r = automation.run_native_script({'run_script_input':i['run_script_input'], 'env':env, 'script_name':'detect'})
-        if r['return'] > 0:
-            return r
+        if r['return'] > 0 and str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE', '')).lower() not in [ "1", "yes", "true" ] and str(env.get('CM_TMP_FAIL_SAFE', '')).lower() not in [ "1", "yes", "true" ]:
+            return {'return': 1, 'error': 'Version detection failed after installation. Please check the provided version command or use env.CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE=yes to ignore the error.'}
 
         r = detect_version(i)
 

diff --git a/script/get-generic-sys-util/detect.sh b/script/get-generic-sys-util/detect.sh
@@ -1,7 +1,14 @@
 #!/bin/bash
 
 if [[ -n "${CM_SYS_UTIL_VERSION_CMD}" ]]; then
-  ${CM_SYS_UTIL_VERSION_CMD}  > tmp-ver.out
+  if [[ "${CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM}" == "yes" ]]; then
+    # Redirect both stdout and stderr to tmp-ver.out
+    cmd="${CM_SYS_UTIL_VERSION_CMD}  > tmp-ver.out 2>&1"
+  else
+    cmd="${CM_SYS_UTIL_VERSION_CMD}  > tmp-ver.out"
+  fi
+  echo $cmd
+  eval $cmd
+  test $? -eq 0 || exit $?
 fi
 
-test $? -eq 0 || exit $?