Merge from go

automation/script/module_misc.py

@@ -1739,7 +1739,7 @@ def docker(i):
 
     image_repo = i.get('docker_image_repo','')
     if image_repo == '':
-        image_repo = 'cknowledge'
+        image_repo = 'local'
 
     # Host system needs to have docker
     r = self_module.cmind.access({'action':'run',

script/app-mlperf-inference-mlcommons-python/_cm.yaml

@@ -1029,21 +1029,21 @@ variations:
     deps:
     - tags: get,generic-python-lib,_package.nibabel
 
-  dlrm-99.9:
+  dlrm-v2-99.9:
     group: models
     base:
-    - dlrm
+    - dlrm-v2_
     env:
       CM_MODEL: dlrm-99.9
 
-  dlrm-99:
+  dlrm-v2-99:
     group: models
     base:
-    - dlrm
+    - dlrm-v2_
     env:
       CM_MODEL: dlrm-99
 
-  dlrm:
+  dlrm-v2_:
     env:
       CM_MLPERF_MODEL_SKIP_BATCHING: true
     deps:

script/build-dockerfile/customize.py

@@ -180,6 +180,8 @@ def preprocess(i):
 
     f.write(EOL+'# Install python packages' + EOL)
     python = get_value(env, config, 'PYTHON', 'CM_DOCKERFILE_PYTHON')
+    f.write('RUN {} -m venv cm-venv'.format(python) + " " + EOL)
+    f.write('RUN source cm-venv/bin/activate' + EOL)
     f.write('RUN {} -m pip install --user '.format(python) + " ".join(get_value(env, config, 'python-packages')) + ' ' + pip_extra_flags + ' ' + EOL)
 
     f.write(EOL+'# Download CM repo for scripts' + EOL)

script/detect-cpu/run.sh

@@ -8,4 +8,15 @@ else
     echo "CM_HOST_MEMORY_CAPACITY=$memory_capacity">>tmp-run-env.out
     disk_capacity=`df -h --total -l |grep total |tr -s ' '|cut -d' ' -f2`
     echo "CM_HOST_DISK_CAPACITY=$disk_capacity">>tmp-run-env.out
+    # extract cpu information which are not there in lscpu
+    cpuinfo=$(cat /proc/cpuinfo)
+    echo "CM_HOST_CPU_WRITE_PROTECT_SUPPORT=$(echo "$cpuinfo" | grep -m 1 "wp" | cut -d ':' -f 2 | xargs)">>tmp-run-env.out
+    echo "CM_HOST_CPU_MICROCODE=$(echo "$cpuinfo" | grep -m 1 "microcode" | cut -d ':' -f 2 | xargs)">>tmp-run-env.out
+    echo "CM_HOST_CPU_FPU_SUPPORT=$(echo "$cpuinfo" | grep -m 1 "fpu" | cut -d ':' -f 2 | xargs)">>tmp-run-env.out
+    echo "CM_HOST_CPU_FPU_EXCEPTION_SUPPORT=$(echo "$cpuinfo" | grep -m 1 "fpu_exception" | cut -d ':' -f 2 | xargs)">>tmp-run-env.out
+    echo "CM_HOST_CPU_BUGS=$(echo "$cpuinfo" | grep -m 1 "bugs" | cut -d ':' -f 2 | xargs)">>tmp-run-env.out
+    echo "CM_HOST_CPU_TLB_SIZE=$(echo "$cpuinfo" | grep -m 1 "TLB size" | cut -d ':' -f 2 | xargs)">>tmp-run-env.out
+    echo "CM_HOST_CPU_CFLUSH_SIZE=$(echo "$cpuinfo" | grep -m 1 "clush size" | cut -d ':' -f 2 | xargs)">>tmp-run-env.out
+    echo "CM_HOST_CACHE_ALLIGNMENT_SIZE=$(echo "$cpuinfo" | grep -m 1 "cache_alignment" | cut -d ':' -f 2 | xargs)">>tmp-run-env.out
+    echo "CM_HOST_POWER_MANAGEMENT=$(echo "$cpuinfo" | grep -m 1 "power management" | cut -d ':' -f 2 | xargs)">>tmp-run-env.out    
 fi

script/get-generic-sys-util/_cm.json

@@ -50,6 +50,21 @@
         }
       }
     },
+    "linux-tools": {
+      "deps":[
+        {
+          "tags":"detect,os"
+        }
+      ],
+      "env": {
+        "CM_SYS_UTIL_NAME": "linux-tools"
+      },
+      "state": {
+        "linux-tools": {
+          "apt": "linux-tools-<<<CM_HOST_OS_KERNEL_VERSION>>>"
+        }
+      }
+    },
     "sox": {
       "env": {
         "CM_SYS_UTIL_NAME": "sox"
@@ -128,6 +143,45 @@
         }
       }
     },
+    "psmisc": {
+      "env": {
+        "CM_SYS_UTIL_NAME": "psmisc"
+      },
+      "state": {
+        "psmisc": {
+          "apt": "psmisc",
+          "dnf": "psmisc",
+          "yum": "psmisc",
+          "brew": "pstree"
+        }
+      }
+    },
+    "systemd": {
+      "env": {
+        "CM_SYS_UTIL_NAME": "systemd"
+      },
+      "state": {
+        "systemd": {
+          "apt": "systemd",
+          "dnf": "systemd",
+          "yum": "systemd",
+          "brew": ""
+        }
+      }
+    },
+    "dmidecode": {
+      "env": {
+        "CM_SYS_UTIL_NAME": "dmidecode"
+      },
+      "state": {
+        "dmidecode": {
+          "apt": "dmidecode",
+          "dnf": "dmidecode",
+          "yum": "dmidecode",
+          "brew": ""
+        }
+      }
+    },
     "libpng-dev": {
       "env": {
         "CM_SYS_UTIL_NAME": "libpng-dev"

script/get-generic-sys-util/customize.py

@@ -1,5 +1,6 @@
 from cmind import utils
 import os
+import re
 
 def preprocess(i):
 
@@ -30,6 +31,15 @@ def preprocess(i):
     package_name = package.get(pm)
     if not package_name:
         return {'return': 1, 'error': 'No package name specified for {} and util name {}'.format(pm, util)}
+
+    # Temporary handling of dynamic state variables
+    tmp_values = re.findall(r'<<<(.*?)>>>', str(package_name))
+    for tmp_value in tmp_values:
+            if tmp_value not in env:
+                return {'return':1, 'error':'variable {} is not in env'.format(tmp_value)}
+            if tmp_value in env:
+                if type(package_name) == str:
+                    package_name = package_name.replace("<<<"+tmp_value+">>>", str(env[tmp_value]))
 
     install_cmd = env.get('CM_HOST_OS_PACKAGE_MANAGER_INSTALL_CMD')
     if not install_cmd:

script/get-platform-details/_cm.json

@@ -0,0 +1,46 @@
+{
+  "alias": "get-platform-details",
+  "automation_alias": "script",
+  "automation_uid": "5b4e0237da074764",
+  "cache": false,
+  "category": "Platform information",
+  "deps": [
+    {
+      "tags": "detect,os"
+    },
+    {
+      "tags": "get,sys-util,generic,_psmisc"
+    },
+    {
+      "tags": "get,sys-util,generic,_systemd"
+    },
+    {
+      "tags": "get,sys-util,generic,_dmidecode"
+    }
+  ],
+  "prehook_deps": [
+    {
+      "enable_if_env": {
+        "CM_INSTALL_NUMACTL": [
+          "True"
+        ]
+      },
+      "tags": "get,sys-util,generic,_numactl"
+    },
+    {
+      "enable_if_env": {
+        "CM_INSTALL_CPUPOWER": [
+          "True"
+        ]
+      },
+      "tags": "get,sys-util,generic,_linux-tools"
+    }
+  ],
+  "tags": [
+    "get",
+    "platform",
+    "details",
+    "platform-details"
+  ],
+  "uid": "f0801943c17f4e48"
+}

script/get-platform-details/customize.py

@@ -0,0 +1,40 @@
+from cmind import utils
+import os
+import subprocess
+
+def check_installation(command, os_info):
+    if os_info['platform'] == "windows":
+        return subprocess.call([command, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) == 0
+    elif os_info['platform'] == "linux":
+        return subprocess.call(['which',command], stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0 #0 means the package is there
+
+def preprocess(i):
+
+    os_info = i['os_info']
+    env = i['env']
+
+    if os_info['platform'] == "windows":
+        return {'return':1, 'error':'get-platform-details script not yet supported in windows!'}
+
+    print(env['CM_HOST_OS_KERNEL_VERSION'])
+
+    if not check_installation("numactl",os_info):
+        env['CM_INSTALL_NUMACTL'] = 'True'
+
+    #if not check_installation("cpupower",os_info):
+    env['CM_INSTALL_CPUPOWER'] = 'True'
+
+    return {'return':0}
+
+
+def postprocess(i):
+
+    state = i['state']
+
+    env = i['env']
+
+    os_info = i['os_info']
+
+    automation = i['automation']
+
+    return {'return':0}

script/get-platform-details/run.bat

@@ -0,0 +1 @@
+echo "This CM script not supported for windows yet"

script/get-platform-details/run.sh

@@ -0,0 +1,123 @@
+#!/bin/bash
+
+OUTPUT_FILE="system_info.txt"
+
+echo "WARNING: sudo permission is needed to some packages for measuring the platform details"
+
+if [[ ${CM_HOST_OS_FLAVOR} == "macos" ]]; then
+    echo "WARNING: To be done for the mac os"
+else
+    echo "Platform Details" > $OUTPUT_FILE
+    echo "" >> $OUTPUT_FILE
+    echo "------------------------------------------------------------" >> $OUTPUT_FILE
+    echo "1. uname -a" >> $OUTPUT_FILE
+    eval "uname -a" >> $OUTPUT_FILE
+    test $? -eq 0 || exit $?
+    echo "------------------------------------------------------------" >> $OUTPUT_FILE
+
+    echo "2. w" >> $OUTPUT_FILE
+    eval "w" >> $OUTPUT_FILE
+    test $? -eq 0 || exit $?
+    echo "------------------------------------------------------------" >> $OUTPUT_FILE
+
+    echo "3. Username" >> $OUTPUT_FILE
+    echo "From environment variable \$USER: $USER" >> $OUTPUT_FILE
+    echo "------------------------------------------------------------" >> $OUTPUT_FILE
+
+    echo "4. ulimit -a" >> $OUTPUT_FILE
+    eval "ulimit -a" >> $OUTPUT_FILE
+    test $? -eq 0 || exit $?
+    echo "------------------------------------------------------------" >> $OUTPUT_FILE
+
+    echo "5. sysinfo process ancestry" >> $OUTPUT_FILE
+    eval "pstree" >> $OUTPUT_FILE
+    test $? -eq 0 || exit $?
+    echo "------------------------------------------------------------" >> $OUTPUT_FILE
+
+    echo "6. /proc/cpuinfo" >> $OUTPUT_FILE
+    eval "cat /proc/cpuinfo" >> $OUTPUT_FILE
+    test $? -eq 0 || exit $?
+    echo "------------------------------------------------------------" >> $OUTPUT_FILE
+
+    echo "7. lscpu" >> $OUTPUT_FILE
+    eval "lscpu" >> $OUTPUT_FILE
+    test $? -eq 0 || exit $?
+    echo "------------------------------------------------------------" >> $OUTPUT_FILE
+
+    echo "8. numactl --hardware" >> $OUTPUT_FILE
+    eval "numactl --hardware" >> $OUTPUT_FILE
+    test $? -eq 0 || exit $?
+    echo "------------------------------------------------------------" >> $OUTPUT_FILE
+
+    echo "9. /proc/meminfo" >> $OUTPUT_FILE
+    eval "cat /proc/meminfo" >> $OUTPUT_FILE
+    test $? -eq 0 || exit $?
+    echo "------------------------------------------------------------" >> $OUTPUT_FILE
+
+    echo "10. who -r" >> $OUTPUT_FILE
+    eval "who -r" >> $OUTPUT_FILE
+    test $? -eq 0 || exit $?
+    echo "------------------------------------------------------------" >> $OUTPUT_FILE
+
+    echo "11. Systemd service manager version" >> $OUTPUT_FILE
+    eval "systemctl --version | head -n 1" >> $OUTPUT_FILE
+    test $? -eq 0 || exit $?
+    echo "------------------------------------------------------------" >> $OUTPUT_FILE
+
+    echo "12. Services, from systemctl list-unit-files" >> $OUTPUT_FILE
+    eval "systemctl list-unit-files" >> $OUTPUT_FILE
+    test $? -eq 0 || exit $?
+    echo "------------------------------------------------------------" >> $OUTPUT_FILE
+
+    echo "13. Linux kernel boot-time arguments, from /proc/cmdline" >> $OUTPUT_FILE
+    eval "cat /proc/cmdline" >> $OUTPUT_FILE
+    test $? -eq 0 || exit $?
+    echo "------------------------------------------------------------" >> $OUTPUT_FILE
+
+    echo "14. cpupower frequency-info" >> $OUTPUT_FILE
+    eval "cpupower frequency-info" >> $OUTPUT_FILE
+    test $? -eq 0 || exit $?
+    echo "------------------------------------------------------------" >> $OUTPUT_FILE
+
+    echo "15. sysctl" >> $OUTPUT_FILE
+    eval "sudo sysctl -a" >> $OUTPUT_FILE
+    test $? -eq 0 || exit $?
+    echo "------------------------------------------------------------" >> $OUTPUT_FILE
+
+    echo "16. /sys/kernel/mm/transparent_hugepage" >> $OUTPUT_FILE
+    eval "cat /sys/kernel/mm/transparent_hugepage/enabled" >> $OUTPUT_FILE
+    test $? -eq 0 || exit $?
+    echo "------------------------------------------------------------" >> $OUTPUT_FILE
+
+    echo "17. /sys/kernel/mm/transparent_hugepage/khugepaged" >> $OUTPUT_FILE
+    eval "cat /sys/kernel/mm/transparent_hugepage/khugepaged/defrag" >> $OUTPUT_FILE
+    test $? -eq 0 || exit $?
+    echo "------------------------------------------------------------" >> $OUTPUT_FILE
+
+    echo "18. OS release" >> $OUTPUT_FILE
+    eval "cat /etc/os-release" >> $OUTPUT_FILE
+    test $? -eq 0 || exit $?
+    echo "------------------------------------------------------------" >> $OUTPUT_FILE
+
+    echo "19. Disk information" >> $OUTPUT_FILE
+    eval "lsblk" >> $OUTPUT_FILE
+    test $? -eq 0 || exit $?
+    echo "------------------------------------------------------------" >> $OUTPUT_FILE
+
+    echo "20. /sys/devices/virtual/dmi/id" >> $OUTPUT_FILE
+    eval "ls /sys/devices/virtual/dmi/id" >> $OUTPUT_FILE
+    test $? -eq 0 || exit $?
+    echo "------------------------------------------------------------" >> $OUTPUT_FILE
+
+    echo "21. dmidecode" >> $OUTPUT_FILE
+    eval "sudo dmidecode" >> $OUTPUT_FILE
+    test $? -eq 0 || exit $?
+    echo "------------------------------------------------------------" >> $OUTPUT_FILE
+
+    echo "22. BIOS" >> $OUTPUT_FILE
+    eval "sudo dmidecode -t bios" >> $OUTPUT_FILE
+    test $? -eq 0 || exit $?
+    echo "------------------------------------------------------------" >> $OUTPUT_FILE
+
+    echo "System information has been saved to $PWD/$OUTPUT_FILE"
+fi