From cc3d2516a5cdf927274e62f7e6e229cba2fab769 Mon Sep 17 00:00:00 2001 From: Oseltamivir Date: Mon, 28 Oct 2024 15:42:56 +0800 Subject: [PATCH 01/56] Updated fix for tests --- script/get-generic-sys-util/README.md | 424 ++++++++++++++++++++++++++ script/get-generic-sys-util/_cm.json | 357 +++++++++++++++------- 2 files changed, 677 insertions(+), 104 deletions(-) diff --git a/script/get-generic-sys-util/README.md b/script/get-generic-sys-util/README.md index b1fc1a160e..bf09f83053 100644 --- a/script/get-generic-sys-util/README.md +++ b/script/get-generic-sys-util/README.md @@ -1 +1,425 @@ Please see [https://docs.mlcommons.org/cm4mlops/scripts/Detection-or-installation-of-tools-and-artifacts/get-generic-sys-util](https://docs.mlcommons.org/cm4mlops/scripts/Detection-or-installation-of-tools-and-artifacts/get-generic-sys-util) for the documentation of this CM script. + +# get-generic-sys-util +Below are the specific regexes and the format of output that they are expecting for each command used to check for versions. + +All commands are tested to be working on Ubuntu. + +Format: + +## Utility name +`regex` + +`command to obtain version` + +command output + +---- + +## g++-12 +`^.*([0-9]+(\\.[0-9]+)+).*` + +`g++-9 --version` + +g++-9 (Ubuntu 9.5.0-1ubuntu1~22.04) 9.5.0
+Copyright (C) 2019 Free Software Foundation, Inc.
+This is free software; see the source for copying conditions. There is NO
+warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ +## g++-11 +`^.*([0-9]+(\\.[0-9]+)+).*` + +`g++-11 --version` + +g++-11 (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0
+Copyright (C) 2021 Free Software Foundation, Inc.
+This is free software; see the source for copying conditions. There is NO
+warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ +## g++-12 +`^.*([0-9]+(\\.[0-9]+)+).*` + +`g++-12 --version` + +g++-12 (Ubuntu 12.3.0-1ubuntu1~22.04) 12.3.0
+Copyright (C) 2022 Free Software Foundation, Inc.
+This is free software; see the source for copying conditions. There is NO
+warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ + +## gcc-9 +`^.*([0-9]+(\\.[0-9]+)+).*` + +`gcc-9 --version` + +gcc-9 (Ubuntu 9.5.0-1ubuntu1~22.04) 9.5.0
+Copyright (C) 2019 Free Software Foundation, Inc.
+This is free software; see the source for copying conditions. There is NO
+warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ +## gcc-11 +`^.*([0-9]+(\\.[0-9]+)+).*` + +`gcc-11 --version` + +gcc-11 (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0
+Copyright (C) 2021 Free Software Foundation, Inc.
+This is free software; see the source for copying conditions. There is NO
+warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ + +## libgflags-dev +`([\d.]+)` + +`pkg-config --modversion gflags` + +2.2.2 + +## libglog-dev +`([\d.]+)` + +`pkg-config --modversion libglog` + +0.4.0 + +## libboost-all-dev +`([0-9]+(\w.[0-9]+)+)` + +`dpkg -s libboost-dev | grep 'Version'` + +Version: 1.74.0.3ubuntu7 + + +## libpng-dev +`([\d.]+)` + +`pkg-config --modversion libpng` + +1.6.37 + +## libre2-dev +`([\d.]+)` + +`pkg-config --modversion libre2` + +0.0.0 + +## libpci-dev +`([\d.]+)` + +`pkg-config --modversion libpci` + +3.7.0 + + +## libreadline_dev +`([\d.]+)` + +`pkg-config --modversion readline` + +8.1 + +## zlib +`([\d.]+)` + +`pkg-config --modversion zlib` + +1.2.11 + + +## libsqlite3_dev +`([\d.]+)` + +`pkg-config --modversion sqlite3` + +3.37.2 + +## libssl_dev +`OpenSSL\s+([\d.]+)` + +`openssl version` + +OpenSSL 3.0.2 15 Mar 2022 (Library: OpenSSL 3.0.2 15 Mar 2022) + +## libudev-dev +`([\d.]+)` + +`pkg-config --modversion libudev` + +249 + + +## libbz2_dev +`Version ([A-Za-z0-9]+(\.[A-Za-z0-9]+)+)` + +`bzcat --version` + +bzip2, a block-sorting file compressor. Version 1.0.8, 13-Jul-2019. + +## libev_dev +dpkg here should be fine as only apt install is supported +`Version ([A-Za-z0-9]+(\.[A-Za-z0-9]+)+)` + +`dpkg -s libev-dev | grep 'Version'` + +Version: 1:4.33-1 + +## libffi-dev +`([\d.]+)` + +`pkg-config --modversion libffi` + +3.4.2 + +## libffi_dev +`([\d.]+)` + +`pkg-config --modversion libffi` + +3.4.2 + +## libffi7 +`\d\.\d-[0-9]+` + +`dpkg -l libffi7 2>/dev/null | grep '^ii' | awk '{print $3}' || rpm -q libffi7 2>/dev/null || pacman -Q libffi7 2>/dev/null` + +3.3-5ubuntu1 + +## libffi8 +`\d\.\d\.\d-\d` + +`pkg-config --modversion libffi8"` + +3.4.2-4 + +## libgdbm_dev +dpkg here should be fine as only apt install is supported +`dpkg -s libgdbm-dev | grep 'Version'` + +`([\d]+\.[\d\.-]+)` + +Version: 1.23-1 + + +## libgmock +`([\d.]+)` + +`pkg-config --modversion libgmock` + +1.11.0 + +## liblzma_dev +`[A-Za-z]+\s\d\.\d\.\d` + +`xz --version` + +xz (XZ Utils) 5.2.5 +liblzma 5.2.5 + + +## libmpfr_dev +`([\d.]+)` + +`pkg-config --modversion mpfr` + +`4.1.0` + +## libncurses_dev +`([0-9]+(\.[0-9]+)+)` + +`ncurses5-config --version` + +6.3.20211021 + + + +## ninja-build +`([\d.]+)` + +`ninja --version` + +1.11.1 + +## md5sha1sum +`md5sum \(GNU coreutils\) ([\d.]+)` + +`md5sum --version` or `sha1sum --version` + +md5sum (GNU coreutils) 9.5 + +sha1sum (GNU coreutils) 9.5 + + +## nlohmann-json3-dev +`([\d.]+)` + +`pkg-config --modversion nlohmann_json` + +`3.10.5` + +## ntpdate +`([A-Za-z0-9]+(\.[A-Za-z0-9]+)+)` + +`dpkg -l ntpdate 2>/dev/null | grep ^ii | awk '{print $3}'` + +1:4.2.8p15+dfsg-1ubuntu2 + +## nvidia-cuda-toolkit +`release ([\d.]+)` + +`nvcc --version` + +nvcc: NVIDIA (R) Cuda compiler driver
+Copyright (c) 2005-2021 NVIDIA Corporation
+Built on Thu_Nov_18_09:45:25_PST_2021
+Cuda compilation tools, release 11.5, V11.5.119
+Build cuda_11.5.r11.5/compiler.30672275_0
+ + +## psmisc +`\(PSmisc\) ([\d.]+)` + +`pstree --version` + +pstree (PSmisc) 23.4 + +## rapidjson-dev +`([\d.]+)` + +`pkg-config --modversion RapidJSON` + +1.1.0 + +## cmake +`cmake version ([\d.]+)` + +`cmake --version` + +cmake version 3.30.4 + +## libnuma-dev +`([\d.]+)` + +`pkg-config --modversion numa` + +2.0.14 + + +## numactl +`([\d.]+)` + +`pkg-config --modversion numa` + +2.0.14 + +## wget +`Wget\s*([\d.]+)` + +`wget --version` + +GNU Wget 1.21.2 built on linux-gnu. + +## screen +`Screen version ([\d.]+)` + +`screen --version` + +Screen version 4.00.020 (FAU) 23-Oct-06 + +## xz +`xz \(XZ Utils\) ([\d.]+)` + +`xz --version` + +xz (XZ Utils) 5.2.5 +liblzma 5.2.5 + +## VIM +`VIM - Vi IMproved ([\d.]+` + +`vim --version` + +VIM - Vi IMproved 9.0 (2022 Jun 28, compiled Aug 3 2024 14:50:46) + +## rsync +`rsync\s+version\s+([\d.]+)` + +`rsync --version` + +rsync version 3.2.7 protocol version 31 + +## sox +`sox:\s+SoX\s+v([\d.]+)` + +`sox --version` + +sox: SoX v14.4.2 + + +## systemd +`systemd ([\d]+)` + +`systemctl --version` + +systemd 249 (249.11-0ubuntu3.12) + +## tk-dev +Probably fine to use `dpkg` here as only installation supported is for ubuntu + +`([0-9]+(\.[0-9]+)+)` + +`dpkg -s tk-dev | grep Version` + +Version: 8.6.11+1build2 + + +## transmission +`transmission-daemon ([\d.]+)` + +`transmission-daemon --version` + +transmission-daemon 3.00 (bb6b5a062e) + + +## wkhtmltopdf +`wkhtmltopdf ([\d.]+)` + +`wkhtmltopdf --version` + +wkhtmltopdf 0.12.6 + +## systemd +`systemd ([\d]+)` + +`systemd --version` + +systemd 255 (255.4-1ubuntu8.4) + + +## dmidecode +`([\d.]+)` + +`dmidecode --version` + +3.3 + +## git-lfs +`git-lfs/([\d.]+)` + +`git-lfs --version` + +git-lfs/3.4.1 (GitHub; linux arm64; go 1.22.2) + +## zlib1g +`([\d.]+)` + +`pkg-config --modversion zlib` + +1.2.11 + +## zlib1g_dev +`([\d.]+)` + +`pkg-config --modversion zlib` + +1.2.11 \ No newline at end of file diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index a1f9d09918..7826a681b9 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -64,7 +64,9 @@ }, "cmake": { "env": { - "CM_SYS_UTIL_NAME": "cmake" + "CM_SYS_UTIL_NAME": "cmake", + "CM_SYS_UTIL_VERSION_CMD": "cmake --version", + "CM_SYS_UTIL_VERSION_RE": "cmake version ([\\d.]+)" }, "state": { "cmake": { @@ -73,11 +75,14 @@ "dnf": "cmake", "yum": "cmake" } - } + }, + "new_env_keys": ["CM_CMAKE_VERSION"] }, "dmidecode": { "env": { - "CM_SYS_UTIL_NAME": "dmidecode" + "CM_SYS_UTIL_NAME": "dmidecode", + "CM_SYS_UTIL_VERSION_CMD": "dmidecode --version", + "CM_SYS_UTIL_VERSION_RE": "version\\s+([\\d.]+)" }, "state": { "dmidecode": { @@ -86,64 +91,82 @@ "dnf": "dmidecode", "yum": "dmidecode" } - } + }, + "new_env_keys": ["CM_DMIDECODE_VERSION"] }, "g++-11": { "env": { - "CM_SYS_UTIL_NAME": "g++11" + "CM_SYS_UTIL_NAME": "g++11", + "CM_SYS_UTIL_VERSION_CMD": "g++-11 --version", + "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*" }, "state": { "g++11": { "apt": "g++-11", "dnf": "gcc-toolset-11-gcc-c++" } - } + }, + "new_env_keys": ["CM_GPP11_VERSION"] }, "g++-12": { "env": { - "CM_SYS_UTIL_NAME": "g++12" + "CM_SYS_UTIL_NAME": "g++12", + "CM_SYS_UTIL_VERSION_CMD": "g++-12 --version", + "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*" }, "state": { "g++12": { "apt": "g++-12", "dnf": "gcc-toolset-12-gcc-c++" } - } + }, + "new_env_keys": ["CM_GPP12_VERSION"] }, "g++-9": { "env": { - "CM_SYS_UTIL_NAME": "g++9" + "CM_SYS_UTIL_NAME": "g++9", + "CM_SYS_UTIL_VERSION_CMD": "g++-9 --version", + "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*" }, "state": { "g++9": { "apt": "g++-9", "dnf": "gcc-toolset-9-gcc-c++" } - } + }, + "new_env_keys": ["CM_GPP9_VERSION"] }, "gcc-11": { "env": { - "CM_SYS_UTIL_NAME": "gcc11" + "CM_SYS_UTIL_NAME": "gcc11", + "CM_SYS_UTIL_VERSION_CMD": "gcc-11 --version", + "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*" }, "state": { "gcc11": { "apt": "gcc-11" } - } + }, + "new_env_keys": ["CM_GCC11_VERSION"] }, "gcc-9": { "env": { - "CM_SYS_UTIL_NAME": "gcc9" + "CM_SYS_UTIL_NAME": "gcc9", + "CM_SYS_UTIL_VERSION_CMD": "gcc-9 --version", + "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*" }, "state": { "gcc9": { "apt": "gcc-9" } - } + }, + "new_env_keys": ["CM_GCC9_VERSION"] }, "gflags-dev": { "env": { - "CM_SYS_UTIL_NAME": "gflags-dev" + "CM_SYS_UTIL_NAME": "gflags-dev", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion gflags", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "gflags-dev": { @@ -152,12 +175,14 @@ "dnf": "gflags-devel", "yum": "gflags-devel" } - } + }, + "new_env_keys": ["CM_GFLAGS_DEV_VERSION"] }, "git-lfs": { "env": { "CM_SYS_UTIL_NAME": "git-lfs", - "CM_SYS_UTIL_CHECK_CMD": "git-lfs --version" + "CM_SYS_UTIL_VERSION_CMD": "git-lfs --version", + "CM_SYS_UTIL_VERSION_RE": "git-lfs\\/([\\d.]+)" }, "state": { "git-lfs": { @@ -166,11 +191,14 @@ "dnf": "git-lfs", "yum": "git-lfs" } - } + }, + "new_env_keys": ["CM_GIT_LFS_VERSION"] }, "glog-dev": { "env": { - "CM_SYS_UTIL_NAME": "glog-dev" + "CM_SYS_UTIL_NAME": "glog-dev", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libglog", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "glog-dev": { @@ -179,11 +207,14 @@ "dnf": "glog-devel", "yum": "glog-devel" } - } + }, + "new_env_keys": ["CM_GLOG_DEV_VERSION"] }, "libboost-all-dev": { "env": { - "CM_SYS_UTIL_NAME": "libboost-all-dev" + "CM_SYS_UTIL_NAME": "libboost-all-dev", + "CM_SYS_UTIL_VERSION_CMD": "dpkg -s libboost-dev | grep 'Version'", + "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)" }, "state": { "libboost-all-dev": { @@ -192,11 +223,14 @@ "dnf": "boost-devel", "yum": "boost-devel" } - } + }, + "new_env_keys": ["CM_LIBBOOST_ALL_DEV_VERSION"] }, "libbz2-dev": { "env": { - "CM_SYS_UTIL_NAME": "libbz2_dev" + "CM_SYS_UTIL_NAME": "libbz2_dev", + "CM_SYS_UTIL_VERSION_CMD": "bzcat --version", + "CM_SYS_UTIL_VERSION_RE": "Version ([A-Za-z0-9]+(\\.[A-Za-z0-9]+)+)" }, "state": { "libbz2_dev": { @@ -206,31 +240,40 @@ "yum": "libbzip2-devel", "zlib-devel": "libbz2-devel" } - } + }, + "new_env_keys": ["CM_LIBBZ2_DEV_VERSION"] }, "libev-dev": { "env": { - "CM_SYS_UTIL_NAME": "libev_dev" + "CM_SYS_UTIL_NAME": "libev_dev", + "CM_SYS_UTIL_VERSION_CMD": "dpkg -s libev-dev | grep 'Version'", + "CM_SYS_UTIL_VERSION_RE": "([\\d:]+\\.[\\d\\.-]+)" }, "state": { "libev_dev": { "apt": "libev-dev" } - } + }, + "new_env_keys": ["CM_LIBEV_DEV_VERSION"] }, "libffi": { "env": { - "CM_SYS_UTIL_NAME": "libffi" + "CM_SYS_UTIL_NAME": "libffi", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libffi", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "libffi7": { "apt": "libffi" } - } + }, + "new_env_keys": ["CM_LIBFFI_VERSION"] }, "libffi-dev": { "env": { - "CM_SYS_UTIL_NAME": "libffi_dev" + "CM_SYS_UTIL_NAME": "libffi_dev", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libffi", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "libffi_dev": { @@ -239,41 +282,53 @@ "dnf": "libffi-devel", "yum": "libffi-devel" } - } + }, + "new_env_keys": ["CM_LIBFFI_DEV_VERSION"] }, "libffi7": { "env": { - "CM_SYS_UTIL_NAME": "libffi7" + "CM_SYS_UTIL_NAME": "libffi7", + "CM_SYS_UTIL_VERSION_CMD": "dpkg -l libffi7 2>/dev/null | grep '^ii' | awk '{print $3}' || rpm -q libffi7 2>/dev/null || pacman -Q libffi7 2>/dev/null", + "CM_SYS_UTIL_VERSION_RE": "\\d\\.\\d-[0-9]+" }, "state": { "libffi7": { "apt": "libffi7" } - } + }, + "new_env_keys": ["CM_LIBFFI7_VERSION"] }, "libffi8": { "env": { - "CM_SYS_UTIL_NAME": "libffi8" + "CM_SYS_UTIL_NAME": "libffi8", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libffi8", + "CM_SYS_UTIL_VERSION_RE": "\\d\\.\\d\\.\\d-\\d" }, "state": { "libffi8": { "apt": "libffi8" } - } + }, + "new_env_keys": ["CM_LIBFFI8_VERSION"] }, "libgdbm-dev": { "env": { - "CM_SYS_UTIL_NAME": "libgdbm_dev" + "CM_SYS_UTIL_NAME": "libgdbm_dev", + "CM_SYS_UTIL_VERSION_CMD": "dpkg -s libgdbm-dev | grep 'Version'", + "CM_SYS_UTIL_VERSION_RE": "([\\d]+\\.[\\d\\.-]+)" }, "state": { "libgdbm_dev": { "apt": "libgdbm-dev" } - } + }, + "new_env_keys": ["CM_LIBGDBM_DEV_VERSION"] }, "libgmock-dev": { "env": { - "CM_SYS_UTIL_NAME": "libgmock-dev" + "CM_SYS_UTIL_NAME": "libgmock-dev", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion gmock", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "libgmock-dev": { @@ -282,21 +337,27 @@ "dnf": "gmock-devel", "yum": "gmock-devel" } - } + }, + "new_env_keys": ["CM_LIBGMOCK_DEV_VERSION"] }, "liblzma-dev": { "env": { - "CM_SYS_UTIL_NAME": "liblzma_dev" + "CM_SYS_UTIL_NAME": "liblzma_dev", + "CM_SYS_UTIL_VERSION_CMD": "xz --version", + "CM_SYS_UTIL_VERSION_RE": "[A-Za-z]+\\s\\d\\.\\d\\.\\d" }, "state": { "liblzma_dev": { "apt": "liblzma-dev" } - } + }, + "new_env_keys": ["CM_LIBLZMA_DEV_VERSION"] }, "libmkl-dev": { "env": { - "CM_SYS_UTIL_NAME": "libmkl-dev" + "CM_SYS_UTIL_NAME": "libmkl-dev", + "CM_SYS_UTIL_VERSION_CMD": "ldconfig -p | grep libmkl", + "CM_SYS_UTIL_VERSION_RE": "libmkl[^ ]+ => /[^ ]+/libmkl[^ ]+\\.so\\.([\\d.]+)" }, "state": { "libmkl-dev": { @@ -305,11 +366,14 @@ "dnf": "", "yum": "" } - } + }, + "new_env_keys": ["CM_LIBMKL_DEV_VERSION"] }, "libmpfr-dev": { "env": { - "CM_SYS_UTIL_NAME": "libmpfr-dev" + "CM_SYS_UTIL_NAME": "libmpfr-dev", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion mpfr", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "libmpfr-dev": { @@ -319,11 +383,14 @@ "yum": "mpfr-devel.x86_64", "zypper": "mpfr-devel" } - } + }, + "new_env_keys": ["CM_LIBMPFR_DEV_VERSION"] }, "libncurses-dev": { "env": { - "CM_SYS_UTIL_NAME": "libncurses_dev" + "CM_SYS_UTIL_NAME": "libncurses_dev", + "CM_SYS_UTIL_VERSION_CMD": "ncurses5-config --version", + "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)" }, "state": { "libncurses_dev": { @@ -331,11 +398,14 @@ "dnf": "libncurses-devel", "yum": "libncurses-devel" } - } + }, + "new_env_keys": ["CM_LIBNCURSES_DEV_VERSION"] }, "libnuma-dev": { "env": { - "CM_SYS_UTIL_NAME": "libnuma-dev" + "CM_SYS_UTIL_NAME": "libnuma-dev", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion numa", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "libnuma-dev": { @@ -344,11 +414,14 @@ "dnf": "numactl-libs", "yum": "numactl-libs" } - } + }, + "new_env_keys": ["CM_LIBNUMA_DEV_VERSION"] }, "libpci-dev": { "env": { - "CM_SYS_UTIL_NAME": "libpci-dev" + "CM_SYS_UTIL_NAME": "libpci-dev", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libpci", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "libpci-dev": { @@ -357,11 +430,14 @@ "dnf": "pciutils-devel", "yum": "pciutils-devel" } - } + }, + "new_env_keys": ["CM_LIBPCI_DEV_VERSION"] }, "libpng-dev": { "env": { - "CM_SYS_UTIL_NAME": "libpng-dev" + "CM_SYS_UTIL_NAME": "libpng-dev", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libpng", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "libpng-dev": { @@ -370,11 +446,14 @@ "dnf": "libpng-devel", "yum": "libpng-devel" } - } + }, + "new_env_keys": ["CM_LIBPNG_DEV_VERSION"] }, "libre2-dev": { "env": { - "CM_SYS_UTIL_NAME": "libre2-dev" + "CM_SYS_UTIL_NAME": "libre2-dev", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion re2", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "libre2-dev": { @@ -383,33 +462,42 @@ "dnf": "libre-devel", "yum": "libre-devel" } - } + }, + "new_env_keys": ["CM_LIBRE2_DEV_VERSION"] }, "libreadline-dev": { "env": { - "CM_SYS_UTIL_NAME": "libreadline_dev" + "CM_SYS_UTIL_NAME": "libreadline_dev", + "CM_SYS_UTIL_VERSION_CMD": "readline --version", + "CM_SYS_UTIL_VERSION_RE": "readline, version ([\\d.]+)" }, "state": { "libreadline_dev": { "apt": "libreadline-dev", "dnf": "libreadline-devel", - "yum": "lireadline-devel" + "yum": "readline-devel" } - } + }, + "new_env_keys": ["CM_LIBREADLINE_DEV_VERSION"] }, "libsqlite3-dev": { "env": { - "CM_SYS_UTIL_NAME": "libsqlite3_dev" + "CM_SYS_UTIL_NAME": "libsqlite3_dev", + "CM_SYS_UTIL_VERSION_CMD": "sqlite3 --version", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "libsqlite3_dev": { "apt": "libsqlite3-dev" } - } + }, + "new_env_keys": ["CM_LIBSQLITE3_DEV_VERSION"] }, "libssl-dev": { "env": { - "CM_SYS_UTIL_NAME": "libssl_dev" + "CM_SYS_UTIL_NAME": "libssl_dev", + "CM_SYS_UTIL_VERSION_CMD": "openssl version", + "CM_SYS_UTIL_VERSION_RE": "OpenSSL\\s+([\\d.]+)" }, "state": { "libssl_dev": { @@ -418,11 +506,14 @@ "dnf": "libssl-devel", "yum": "libssl-devel" } - } + }, + "new_env_keys": ["CM_LIBSSL_DEV_VERSION"] }, "libudev-dev": { "env": { - "CM_SYS_UTIL_NAME": "libudev-dev" + "CM_SYS_UTIL_NAME": "libudev-dev", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libudev", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "libudev-dev": { @@ -431,7 +522,8 @@ "dnf": "libudev-devl", "yum": "libudev-devel" } - } + }, + "new_env_keys": ["CM_LIBUDEV_DEV_VERSION"] }, "linux-tools": { "deps": [ @@ -440,28 +532,36 @@ } ], "env": { - "CM_SYS_UTIL_NAME": "linux-tools" + "CM_SYS_UTIL_NAME": "linux-tools", + "CM_SYS_UTIL_VERSION_CMD": "linux-tools --version", + "CM_SYS_UTIL_VERSION_RE": "Linux Tools version ([\\d.]+)" }, "state": { "linux-tools": { "apt": "linux-tools-<<>>" } - } + }, + "new_env_keys": ["CM_LINUX_TOOLS_VERSION"] }, "md5sha1sum": { "env": { - "CM_SYS_UTIL_NAME": "md5sha1sum" + "CM_SYS_UTIL_NAME": "md5sha1sum", + "CM_SYS_UTIL_VERSION_CMD": "md5sum --version", + "CM_SYS_UTIL_VERSION_RE": "md5sum \\(GNU coreutils\\) ([\\d.]+)" }, "state": { "md5sha1sum": { "apt": "", "brew": "md5sha1sum" } - } + }, + "new_env_keys": ["CM_MD5SHA1SUM_VERSION"] }, "ninja-build": { "env": { - "CM_SYS_UTIL_NAME": "ninja-build" + "CM_SYS_UTIL_NAME": "ninja-build", + "CM_SYS_UTIL_VERSION_CMD": "ninja --version", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "ninja-build": { @@ -471,22 +571,28 @@ "yum": "ninja-build", "zypper": "ninja-build" } - } + }, + "new_env_keys": ["CM_NINJA_BUILD_VERSION"] }, "nlohmann-json3-dev": { "env": { - "CM_SYS_UTIL_NAME": "nlohmann_json3_dev" + "CM_SYS_UTIL_NAME": "nlohmann_json3_dev", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion nlohmann_json", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "nlohmann_json3_dev": { "apt": "nlohmann-json3-dev", - "dnf": "nlohmann-json-dev" + "dnf": "nlohmann-json-devel" } - } + }, + "new_env_keys": ["CM_NLOHMANN_JSON3_DEV_VERSION"] }, "ntpdate": { "env": { - "CM_SYS_UTIL_NAME": "ntpdate" + "CM_SYS_UTIL_NAME": "ntpdate", + "CM_SYS_UTIL_VERSION_CMD": "dpkg -l ntpdate 2>/dev/null | grep ^ii | awk '{print $3}'", + "CM_SYS_UTIL_VERSION_RE": "([A-Za-z0-9]+(\\.[A-Za-z0-9]+)+)" }, "state": { "ntpdate": { @@ -495,7 +601,8 @@ "dnf": "ntpdate", "yum": "ntpdate" } - } + }, + "new_env_keys": ["CM_NTPDATE_VERSION"] }, "numactl": { "deps": [ @@ -514,7 +621,9 @@ } ], "env": { - "CM_SYS_UTIL_NAME": "numactl" + "CM_SYS_UTIL_NAME": "numactl", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion numa", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "numactl": { @@ -522,11 +631,14 @@ "dnf": "numactl-devel", "yum": "numactl-devel" } - } + }, + "new_env_keys": ["CM_NUMACTL_VERSION"] }, "nvidia-cuda-toolkit": { "env": { - "CM_SYS_UTIL_NAME": "nvidia-cuda-toolkit" + "CM_SYS_UTIL_NAME": "nvidia-cuda-toolkit", + "CM_SYS_UTIL_VERSION_CMD": "nvcc --version", + "CM_SYS_UTIL_VERSION_RE": "release ([\\d.]+)" }, "state": { "nvidia-cuda-toolkit": { @@ -535,11 +647,14 @@ "dnf": "nvidia-cuda-toolkit", "yum": "nvidia-cuda-toolkit" } - } + }, + "new_env_keys": ["CM_NVIDIA_CUDA_TOOLKIT_VERSION"] }, "psmisc": { "env": { - "CM_SYS_UTIL_NAME": "psmisc" + "CM_SYS_UTIL_NAME": "psmisc", + "CM_SYS_UTIL_VERSION_CMD": "pstree --version", + "CM_SYS_UTIL_VERSION_RE": "pstree \\(PSmisc\\) ([\\d.]+)" }, "state": { "psmisc": { @@ -548,11 +663,14 @@ "dnf": "psmisc", "yum": "psmisc" } - } + }, + "new_env_keys": ["CM_PSMISC_VERSION"] }, "rapidjson-dev": { "env": { - "CM_SYS_UTIL_NAME": "rapidjson-dev" + "CM_SYS_UTIL_NAME": "rapidjson-dev", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion RapidJSON", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "rapidjson-dev": { @@ -561,12 +679,14 @@ "dnf": "rapidjson-devel", "yum": "rapidjson-devel" } - } + }, + "new_env_keys": ["CM_RAPIDJSON_DEV_VERSION"] }, "rsync": { "env": { "CM_SYS_UTIL_NAME": "rsync", - "CM_SYS_UTIL_CHECK_CMD": "rsync --version" + "CM_SYS_UTIL_VERSION_CMD": "rsync --version", + "CM_SYS_UTIL_VERSION_RE": "rsync\\s+version\\s+([\\d.]+)" }, "state": { "rsync": { @@ -576,11 +696,14 @@ "yum": "rsync", "zypper": "rsync" } - } + }, + "new_env_keys": ["CM_RSYNC_VERSION"] }, "screen": { "env": { - "CM_SYS_UTIL_NAME": "screen" + "CM_SYS_UTIL_NAME": "screen", + "CM_SYS_UTIL_VERSION_CMD": "screen --version", + "CM_SYS_UTIL_VERSION_RE": "Screen version ([\\d.]+)" }, "state": { "screen": { @@ -590,11 +713,14 @@ "yum": "screen", "zypper": "rsync" } - } + }, + "new_env_keys": ["CM_SCREEN_VERSION"] }, "sox": { "env": { - "CM_SYS_UTIL_NAME": "sox" + "CM_SYS_UTIL_NAME": "sox", + "CM_SYS_UTIL_VERSION_CMD": "sox --version", + "CM_SYS_UTIL_VERSION_RE": "sox:\\s+SoX\\s+v([\\d.]+)" }, "state": { "sox": { @@ -602,11 +728,14 @@ "brew": "sox", "dnf": "sox" } - } + }, + "new_env_keys": ["CM_SOX_VERSION"] }, "systemd": { "env": { - "CM_SYS_UTIL_NAME": "systemd" + "CM_SYS_UTIL_NAME": "systemd", + "CM_SYS_UTIL_VERSION_CMD": "systemctl --version", + "CM_SYS_UTIL_VERSION_RE": "systemd ([\\d]+)" }, "state": { "systemd": { @@ -615,21 +744,27 @@ "dnf": "systemd", "yum": "systemd" } - } + }, + "new_env_keys": ["CM_SYSTEMD_VERSION"] }, "tk-dev": { "env": { - "CM_SYS_UTIL_NAME": "tk_dev" + "CM_SYS_UTIL_NAME": "tk_dev", + "CM_SYS_UTIL_VERSION_CMD": "dpkg -s tk-dev | grep Version", + "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)" }, "state": { "tk_dev": { "apt": "tk-dev" } - } + }, + "new_env_keys": ["CM_TK_DEV_VERSION"] }, "transmission": { "env": { - "CM_SYS_UTIL_NAME": "transmission" + "CM_SYS_UTIL_NAME": "transmission", + "CM_SYS_UTIL_VERSION_CMD": "transmission-daemon --version", + "CM_SYS_UTIL_VERSION_RE": "transmission-daemon ([\\d.]+)" }, "state": { "transmission": { @@ -638,7 +773,8 @@ "dnf": "transmission-daemon", "yum": "transmission-daemon" } - } + }, + "new_env_keys": ["CM_TRANSMISSION_VERSION"] }, "wget": { "env": { @@ -659,19 +795,22 @@ "wkhtmltopdf": { "env": { "CM_SYS_UTIL_NAME": "wkhtmltopdf", - "CM_SYS_UTIL_CHECK_CMD": "wkhtmltopdf --version" + "CM_SYS_UTIL_VERSION_CMD": "wkhtmltopdf --version", + "CM_SYS_UTIL_VERSION_RE": "wkhtmltopdf ([\\d.]+)" }, "state": { "wkhtmltopdf": { "apt": "wkhtmltopdf", "brew": "wkhtmltopdf" } - } + }, + "new_env_keys": ["CM_WKHTMLTOPDF_VERSION"] }, "vim-common": { "env": { "CM_SYS_UTIL_NAME": "vim_common", - "CM_SYS_UTIL_CHECK_CMD": "xxd --version" + "CM_SYS_UTIL_VERSION_CMD": "vim --version", + "CM_SYS_UTIL_VERSION_RE": "VIM - Vi IMproved ([\\d.]+)" }, "state": { "vim_common": { @@ -681,11 +820,14 @@ "yum": "vim-common", "choco": "vim" } - } + }, + "new_env_keys": ["CM_VIM_COMMON_VERSION"] }, "xz": { "env": { - "CM_SYS_UTIL_NAME": "xz" + "CM_SYS_UTIL_NAME": "xz", + "CM_SYS_UTIL_VERSION_CMD": "xz --version", + "CM_SYS_UTIL_VERSION_RE": "xz \\(XZ Utils\\) ([\\d.]+)" }, "state": { "xz": { @@ -695,22 +837,28 @@ "yum": "xz", "choco": "xz" } - } + }, + "new_env_keys": ["CM_XZ_VERSION"] }, "zlib": { "env": { - "CM_SYS_UTIL_NAME": "zlib" + "CM_SYS_UTIL_NAME": "zlib", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion zlib", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "zlib": { "apt": "zlib1g", "choco": "zlib" } - } + }, + "new_env_keys": ["CM_ZLIB_VERSION"] }, "zlib1g-dev": { "env": { - "CM_SYS_UTIL_NAME": "zlib1g_dev" + "CM_SYS_UTIL_NAME": "zlib1g_dev", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion zlib", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "zlib1g_dev": { @@ -719,7 +867,8 @@ "yum": "zlib-devel", "zypper": "zlib-devel" } - } + }, + "new_env_keys": ["CM_ZLIB1G_DEV_VERSION"] } } } From 23d692b7a3176498109fabd73574326dd1680ed1 Mon Sep 17 00:00:00 2001 From: Oseltamivir Date: Thu, 31 Oct 2024 23:06:52 +0800 Subject: [PATCH 02/56] - fixed regex for md5sum, pstree, and bzcat which caused failed tests --- script/get-generic-sys-util/_cm.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 39e1322d8e..a95d618b3b 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -237,7 +237,7 @@ "env": { "CM_SYS_UTIL_NAME": "libbz2_dev", "CM_SYS_UTIL_VERSION_CMD": "bzcat --version", - "CM_SYS_UTIL_VERSION_RE": "Version ([A-Za-z0-9]+(\\.[A-Za-z0-9]+)+)" + "CM_SYS_UTIL_VERSION_RE": "\\b\\d+\\.\\d+(?:\\.\\d+)?\\b" }, "state": { "libbz2_dev": { @@ -554,7 +554,7 @@ "env": { "CM_SYS_UTIL_NAME": "md5sha1sum", "CM_SYS_UTIL_VERSION_CMD": "md5sum --version", - "CM_SYS_UTIL_VERSION_RE": "md5sum \\(GNU coreutils\\) ([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "\\b\\d+\\.\\d+(?:\\.\\d+)?\\b" }, "state": { "md5sha1sum": { @@ -661,7 +661,7 @@ "env": { "CM_SYS_UTIL_NAME": "psmisc", "CM_SYS_UTIL_VERSION_CMD": "pstree --version", - "CM_SYS_UTIL_VERSION_RE": "pstree \\(PSmisc\\) ([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "\\b\\d+\\.\\d+(?:\\.\\d+)?\\b" }, "state": { "psmisc": { From 6723d2c2f69329cb84654e38c8670db7ee2a0b93 Mon Sep 17 00:00:00 2001 From: Oseltamvir Date: Thu, 31 Oct 2024 17:24:15 +0000 Subject: [PATCH 03/56] - fixed an annoying psmisc/pstree bug --- script/get-generic-sys-util/_cm.json | 12 ++++++------ script/get-generic-sys-util/customize.py | 23 +++++++++++++++-------- script/get-generic-sys-util/detect.sh | 9 ++++++++- 3 files changed, 29 insertions(+), 15 deletions(-) mode change 100644 => 100755 script/get-generic-sys-util/detect.sh diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index a95d618b3b..51fe526c2a 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -89,7 +89,7 @@ "env": { "CM_SYS_UTIL_NAME": "dmidecode", "CM_SYS_UTIL_VERSION_CMD": "dmidecode --version", - "CM_SYS_UTIL_VERSION_RE": "version\\s+([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "dmidecode": { @@ -411,8 +411,8 @@ "libnuma-dev": { "env": { "CM_SYS_UTIL_NAME": "libnuma-dev", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion numa", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_CMD": "dpkg -l | grep libnuma-dev", + "CM_SYS_UTIL_VERSION_RE": "\\b\\d+\\.\\d+(?:\\.\\d+)?\\b" }, "state": { "libnuma-dev": { @@ -629,8 +629,8 @@ ], "env": { "CM_SYS_UTIL_NAME": "numactl", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion numa", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_CMD": "dpkg -l | grep libnuma-dev", + "CM_SYS_UTIL_VERSION_RE": "\\b\\d+\\.\\d+(?:\\.\\d+)?\\b" }, "state": { "numactl": { @@ -661,7 +661,7 @@ "env": { "CM_SYS_UTIL_NAME": "psmisc", "CM_SYS_UTIL_VERSION_CMD": "pstree --version", - "CM_SYS_UTIL_VERSION_RE": "\\b\\d+\\.\\d+(?:\\.\\d+)?\\b" + "CM_SYS_UTIL_VERSION_RE": "pstree \\(PSmisc\\) (\\d+\\.\\d+(?:\\.\\d+)?)" }, "state": { "psmisc": { diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index 4996459351..5121708343 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -100,25 +100,32 @@ def preprocess(i): return {'return':0} + + def detect_version(i): env = i['env'] version_env_key = f"CM_{env['CM_SYS_UTIL_NAME'].upper()}_VERSION" version_check_re = env.get('CM_SYS_UTIL_VERSION_RE', '') + # Confirm that the regex pattern and file are present if version_check_re == '' or not os.path.exists("tmp-ver.out"): version = "undetected" - else: - r = i['automation'].parse_version({'match_text': version_check_re, - 'group_number': 1, - 'env_key': version_env_key, - 'which_env': env}) - if r['return'] >0: return r + r = i['automation'].parse_version({ + 'match_text': version_check_re, + 'group_number': 1, + 'env_key': version_env_key, + 'which_env': env, + }) + + if r['return'] > 0: + return r version = r['version'] + print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + + return {'return': 0, 'version': version} - print (i['recursion_spaces'] + ' Detected version: {}'.format(version)) - return {'return':0, 'version':version} def postprocess(i): env = i['env'] diff --git a/script/get-generic-sys-util/detect.sh b/script/get-generic-sys-util/detect.sh old mode 100644 new mode 100755 index 6f2e3759bd..42de864cde --- a/script/get-generic-sys-util/detect.sh +++ b/script/get-generic-sys-util/detect.sh @@ -1,7 +1,14 @@ #!/bin/bash if [[ -n "${CM_SYS_UTIL_VERSION_CMD}" ]]; then - ${CM_SYS_UTIL_VERSION_CMD} > tmp-ver.out + if [[ "${CM_SYS_UTIL_VERSION_CMD}" == *"pstree --version"* ]]; then + # Redirect both stdout and stderr to tmp-ver.out + ${CM_SYS_UTIL_VERSION_CMD} > tmp-ver.out 2>&1 + else + ${CM_SYS_UTIL_VERSION_CMD} > tmp-ver.out + fi + echo ${CM_SYS_UTIL_VERSION_CMD} + echo $PWD fi test $? -eq 0 || exit $? From 737024bd447cfdbc7644d9057ad092138b0eef03 Mon Sep 17 00:00:00 2001 From: Oseltamvir Date: Fri, 1 Nov 2024 08:00:45 +0000 Subject: [PATCH 04/56] - changed to use env var to detect stderr, fixed some regexes --- automation/script/module.py | 8 ++++++-- script/get-generic-sys-util/_cm.json | 12 +++++++----- script/get-generic-sys-util/customize.py | 2 +- script/get-generic-sys-util/detect.sh | 2 +- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index 64226f5742..4a1b91f233 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -8,7 +8,7 @@ # TBD: when we have bandwidth and resources, we should refactor it # and make it cleaner and simpler while keeping full backwards compatibility. # - +import re import os import logging @@ -3979,7 +3979,11 @@ def parse_version(self, i): return r string = r['string'] - + match = re.search(match_text, string) + if debug: + print(f"Regex Pattern: {match_text}") + print(f"Matched String: {string}") + print(f"Match Groups: {match.groups()}") version = r['match'].group(group_number) which_env[env_key] = version diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 51fe526c2a..345673d364 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -235,9 +235,10 @@ }, "libbz2-dev": { "env": { + "CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM": "yes", "CM_SYS_UTIL_NAME": "libbz2_dev", "CM_SYS_UTIL_VERSION_CMD": "bzcat --version", - "CM_SYS_UTIL_VERSION_RE": "\\b\\d+\\.\\d+(?:\\.\\d+)?\\b" + "CM_SYS_UTIL_VERSION_RE": "([\\d:]+\\.[\\d\\.-]+)" }, "state": { "libbz2_dev": { @@ -351,7 +352,7 @@ "env": { "CM_SYS_UTIL_NAME": "liblzma_dev", "CM_SYS_UTIL_VERSION_CMD": "xz --version", - "CM_SYS_UTIL_VERSION_RE": "[A-Za-z]+\\s\\d\\.\\d\\.\\d" + "CM_SYS_UTIL_VERSION_RE": "(\\d(\\.\\d)+)" }, "state": { "liblzma_dev": { @@ -475,8 +476,8 @@ "libreadline-dev": { "env": { "CM_SYS_UTIL_NAME": "libreadline_dev", - "CM_SYS_UTIL_VERSION_CMD": "readline --version", - "CM_SYS_UTIL_VERSION_RE": "readline, version ([\\d.]+)" + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion readline", + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { "libreadline_dev": { @@ -490,7 +491,7 @@ "libsqlite3-dev": { "env": { "CM_SYS_UTIL_NAME": "libsqlite3_dev", - "CM_SYS_UTIL_VERSION_CMD": "sqlite3 --version", + "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion sqlite3", "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" }, "state": { @@ -659,6 +660,7 @@ }, "psmisc": { "env": { + "CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM": "yes", "CM_SYS_UTIL_NAME": "psmisc", "CM_SYS_UTIL_VERSION_CMD": "pstree --version", "CM_SYS_UTIL_VERSION_RE": "pstree \\(PSmisc\\) (\\d+\\.\\d+(?:\\.\\d+)?)" diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index 5121708343..7480c77991 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -115,7 +115,7 @@ def detect_version(i): 'match_text': version_check_re, 'group_number': 1, 'env_key': version_env_key, - 'which_env': env, + 'which_env': env, #'debug': 1 }) if r['return'] > 0: diff --git a/script/get-generic-sys-util/detect.sh b/script/get-generic-sys-util/detect.sh index 42de864cde..dff0d91fd7 100755 --- a/script/get-generic-sys-util/detect.sh +++ b/script/get-generic-sys-util/detect.sh @@ -1,7 +1,7 @@ #!/bin/bash if [[ -n "${CM_SYS_UTIL_VERSION_CMD}" ]]; then - if [[ "${CM_SYS_UTIL_VERSION_CMD}" == *"pstree --version"* ]]; then + if [[ "${CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM}" == "yes" ]]; then # Redirect both stdout and stderr to tmp-ver.out ${CM_SYS_UTIL_VERSION_CMD} > tmp-ver.out 2>&1 else From ffa3966c10932a5868c5876b36fc50edc8c8cfc7 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 1 Nov 2024 14:55:14 +0000 Subject: [PATCH 05/56] Use raw string in build-docker-image --- script/build-docker-image/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/build-docker-image/customize.py b/script/build-docker-image/customize.py index e66eddbd8b..a3a3bc8df7 100644 --- a/script/build-docker-image/customize.py +++ b/script/build-docker-image/customize.py @@ -54,7 +54,7 @@ def preprocess(i): image_name = get_image_name(env) if build_dockerfile: - dockerfile_path = "\${CM_DOCKERFILE_WITH_PATH}" + dockerfile_path = r"\${CM_DOCKERFILE_WITH_PATH}" # Write .dockerignore with open('.dockerignore', 'w') as f: From 6d9d67571f2a5451e8d50f08ffb6c0b6ab06d0a6 Mon Sep 17 00:00:00 2001 From: Oseltamivir Date: Sun, 3 Nov 2024 00:58:31 +0800 Subject: [PATCH 06/56] Change numactl command to be more inclusive libnuma-dev => libnuma --- script/get-generic-sys-util/_cm.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 2246bc6a2b..fb1fd4c86c 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -412,7 +412,7 @@ "libnuma-dev": { "env": { "CM_SYS_UTIL_NAME": "libnuma-dev", - "CM_SYS_UTIL_VERSION_CMD": "dpkg -l | grep libnuma-dev", + "CM_SYS_UTIL_VERSION_CMD": "dpkg -l | grep libnuma", "CM_SYS_UTIL_VERSION_RE": "\\b\\d+\\.\\d+(?:\\.\\d+)?\\b" }, "state": { @@ -632,7 +632,7 @@ ], "env": { "CM_SYS_UTIL_NAME": "numactl", - "CM_SYS_UTIL_VERSION_CMD": "dpkg -l | grep libnuma-dev", + "CM_SYS_UTIL_VERSION_CMD": "dpkg -l | grep libnuma", "CM_SYS_UTIL_VERSION_RE": "\\b\\d+\\.\\d+(?:\\.\\d+)?\\b" }, "state": { From f1f084d71d588f1fa62b00242f929cafa3d01582 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 2 Nov 2024 23:26:26 +0530 Subject: [PATCH 07/56] Added sympy dependency for retinanet --- script/app-mlperf-inference-nvidia/_cm.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/script/app-mlperf-inference-nvidia/_cm.yaml b/script/app-mlperf-inference-nvidia/_cm.yaml index 3c3ea70733..a997ef8156 100644 --- a/script/app-mlperf-inference-nvidia/_cm.yaml +++ b/script/app-mlperf-inference-nvidia/_cm.yaml @@ -436,6 +436,7 @@ variations: - tags: get,generic-python-lib,_onnx-graphsurgeon - tags: get,generic-python-lib,_package.onnx version: 1.13.1 + - tags: get,generic-python-lib,_package.sympy sdxl: new_env_keys: From b9eb3d7db08e890b5b8e75759cfd8e65f758cf9c Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 2 Nov 2024 18:04:13 +0000 Subject: [PATCH 08/56] Update _cm.json | Fix libnuma re --- script/get-generic-sys-util/_cm.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index fb1fd4c86c..2579713152 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -413,7 +413,8 @@ "env": { "CM_SYS_UTIL_NAME": "libnuma-dev", "CM_SYS_UTIL_VERSION_CMD": "dpkg -l | grep libnuma", - "CM_SYS_UTIL_VERSION_RE": "\\b\\d+\\.\\d+(?:\\.\\d+)?\\b" + "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libnuma-dev": { From 3486ef92ab4488ec76ec5edd8d84f3a92c79a969 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 2 Nov 2024 18:45:50 +0000 Subject: [PATCH 09/56] Update default-config.yaml --- .../framework-version-default/default-config.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/script/get-mlperf-inference-sut-configs/configs/RTX4090x2/nvidia_original-implementation/gpu-device/tensorrt-framework/framework-version-default/default-config.yaml b/script/get-mlperf-inference-sut-configs/configs/RTX4090x2/nvidia_original-implementation/gpu-device/tensorrt-framework/framework-version-default/default-config.yaml index d2c66a716c..cffda0ea0c 100644 --- a/script/get-mlperf-inference-sut-configs/configs/RTX4090x2/nvidia_original-implementation/gpu-device/tensorrt-framework/framework-version-default/default-config.yaml +++ b/script/get-mlperf-inference-sut-configs/configs/RTX4090x2/nvidia_original-implementation/gpu-device/tensorrt-framework/framework-version-default/default-config.yaml @@ -12,9 +12,13 @@ bert-99: Offline: target_qps: 8000 + Server: + target_qps: 6000 bert-99.9: Offline: - target_qps: 8000 + target_qps: 3500 + Server: + target_qps: 3000 3d-unet-99: Offline: target_qps: 8.0 From f1010d615bc8e7ad25b9c708ffe04c3f66cae8cd Mon Sep 17 00:00:00 2001 From: Oseltamivir Date: Sun, 3 Nov 2024 02:52:49 +0800 Subject: [PATCH 10/56] pstree regex change to be more lax for brew version --- script/get-generic-sys-util/_cm.json | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 2579713152..ef849a3246 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -414,7 +414,7 @@ "CM_SYS_UTIL_NAME": "libnuma-dev", "CM_SYS_UTIL_VERSION_CMD": "dpkg -l | grep libnuma", "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libnuma-dev": { @@ -559,7 +559,6 @@ "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, - "new_env_keys": ["CM_MD5SHA1SUM_VERSION"], "state": { "md5sha1sum": { "apt": "", @@ -663,10 +662,10 @@ }, "psmisc": { "env": { - "CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM": "yes", + "CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM": "yes", "CM_SYS_UTIL_NAME": "psmisc", "CM_SYS_UTIL_VERSION_CMD": "pstree --version", - "CM_SYS_UTIL_VERSION_RE": "pstree \\(PSmisc\\) (\\d+\\.\\d+(?:\\.\\d+)?)" + "CM_SYS_UTIL_VERSION_RE": "\b\\d+\\.\\d+(?:\\.\\d+)?\\b" }, "state": { "psmisc": { From 8fef3edd4dda66661598a89500d075ce49be8756 Mon Sep 17 00:00:00 2001 From: Oseltamivir Date: Sun, 3 Nov 2024 18:35:54 +0800 Subject: [PATCH 11/56] Added CM_TMP_VERSION_DETECT_GROUP_NUMBER for all packages --- script/get-generic-sys-util/_cm.json | 151 ++++++++++++++++++--------- 1 file changed, 101 insertions(+), 50 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index ef849a3246..9f63ca943e 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -73,7 +73,8 @@ "env": { "CM_SYS_UTIL_NAME": "cmake", "CM_SYS_UTIL_VERSION_CMD": "cmake --version", - "CM_SYS_UTIL_VERSION_RE": "cmake version ([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "cmake version ([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "cmake": { @@ -89,7 +90,8 @@ "env": { "CM_SYS_UTIL_NAME": "dmidecode", "CM_SYS_UTIL_VERSION_CMD": "dmidecode --version", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "dmidecode": { @@ -105,7 +107,8 @@ "env": { "CM_SYS_UTIL_NAME": "g++11", "CM_SYS_UTIL_VERSION_CMD": "g++-11 --version", - "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*" + "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "g++11": { @@ -119,7 +122,8 @@ "env": { "CM_SYS_UTIL_NAME": "g++12", "CM_SYS_UTIL_VERSION_CMD": "g++-12 --version", - "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*" + "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "g++12": { @@ -133,7 +137,8 @@ "env": { "CM_SYS_UTIL_NAME": "g++9", "CM_SYS_UTIL_VERSION_CMD": "g++-9 --version", - "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*" + "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "g++9": { @@ -147,7 +152,8 @@ "env": { "CM_SYS_UTIL_NAME": "gcc11", "CM_SYS_UTIL_VERSION_CMD": "gcc-11 --version", - "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*" + "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "gcc11": { @@ -160,7 +166,8 @@ "env": { "CM_SYS_UTIL_NAME": "gcc9", "CM_SYS_UTIL_VERSION_CMD": "gcc-9 --version", - "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*" + "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "gcc9": { @@ -173,7 +180,9 @@ "env": { "CM_SYS_UTIL_NAME": "gflags-dev", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion gflags", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + }, "state": { "gflags-dev": { @@ -189,7 +198,8 @@ "env": { "CM_SYS_UTIL_NAME": "git-lfs", "CM_SYS_UTIL_VERSION_CMD": "git-lfs --version", - "CM_SYS_UTIL_VERSION_RE": "git-lfs\\/([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "git-lfs\\/([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "git-lfs": { @@ -205,7 +215,8 @@ "env": { "CM_SYS_UTIL_NAME": "glog-dev", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libglog", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "glog-dev": { @@ -221,7 +232,8 @@ "env": { "CM_SYS_UTIL_NAME": "libboost-all-dev", "CM_SYS_UTIL_VERSION_CMD": "dpkg -s libboost-dev | grep 'Version'", - "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)" + "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libboost-all-dev": { @@ -238,7 +250,8 @@ "CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM": "yes", "CM_SYS_UTIL_NAME": "libbz2_dev", "CM_SYS_UTIL_VERSION_CMD": "bzcat --version", - "CM_SYS_UTIL_VERSION_RE": "([\\d:]+\\.[\\d\\.-]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d:]+\\.[\\d\\.-]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libbz2_dev": { @@ -255,7 +268,8 @@ "env": { "CM_SYS_UTIL_NAME": "libev_dev", "CM_SYS_UTIL_VERSION_CMD": "dpkg -s libev-dev | grep 'Version'", - "CM_SYS_UTIL_VERSION_RE": "([\\d:]+\\.[\\d\\.-]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d:]+\\.[\\d\\.-]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libev_dev": { @@ -268,7 +282,8 @@ "env": { "CM_SYS_UTIL_NAME": "libffi", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libffi", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libffi": { @@ -281,7 +296,8 @@ "env": { "CM_SYS_UTIL_NAME": "libffi_dev", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libffi", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libffi_dev": { @@ -297,7 +313,8 @@ "env": { "CM_SYS_UTIL_NAME": "libffi7", "CM_SYS_UTIL_VERSION_CMD": "dpkg -l libffi7 2>/dev/null | grep '^ii' | awk '{print $3}' || rpm -q libffi7 2>/dev/null || pacman -Q libffi7 2>/dev/null", - "CM_SYS_UTIL_VERSION_RE": "\\d\\.\\d-[0-9]+" + "CM_SYS_UTIL_VERSION_RE": "\\d\\.\\d-[0-9]+", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libffi7": { @@ -310,7 +327,8 @@ "env": { "CM_SYS_UTIL_NAME": "libffi8", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libffi8", - "CM_SYS_UTIL_VERSION_RE": "\\d\\.\\d\\.\\d-\\d" + "CM_SYS_UTIL_VERSION_RE": "\\d\\.\\d\\.\\d-\\d", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libffi8": { @@ -323,7 +341,8 @@ "env": { "CM_SYS_UTIL_NAME": "libgdbm_dev", "CM_SYS_UTIL_VERSION_CMD": "dpkg -s libgdbm-dev | grep 'Version'", - "CM_SYS_UTIL_VERSION_RE": "([\\d]+\\.[\\d\\.-]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d]+\\.[\\d\\.-]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libgdbm_dev": { @@ -336,7 +355,8 @@ "env": { "CM_SYS_UTIL_NAME": "libgmock-dev", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion gmock", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libgmock-dev": { @@ -352,7 +372,8 @@ "env": { "CM_SYS_UTIL_NAME": "liblzma_dev", "CM_SYS_UTIL_VERSION_CMD": "xz --version", - "CM_SYS_UTIL_VERSION_RE": "(\\d(\\.\\d)+)" + "CM_SYS_UTIL_VERSION_RE": "(\\d(\\.\\d)+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "liblzma_dev": { @@ -365,7 +386,8 @@ "env": { "CM_SYS_UTIL_NAME": "libmkl-dev", "CM_SYS_UTIL_VERSION_CMD": "ldconfig -p | grep libmkl", - "CM_SYS_UTIL_VERSION_RE": "libmkl[^ ]+ => /[^ ]+/libmkl[^ ]+\\.so\\.([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "libmkl[^ ]+ => /[^ ]+/libmkl[^ ]+\\.so\\.([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "libmkl-dev": { @@ -381,7 +403,8 @@ "env": { "CM_SYS_UTIL_NAME": "libmpfr-dev", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion mpfr", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libmpfr-dev": { @@ -398,7 +421,8 @@ "env": { "CM_SYS_UTIL_NAME": "libncurses_dev", "CM_SYS_UTIL_VERSION_CMD": "ncurses5-config --version", - "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)" + "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libncurses_dev": { @@ -430,7 +454,8 @@ "env": { "CM_SYS_UTIL_NAME": "libpci-dev", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libpci", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libpci-dev": { @@ -446,7 +471,8 @@ "env": { "CM_SYS_UTIL_NAME": "libpng-dev", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libpng", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libpng-dev": { @@ -462,7 +488,8 @@ "env": { "CM_SYS_UTIL_NAME": "libre2-dev", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion re2", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libre2-dev": { @@ -478,7 +505,8 @@ "env": { "CM_SYS_UTIL_NAME": "libreadline_dev", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion readline", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libreadline_dev": { @@ -493,7 +521,8 @@ "env": { "CM_SYS_UTIL_NAME": "libsqlite3_dev", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion sqlite3", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libsqlite3_dev": { @@ -506,7 +535,8 @@ "env": { "CM_SYS_UTIL_NAME": "libssl_dev", "CM_SYS_UTIL_VERSION_CMD": "openssl version", - "CM_SYS_UTIL_VERSION_RE": "OpenSSL\\s+([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "OpenSSL\\s+([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "libssl_dev": { @@ -522,7 +552,8 @@ "env": { "CM_SYS_UTIL_NAME": "libudev-dev", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libudev", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "libudev-dev": { @@ -543,7 +574,8 @@ "env": { "CM_SYS_UTIL_NAME": "linux-tools", "CM_SYS_UTIL_VERSION_CMD": "linux-tools --version", - "CM_SYS_UTIL_VERSION_RE": "Linux Tools version ([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "Linux Tools version ([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "linux-tools": { @@ -571,7 +603,8 @@ "env": { "CM_SYS_UTIL_NAME": "ninja-build", "CM_SYS_UTIL_VERSION_CMD": "ninja --version", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "ninja-build": { @@ -588,7 +621,8 @@ "env": { "CM_SYS_UTIL_NAME": "nlohmann_json3_dev", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion nlohmann_json", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "nlohmann_json3_dev": { @@ -602,7 +636,8 @@ "env": { "CM_SYS_UTIL_NAME": "ntpdate", "CM_SYS_UTIL_VERSION_CMD": "dpkg -l ntpdate 2>/dev/null | grep ^ii | awk '{print $3}'", - "CM_SYS_UTIL_VERSION_RE": "([A-Za-z0-9]+(\\.[A-Za-z0-9]+)+)" + "CM_SYS_UTIL_VERSION_RE": "([A-Za-z0-9]+(\\.[A-Za-z0-9]+)+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "ntpdate": { @@ -633,7 +668,8 @@ "env": { "CM_SYS_UTIL_NAME": "numactl", "CM_SYS_UTIL_VERSION_CMD": "dpkg -l | grep libnuma", - "CM_SYS_UTIL_VERSION_RE": "\\b\\d+\\.\\d+(?:\\.\\d+)?\\b" + "CM_SYS_UTIL_VERSION_RE": "\\b\\d+\\.\\d+(?:\\.\\d+)?\\b", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "numactl": { @@ -648,7 +684,8 @@ "env": { "CM_SYS_UTIL_NAME": "nvidia-cuda-toolkit", "CM_SYS_UTIL_VERSION_CMD": "nvcc --version", - "CM_SYS_UTIL_VERSION_RE": "release ([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "release ([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "nvidia-cuda-toolkit": { @@ -665,7 +702,8 @@ "CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM": "yes", "CM_SYS_UTIL_NAME": "psmisc", "CM_SYS_UTIL_VERSION_CMD": "pstree --version", - "CM_SYS_UTIL_VERSION_RE": "\b\\d+\\.\\d+(?:\\.\\d+)?\\b" + "CM_SYS_UTIL_VERSION_RE": "\b\\d+\\.\\d+(?:\\.\\d+)?\\b", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "psmisc": { @@ -681,7 +719,8 @@ "env": { "CM_SYS_UTIL_NAME": "rapidjson-dev", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion RapidJSON", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "rapidjson-dev": { @@ -697,7 +736,8 @@ "env": { "CM_SYS_UTIL_NAME": "rsync", "CM_SYS_UTIL_VERSION_CMD": "rsync --version", - "CM_SYS_UTIL_VERSION_RE": "rsync\\s+version\\s+([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "rsync\\s+version\\s+([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "rsync": { @@ -714,7 +754,8 @@ "env": { "CM_SYS_UTIL_NAME": "screen", "CM_SYS_UTIL_VERSION_CMD": "screen --version", - "CM_SYS_UTIL_VERSION_RE": "Screen version ([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "Screen version ([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "screen": { @@ -731,7 +772,8 @@ "env": { "CM_SYS_UTIL_NAME": "sox", "CM_SYS_UTIL_VERSION_CMD": "sox --version", - "CM_SYS_UTIL_VERSION_RE": "sox:\\s+SoX\\s+v([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "sox:\\s+SoX\\s+v([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "sox": { @@ -746,7 +788,8 @@ "env": { "CM_SYS_UTIL_NAME": "systemd", "CM_SYS_UTIL_VERSION_CMD": "systemctl --version", - "CM_SYS_UTIL_VERSION_RE": "systemd ([\\d]+)" + "CM_SYS_UTIL_VERSION_RE": "systemd ([\\d]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "systemd": { @@ -762,7 +805,8 @@ "env": { "CM_SYS_UTIL_NAME": "tk_dev", "CM_SYS_UTIL_VERSION_CMD": "dpkg -s tk-dev | grep Version", - "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)" + "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "tk_dev": { @@ -775,7 +819,8 @@ "env": { "CM_SYS_UTIL_NAME": "transmission", "CM_SYS_UTIL_VERSION_CMD": "transmission-daemon --version", - "CM_SYS_UTIL_VERSION_RE": "transmission-daemon ([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "transmission-daemon ([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "transmission": { @@ -791,7 +836,8 @@ "env": { "CM_SYS_UTIL_NAME": "wget", "CM_SYS_UTIL_VERSION_CMD": "wget --version", - "CM_SYS_UTIL_VERSION_RE": "Wget\\s*([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "Wget\\s*([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "wget": { @@ -807,7 +853,8 @@ "env": { "CM_SYS_UTIL_NAME": "wkhtmltopdf", "CM_SYS_UTIL_VERSION_CMD": "wkhtmltopdf --version", - "CM_SYS_UTIL_VERSION_RE": "wkhtmltopdf ([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "wkhtmltopdf ([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "wkhtmltopdf": { @@ -821,7 +868,8 @@ "env": { "CM_SYS_UTIL_NAME": "vim_common", "CM_SYS_UTIL_VERSION_CMD": "vim --version", - "CM_SYS_UTIL_VERSION_RE": "VIM - Vi IMproved ([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "VIM - Vi IMproved ([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "vim_common": { @@ -838,7 +886,8 @@ "env": { "CM_SYS_UTIL_NAME": "xz", "CM_SYS_UTIL_VERSION_CMD": "xz --version", - "CM_SYS_UTIL_VERSION_RE": "xz \\(XZ Utils\\) ([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "xz \\(XZ Utils\\) ([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "xz": { @@ -855,7 +904,8 @@ "env": { "CM_SYS_UTIL_NAME": "zlib", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion zlib", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "zlib": { @@ -869,7 +919,8 @@ "env": { "CM_SYS_UTIL_NAME": "zlib1g_dev", "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion zlib", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)" + "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "zlib1g_dev": { From 7da8b3ec9038199237e3704e71295260549579de Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 14:25:18 +0000 Subject: [PATCH 12/56] Update _cm.json --- script/get-generic-sys-util/_cm.json | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 9f63ca943e..49ad809357 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -668,7 +668,7 @@ "env": { "CM_SYS_UTIL_NAME": "numactl", "CM_SYS_UTIL_VERSION_CMD": "dpkg -l | grep libnuma", - "CM_SYS_UTIL_VERSION_RE": "\\b\\d+\\.\\d+(?:\\.\\d+)?\\b", + "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { @@ -699,10 +699,10 @@ }, "psmisc": { "env": { - "CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM": "yes", + "CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM": "yes", "CM_SYS_UTIL_NAME": "psmisc", "CM_SYS_UTIL_VERSION_CMD": "pstree --version", - "CM_SYS_UTIL_VERSION_RE": "\b\\d+\\.\\d+(?:\\.\\d+)?\\b", + "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { @@ -737,7 +737,7 @@ "CM_SYS_UTIL_NAME": "rsync", "CM_SYS_UTIL_VERSION_CMD": "rsync --version", "CM_SYS_UTIL_VERSION_RE": "rsync\\s+version\\s+([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "rsync": { @@ -755,7 +755,7 @@ "CM_SYS_UTIL_NAME": "screen", "CM_SYS_UTIL_VERSION_CMD": "screen --version", "CM_SYS_UTIL_VERSION_RE": "Screen version ([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "screen": { @@ -773,7 +773,7 @@ "CM_SYS_UTIL_NAME": "sox", "CM_SYS_UTIL_VERSION_CMD": "sox --version", "CM_SYS_UTIL_VERSION_RE": "sox:\\s+SoX\\s+v([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "sox": { @@ -789,7 +789,7 @@ "CM_SYS_UTIL_NAME": "systemd", "CM_SYS_UTIL_VERSION_CMD": "systemctl --version", "CM_SYS_UTIL_VERSION_RE": "systemd ([\\d]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "systemd": { @@ -806,7 +806,7 @@ "CM_SYS_UTIL_NAME": "tk_dev", "CM_SYS_UTIL_VERSION_CMD": "dpkg -s tk-dev | grep Version", "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "tk_dev": { @@ -820,7 +820,7 @@ "CM_SYS_UTIL_NAME": "transmission", "CM_SYS_UTIL_VERSION_CMD": "transmission-daemon --version", "CM_SYS_UTIL_VERSION_RE": "transmission-daemon ([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "transmission": { @@ -837,7 +837,7 @@ "CM_SYS_UTIL_NAME": "wget", "CM_SYS_UTIL_VERSION_CMD": "wget --version", "CM_SYS_UTIL_VERSION_RE": "Wget\\s*([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "wget": { @@ -854,7 +854,7 @@ "CM_SYS_UTIL_NAME": "wkhtmltopdf", "CM_SYS_UTIL_VERSION_CMD": "wkhtmltopdf --version", "CM_SYS_UTIL_VERSION_RE": "wkhtmltopdf ([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "wkhtmltopdf": { @@ -869,7 +869,7 @@ "CM_SYS_UTIL_NAME": "vim_common", "CM_SYS_UTIL_VERSION_CMD": "vim --version", "CM_SYS_UTIL_VERSION_RE": "VIM - Vi IMproved ([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "vim_common": { From 53b1f7a87969464e1a475d5378d6444a841a9c57 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 14:30:59 +0000 Subject: [PATCH 13/56] Update _cm.json --- script/get-generic-sys-util/_cm.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 49ad809357..2ce5249c41 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -108,7 +108,7 @@ "CM_SYS_UTIL_NAME": "g++11", "CM_SYS_UTIL_VERSION_CMD": "g++-11 --version", "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "g++11": { @@ -123,7 +123,7 @@ "CM_SYS_UTIL_NAME": "g++12", "CM_SYS_UTIL_VERSION_CMD": "g++-12 --version", "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "g++12": { @@ -138,7 +138,7 @@ "CM_SYS_UTIL_NAME": "g++9", "CM_SYS_UTIL_VERSION_CMD": "g++-9 --version", "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "g++9": { @@ -153,7 +153,7 @@ "CM_SYS_UTIL_NAME": "gcc11", "CM_SYS_UTIL_VERSION_CMD": "gcc-11 --version", "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "gcc11": { @@ -199,7 +199,7 @@ "CM_SYS_UTIL_NAME": "git-lfs", "CM_SYS_UTIL_VERSION_CMD": "git-lfs --version", "CM_SYS_UTIL_VERSION_RE": "git-lfs\\/([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "git-lfs": { @@ -233,7 +233,7 @@ "CM_SYS_UTIL_NAME": "libboost-all-dev", "CM_SYS_UTIL_VERSION_CMD": "dpkg -s libboost-dev | grep 'Version'", "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "libboost-all-dev": { @@ -249,7 +249,7 @@ "env": { "CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM": "yes", "CM_SYS_UTIL_NAME": "libbz2_dev", - "CM_SYS_UTIL_VERSION_CMD": "bzcat --version", + "CM_SYS_UTIL_VERSION_CMD": "bzcat --version | grep Version", "CM_SYS_UTIL_VERSION_RE": "([\\d:]+\\.[\\d\\.-]+)", "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, From 9e3964226a195755307b17e112e6904e953c4873 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 14:34:14 +0000 Subject: [PATCH 14/56] Update _cm.json --- script/get-generic-sys-util/_cm.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 2ce5249c41..e6984d6ab6 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -250,8 +250,8 @@ "CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM": "yes", "CM_SYS_UTIL_NAME": "libbz2_dev", "CM_SYS_UTIL_VERSION_CMD": "bzcat --version | grep Version", - "CM_SYS_UTIL_VERSION_RE": "([\\d:]+\\.[\\d\\.-]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "libbz2_dev": { From 68596062b031f127b5df5283f035d09797ba2b05 Mon Sep 17 00:00:00 2001 From: Oseltamivir Date: Sun, 3 Nov 2024 23:02:17 +0800 Subject: [PATCH 15/56] change g++/gcc regex --- script/get-generic-sys-util/_cm.json | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index e6984d6ab6..0dd6dfc55e 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -107,8 +107,8 @@ "env": { "CM_SYS_UTIL_NAME": "g++11", "CM_SYS_UTIL_VERSION_CMD": "g++-11 --version", - "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "g++11": { @@ -122,8 +122,8 @@ "env": { "CM_SYS_UTIL_NAME": "g++12", "CM_SYS_UTIL_VERSION_CMD": "g++-12 --version", - "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "g++12": { @@ -137,8 +137,8 @@ "env": { "CM_SYS_UTIL_NAME": "g++9", "CM_SYS_UTIL_VERSION_CMD": "g++-9 --version", - "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "g++9": { @@ -152,8 +152,8 @@ "env": { "CM_SYS_UTIL_NAME": "gcc11", "CM_SYS_UTIL_VERSION_CMD": "gcc-11 --version", - "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "gcc11": { @@ -166,7 +166,7 @@ "env": { "CM_SYS_UTIL_NAME": "gcc9", "CM_SYS_UTIL_VERSION_CMD": "gcc-9 --version", - "CM_SYS_UTIL_VERSION_RE": "^.*([0-9]+(\\.[0-9]+)+).*", + "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { From 646252a5f5dc20c34c1200437c361fe5c08198d5 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 15:24:55 +0000 Subject: [PATCH 16/56] Update _cm.json --- script/get-generic-sys-util/_cm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 0dd6dfc55e..f2205b16ca 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -249,7 +249,7 @@ "env": { "CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM": "yes", "CM_SYS_UTIL_NAME": "libbz2_dev", - "CM_SYS_UTIL_VERSION_CMD": "bzcat --version | grep Version", + "CM_SYS_UTIL_VERSION_CMD": "bzcat --version", "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)", "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, From 90d17f00f5dc77ce01b2b1786049761c309b8ee8 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 16:20:41 +0000 Subject: [PATCH 17/56] Added pkg-config deps for get-generic-sys-util --- script/get-generic-sys-util/_cm.json | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 85a5b84503..9c7e7d559c 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -11,6 +11,14 @@ "deps": [ { "tags": "detect,os" + }, + { + "tags": "get,generic-sys-util,_package.pkg-config", + "skip_if_env": { + "CM_SYS_UTIL_NAME": [ + "pkg-config" + ] + } } ], "new_env_keys": [ @@ -548,6 +556,19 @@ } } }, + "pkg-config": { + "env": { + "CM_SYS_UTIL_NAME": "pkg_config" + }, + "state": { + "pkg_config": { + "apt": "pkg-config", + "brew": "pkg-config", + "dnf": "pkg-config", + "yum": "pkg-config" + } + } + }, "psmisc": { "env": { "CM_SYS_UTIL_NAME": "psmisc" From 1f961608298db9df3e25b28e6c31772dfa376ecf Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 16:23:38 +0000 Subject: [PATCH 18/56] Added pkg-config deps for get-generic-sys-util --- script/get-generic-sys-util/_cm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 9c7e7d559c..0944a82ab1 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -13,7 +13,7 @@ "tags": "detect,os" }, { - "tags": "get,generic-sys-util,_package.pkg-config", + "tags": "get,generic-sys-util,_pkg-config", "skip_if_env": { "CM_SYS_UTIL_NAME": [ "pkg-config" From 26f9eb332aa2ba200a19cc3316085bc1b0730170 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 16:28:38 +0000 Subject: [PATCH 19/56] Added pkg-config deps for get-generic-sys-util --- script/get-generic-sys-util/_cm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 0944a82ab1..941ebfb9e6 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -16,7 +16,7 @@ "tags": "get,generic-sys-util,_pkg-config", "skip_if_env": { "CM_SYS_UTIL_NAME": [ - "pkg-config" + "pkg_config" ] } } From 57b659a416815990b19f3269af64bd04e36fa6da Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 17:58:20 +0000 Subject: [PATCH 20/56] Added pkg-config deps for get-generic-sys-util --- script/get-generic-sys-util/_cm.json | 4 +++- script/get-generic-sys-util/detect.sh | 24 ++++++++++++++---------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 941ebfb9e6..be44441a9a 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -558,7 +558,9 @@ }, "pkg-config": { "env": { - "CM_SYS_UTIL_NAME": "pkg_config" + "CM_SYS_UTIL_NAME": "pkg_config", + "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "pkg_config": { diff --git a/script/get-generic-sys-util/detect.sh b/script/get-generic-sys-util/detect.sh index 4095d0fbdf..330a10806f 100644 --- a/script/get-generic-sys-util/detect.sh +++ b/script/get-generic-sys-util/detect.sh @@ -1,14 +1,18 @@ #!/bin/bash - -if [[ -n "${CM_SYS_UTIL_VERSION_CMD}" ]]; then - if [[ "${CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM}" == "yes" ]]; then - # Redirect both stdout and stderr to tmp-ver.out - cmd="${CM_SYS_UTIL_VERSION_CMD} > tmp-ver.out 2>&1" - else - cmd="${CM_SYS_UTIL_VERSION_CMD} > tmp-ver.out" - fi - echo $cmd - eval $cmd +if [[ -n "${CM_SYS_UTIL_VERSION_CMD_OVERRIDE}" ]]; then + eval "${CM_SYS_UTIL_VERSION_CMD_OVERRIDE}" test $? -eq 0 || exit $? +else + if [[ -n "${CM_SYS_UTIL_VERSION_CMD}" ]]; then + if [[ "${CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM}" == "yes" ]]; then + # Redirect both stdout and stderr to tmp-ver.out + cmd="${CM_SYS_UTIL_VERSION_CMD} > tmp-ver.out 2>&1" + else + cmd="${CM_SYS_UTIL_VERSION_CMD} > tmp-ver.out" + fi + echo $cmd + eval $cmd + test $? -eq 0 || exit $? + fi fi From f0497658b825e69def4baf5648c66382bc351ccf Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 18:10:13 +0000 Subject: [PATCH 21/56] Added pkg-config deps for get-generic-sys-util --- script/get-generic-sys-util/_cm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index be44441a9a..6fc9c004ee 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -13,7 +13,7 @@ "tags": "detect,os" }, { - "tags": "get,generic-sys-util,_pkg-config", + "tags": "get,generic-sys-util,_pkg-config,_detect", "skip_if_env": { "CM_SYS_UTIL_NAME": [ "pkg_config" From 5e02bad6c89399f1b0778a9bb5c3d2589a7775c3 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 18:25:58 +0000 Subject: [PATCH 22/56] Update _cm.json --- script/get-generic-sys-util/_cm.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 38907e7a59..dfabebeb9f 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -255,9 +255,8 @@ }, "libbz2-dev": { "env": { - "CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM": "yes", "CM_SYS_UTIL_NAME": "libbz2_dev", - "CM_SYS_UTIL_VERSION_CMD": "bzcat --version", + "CM_SYS_UTIL_VERSION_CMD_OVERRIDE": "bzcat --version > tmp_ver.out 2>&1 | grep bzip", "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)", "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, From f7ccd2cb5a81dc88764816924849d789000ef66f Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 18:43:10 +0000 Subject: [PATCH 23/56] Fix pstree version detection on macos --- script/get-generic-sys-util/customize.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index b020841eac..c429066b6e 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -10,6 +10,8 @@ def preprocess(i): state = i['state'] automation = i['automation'] + if env['CM_SYS_UTIL_NAME'] == "psmisc" and env.get('CM_HOST_OS_PACKAGE_MANAGER', '') == "brew": + env['CM_SYS_UTIL_VERSION_CMD_OVERRIDE'] = "brew info pstree > tmp-run.out 2>&1 | grep pstree" #Use VERSION_CMD and CHECK_CMD if no CHECK_CMD is set if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' and env.get('CM_SYS_UTIL_CHECK_CMD', '') == '': env['CM_SYS_UTIL_CHECK_CMD'] = env['CM_SYS_UTIL_VERSION_CMD'] From c0e92ac1ee924f75895b9aec40bb4460b32626db Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 19:11:54 +0000 Subject: [PATCH 24/56] Fix pstree version detection on macos --- script/get-generic-sys-util/customize.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index c429066b6e..8e3598f1ec 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -11,7 +11,8 @@ def preprocess(i): automation = i['automation'] if env['CM_SYS_UTIL_NAME'] == "psmisc" and env.get('CM_HOST_OS_PACKAGE_MANAGER', '') == "brew": - env['CM_SYS_UTIL_VERSION_CMD_OVERRIDE'] = "brew info pstree > tmp-run.out 2>&1 | grep pstree" + env['CM_SYS_UTIL_VERSION_CMD_OVERRIDE'] = "brew info pstree > tmp-run.out 2>&1 | grep pstree:" + env['CM_SYS_UTIL_VERSION_RE'] = r"\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b" #Use VERSION_CMD and CHECK_CMD if no CHECK_CMD is set if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' and env.get('CM_SYS_UTIL_CHECK_CMD', '') == '': env['CM_SYS_UTIL_CHECK_CMD'] = env['CM_SYS_UTIL_VERSION_CMD'] From 5df4e4a54a4405c685d1a4c8614b41e65bfe059f Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 19:16:30 +0000 Subject: [PATCH 25/56] Update _cm.json --- script/get-generic-sys-util/_cm.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index dfabebeb9f..6b671db54f 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -579,10 +579,7 @@ } ], "env": { - "CM_SYS_UTIL_NAME": "linux-tools", - "CM_SYS_UTIL_VERSION_CMD": "linux-tools --version", - "CM_SYS_UTIL_VERSION_RE": "Linux Tools version ([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_SYS_UTIL_NAME": "linux-tools" }, "state": { "linux-tools": { From 93e8d947d81dbbfa24c3cc15de00cd7be499accb Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 19:23:48 +0000 Subject: [PATCH 26/56] Update _cm.json | Dont do version check for numactl --- script/get-generic-sys-util/_cm.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 6b671db54f..f0a79e3b22 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -671,8 +671,6 @@ ], "env": { "CM_SYS_UTIL_NAME": "numactl", - "CM_SYS_UTIL_VERSION_CMD": "dpkg -l | grep libnuma", - "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { From 68fec0794f91aa2e912c4036d7e357144b1339e1 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 19:36:39 +0000 Subject: [PATCH 27/56] Update _cm.json --- script/get-generic-sys-util/_cm.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index f0a79e3b22..d0a1320eca 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -75,7 +75,10 @@ "group": "mode", "env": { "CM_GENERIC_SYS_UTIL_RUN_MODE": "install" - } + }, + "new_env_keys": [ + "CM_TMP_GENERIC_SYS_UTIL_PACKAGE_INSTALL_IGNORED" + ] }, "cmake": { "env": { @@ -332,10 +335,7 @@ }, "libffi8": { "env": { - "CM_SYS_UTIL_NAME": "libffi8", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libffi8", - "CM_SYS_UTIL_VERSION_RE": "\\d\\.\\d\\.\\d-\\d", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "libffi8" }, "state": { "libffi8": { From 00f3d26e85182fa584dd535035900013f7d93a0d Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 19:40:49 +0000 Subject: [PATCH 28/56] Remove version detection for libffi --- script/get-generic-sys-util/_cm.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index d0a1320eca..e63a05ebba 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -290,10 +290,7 @@ }, "libffi": { "env": { - "CM_SYS_UTIL_NAME": "libffi", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libffi", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "libffi" }, "state": { "libffi": { From bc9e102ee7cbf31b16af33c3c947fd2b4e587ba5 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 4 Nov 2024 01:11:59 +0530 Subject: [PATCH 29/56] Fix onnx version for nvidia mlperf inference retinanet --- script/app-mlperf-inference-nvidia/_cm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/app-mlperf-inference-nvidia/_cm.yaml b/script/app-mlperf-inference-nvidia/_cm.yaml index a997ef8156..9102732e31 100644 --- a/script/app-mlperf-inference-nvidia/_cm.yaml +++ b/script/app-mlperf-inference-nvidia/_cm.yaml @@ -435,7 +435,7 @@ variations: - tags: get,generic-python-lib,_pycocotools - tags: get,generic-python-lib,_onnx-graphsurgeon - tags: get,generic-python-lib,_package.onnx - version: 1.13.1 + version: 1.14.1 - tags: get,generic-python-lib,_package.sympy sdxl: From fa071a00d2fa1fa1a1846b62a92f22b1c18ba8da Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 19:56:25 +0000 Subject: [PATCH 30/56] Fix pstree version detect on macos --- script/get-generic-sys-util/customize.py | 5 +++-- script/get-generic-sys-util/detect.sh | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index 8e3598f1ec..65c41eaae1 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -13,12 +13,13 @@ def preprocess(i): if env['CM_SYS_UTIL_NAME'] == "psmisc" and env.get('CM_HOST_OS_PACKAGE_MANAGER', '') == "brew": env['CM_SYS_UTIL_VERSION_CMD_OVERRIDE'] = "brew info pstree > tmp-run.out 2>&1 | grep pstree:" env['CM_SYS_UTIL_VERSION_RE'] = r"\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b" + #Use VERSION_CMD and CHECK_CMD if no CHECK_CMD is set if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' and env.get('CM_SYS_UTIL_CHECK_CMD', '') == '': env['CM_SYS_UTIL_CHECK_CMD'] = env['CM_SYS_UTIL_VERSION_CMD'] if env.get('CM_GENERIC_SYS_UTIL_RUN_MODE', '') == "detect": - if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '': + if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' or env.get('CM_SYS_UTIL_VERSION_CMD_OVERRIDE', '') != '': r = automation.run_native_script({'run_script_input':i['run_script_input'], 'env':env, 'script_name':'detect'}) if r['return'] != 0: #detection failed, do install via prehook_deps print("detection failed, going for installation") @@ -131,7 +132,7 @@ def postprocess(i): version_env_key = f"CM_{env['CM_SYS_UTIL_NAME'].upper()}_VERSION" - if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' and (env['CM_GENERIC_SYS_UTIL_RUN_MODE'] == "install" or env.get(version_env_key, '') == '') and str(env.get('CM_TMP_GENERIC_SYS_UTIL_PACKAGE_INSTALL_IGNORED', '')).lower() not in ["yes", "1", "true"]: + if (env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' or env.get('CM_SYS_UTIL_VERSION_CMD_OVERRIDE', '') != '') and (env['CM_GENERIC_SYS_UTIL_RUN_MODE'] == "install" or env.get(version_env_key, '') == '') and str(env.get('CM_TMP_GENERIC_SYS_UTIL_PACKAGE_INSTALL_IGNORED', '')).lower() not in ["yes", "1", "true"]: automation = i['automation'] r = automation.run_native_script({'run_script_input':i['run_script_input'], 'env':env, 'script_name':'detect'}) if r['return'] > 0 and str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE', '')).lower() not in [ "1", "yes", "true" ] and str(env.get('CM_TMP_FAIL_SAFE', '')).lower() not in [ "1", "yes", "true" ]: diff --git a/script/get-generic-sys-util/detect.sh b/script/get-generic-sys-util/detect.sh index 330a10806f..90ef1312ba 100644 --- a/script/get-generic-sys-util/detect.sh +++ b/script/get-generic-sys-util/detect.sh @@ -1,6 +1,8 @@ #!/bin/bash if [[ -n "${CM_SYS_UTIL_VERSION_CMD_OVERRIDE}" ]]; then - eval "${CM_SYS_UTIL_VERSION_CMD_OVERRIDE}" + cmd="${CM_SYS_UTIL_VERSION_CMD_OVERRIDE}" + echo $cmd + eval $cmd test $? -eq 0 || exit $? else if [[ -n "${CM_SYS_UTIL_VERSION_CMD}" ]]; then From 62fe3a6bbfcd7c1d92231d03e99724fa87cb35b7 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 20:04:58 +0000 Subject: [PATCH 31/56] Fix version check for get-generic-sys-util install failures --- script/get-generic-sys-util/customize.py | 4 ++-- script/get-generic-sys-util/run.sh | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index 65c41eaae1..0d15e14d2c 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -11,7 +11,7 @@ def preprocess(i): automation = i['automation'] if env['CM_SYS_UTIL_NAME'] == "psmisc" and env.get('CM_HOST_OS_PACKAGE_MANAGER', '') == "brew": - env['CM_SYS_UTIL_VERSION_CMD_OVERRIDE'] = "brew info pstree > tmp-run.out 2>&1 | grep pstree:" + env['CM_SYS_UTIL_VERSION_CMD_OVERRIDE'] = "brew info pstree > tmp-ver.out 2>&1 | grep pstree:" env['CM_SYS_UTIL_VERSION_RE'] = r"\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b" #Use VERSION_CMD and CHECK_CMD if no CHECK_CMD is set @@ -132,7 +132,7 @@ def postprocess(i): version_env_key = f"CM_{env['CM_SYS_UTIL_NAME'].upper()}_VERSION" - if (env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' or env.get('CM_SYS_UTIL_VERSION_CMD_OVERRIDE', '') != '') and (env['CM_GENERIC_SYS_UTIL_RUN_MODE'] == "install" or env.get(version_env_key, '') == '') and str(env.get('CM_TMP_GENERIC_SYS_UTIL_PACKAGE_INSTALL_IGNORED', '')).lower() not in ["yes", "1", "true"]: + if (env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' or env.get('CM_SYS_UTIL_VERSION_CMD_OVERRIDE', '') != '') and (env['CM_GENERIC_SYS_UTIL_RUN_MODE'] == "install" or env.get(version_env_key, '') == '') and str(env.get('CM_TMP_GENERIC_SYS_UTIL_PACKAGE_INSTALL_IGNORED', '')).lower() not in ["yes", "1", "true"] and env.get('CM_GET_GENERIC_SYS_UTIL_INSTALL_FAILED', '') != 'yes': automation = i['automation'] r = automation.run_native_script({'run_script_input':i['run_script_input'], 'env':env, 'script_name':'detect'}) if r['return'] > 0 and str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE', '')).lower() not in [ "1", "yes", "true" ] and str(env.get('CM_TMP_FAIL_SAFE', '')).lower() not in [ "1", "yes", "true" ]: diff --git a/script/get-generic-sys-util/run.sh b/script/get-generic-sys-util/run.sh index 7e8afaf10a..de14fe9d03 100644 --- a/script/get-generic-sys-util/run.sh +++ b/script/get-generic-sys-util/run.sh @@ -8,6 +8,7 @@ if ! eval "$cmd"; then echo "Command failed with status $?" if [[ "${CM_TMP_FAIL_SAFE}" == 'yes' ]]; then # Exit safely if fail-safe is enabled + echo "CM_GET_GENERIC_SYS_UTIL_INSTALL_FAILED=yes" > tmp-run.out echo "Fail-safe is enabled, exiting with status 0" exit 0 else From 12b70e646c7fdacd7fec035eb5998e141153b3ce Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 20:26:57 +0000 Subject: [PATCH 32/56] Fix version detection only for install mode --- script/get-generic-sys-util/customize.py | 7 ++++--- script/get-generic-sys-util/detect.sh | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index 0d15e14d2c..df852f624d 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -11,8 +11,9 @@ def preprocess(i): automation = i['automation'] if env['CM_SYS_UTIL_NAME'] == "psmisc" and env.get('CM_HOST_OS_PACKAGE_MANAGER', '') == "brew": - env['CM_SYS_UTIL_VERSION_CMD_OVERRIDE'] = "brew info pstree > tmp-ver.out 2>&1 | grep pstree:" + env['CM_SYS_UTIL_VERSION_CMD_OVERRIDE'] = "brew info pstree > tmp-ver.out 2>&1" env['CM_SYS_UTIL_VERSION_RE'] = r"\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b" + env['CM_TMP_VERSION_DETECT_GROUP_NUMBER'] = 0 #Use VERSION_CMD and CHECK_CMD if no CHECK_CMD is set if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' and env.get('CM_SYS_UTIL_CHECK_CMD', '') == '': @@ -132,10 +133,10 @@ def postprocess(i): version_env_key = f"CM_{env['CM_SYS_UTIL_NAME'].upper()}_VERSION" - if (env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' or env.get('CM_SYS_UTIL_VERSION_CMD_OVERRIDE', '') != '') and (env['CM_GENERIC_SYS_UTIL_RUN_MODE'] == "install" or env.get(version_env_key, '') == '') and str(env.get('CM_TMP_GENERIC_SYS_UTIL_PACKAGE_INSTALL_IGNORED', '')).lower() not in ["yes", "1", "true"] and env.get('CM_GET_GENERIC_SYS_UTIL_INSTALL_FAILED', '') != 'yes': + if (env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' or env.get('CM_SYS_UTIL_VERSION_CMD_OVERRIDE', '') != '') and (env['CM_GENERIC_SYS_UTIL_RUN_MODE'] == "install" and env.get(version_env_key, '') == '') and str(env.get('CM_TMP_GENERIC_SYS_UTIL_PACKAGE_INSTALL_IGNORED', '')).lower() not in ["yes", "1", "true"] and env.get('CM_GET_GENERIC_SYS_UTIL_INSTALL_FAILED', '') != 'yes': automation = i['automation'] r = automation.run_native_script({'run_script_input':i['run_script_input'], 'env':env, 'script_name':'detect'}) - if r['return'] > 0 and str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE', '')).lower() not in [ "1", "yes", "true" ] and str(env.get('CM_TMP_FAIL_SAFE', '')).lower() not in [ "1", "yes", "true" ]: + if r['return'] > 0 and str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE', '')).lower() not in [ "1", "yes", "true" ]: return {'return': 1, 'error': 'Version detection failed after installation. Please check the provided version command or use env.CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE=yes to ignore the error.'} r = detect_version(i) diff --git a/script/get-generic-sys-util/detect.sh b/script/get-generic-sys-util/detect.sh index 90ef1312ba..2c35837992 100644 --- a/script/get-generic-sys-util/detect.sh +++ b/script/get-generic-sys-util/detect.sh @@ -1,4 +1,5 @@ #!/bin/bash + if [[ -n "${CM_SYS_UTIL_VERSION_CMD_OVERRIDE}" ]]; then cmd="${CM_SYS_UTIL_VERSION_CMD_OVERRIDE}" echo $cmd From 9711d26db222fc254aa4ff9cc755c47d9e70ea1e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 4 Nov 2024 02:11:58 +0530 Subject: [PATCH 33/56] Support detached mode for nvidia-mlperf-inference-gptj --- script/get-ml-model-gptj/customize.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/get-ml-model-gptj/customize.py b/script/get-ml-model-gptj/customize.py index ab475f0454..b3c61086d1 100644 --- a/script/get-ml-model-gptj/customize.py +++ b/script/get-ml-model-gptj/customize.py @@ -24,6 +24,8 @@ def preprocess(i): elif env.get('CM_TMP_ML_MODEL_PROVIDER', '') == 'nvidia': i['run_script_input']['script_name'] = 'run-nvidia' + if str(env.get('CM_DOCKER_DETACHED_MODE','')).lower() in ['yes', 'true', "1"]: + env['DOCKER_RUN_OPTS'] = "--rm -dt --ipc=host --ulimit memlock=-1 --ulimit stack=67108864" gpu_arch = int(float(env['CM_CUDA_DEVICE_PROP_GPU_COMPUTE_CAPABILITY']) * 10) env['CM_GPU_ARCH'] = gpu_arch env['CM_TMP_REQUIRE_DOWNLOAD'] = 'no' From 4d7195b8f664ac52c64b5fa81b7ffbfa2f63ff1b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 21:54:55 +0000 Subject: [PATCH 34/56] Cleanup for get-generic-sys-util --- script/get-generic-sys-util/customize.py | 22 +++++++++++++------ .../{run.sh => install.sh} | 0 2 files changed, 15 insertions(+), 7 deletions(-) rename script/get-generic-sys-util/{run.sh => install.sh} (100%) diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index df852f624d..25ba9ec575 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -11,14 +11,17 @@ def preprocess(i): automation = i['automation'] if env['CM_SYS_UTIL_NAME'] == "psmisc" and env.get('CM_HOST_OS_PACKAGE_MANAGER', '') == "brew": - env['CM_SYS_UTIL_VERSION_CMD_OVERRIDE'] = "brew info pstree > tmp-ver.out 2>&1" - env['CM_SYS_UTIL_VERSION_RE'] = r"\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b" - env['CM_TMP_VERSION_DETECT_GROUP_NUMBER'] = 0 + env['CM_SYS_UTIL_VERSION_CMD_OVERRIDE'] = "brew info pstree | grep pstree: > tmp-ver.out 2>&1" + env['CM_SYS_UTIL_VERSION_RE'] = r"(?:stable\s+)(\d+\.\d+(?:\.\d+)?)" + env['CM_TMP_VERSION_DETECT_GROUP_NUMBER'] = 1 #Use VERSION_CMD and CHECK_CMD if no CHECK_CMD is set if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' and env.get('CM_SYS_UTIL_CHECK_CMD', '') == '': env['CM_SYS_UTIL_CHECK_CMD'] = env['CM_SYS_UTIL_VERSION_CMD'] + if env.get('CM_GENERIC_SYS_UTIL_RUN_MODE', '') == "install": + i['run_script_input']['name'] = "install" + if env.get('CM_GENERIC_SYS_UTIL_RUN_MODE', '') == "detect": if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' or env.get('CM_SYS_UTIL_VERSION_CMD_OVERRIDE', '') != '': r = automation.run_native_script({'run_script_input':i['run_script_input'], 'env':env, 'script_name':'detect'}) @@ -27,9 +30,11 @@ def preprocess(i): env['CM_GENERIC_SYS_UTIL_INSTALL_NEEDED'] = "yes" return {'return': 0} else: #detection is successful, no need to install + #print("detection success") env['CM_SYS_UTIL_INSTALL_CMD'] = "" return {'return': 0} else: #No detction command available, just install + #print("No detection possible, going for installation") env['CM_GENERIC_SYS_UTIL_INSTALL_NEEDED'] = "yes" return {'return': 0} @@ -121,6 +126,7 @@ def detect_version(i): 'group_number': group_number, 'env_key': version_env_key, 'which_env': env}) + if r['return'] >0: return r version = r['version'] @@ -133,11 +139,13 @@ def postprocess(i): version_env_key = f"CM_{env['CM_SYS_UTIL_NAME'].upper()}_VERSION" - if (env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' or env.get('CM_SYS_UTIL_VERSION_CMD_OVERRIDE', '') != '') and (env['CM_GENERIC_SYS_UTIL_RUN_MODE'] == "install" and env.get(version_env_key, '') == '') and str(env.get('CM_TMP_GENERIC_SYS_UTIL_PACKAGE_INSTALL_IGNORED', '')).lower() not in ["yes", "1", "true"] and env.get('CM_GET_GENERIC_SYS_UTIL_INSTALL_FAILED', '') != 'yes': + if (env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' or env.get('CM_SYS_UTIL_VERSION_CMD_OVERRIDE', '') != '') and str(env.get('CM_TMP_GENERIC_SYS_UTIL_PACKAGE_INSTALL_IGNORED', '')).lower() not in ["yes", "1", "true"] and env.get('CM_GET_GENERIC_SYS_UTIL_INSTALL_FAILED', '') != 'yes': automation = i['automation'] - r = automation.run_native_script({'run_script_input':i['run_script_input'], 'env':env, 'script_name':'detect'}) - if r['return'] > 0 and str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE', '')).lower() not in [ "1", "yes", "true" ]: - return {'return': 1, 'error': 'Version detection failed after installation. Please check the provided version command or use env.CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE=yes to ignore the error.'} + + if (env['CM_GENERIC_SYS_UTIL_RUN_MODE'] == "install" and env.get(version_env_key, '') == ''): + r = automation.run_native_script({'run_script_input':i['run_script_input'], 'env':env, 'script_name':'detect'}) + if r['return'] > 0 and str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE', '')).lower() not in [ "1", "yes", "true" ]: + return {'return': 1, 'error': 'Version detection failed after installation. Please check the provided version command or use env.CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE=yes to ignore the error.'} r = detect_version(i) diff --git a/script/get-generic-sys-util/run.sh b/script/get-generic-sys-util/install.sh similarity index 100% rename from script/get-generic-sys-util/run.sh rename to script/get-generic-sys-util/install.sh From 92391b3a8960b3833c74c9e1986cf08d6847bfc5 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 22:14:36 +0000 Subject: [PATCH 35/56] Cleanup for get-generic-sys-util --- script/get-generic-sys-util/customize.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index 25ba9ec575..e6aac8b763 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -20,7 +20,7 @@ def preprocess(i): env['CM_SYS_UTIL_CHECK_CMD'] = env['CM_SYS_UTIL_VERSION_CMD'] if env.get('CM_GENERIC_SYS_UTIL_RUN_MODE', '') == "install": - i['run_script_input']['name'] = "install" + i['run_script_input']['script_name'] = "install" if env.get('CM_GENERIC_SYS_UTIL_RUN_MODE', '') == "detect": if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' or env.get('CM_SYS_UTIL_VERSION_CMD_OVERRIDE', '') != '': @@ -139,13 +139,12 @@ def postprocess(i): version_env_key = f"CM_{env['CM_SYS_UTIL_NAME'].upper()}_VERSION" - if (env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' or env.get('CM_SYS_UTIL_VERSION_CMD_OVERRIDE', '') != '') and str(env.get('CM_TMP_GENERIC_SYS_UTIL_PACKAGE_INSTALL_IGNORED', '')).lower() not in ["yes", "1", "true"] and env.get('CM_GET_GENERIC_SYS_UTIL_INSTALL_FAILED', '') != 'yes': + if (env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' or env.get('CM_SYS_UTIL_VERSION_CMD_OVERRIDE', '') != '') and env.get(version_env_key, '') == '' and str(env.get('CM_TMP_GENERIC_SYS_UTIL_PACKAGE_INSTALL_IGNORED', '')).lower() not in ["yes", "1", "true"] and env.get('CM_GET_GENERIC_SYS_UTIL_INSTALL_FAILED', '') != 'yes': automation = i['automation'] - if (env['CM_GENERIC_SYS_UTIL_RUN_MODE'] == "install" and env.get(version_env_key, '') == ''): - r = automation.run_native_script({'run_script_input':i['run_script_input'], 'env':env, 'script_name':'detect'}) - if r['return'] > 0 and str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE', '')).lower() not in [ "1", "yes", "true" ]: - return {'return': 1, 'error': 'Version detection failed after installation. Please check the provided version command or use env.CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE=yes to ignore the error.'} + r = automation.run_native_script({'run_script_input':i['run_script_input'], 'env':env, 'script_name':'detect'}) + if r['return'] > 0 and str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE', '')).lower() not in [ "1", "yes", "true" ]: + return {'return': 1, 'error': 'Version detection failed after installation. Please check the provided version command or use env.CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE=yes to ignore the error.'} r = detect_version(i) From 196c9b800fd9e035e4b864918a5311688858b986 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 22:21:50 +0000 Subject: [PATCH 36/56] No version detection for psmisc --- script/get-generic-sys-util/customize.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index e6aac8b763..673cb7f89f 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -10,11 +10,6 @@ def preprocess(i): state = i['state'] automation = i['automation'] - if env['CM_SYS_UTIL_NAME'] == "psmisc" and env.get('CM_HOST_OS_PACKAGE_MANAGER', '') == "brew": - env['CM_SYS_UTIL_VERSION_CMD_OVERRIDE'] = "brew info pstree | grep pstree: > tmp-ver.out 2>&1" - env['CM_SYS_UTIL_VERSION_RE'] = r"(?:stable\s+)(\d+\.\d+(?:\.\d+)?)" - env['CM_TMP_VERSION_DETECT_GROUP_NUMBER'] = 1 - #Use VERSION_CMD and CHECK_CMD if no CHECK_CMD is set if env.get('CM_SYS_UTIL_VERSION_CMD', '') != '' and env.get('CM_SYS_UTIL_CHECK_CMD', '') == '': env['CM_SYS_UTIL_CHECK_CMD'] = env['CM_SYS_UTIL_VERSION_CMD'] From f5251950d17991440b7b0448836b57dd1dbb533e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 22:23:02 +0000 Subject: [PATCH 37/56] No version detection for psmisc --- script/get-generic-sys-util/_cm.json | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index e63a05ebba..d27ea3720e 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -713,11 +713,7 @@ }, "psmisc": { "env": { - "CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM": "yes", - "CM_SYS_UTIL_NAME": "psmisc", - "CM_SYS_UTIL_VERSION_CMD": "pstree --version", - "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "psmisc" }, "state": { "psmisc": { From 94ac51c299af8fd49581f354eebea836615d062e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 22:30:52 +0000 Subject: [PATCH 38/56] Fix tmp-run-env.out name --- script/get-generic-sys-util/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/install.sh b/script/get-generic-sys-util/install.sh index de14fe9d03..145d0b7451 100644 --- a/script/get-generic-sys-util/install.sh +++ b/script/get-generic-sys-util/install.sh @@ -8,7 +8,7 @@ if ! eval "$cmd"; then echo "Command failed with status $?" if [[ "${CM_TMP_FAIL_SAFE}" == 'yes' ]]; then # Exit safely if fail-safe is enabled - echo "CM_GET_GENERIC_SYS_UTIL_INSTALL_FAILED=yes" > tmp-run.out + echo "CM_GET_GENERIC_SYS_UTIL_INSTALL_FAILED=yes" > tmp-run-env.out echo "Fail-safe is enabled, exiting with status 0" exit 0 else From 2da6bef79767e66ef19917b537dc88981d34e998 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 22:50:00 +0000 Subject: [PATCH 39/56] Update _cm.json | Fix bzip2 version cmd --- script/get-generic-sys-util/_cm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index d27ea3720e..1d5f590b2b 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -259,7 +259,7 @@ "libbz2-dev": { "env": { "CM_SYS_UTIL_NAME": "libbz2_dev", - "CM_SYS_UTIL_VERSION_CMD_OVERRIDE": "bzcat --version > tmp_ver.out 2>&1 | grep bzip", + "CM_SYS_UTIL_VERSION_CMD_OVERRIDE": "bzcat --version 2>&1 | grep bzip > tmp-ver.out", "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)", "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, From 000a799893e63514f80101891c4cdd1daaa5ee68 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 23:05:22 +0000 Subject: [PATCH 40/56] Fix tmp-run-env.out name --- script/get-generic-sys-util/_cm.json | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 6fc9c004ee..415def53ca 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -43,9 +43,6 @@ "detect": { "group": "mode", "default": true, - "names": [ - "detect-sys-util" - ], "env": { "CM_GENERIC_SYS_UTIL_RUN_MODE": "detect" }, @@ -75,7 +72,10 @@ "group": "mode", "env": { "CM_GENERIC_SYS_UTIL_RUN_MODE": "install" - } + }, + "new_env_keys": [ + "CM_GET_GENERIC_SYS_UTIL_INSTALL_FAILED" + ] }, "cmake": { "env": { @@ -105,7 +105,10 @@ }, "g++-11": { "env": { - "CM_SYS_UTIL_NAME": "g++11" + "CM_SYS_UTIL_NAME": "g++11", + "CM_SYS_UTIL_VERSION_CMD": "g++-11 --version", + "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 }, "state": { "g++11": { From 4795f0b651487829156da64673c73e50c736cd2e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 23:14:36 +0000 Subject: [PATCH 41/56] Skip g++11 version detection for ubuntu 20.04 --- script/get-generic-sys-util/_cm.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 415def53ca..90f7107955 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -108,7 +108,8 @@ "CM_SYS_UTIL_NAME": "g++11", "CM_SYS_UTIL_VERSION_CMD": "g++-11 --version", "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0, + "CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE": "yes" }, "state": { "g++11": { From dd2c715011007c856025287029495c3a99559026 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 23:20:24 +0000 Subject: [PATCH 42/56] Skip g++11 version detection for ubuntu 20.04 --- script/get-generic-sys-util/customize.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index 673cb7f89f..80cdd1e96b 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -141,16 +141,17 @@ def postprocess(i): if r['return'] > 0 and str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE', '')).lower() not in [ "1", "yes", "true" ]: return {'return': 1, 'error': 'Version detection failed after installation. Please check the provided version command or use env.CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE=yes to ignore the error.'} - r = detect_version(i) + elif r['return'] == 0:: + r = detect_version(i) - if r['return'] >0: return r + if r['return'] >0: return r - version = r['version'] + version = r['version'] - env[version_env_key] = version + env[version_env_key] = version - #Not used now - env['CM_GENERIC_SYS_UTIL_'+env['CM_SYS_UTIL_NAME'].upper()+'_CACHE_TAGS'] = 'version-'+version + #Not used now + env['CM_GENERIC_SYS_UTIL_'+env['CM_SYS_UTIL_NAME'].upper()+'_CACHE_TAGS'] = 'version-'+version if env.get(version_env_key, '') == '': env[version_env_key] = "undetected" From f1c9e31e2e8dd41dbbd63ee44c4bda90157e1f2c Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 23:22:34 +0000 Subject: [PATCH 43/56] Skip g++11 version detection for ubuntu 20.04 --- script/get-generic-sys-util/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index 80cdd1e96b..ae707684d5 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -141,7 +141,7 @@ def postprocess(i): if r['return'] > 0 and str(env.get('CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE', '')).lower() not in [ "1", "yes", "true" ]: return {'return': 1, 'error': 'Version detection failed after installation. Please check the provided version command or use env.CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE=yes to ignore the error.'} - elif r['return'] == 0:: + elif r['return'] == 0: r = detect_version(i) if r['return'] >0: return r From 9990d9886bc7ab6bdec05c0bcd840ab3b50b2a7c Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 23:40:08 +0000 Subject: [PATCH 44/56] Update _cm.json --- script/get-generic-sys-util/_cm.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index b0b561d2bf..afdf341dd9 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -133,7 +133,8 @@ "CM_SYS_UTIL_NAME": "g++12", "CM_SYS_UTIL_VERSION_CMD": "g++-12 --version", "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0, + "CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE": "yes" }, "state": { "g++12": { From 432a6ed4811a5d3d5aa6c241850abfba471d903b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 23:47:22 +0000 Subject: [PATCH 45/56] Remove version detect for libnuma-dev --- script/get-generic-sys-util/_cm.json | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index afdf341dd9..98d07f5c0c 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -388,10 +388,7 @@ }, "libmkl-dev": { "env": { - "CM_SYS_UTIL_NAME": "libmkl-dev", - "CM_SYS_UTIL_VERSION_CMD": "ldconfig -p | grep libmkl", - "CM_SYS_UTIL_VERSION_RE": "libmkl[^ ]+ => /[^ ]+/libmkl[^ ]+\\.so\\.([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_SYS_UTIL_NAME": "libmkl-dev" }, "state": { "libmkl-dev": { @@ -439,10 +436,7 @@ }, "libnuma-dev": { "env": { - "CM_SYS_UTIL_NAME": "libnuma-dev", - "CM_SYS_UTIL_VERSION_CMD": "dpkg -l | grep libnuma", - "CM_SYS_UTIL_VERSION_RE": "\\b(\\d+\\.\\d+(?:\\.\\d+)?)\\b", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "libnuma-dev" }, "state": { "libnuma-dev": { From 4478fde1ed0f508f86b67cbb5f91087f488cfc76 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Nov 2024 23:52:57 +0000 Subject: [PATCH 46/56] Use berkeley link for imagenet-aux by default --- script/get-dataset-imagenet-aux/_cm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-dataset-imagenet-aux/_cm.json b/script/get-dataset-imagenet-aux/_cm.json index 97e10571cd..7093e8b2b6 100644 --- a/script/get-dataset-imagenet-aux/_cm.json +++ b/script/get-dataset-imagenet-aux/_cm.json @@ -39,6 +39,7 @@ }, "from.berkeleyvision": { "group": "download-source", + "default": true, "base": [ "2012" ], @@ -51,7 +52,6 @@ }, "from.dropbox": { "group": "download-source", - "default": true, "base": [ "2012" ], From c54eaaa0c48692edaa418e5fbc2615c61fa72693 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 4 Nov 2024 00:12:40 +0000 Subject: [PATCH 47/56] Remove ntpdate version detect --- script/get-generic-sys-util/_cm.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 98d07f5c0c..538851ac04 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -629,10 +629,7 @@ }, "ntpdate": { "env": { - "CM_SYS_UTIL_NAME": "ntpdate", - "CM_SYS_UTIL_VERSION_CMD": "dpkg -l ntpdate 2>/dev/null | grep ^ii | awk '{print $3}'", - "CM_SYS_UTIL_VERSION_RE": "([A-Za-z0-9]+(\\.[A-Za-z0-9]+)+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "ntpdate" }, "state": { "ntpdate": { From 9739fafe5f5eaf1dab8c0fc35e75fd6f045c62a3 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 4 Nov 2024 00:19:01 +0000 Subject: [PATCH 48/56] Update _cm.json --- script/get-generic-sys-util/_cm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 538851ac04..a569e6e1b9 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -423,7 +423,7 @@ "CM_SYS_UTIL_NAME": "libncurses_dev", "CM_SYS_UTIL_VERSION_CMD": "ncurses5-config --version", "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 }, "state": { "libncurses_dev": { From 3c7904bdff12ddef9ba34e88c95ac1464ee67a1f Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 4 Nov 2024 00:24:44 +0000 Subject: [PATCH 49/56] Update _cm.json --- script/get-generic-sys-util/_cm.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index a569e6e1b9..e231f49bce 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -423,7 +423,8 @@ "CM_SYS_UTIL_NAME": "libncurses_dev", "CM_SYS_UTIL_VERSION_CMD": "ncurses5-config --version", "CM_SYS_UTIL_VERSION_RE": "([0-9]+(\\.[0-9]+)+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 1, + "CM_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE": "yes" }, "state": { "libncurses_dev": { From 9501b18db588b779ba4372c88015942ec3c6c342 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 4 Nov 2024 00:33:17 +0000 Subject: [PATCH 50/56] Update _cm.json | Remove failing version detects --- script/get-generic-sys-util/_cm.json | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index e231f49bce..3135863a59 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -11,14 +11,6 @@ "deps": [ { "tags": "detect,os" - }, - { - "tags": "get,generic-sys-util,_pkg-config,_detect", - "skip_if_env": { - "CM_SYS_UTIL_NAME": [ - "pkg_config" - ] - } } ], "new_env_keys": [ @@ -615,10 +607,7 @@ }, "nlohmann-json3-dev": { "env": { - "CM_SYS_UTIL_NAME": "nlohmann_json3_dev", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion nlohmann_json", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "nlohmann_json3_dev" }, "state": { "nlohmann_json3_dev": { @@ -719,10 +708,7 @@ }, "rapidjson-dev": { "env": { - "CM_SYS_UTIL_NAME": "rapidjson-dev", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion RapidJSON", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "rapidjson-dev" }, "state": { "rapidjson-dev": { @@ -904,10 +890,7 @@ }, "zlib": { "env": { - "CM_SYS_UTIL_NAME": "zlib", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion zlib", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "zlib" }, "state": { "zlib": { @@ -919,10 +902,7 @@ }, "zlib1g-dev": { "env": { - "CM_SYS_UTIL_NAME": "zlib1g_dev", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion zlib", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "zlib1g_dev" }, "state": { "zlib1g_dev": { From c9edadaa7c7e104258628178d1d807dd5093310e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 4 Nov 2024 00:40:09 +0000 Subject: [PATCH 51/56] Removed pkg detection from pkg_config --- script/get-generic-sys-util/_cm.json | 56 ++++++---------------------- 1 file changed, 11 insertions(+), 45 deletions(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 3135863a59..023bfabb01 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -181,11 +181,7 @@ }, "gflags-dev": { "env": { - "CM_SYS_UTIL_NAME": "gflags-dev", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion gflags", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 - + "CM_SYS_UTIL_NAME": "gflags-dev" }, "state": { "gflags-dev": { @@ -216,10 +212,7 @@ }, "glog-dev": { "env": { - "CM_SYS_UTIL_NAME": "glog-dev", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libglog", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "glog-dev" }, "state": { "glog-dev": { @@ -293,10 +286,7 @@ }, "libffi-dev": { "env": { - "CM_SYS_UTIL_NAME": "libffi_dev", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libffi", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "libffi_dev" }, "state": { "libffi_dev": { @@ -349,10 +339,7 @@ }, "libgmock-dev": { "env": { - "CM_SYS_UTIL_NAME": "libgmock-dev", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion gmock", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "libgmock-dev" }, "state": { "libgmock-dev": { @@ -394,10 +381,7 @@ }, "libmpfr-dev": { "env": { - "CM_SYS_UTIL_NAME": "libmpfr-dev", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion mpfr", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "libmpfr-dev" }, "state": { "libmpfr-dev": { @@ -443,10 +427,7 @@ }, "libpci-dev": { "env": { - "CM_SYS_UTIL_NAME": "libpci-dev", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libpci", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "libpci-dev" }, "state": { "libpci-dev": { @@ -460,10 +441,7 @@ }, "libpng-dev": { "env": { - "CM_SYS_UTIL_NAME": "libpng-dev", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libpng", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "libpng-dev" }, "state": { "libpng-dev": { @@ -477,10 +455,7 @@ }, "libre2-dev": { "env": { - "CM_SYS_UTIL_NAME": "libre2-dev", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion re2", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "libre2-dev" }, "state": { "libre2-dev": { @@ -494,10 +469,7 @@ }, "libreadline-dev": { "env": { - "CM_SYS_UTIL_NAME": "libreadline_dev", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion readline", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "libreadline_dev" }, "state": { "libreadline_dev": { @@ -510,10 +482,7 @@ }, "libsqlite3-dev": { "env": { - "CM_SYS_UTIL_NAME": "libsqlite3_dev", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion sqlite3", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "libsqlite3_dev" }, "state": { "libsqlite3_dev": { @@ -541,10 +510,7 @@ }, "libudev-dev": { "env": { - "CM_SYS_UTIL_NAME": "libudev-dev", - "CM_SYS_UTIL_VERSION_CMD": "pkg-config --modversion libudev", - "CM_SYS_UTIL_VERSION_RE": "([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_SYS_UTIL_NAME": "libudev-dev" }, "state": { "libudev-dev": { From 63859985ee17d2a9227345fd665fa528d5c562e2 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 4 Nov 2024 01:13:51 +0000 Subject: [PATCH 52/56] Update _cm.json --- script/get-generic-sys-util/_cm.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 023bfabb01..100dfbc4b8 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -774,7 +774,8 @@ "CM_SYS_UTIL_NAME": "transmission", "CM_SYS_UTIL_VERSION_CMD": "transmission-daemon --version", "CM_SYS_UTIL_VERSION_RE": "transmission-daemon ([\\d.]+)", - "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0 + "CM_TMP_VERSION_DETECT_GROUP_NUMBER": 0, + "CM_SYS_UTIL_VERSION_CMD_USE_ERROR_STREAM": "yes" }, "state": { "transmission": { From cb0c19a777f5d4048d082dad3f4e3c5b480002ec Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 4 Nov 2024 08:50:29 +0530 Subject: [PATCH 53/56] Generalised the code --- .../test-mlperf-inference-abtf-poc.yml | 67 ++++--------------- 1 file changed, 13 insertions(+), 54 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-abtf-poc.yml b/.github/workflows/test-mlperf-inference-abtf-poc.yml index bb8403ed9f..5c1b3c750b 100644 --- a/.github/workflows/test-mlperf-inference-abtf-poc.yml +++ b/.github/workflows/test-mlperf-inference-abtf-poc.yml @@ -16,13 +16,24 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-22.04, ubuntu-20.04, ubuntu-24.04] + os: [ubuntu-22.04, ubuntu-20.04, ubuntu-24.04, macos-latest, macos-13, windows-latest] python-version: [ "3.8", "3.12" ] backend: [ "pytorch" ] implementation: [ "python" ] + extra-args: [ "--adr.compiler.tags=gcc" ] exclude: - os: ubuntu-24.04 python-version: "3.8" + - os: windows-latest + python-version: "3.8" + extra-args: "--adr.compiler.tags=gcc" + - os: macos-latest + python-version: "3.8" + - os: macos-13 + python-version: "3.8" + include: + - os: windows-latest + extra-args: "--env.CM_MLPERF_LOADGEN_BUILD_FROM_SRC=off" steps: - uses: actions/checkout@v3 @@ -35,58 +46,6 @@ jobs: pip install cmind cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} cm pull repo mlcommons@cm4abtf --branch=poc - - name: Test MLPerf Inference ABTF POC using ${{ matrix.backend }} on docker - run: | - cm run script --tags=run-abtf,inference,_poc-demo --test_query_count=2 --adr.compiler.tags=gcc --adr.cocoeval.version_max=1.5.7 --adr.cocoeval.version_max_usable=1.5.7 --quiet -v - - build2: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [macos-latest, macos-13] - python-version: [ "3.12" ] - backend: [ "pytorch" ] - implementation: [ "python" ] - - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python3 -m pip install cmind - cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - cm pull repo mlcommons@cm4abtf --branch=poc - - name: Test MLPerf Inference ABTF POC using ${{ matrix.backend }} on ${{ matrix.os }} - run: | - cm run script --tags=run-abtf,inference,_poc-demo --test_query_count=2 --adr.compiler.tags=gcc --adr.cocoeval.version_max=1.5.7 --adr.cocoeval.version_max_usable=1.5.7 --quiet -v - - build3: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [windows-latest] - python-version: [ "3.12" ] - backend: [ "pytorch" ] - implementation: [ "python" ] - exclude: - - python-version: "3.8" - - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python3 -m pip install cmind - cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - cm pull repo mlcommons@cm4abtf --branch=poc - name: Test MLPerf Inference ABTF POC using ${{ matrix.backend }} on ${{ matrix.os }} run: | - cm run script --tags=run-abtf,inference,_poc-demo --test_query_count=2 --quiet --env.CM_MLPERF_LOADGEN_BUILD_FROM_SRC=off --adr.cocoeval.version_max=1.5.7 --adr.cocoeval.version_max_usable=1.5.7 -v + cm run script --tags=run-abtf,inference,_poc-demo --test_query_count=2 --adr.cocoeval.version_max=1.5.7 --adr.cocoeval.version_max_usable=1.5.7 --quiet ${{ matrix.extra-args }} -v From b206baf2ea7cb1b260c43fcceb395b334aa7a49e Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 4 Nov 2024 08:58:18 +0530 Subject: [PATCH 54/56] split exclude condition --- .github/workflows/test-mlperf-inference-abtf-poc.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-mlperf-inference-abtf-poc.yml b/.github/workflows/test-mlperf-inference-abtf-poc.yml index 5c1b3c750b..8a4ef20171 100644 --- a/.github/workflows/test-mlperf-inference-abtf-poc.yml +++ b/.github/workflows/test-mlperf-inference-abtf-poc.yml @@ -26,6 +26,7 @@ jobs: python-version: "3.8" - os: windows-latest python-version: "3.8" + - os: windows-latest extra-args: "--adr.compiler.tags=gcc" - os: macos-latest python-version: "3.8" From 16a88afb679d19fc21171c419bf82e171f42ef5a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 4 Nov 2024 11:55:25 +0000 Subject: [PATCH 55/56] Rename README.md to README-extra.md --- script/get-generic-sys-util/{README.md => README-extra.md} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename script/get-generic-sys-util/{README.md => README-extra.md} (99%) diff --git a/script/get-generic-sys-util/README.md b/script/get-generic-sys-util/README-extra.md similarity index 99% rename from script/get-generic-sys-util/README.md rename to script/get-generic-sys-util/README-extra.md index bf09f83053..d8f0015ae8 100644 --- a/script/get-generic-sys-util/README.md +++ b/script/get-generic-sys-util/README-extra.md @@ -422,4 +422,4 @@ wkhtmltopdf 0.12.6 `pkg-config --modversion zlib` -1.2.11 \ No newline at end of file +1.2.11 From 8470530393f1ec0ad262f5532d8ad17e03fa8507 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 6 Nov 2024 14:26:53 +0000 Subject: [PATCH 56/56] Update module.py --- automation/script/module.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index 8b11ff45e1..9fd16dfd2d 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -8,7 +8,6 @@ # TBD: when we have bandwidth and resources, we should refactor it # and make it cleaner and simpler while keeping full backwards compatibility. # -import re import os import logging @@ -3979,11 +3978,6 @@ def parse_version(self, i): return r string = r['string'] - match = re.search(match_text, string) - if debug: - print(f"Regex Pattern: {match_text}") - print(f"Matched String: {string}") - print(f"Match Groups: {match.groups()}") if r['match'].lastindex and r['match'].lastindex >= group_number: version = r['match'].group(group_number)