Merge pull request #232 from mlcommons/mlperf-inference

Mlperf inference

.github/workflows/test-cm-script-features.yml

@@ -13,14 +13,15 @@ on:
 jobs:
   build:
 
-    runs-on: ubuntu-latest
+    runs-on: S{{ matrix.on }}
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.12", "3.11", "3.10", "3.9", "3.8"]
+        python-version: ["3.12", "3.8"]
+        on: ["ubuntu-latest", "windows-latest", "macos-latest"]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v3
       with:
@@ -32,6 +33,7 @@ jobs:
         cm run script --quiet --tags=get,sys-utils-cm
     - name: Test CM Script Features
       run: |
+        python script/test-cm-core/src/script/test_deps.py
         python script/test-cm-core/src/script/test_install.py
         python script/test-cm-core/src/script/test_docker.py
         python script/test-cm-core/src/script/test_features.py

.github/workflows/test-cm4mlops-wheel-macos.yml

@@ -6,6 +6,9 @@ on:
       - main
       - dev
       - mlperf-inference
+    paths:
+      - '.github/workflows/test-cm4mlops-wheel-macos.yml'
+      - 'setup.py'
 
 jobs:
   build:

.github/workflows/test-cm4mlops-wheel-ubuntu.yml

@@ -6,14 +6,17 @@ on:
       - main
       - dev
       - mlperf-inference
+    paths:
+      - '.github/workflows/test-cm4mlops-wheel-ubuntu.yml'
+      - 'setup.py'
 
 jobs:
   build:
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, ubuntu-20.04]
-        python-version: ['3.8', '3.11', '3.12']
+        python-version: ['3.7', '3.8', '3.11', '3.12']
         exclude:
         - os: ubuntu-latest
           python-version: "3.8"

.github/workflows/test-cm4mlops-wheel-windows.yml

@@ -6,6 +6,9 @@ on:
       - main
       - dev
       - mlperf-inference
+    paths:
+      - '.github/workflows/test-cm4mlops-wheel-windows.yml'
+      - 'setup.py'
 
 jobs:
   build:

.github/workflows/test-mlperf-inference-resnet50.yml

@@ -27,8 +27,6 @@ jobs:
             implementation: cpp
           - os: macos-latest
             backend: tf
-          - os: macos-latest
-            python-version: "3.9"
           - os: windows-latest
 
     steps:

script/get-generic-sys-util/_cm.json

@@ -162,6 +162,7 @@
       "state": {
         "libbz2_dev": {
           "apt": "libbz2-dev",
+          "brew": "bzip2",
           "dnf": "libbzip2-devel",
           "yum": "libbzip2-devel",
           "zlib-devel": "libbz2-devel"

setup.py

@@ -8,6 +8,21 @@
 import platform
 import os
 
+# Try to use importlib.metadata for Python 3.8+ 
+try:
+    if sys.version_info >= (3, 8):
+        from importlib.metadata import version, PackageNotFoundError
+    else:
+        # Fallback to pkg_resources for Python < 3.8
+        import pkg_resources
+        PackageNotFoundError = pkg_resources.DistributionNotFound
+except ImportError:
+    # If importlib.metadata is unavailable, fall back to pkg_resources
+    import pkg_resources
+    PackageNotFoundError = pkg_resources.DistributionNotFound
+
+
+
 class CustomInstallCommand(install):
     def run(self):
         self.get_sys_platform()
@@ -19,6 +34,16 @@ def run(self):
         # Call the custom function
         return self.custom_function()
 
+    def is_package_installed(self, package_name):
+        try:
+            if sys.version_info >= (3, 8):
+                version(package_name)  # Tries to get the version of the package
+            else:
+                pkg_resources.get_distribution(package_name)  # Fallback for < 3.8
+            return True
+        except PackageNotFoundError:
+            return False
+
     def install_system_packages(self):
         # List of packages to install via system package manager
         packages = []
@@ -37,11 +62,8 @@ def install_system_packages(self):
 
         if name in sys.modules:
             pass #nothing needed
-        elif (spec := importlib.util.find_spec(name)) is not None:
-            module = importlib.util.module_from_spec(spec)
-            sys.modules[name] = module
-            spec.loader.exec_module(module)
-            #print(f"{name} has been imported")
+        elif self.is_package_installed(name):
+            pass
         else:
             packages.append("python3-venv")