diff --git a/.github/workflows/test-cm-script-features.yml b/.github/workflows/test-cm-script-features.yml index df156832b7..b72aa4fc83 100644 --- a/.github/workflows/test-cm-script-features.yml +++ b/.github/workflows/test-cm-script-features.yml @@ -13,14 +13,15 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: S{{ matrix.on }} strategy: fail-fast: false matrix: - python-version: ["3.12", "3.11", "3.10", "3.9", "3.8"] + python-version: ["3.12", "3.8"] + on: ["ubuntu-latest", "windows-latest", "macos-latest"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v3 with: @@ -32,6 +33,7 @@ jobs: cm run script --quiet --tags=get,sys-utils-cm - name: Test CM Script Features run: | + python script/test-cm-core/src/script/test_deps.py python script/test-cm-core/src/script/test_install.py python script/test-cm-core/src/script/test_docker.py python script/test-cm-core/src/script/test_features.py diff --git a/.github/workflows/test-cm-scripts.yml b/.github/workflows/test-cm-scripts.yml deleted file mode 100644 index e3c0c11a6f..0000000000 --- a/.github/workflows/test-cm-scripts.yml +++ /dev/null @@ -1,37 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -name: CM script automation test - -on: - pull_request: - branches: [ "main", "dev", "mlperf-inference" ] - paths: - - '.github/workflows/test-cm-scripts.yml' - - '**' - - '!**.md' - -jobs: - build: - - strategy: - fail-fast: false - matrix: - on: [ubuntu-latest, windows-latest, macos-latest] - python-version: ["3.12", "3.9"] - runs-on: "${{ matrix.on }}" - - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install cmind - cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - cm run script --quiet --tags=get,sys-utils-cm - - name: Test CM Script Automation - run: | - python script/test-cm-core/src/script/test_deps.py diff --git a/.github/workflows/test-cm4mlops-wheel-macos.yml b/.github/workflows/test-cm4mlops-wheel-macos.yml index a37395802a..fede3adb9f 100644 --- a/.github/workflows/test-cm4mlops-wheel-macos.yml +++ b/.github/workflows/test-cm4mlops-wheel-macos.yml @@ -6,6 +6,9 @@ on: - main - dev - mlperf-inference + paths: + - '.github/workflows/test-cm4mlops-wheel-macos.yml' + - 'setup.py' jobs: build: diff --git a/.github/workflows/test-cm4mlops-wheel-ubuntu.yml b/.github/workflows/test-cm4mlops-wheel-ubuntu.yml index f2e41371c5..4a37a5290e 100644 --- a/.github/workflows/test-cm4mlops-wheel-ubuntu.yml +++ b/.github/workflows/test-cm4mlops-wheel-ubuntu.yml @@ -6,6 +6,9 @@ on: - main - dev - mlperf-inference + paths: + - '.github/workflows/test-cm4mlops-wheel-ubuntu.yml' + - 'setup.py' jobs: build: @@ -13,7 +16,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, ubuntu-20.04] - python-version: ['3.8', '3.11', '3.12'] + python-version: ['3.7', '3.8', '3.11', '3.12'] exclude: - os: ubuntu-latest python-version: "3.8" diff --git a/.github/workflows/test-cm4mlops-wheel-windows.yml b/.github/workflows/test-cm4mlops-wheel-windows.yml index 0978ef5d59..68bf199a7e 100644 --- a/.github/workflows/test-cm4mlops-wheel-windows.yml +++ b/.github/workflows/test-cm4mlops-wheel-windows.yml @@ -6,6 +6,9 @@ on: - main - dev - mlperf-inference + paths: + - '.github/workflows/test-cm4mlops-wheel-windows.yml' + - 'setup.py' jobs: build: diff --git a/.github/workflows/test-mlperf-inference-resnet50.yml b/.github/workflows/test-mlperf-inference-resnet50.yml index 12797a29ef..797c6f2a0b 100644 --- a/.github/workflows/test-mlperf-inference-resnet50.yml +++ b/.github/workflows/test-mlperf-inference-resnet50.yml @@ -27,8 +27,6 @@ jobs: implementation: cpp - os: macos-latest backend: tf - - os: macos-latest - python-version: "3.9" - os: windows-latest steps: diff --git a/script/get-generic-sys-util/_cm.json b/script/get-generic-sys-util/_cm.json index 856ed0b659..ff7378b936 100644 --- a/script/get-generic-sys-util/_cm.json +++ b/script/get-generic-sys-util/_cm.json @@ -162,6 +162,7 @@ "state": { "libbz2_dev": { "apt": "libbz2-dev", + "brew": "bzip2", "dnf": "libbzip2-devel", "yum": "libbzip2-devel", "zlib-devel": "libbz2-devel" diff --git a/setup.py b/setup.py index 45f577ed51..e7c96a97cd 100644 --- a/setup.py +++ b/setup.py @@ -8,6 +8,21 @@ import platform import os +# Try to use importlib.metadata for Python 3.8+ +try: + if sys.version_info >= (3, 8): + from importlib.metadata import version, PackageNotFoundError + else: + # Fallback to pkg_resources for Python < 3.8 + import pkg_resources + PackageNotFoundError = pkg_resources.DistributionNotFound +except ImportError: + # If importlib.metadata is unavailable, fall back to pkg_resources + import pkg_resources + PackageNotFoundError = pkg_resources.DistributionNotFound + + + class CustomInstallCommand(install): def run(self): self.get_sys_platform() @@ -19,6 +34,16 @@ def run(self): # Call the custom function return self.custom_function() + def is_package_installed(self, package_name): + try: + if sys.version_info >= (3, 8): + version(package_name) # Tries to get the version of the package + else: + pkg_resources.get_distribution(package_name) # Fallback for < 3.8 + return True + except PackageNotFoundError: + return False + def install_system_packages(self): # List of packages to install via system package manager packages = [] @@ -37,11 +62,8 @@ def install_system_packages(self): if name in sys.modules: pass #nothing needed - elif (spec := importlib.util.find_spec(name)) is not None: - module = importlib.util.module_from_spec(spec) - sys.modules[name] = module - spec.loader.exec_module(module) - #print(f"{name} has been imported") + elif self.is_package_installed(name): + pass else: packages.append("python3-venv")