Dependency cleanup and Python 3.12 support

.github/workflows/ci-slow.yml

@@ -164,7 +164,8 @@ jobs:
         run: |
           source .venv/bin/activate
           uv pip install -e .
-          uv pip install mlstacks
+          # TODO: Remove and add mlstacks back in
+          uv pip install git+https://github.com/zenml-io/mlstacks.git@feature/upgrade-python-3.12
       - name: Check for broken dependencies
         run: |
           source .venv/bin/activate
@@ -199,7 +200,7 @@ jobs:
     strategy:
       matrix:
         os: [arc-runner-set]
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
       fail-fast: false
     uses: ./.github/workflows/unit-test.yml
     with:
@@ -212,7 +213,7 @@ jobs:
     strategy:
       matrix:
         os: [windows-latest]
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
       fail-fast: false
     uses: ./.github/workflows/unit-test.yml
     with:
@@ -225,7 +226,7 @@ jobs:
     strategy:
       matrix:
         os: [macos-latest]
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
         # Python 3.8 and 3.9 is on macos-13 but not macos-latest (macos-14-arm64)
         # https://github.com/actions/setup-python/issues/696#issuecomment-1637587760
         exclude:
@@ -246,7 +247,7 @@ jobs:
     strategy:
       matrix:
         os: [windows-latest]
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
         test_environment: [default]
       fail-fast: false
     uses: ./.github/workflows/integration-test-slow.yml
@@ -261,7 +262,7 @@ jobs:
     strategy:
       matrix:
         os: [macos-13]
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
         # Python 3.8 and 3.9 is on macos-13 but not macos-latest (macos-14-arm64)
         # https://github.com/actions/setup-python/issues/696#issuecomment-1637587760
         exclude:
@@ -284,7 +285,7 @@ jobs:
     strategy:
       matrix:
         os: [arc-runner-set]
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
         test_environment:
           - default
           - docker-server-docker-orchestrator-mysql

.github/workflows/integration-test-fast.yml

@@ -36,7 +36,7 @@ on:
       python-version:
         description: Python version
         type: choice
-        options: ['3.8', '3.9', '3.10', '3.11']
+        options: ['3.8', '3.9', '3.10', '3.11', '3.12']
         required: false
         default: '3.8'
       test_environment:

.github/workflows/integration-test-slow.yml

@@ -36,7 +36,7 @@ on:
       python-version:
         description: Python version
         type: choice
-        options: ['3.8', '3.9', '3.10', '3.11']
+        options: ['3.8', '3.9', '3.10', '3.11', '3.12']
         required: false
         default: '3.8'
       test_environment:
@@ -94,9 +94,9 @@ jobs:
       GCP_US_EAST4_SERVER_URL: ${{ secrets.GCP_US_EAST4_SERVER_URL }}
       GCP_US_EAST4_SERVER_USERNAME: ${{ secrets.GCP_US_EAST4_SERVER_USERNAME }}
       GCP_US_EAST4_SERVER_PASSWORD: ${{ secrets.GCP_US_EAST4_SERVER_PASSWORD }}
-    # TODO: add Windows testing for Python 3.11 back in
+    # TODO: add Windows testing for Python 3.11 and 3.12 back in
     # TODO: add macos testing back in
-    if: ${{ ! startsWith(github.event.head_commit.message, 'GitBook:') && ! (inputs.os == 'windows-latest' && inputs.python-version == '3.11') && ! (inputs.os == 'macos-13' || inputs.os == 'macos-latest') }}
+    if: ${{ ! startsWith(github.event.head_commit.message, 'GitBook:') && ! (inputs.os == 'windows-latest' && inputs.python-version == '3.11') && ! (inputs.os == 'windows-latest' && inputs.python-version == '3.12') && ! (inputs.os == 'macos-13' || inputs.os == 'macos-latest')  }}
     defaults:
       run:
         shell: bash

.github/workflows/templates-test.yml

@@ -22,7 +22,7 @@ on:
       python-version:
         description: Python version
         type: choice
-        options: ['3.8', '3.9', '3.10', '3.11']
+        options: ['3.8', '3.9', '3.10', '3.11', '3.12']
         required: false
         default: '3.8'
 jobs:

.github/workflows/unit-test.yml

@@ -37,7 +37,7 @@ on:
       python-version:
         description: Python version
         type: choice
-        options: ['3.8', '3.9', '3.10', '3.11']
+        options: ['3.8', '3.9', '3.10', '3.11', '3.12']
         required: false
         default: '3.8'
       enable_tmate:
@@ -65,9 +65,9 @@ jobs:
       # to fix problems with the fork() calls (see this thread
       # for more information: http://sealiesoftware.com/blog/archive/2017/6/5/Objective-C_and_fork_in_macOS_1013.html)
       OBJC_DISABLE_INITIALIZE_FORK_SAFETY: 'YES'
-    # TODO: add Windows testing for Python 3.11 back in
+    # TODO: add Windows testing for Python 3.11 and 3.12 back in
     # Exit if it's a commit from Gitbook
-    if: ${{ ! startsWith(github.event.head_commit.message, 'GitBook:') && ! (inputs.os == 'windows-latest' && inputs.python-version == '3.11') }}
+    if: ${{ ! startsWith(github.event.head_commit.message, 'GitBook:') && ! (inputs.os == 'windows-latest' && inputs.python-version == '3.11') && ! (inputs.os == 'windows-latest' && inputs.python-version == '3.12') }}
     defaults:
       run:
         shell: bash

.github/workflows/update-templates-to-examples.yml

@@ -22,7 +22,7 @@ on:
       python-version:
         description: Python version
         type: choice
-        options: ['3.8', '3.9', '3.10', '3.11']
+        options: ['3.8', '3.9', '3.10', '3.11', '3.12']
         required: false
         default: '3.8'
 jobs:
@@ -46,7 +46,7 @@ jobs:
           python-version: ${{ inputs.python-version }}
           stack-name: local
           ref-zenml: ${{ github.ref }}
-          ref-template: 2024.06.06  # Make sure it is aligned with ZENML_PROJECT_TEMPLATES from src/zenml/cli/base.py
+          ref-template: 2024.08.29  # Make sure it is aligned with ZENML_PROJECT_TEMPLATES from src/zenml/cli/base.py
       - name: Clean-up
         run: |
           rm -rf ./local_checkout
@@ -118,7 +118,7 @@ jobs:
           python-version: ${{ inputs.python-version }}
           stack-name: local
           ref-zenml: ${{ github.ref }}
-          ref-template: 2024.08.07  # Make sure it is aligned with ZENML_PROJECT_TEMPLATES from src/zenml/cli/base.py
+          ref-template: 2024.08.29  # Make sure it is aligned with ZENML_PROJECT_TEMPLATES from src/zenml/cli/base.py
       - name: Clean-up
         run: |
           rm -rf ./local_checkout
@@ -189,7 +189,7 @@ jobs:
           python-version: ${{ inputs.python-version }}
           stack-name: local
           ref-zenml: ${{ github.ref }}
-          ref-template: 2024.07.26  # Make sure it is aligned with ZENML_PROJECT_TEMPLATES from src/zenml/cli/base.py
+          ref-template: 2024.08.28  # Make sure it is aligned with ZENML_PROJECT_TEMPLATES from src/zenml/cli/base.py
       - name: Clean-up
         run: |
           rm -rf ./local_checkout
@@ -261,7 +261,7 @@ jobs:
         with:
           python-version: ${{ inputs.python-version }}
           ref-zenml: ${{ github.ref }}
-          ref-template: 2024.06.20  # Make sure it is aligned with ZENML_PROJECT_TEMPLATES from src/zenml/cli/base.py
+          ref-template: 2024.08.29  # Make sure it is aligned with ZENML_PROJECT_TEMPLATES from src/zenml/cli/base.py
       - name: Clean-up
         run: |
           rm -rf ./local_checkout

docs/mocked_libs.json

@@ -162,7 +162,9 @@
     "model_archiver.model_packaging_utils",
     "neptune",
     "neuralprophet",
+    "numpy",
     "openai",
+    "pandas",
     "paramiko",
     "polars",
     "pyarrow",

examples/e2e/.copier-answers.yml

@@ -1,5 +1,5 @@
 # Changes here will be overwritten by Copier
-_commit: 2024.06.06
+_commit: 2024.08.29
 _src_path: gh:zenml-io/template-e2e-batch
 data_quality_checks: true
 email: [email protected]

examples/e2e_nlp/.copier-answers.yml

@@ -1,5 +1,5 @@
 # Changes here will be overwritten by Copier
-_commit: 2024.08.07
+_commit: 2024.08.29
 _src_path: gh:zenml-io/template-nlp
 accelerator: cpu
 cloud_of_choice: aws

examples/llm_finetuning/.copier-answers.yml

@@ -1,5 +1,5 @@
 # Changes here will be overwritten by Copier
-_commit: 2024.06.20
+_commit: 2024.08.29
 _src_path: gh:zenml-io/template-llm-finetuning
 bf16: true
 cuda_version: cuda11.8

examples/mlops_starter/.copier-answers.yml

@@ -1,5 +1,5 @@
 # Changes here will be overwritten by Copier
-_commit: 2024.07.26
+_commit: 2024.08.28
 _src_path: gh:zenml-io/template-starter
 email: [email protected]
 full_name: ZenML GmbH

pyproject.toml

@@ -21,6 +21,7 @@ classifiers = [
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "Topic :: System :: Distributed Computing",
     "Topic :: Software Development :: Libraries :: Python Modules",
     "Typing :: Typed",
@@ -44,26 +45,21 @@ zenml = "zenml.cli.cli:cli"
 alembic = { version = "~1.8.1" }
 bcrypt = { version = "4.0.1" }
 click = "^8.0.1,<8.1.4"
-click-params = "^0.3.0"
 cloudpickle = ">=2.0.0,<3"
 distro = "^1.6.0"
 docker = "~7.1.0"
-httplib2 = "<0.20,>=0.19.1"
 gitpython = "^3.1.18"
-numpy = "<2.0.0"
 packaging = ">=24.1"
-pandas = ">=1.1.5"
 passlib = { extras = ["bcrypt"], version = "~1.7.4" }
 psutil = ">=5.0.0"
-pydantic = "~2.7"
-pydantic-settings = "~2.2.1"
+pydantic = "~2.8"
+pydantic-settings = "*"
 pymysql = { version = "~1.1.0,>=1.1.1" }
-pyparsing = ">=2.4.0"
-python = ">=3.8,<3.12"
+python = ">=3.8,<3.13"
 python-dateutil = "^2.8.1"
 pyyaml = ">=6.0.1"
 rich = { extras = ["jupyter"], version = ">=12.0.0" }
-secure = "~0.3.0"
+setuptools = "*"
 sqlalchemy = "^2.0.0"
 sqlalchemy_utils = "*"
 sqlmodel = "0.0.18"
@@ -77,6 +73,7 @@ pyjwt = { extras = ["crypto"], version = "2.7.*", optional = true }
 orjson = { version = "~3.10.0", optional = true }
 Jinja2 = { version = "*", optional = true }
 ipinfo = { version = ">=4.4.3", optional = true }
+secure = { version = "~0.3.0" , optional = true }
 
 # Optional dependencies for project templates
 copier = { version = ">=8.1.0", optional = true }
@@ -137,6 +134,7 @@ typing-extensions = { version = ">=3.7.4", optional = true }
 darglint = { version = "^1.8.1", optional = true }
 ruff = { version = ">=0.1.7", optional = true }
 yamlfix = { version = "^1.16.0", optional = true }
+maison = { version = "<2.0", optional = true }
 
 # pytest
 pytest = { version = "^7.4.0", optional = true }
@@ -153,6 +151,7 @@ mkdocs-material = { version = "^8.1.7", optional = true }
 mkdocs-awesome-pages-plugin = { version = "^2.6.1", optional = true }
 mkdocstrings = { version = "^0.17.0", optional = true }
 mike = { version = "^1.1.2", optional = true }
+
 # mypy type stubs
 types-certifi = { version = "^2021.10.8.0", optional = true }
 types-croniter = { version = "^1.0.2", optional = true }
@@ -186,6 +185,7 @@ server = [
     "orjson",
     "Jinja2",
     "ipinfo",
+    "secure",
 ]
 templates = ["copier", "jinja2-time", "ruff", "pyyaml-include"]
 terraform = ["python-terraform"]
@@ -237,6 +237,7 @@ dev = [
     "mkdocs-awesome-pages-plugin",
     "mkdocstrings",
     "mike",
+    "maison",
     "types-certifi",
     "types-croniter",
     "types-futures",
@@ -462,6 +463,8 @@ module = [
     "matplotlib.*",
     "IPython.*",
     "huggingface_hub.*",
+    "distutils.*",
+    "accelerate.*",
     "label_studio_sdk.*",
     "argilla.*",
     "lightning_sdk.*",

scripts/install-zenml-dev.sh

@@ -28,7 +28,9 @@ parse_args () {
 
 install_zenml() {
     # install ZenML in editable mode
-    uv pip install $PIP_ARGS -e ".[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]"
+    uv pip install $PIP_ARGS -e ".[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev]"
+    # TODO: Remove and add the extra back in
+    uv pip install $PIP_ARGS git+https://github.com/zenml-io/mlstacks.git@feature/upgrade-python-3.12
 }
 
 install_integrations() {
@@ -37,6 +39,11 @@ install_integrations() {
     python_version=$(python -c "import sys; print('.'.join(map(str, sys.version_info[:2])))")
 
     ignore_integrations="feast label_studio bentoml seldon pycaret skypilot_aws skypilot_gcp skypilot_azure pigeon prodigy"
+
+    # Ignore tensorflow and deepchecks only on Python 3.12
+    if [ "$python_version" = "3.12" ]; then
+        ignore_integrations="$ignore_integrations tensorflow deepchecks"
+    fi
 
     # turn the ignore integrations into a list of --ignore-integration args
     ignore_integrations_args=""

scripts/test-migrations.sh

@@ -109,10 +109,18 @@ function run_tests_for_version() {
 
     cd test_starter
 
-    echo "===== Installing sklearn integration ====="
-    zenml integration export-requirements sklearn --output-file sklearn-requirements.txt
-    uv pip install -r sklearn-requirements.txt
-    rm sklearn-requirements.txt
+    echo "===== Installing required integrations ====="
+    # TODO: REMOVE BEFORE MERGE
+    if [ "$VERSION" == "current" ]; then
+        zenml integration export-requirements sklearn pandas --output-file integration-requirements.txt
+    elif [ "$(version_compare "$VERSION" "0.66.0")" == "<" ]; then
+        zenml integration export-requirements sklearn --output-file integration-requirements.txt
+    else
+        zenml integration export-requirements sklearn pandas --output-file integration-requirements.txt
+    fi
+
+    uv pip install -r integration-requirements.txt
+    rm integration-requirements.txt
 
     echo "===== Running starter template pipeline ====="
     # Check if the version supports templates with arguments (> 0.52.0)
@@ -208,13 +216,12 @@ function test_upgrade_to_version() {
 
     if [ "$VERSION" == "current" ]; then
         uv pip install -e ".[templates,server]"
-    elif [ "$(version_compare "$VERSION" "0.58.2")" == ">" ]; then
-        # For releases  >= 0.60.0 we don't want to have those old requirements pinned
-        uv pip install -e ".[templates,server]"
     else
         uv pip install "zenml[templates,server]==$VERSION"
-        # handles unpinned sqlmodel dependency in older versions
-        uv pip install "sqlmodel==0.0.8" "bcrypt==4.0.1" "pyyaml-include<2.0" "numpy<2.0.0" "tenacity!=8.4.0"
+        if [ "$(version_compare "$VERSION" "0.60.0")" == "<" ]; then
+            # handles unpinned sqlmodel dependency in older versions
+            uv pip install "sqlmodel==0.0.8" "bcrypt==4.0.1" "pyyaml-include<2.0" "numpy<2.0.0" "tenacity!=8.4.0"
+        fi
     fi
 
     # Get the major and minor version of Python

src/zenml/cli/base.py

@@ -79,19 +79,19 @@ def copier_github_url(self) -> str:
 ZENML_PROJECT_TEMPLATES = dict(
     e2e_batch=ZenMLProjectTemplateLocation(
         github_url="zenml-io/template-e2e-batch",
-        github_tag="2024.06.06",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
+        github_tag="2024.08.29",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
     ),
     starter=ZenMLProjectTemplateLocation(
         github_url="zenml-io/template-starter",
-        github_tag="2024.07.26",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
+        github_tag="2024.08.28",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
     ),
     nlp=ZenMLProjectTemplateLocation(
         github_url="zenml-io/template-nlp",
-        github_tag="2024.08.07",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
+        github_tag="2024.08.29",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
     ),
     llm_finetuning=ZenMLProjectTemplateLocation(
         github_url="zenml-io/template-llm-finetuning",
-        github_tag="2024.06.20",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
+        github_tag="2024.08.29",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
     ),
 )