From e533609690b9a261781f5cc8c0130fe85b41e7eb Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Sun, 13 Oct 2024 15:19:00 +0100
Subject: [PATCH 01/60] Refactor environment setup and caching

---
 .github/actions/setup_environment/action.yml | 16 ++++++++++++++++
 .github/workflows/ci-fast.yml                |  1 +
 2 files changed, 17 insertions(+)

diff --git a/.github/actions/setup_environment/action.yml b/.github/actions/setup_environment/action.yml
index d5b70818292..c51fc0b23c6 100644
--- a/.github/actions/setup_environment/action.yml
+++ b/.github/actions/setup_environment/action.yml
@@ -29,6 +29,22 @@ inputs:
 runs:
   using: composite
   steps:
+    - name: Restore pip cache
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/pip
+        key: pip-${{ runner.os }}-${{ inputs.python-version }}-${{ hashFiles('src/zenml/integrations/*/__init__.py')
+          }}
+        restore-keys: |
+          pip-${{ runner.os }}-${{ inputs.python-version }}
+    - name: Restore uv cache
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/uv
+        key: uv-${{ runner.os }}-${{ inputs.python-version }}-${{ hashFiles('src/zenml/integrations/*/__init__.py')
+          }}
+        restore-keys: |
+          uv-${{ runner.os }}-${{ inputs.python-version }}
     - name: Set up Python
       uses: actions/setup-python@v5.0.0
       with:
diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml
index b2f0b8626b9..032fd01535f 100644
--- a/.github/workflows/ci-fast.yml
+++ b/.github/workflows/ci-fast.yml
@@ -95,6 +95,7 @@ jobs:
       os: ${{ matrix.os }}
     secrets: inherit
   custom-arc-runner-set-integration-test:
+    needs: custom-ubuntu-setup-and-unit-test
     if: github.event.pull_request.draft == false
     strategy:
       matrix:

From daf146f667a3fe719a0579fba703c506bf6b029a Mon Sep 17 00:00:00 2001
From: GitHub Actions <actions@github.com>
Date: Sun, 13 Oct 2024 14:21:21 +0000
Subject: [PATCH 02/60] Auto-update of LLM Finetuning template

---
 examples/llm_finetuning/.copier-answers.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/llm_finetuning/.copier-answers.yml b/examples/llm_finetuning/.copier-answers.yml
index ba14a720770..09cb600a5fa 100644
--- a/examples/llm_finetuning/.copier-answers.yml
+++ b/examples/llm_finetuning/.copier-answers.yml
@@ -1,5 +1,5 @@
 # Changes here will be overwritten by Copier
-_commit: 2024.08.29-1-ge25e9e0
+_commit: 2024.09.24
 _src_path: gh:zenml-io/template-llm-finetuning
 bf16: true
 cuda_version: cuda11.8

From 764b0fc610b4c857380cb8221a55b6ebf7cf860f Mon Sep 17 00:00:00 2001
From: GitHub Actions <actions@github.com>
Date: Sun, 13 Oct 2024 14:21:43 +0000
Subject: [PATCH 03/60] Auto-update of Starter template

---
 examples/mlops_starter/.copier-answers.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/mlops_starter/.copier-answers.yml b/examples/mlops_starter/.copier-answers.yml
index 0c5cf3dc861..8b1fb8187ed 100644
--- a/examples/mlops_starter/.copier-answers.yml
+++ b/examples/mlops_starter/.copier-answers.yml
@@ -1,5 +1,5 @@
 # Changes here will be overwritten by Copier
-_commit: 2024.09.23-1-g9128c4b
+_commit: 2024.09.24
 _src_path: gh:zenml-io/template-starter
 email: info@zenml.io
 full_name: ZenML GmbH

From 2d532a929e32a29a8220cb3b7daae557f772b80d Mon Sep 17 00:00:00 2001
From: GitHub Actions <actions@github.com>
Date: Sun, 13 Oct 2024 14:28:25 +0000
Subject: [PATCH 04/60] Auto-update of E2E template

---
 examples/e2e/.copier-answers.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/e2e/.copier-answers.yml b/examples/e2e/.copier-answers.yml
index 09faefb8a45..74cc33d8594 100644
--- a/examples/e2e/.copier-answers.yml
+++ b/examples/e2e/.copier-answers.yml
@@ -1,5 +1,5 @@
 # Changes here will be overwritten by Copier
-_commit: 2024.10.09-2-g429e445
+_commit: 2024.10.10
 _src_path: gh:zenml-io/template-e2e-batch
 data_quality_checks: true
 email: info@zenml.io

From 098b3dec6df584815349f13aef8aa55c220f444e Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Sun, 13 Oct 2024 18:38:14 +0100
Subject: [PATCH 05/60] Refactor environment setup and caching in ci-fast.yml

---
 .github/workflows/ci-fast.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml
index 032fd01535f..e86d5d3954f 100644
--- a/.github/workflows/ci-fast.yml
+++ b/.github/workflows/ci-fast.yml
@@ -16,7 +16,7 @@ concurrency:
 jobs:
   docstring-check:
     if: github.event.pull_request.draft == false
-    runs-on: arc-runner-set
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout code
         uses: actions/checkout@v4.1.1
@@ -32,7 +32,7 @@ jobs:
       - name: Check docstrings
         run: bash scripts/docstring.sh
   sqlite-db-migration-testing-random:
-    runs-on: arc-runner-set
+    runs-on: ubuntu-latest
     env:
       ZENML_ANALYTICS_OPT_IN: false
       ZENML_DEBUG: true
@@ -86,7 +86,7 @@ jobs:
     if: github.event.pull_request.draft == false
     strategy:
       matrix:
-        os: [arc-runner-set]
+        os: [ubuntu-latest]
         python-version: ['3.10']
       fail-fast: false
     uses: ./.github/workflows/unit-test.yml

From 7b708d579b950cc32aff317349f170ebd769fb0f Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Sun, 13 Oct 2024 18:47:41 +0100
Subject: [PATCH 06/60] Refactor environment setup and caching in ci-fast.yml

---
 .github/actions/setup_environment/action.yml | 16 ----------------
 .github/workflows/unit-test.yml              | 14 ++++++++++++++
 2 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/.github/actions/setup_environment/action.yml b/.github/actions/setup_environment/action.yml
index c51fc0b23c6..d5b70818292 100644
--- a/.github/actions/setup_environment/action.yml
+++ b/.github/actions/setup_environment/action.yml
@@ -29,22 +29,6 @@ inputs:
 runs:
   using: composite
   steps:
-    - name: Restore pip cache
-      uses: actions/cache@v4
-      with:
-        path: ~/.cache/pip
-        key: pip-${{ runner.os }}-${{ inputs.python-version }}-${{ hashFiles('src/zenml/integrations/*/__init__.py')
-          }}
-        restore-keys: |
-          pip-${{ runner.os }}-${{ inputs.python-version }}
-    - name: Restore uv cache
-      uses: actions/cache@v4
-      with:
-        path: ~/.cache/uv
-        key: uv-${{ runner.os }}-${{ inputs.python-version }}-${{ hashFiles('src/zenml/integrations/*/__init__.py')
-          }}
-        restore-keys: |
-          uv-${{ runner.os }}-${{ inputs.python-version }}
     - name: Set up Python
       uses: actions/setup-python@v5.0.0
       with:
diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
index 9737c8f2ce2..2d34fab4e14 100644
--- a/.github/workflows/unit-test.yml
+++ b/.github/workflows/unit-test.yml
@@ -77,6 +77,20 @@ jobs:
         with:
           repository: ${{ github.repository }}
           ref: ${{ github.event.pull_request.head.sha }}
+      - name: Restore pip cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: pip-${{ runner.os }}-${{ inputs.python-version }}-${{ hashFiles('src/zenml/integrations/*/__init__.py') }}
+          restore-keys: |
+            pip-${{ runner.os }}-${{ inputs.python-version }}
+      - name: Restore uv cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/uv
+          key: uv-${{ runner.os }}-${{ inputs.python-version }}-${{ hashFiles('src/zenml/integrations/*/__init__.py') }}
+          restore-keys: |
+            uv-${{ runner.os }}-${{ inputs.python-version }}
       - name: Install libomp
         if: ${{ inputs.os == 'macos-latest' || inputs.os == 'macos-13' }}
         run: brew install libomp

From 4d4f49ad4e10b93bbd708e92fae41c4abcf96925 Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Sun, 13 Oct 2024 19:11:48 +0100
Subject: [PATCH 07/60] Refactor caching keys in ci-fast.yml

---
 .github/workflows/unit-test.yml | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
index 2d34fab4e14..12c8fdd0d75 100644
--- a/.github/workflows/unit-test.yml
+++ b/.github/workflows/unit-test.yml
@@ -81,16 +81,18 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.cache/pip
-          key: pip-${{ runner.os }}-${{ inputs.python-version }}-${{ hashFiles('src/zenml/integrations/*/__init__.py') }}
+          key: |
+            pip-${{ runner.os }}-${{ hashFiles('src/zenml/integrations/*/__init__.py') }}
           restore-keys: |
-            pip-${{ runner.os }}-${{ inputs.python-version }}
+            pip-${{ runner.os }}-${{ hashFiles('src/zenml/integrations/*/__init__.py') }}
       - name: Restore uv cache
         uses: actions/cache@v4
         with:
           path: ~/.cache/uv
-          key: uv-${{ runner.os }}-${{ inputs.python-version }}-${{ hashFiles('src/zenml/integrations/*/__init__.py') }}
+          key: |
+            uv-${{ runner.os }}-${{ hashFiles('src/zenml/integrations/*/__init__.py') }}
           restore-keys: |
-            uv-${{ runner.os }}-${{ inputs.python-version }}
+            uv-${{ runner.os }}-${{ hashFiles('src/zenml/integrations/*/__init__.py') }}
       - name: Install libomp
         if: ${{ inputs.os == 'macos-latest' || inputs.os == 'macos-13' }}
         run: brew install libomp

From f9c9190f35023ff829328ad647929302fb794996 Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Sun, 13 Oct 2024 19:18:21 +0100
Subject: [PATCH 08/60] Refactor mlflow integration dependencies

---
 src/zenml/integrations/mlflow/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/zenml/integrations/mlflow/__init__.py b/src/zenml/integrations/mlflow/__init__.py
index fcd9c1f0545..017b769e4ae 100644
--- a/src/zenml/integrations/mlflow/__init__.py
+++ b/src/zenml/integrations/mlflow/__init__.py
@@ -58,7 +58,7 @@ def get_requirements(cls, target_os: Optional[str] = None) -> List[str]:
         from zenml.integrations.pandas import PandasIntegration
 
         reqs = [
-            "mlflow>=2.1.1,<3",
+            "mlflow>=2.1.1,<2.17.0",
             # TODO: remove this requirement once rapidjson is fixed
             "python-rapidjson<1.15",
             # When you do:

From e9d860ad063c02f94a474721c6f7475a5996c297 Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Sun, 13 Oct 2024 21:33:16 +0100
Subject: [PATCH 09/60] Refactor environment setup and caching in ci-fast.yml
 and lint.sh

---
 .github/workflows/unit-test.yml | 18 +++++++-----
 scripts/lint.sh                 | 52 +++++++++++++++++++++++++++++----
 2 files changed, 57 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
index 12c8fdd0d75..74f7839ba5d 100644
--- a/.github/workflows/unit-test.yml
+++ b/.github/workflows/unit-test.yml
@@ -60,13 +60,7 @@ jobs:
       ZENML_ANALYTICS_OPT_IN: false
       PYTHONIOENCODING: utf-8
       UV_HTTP_TIMEOUT: 600
-
-      # on MAC OS, we need to set this environment variable
-      # to fix problems with the fork() calls (see this thread
-      # for more information: http://sealiesoftware.com/blog/archive/2017/6/5/Objective-C_and_fork_in_macOS_1013.html)
       OBJC_DISABLE_INITIALIZE_FORK_SAFETY: 'YES'
-    # TODO: add Windows testing for Python 3.11 and 3.12 back in
-    # Exit if it's a commit from Gitbook
     if: ${{ ! startsWith(github.event.head_commit.message, 'GitBook:') && ! (inputs.os == 'windows-latest' && inputs.python-version == '3.11') && ! (inputs.os == 'windows-latest' && inputs.python-version == '3.12') }}
     defaults:
       run:
@@ -77,6 +71,15 @@ jobs:
         with:
           repository: ${{ github.repository }}
           ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0  # Fetch all history for all branches and tags
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v42
+        with:
+          files: |
+            src/**
+            tests/**
+            examples/**
       - name: Restore pip cache
         uses: actions/cache@v4
         with:
@@ -112,8 +115,9 @@ jobs:
       - name: Lint check
         env:
           OS: ${{ inputs.os }}
+          CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
         run: |
-          bash scripts/lint.sh
+          bash scripts/lint.sh --changed-files "$CHANGED_FILES"
       - name: Run unit tests
         run: |
           bash scripts/test-coverage-xml.sh unit
diff --git a/scripts/lint.sh b/scripts/lint.sh
index 26bbcdaa3ed..474cf70492e 100755
--- a/scripts/lint.sh
+++ b/scripts/lint.sh
@@ -3,14 +3,33 @@ set -e
 set -x
 set -o pipefail
 
+# Parse command line arguments
+while [[ $# -gt 0 ]]; do
+  key="$1"
+  case $key in
+    --changed-files)
+      CHANGED_FILES="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    *)
+      # unknown option
+      shift
+      ;;
+  esac
+done
+
 # Default source directories
-SRC=${1:-"src/zenml tests examples"}
-SRC_NO_TESTS=${1:-"src/zenml tests/harness"}
-TESTS_EXAMPLES=${1:-"tests examples"}
+SRC="src/zenml tests examples"
+SRC_NO_TESTS="src/zenml tests/harness"
+TESTS_EXAMPLES="tests examples"
 
 export ZENML_DEBUG=1
 export ZENML_ANALYTICS_OPT_IN=false
+
+# Run ruff check on SRC_NO_TESTS
 ruff check $SRC_NO_TESTS
+
 # TODO: Fix docstrings in tests and examples and remove the `--extend-ignore D` flag
 ruff check $TESTS_EXAMPLES --extend-ignore D --extend-exclude "*.ipynb"
 
@@ -36,7 +55,28 @@ fi
 # autoflake replacement: checks for unused imports and variables
 ruff check $SRC --select F401,F841 --exclude "__init__.py" --exclude "*.ipynb" --isolated
 
-ruff format $SRC  --check
+ruff format $SRC --check
+
+# check type annotations with mypy
+# If CHANGED_FILES is not set, check all files in SRC_NO_TESTS
+if [ -z "$CHANGED_FILES" ]; then
+    mypy $SRC_NO_TESTS
+else
+    # Convert the space-separated list to an array
+    IFS=' ' read -ra CHANGED_FILES_ARRAY <<< "$CHANGED_FILES"
+    
+    MYPY_FILES=()
+
+    for file in "${CHANGED_FILES_ARRAY[@]}"; do
+        if [[ $file == src/zenml* || $file == tests/harness* ]]; then
+            MYPY_FILES+=("$file")
+        fi
+    done
 
-# check type annotations
-mypy $SRC_NO_TESTS
+    # Run mypy only if there are files to check
+    if [ ${#MYPY_FILES[@]} -gt 0 ]; then
+        mypy "${MYPY_FILES[@]}"
+    else
+        echo "No files to check with mypy."
+    fi
+fi
\ No newline at end of file

From 3fc6952e7d86513c76d1ff252807a7e6ecc88055 Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Sun, 13 Oct 2024 22:08:35 +0100
Subject: [PATCH 10/60] Refactor environment setup and caching in ci-fast.yml
 and lint.sh

---
 .github/workflows/ci-fast.yml               |  8 ++++----
 .github/workflows/integration-test-fast.yml | 12 ++++++++++--
 .github/workflows/unit-test.yml             | 12 ++----------
 scripts/lint.sh                             |  4 ++--
 4 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml
index e86d5d3954f..9a2ba7d3084 100644
--- a/.github/workflows/ci-fast.yml
+++ b/.github/workflows/ci-fast.yml
@@ -87,7 +87,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: ['3.10']
+        python-version: ['3.12']
       fail-fast: false
     uses: ./.github/workflows/unit-test.yml
     with:
@@ -99,9 +99,9 @@ jobs:
     if: github.event.pull_request.draft == false
     strategy:
       matrix:
-        os: [arc-runner-set]
-        python-version: ['3.10']
-        test_environment: [default, docker-server-docker-orchestrator-mysql]
+        os: [ubuntu-latest]
+        python-version: ['3.12']
+        test_environment: [default]
       fail-fast: false
     uses: ./.github/workflows/integration-test-fast.yml
     with:
diff --git a/.github/workflows/integration-test-fast.yml b/.github/workflows/integration-test-fast.yml
index cc1e97eaaa9..b78476b5382 100644
--- a/.github/workflows/integration-test-fast.yml
+++ b/.github/workflows/integration-test-fast.yml
@@ -78,7 +78,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
+        shard: [1, 2, 3, 4, 5, 6]
     env:
       ZENML_DEBUG: 1
       ZENML_ANALYTICS_OPT_IN: false
@@ -119,6 +119,14 @@ jobs:
           || contains(inputs.test_environment, 'kubeflow') || contains(inputs.test_environment,
           'airflow') || contains(inputs.test_environment, 'kubernetes'))
       - uses: actions/checkout@v4.1.1
+      - name: Restore uv cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/uv
+          key: |
+            uv-${{ runner.os }}-${{ inputs.python-version }}-${{ hashFiles('src/zenml/integrations/*/__init__.py') }}
+          restore-keys: |
+            uv-${{ runner.os }}-${{ inputs.python-version }}-${{ hashFiles('src/zenml/integrations/*/__init__.py') }}
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:
@@ -236,7 +244,7 @@ jobs:
         # Ubuntu integration tests run as 6 shards
         if: runner.os != 'macOS' && runner.os != 'Windows'
         run: |
-          bash scripts/test-coverage-xml.sh integration ${{ inputs.test_environment }} 12 ${{ matrix.shard }}
+          bash scripts/test-coverage-xml.sh integration ${{ inputs.test_environment }} 6 ${{ matrix.shard }}
       - name: Setup tmate session after tests
         if: ${{ inputs.enable_tmate == 'always' || (inputs.enable_tmate == 'on-failure' && failure()) }}
         uses: mxschmitt/action-tmate@v3.17
diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
index 74f7839ba5d..840d2f79e11 100644
--- a/.github/workflows/unit-test.yml
+++ b/.github/workflows/unit-test.yml
@@ -80,22 +80,14 @@ jobs:
             src/**
             tests/**
             examples/**
-      - name: Restore pip cache
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/pip
-          key: |
-            pip-${{ runner.os }}-${{ hashFiles('src/zenml/integrations/*/__init__.py') }}
-          restore-keys: |
-            pip-${{ runner.os }}-${{ hashFiles('src/zenml/integrations/*/__init__.py') }}
       - name: Restore uv cache
         uses: actions/cache@v4
         with:
           path: ~/.cache/uv
           key: |
-            uv-${{ runner.os }}-${{ hashFiles('src/zenml/integrations/*/__init__.py') }}
+            uv-${{ runner.os }}-${{ inputs.python-version }}-${{ hashFiles('src/zenml/integrations/*/__init__.py') }}
           restore-keys: |
-            uv-${{ runner.os }}-${{ hashFiles('src/zenml/integrations/*/__init__.py') }}
+            uv-${{ runner.os }}-${{ inputs.python-version }}-${{ hashFiles('src/zenml/integrations/*/__init__.py') }}
       - name: Install libomp
         if: ${{ inputs.os == 'macos-latest' || inputs.os == 'macos-13' }}
         run: brew install libomp
diff --git a/scripts/lint.sh b/scripts/lint.sh
index 474cf70492e..ea447c34977 100755
--- a/scripts/lint.sh
+++ b/scripts/lint.sh
@@ -60,7 +60,7 @@ ruff format $SRC --check
 # check type annotations with mypy
 # If CHANGED_FILES is not set, check all files in SRC_NO_TESTS
 if [ -z "$CHANGED_FILES" ]; then
-    mypy $SRC_NO_TESTS
+    mypy --cache-dir='~/.cache/mypy' $SRC_NO_TESTS
 else
     # Convert the space-separated list to an array
     IFS=' ' read -ra CHANGED_FILES_ARRAY <<< "$CHANGED_FILES"
@@ -75,7 +75,7 @@ else
 
     # Run mypy only if there are files to check
     if [ ${#MYPY_FILES[@]} -gt 0 ]; then
-        mypy "${MYPY_FILES[@]}"
+        mypy --cache-dir='~/.cache/mypy' "${MYPY_FILES[@]}"
     else
         echo "No files to check with mypy."
     fi

From e69482045d2c88e903ac221b073383b8fb5e8396 Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Sun, 13 Oct 2024 22:15:28 +0100
Subject: [PATCH 11/60] Refactor mypy cache directory in lint.sh

---
 scripts/lint.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/lint.sh b/scripts/lint.sh
index ea447c34977..474cf70492e 100755
--- a/scripts/lint.sh
+++ b/scripts/lint.sh
@@ -60,7 +60,7 @@ ruff format $SRC --check
 # check type annotations with mypy
 # If CHANGED_FILES is not set, check all files in SRC_NO_TESTS
 if [ -z "$CHANGED_FILES" ]; then
-    mypy --cache-dir='~/.cache/mypy' $SRC_NO_TESTS
+    mypy $SRC_NO_TESTS
 else
     # Convert the space-separated list to an array
     IFS=' ' read -ra CHANGED_FILES_ARRAY <<< "$CHANGED_FILES"
@@ -75,7 +75,7 @@ else
 
     # Run mypy only if there are files to check
     if [ ${#MYPY_FILES[@]} -gt 0 ]; then
-        mypy --cache-dir='~/.cache/mypy' "${MYPY_FILES[@]}"
+        mypy "${MYPY_FILES[@]}"
     else
         echo "No files to check with mypy."
     fi

From 35de763fbe796ec48086adebed8bcd79d7071bc1 Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Sun, 13 Oct 2024 22:22:42 +0100
Subject: [PATCH 12/60] Refactor workflow in ci-fast.yml to remove unnecessary
 job dependency

---
 .github/workflows/ci-fast.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml
index 9a2ba7d3084..726627fcc4a 100644
--- a/.github/workflows/ci-fast.yml
+++ b/.github/workflows/ci-fast.yml
@@ -95,7 +95,6 @@ jobs:
       os: ${{ matrix.os }}
     secrets: inherit
   custom-arc-runner-set-integration-test:
-    needs: custom-ubuntu-setup-and-unit-test
     if: github.event.pull_request.draft == false
     strategy:
       matrix:

From 0903f6578fb058bdf8f54d5d5c6820f799ef6ba8 Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Mon, 14 Oct 2024 01:10:20 +0100
Subject: [PATCH 13/60] crazy expirement

---
 .github/workflows/ci-fast.yml               | 40 +++++++++++++++-
 .github/workflows/integration-test-fast.yml | 20 ++++++++
 .github/workflows/spellcheck.yml            |  2 +-
 scripts/lint.sh                             | 52 +++------------------
 tests/harness/cfg/deployments.yaml          | 11 +++++
 5 files changed, 77 insertions(+), 48 deletions(-)

diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml
index 726627fcc4a..5d91c8dc146 100644
--- a/.github/workflows/ci-fast.yml
+++ b/.github/workflows/ci-fast.yml
@@ -94,7 +94,7 @@ jobs:
       python-version: ${{ matrix.python-version }}
       os: ${{ matrix.os }}
     secrets: inherit
-  custom-arc-runner-set-integration-test:
+  custom-arc-runner-set-integration-test-default:
     if: github.event.pull_request.draft == false
     strategy:
       matrix:
@@ -108,3 +108,41 @@ jobs:
       python-version: ${{ matrix.python-version }}
       test_environment: ${{ matrix.test_environment }}
     secrets: inherit
+  build-and-push:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4.1.1
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: arn:aws:iam::715803424590:role/gh-action-role-zenml-quickstart-ecr
+          aws-region: eu-central-1
+      - name: Login to Amazon ECR
+        id: login-ecr
+        run: |
+          aws ecr get-login-password --region eu-central-1 | docker login --username AWS --password-stdin 715803424590.dkr.ecr.eu-central-1.amazonaws.com
+      - name: Build and push ZenML server image to AWS ECR
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: 715803424590.dkr.ecr.eu-central-1.amazonaws.com/zenml-public-pipelines:${{ github.sha }}
+          file: ./Dockerfile  # Adjust this path if your Dockerfile is located elsewhere
+  custom-arc-runner-set-integration-test-server:
+    needs: build-and-push
+    if: github.event.pull_request.draft == false
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ['3.12']
+        test_environment: [github-actions-server]
+      fail-fast: false
+    uses: ./.github/workflows/integration-test-fast.yml
+    with:
+      os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }}
+      test_environment: ${{ matrix.test_environment }}
+    secrets: inherit
diff --git a/.github/workflows/integration-test-fast.yml b/.github/workflows/integration-test-fast.yml
index b78476b5382..4ffab6c6b00 100644
--- a/.github/workflows/integration-test-fast.yml
+++ b/.github/workflows/integration-test-fast.yml
@@ -100,6 +100,26 @@ jobs:
     defaults:
       run:
         shell: bash
+    services:
+      mysql:
+        image: mysql:5.7
+        env:
+          MYSQL_ROOT_PASSWORD: zenml
+          MYSQL_DATABASE: zenml
+        ports: [3306:3306]
+        options: >-
+          --health-cmd="mysqladmin ping"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=3
+      zenml-server:
+        image: 715803424590.dkr.ecr.eu-central-1.amazonaws.com/zenml-public-pipelines:${{ github.sha }}
+        env:
+          ZENML_STORE_URL: mysql://root:zenml@mysql:3306/zenml
+          ZENML_SERVER_DEPLOYMENT_TYPE: docker
+          ZENML_SERVER_AUTO_ACTIVATE: 'True'
+          ZENML_SERVER_AUTO_CREATE_DEFAULT_USER: 'True'
+        ports: [8080:8080]
     steps:
       - name: Maximize space for Docker
         uses: easimon/maximize-build-space@v10
diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml
index 55f481d1ced..288a2a1f0df 100644
--- a/.github/workflows/spellcheck.yml
+++ b/.github/workflows/spellcheck.yml
@@ -16,7 +16,7 @@ concurrency:
 jobs:
   spellcheck:
     if: github.event.pull_request.draft == false && github.repository == 'zenml-io/zenml'
-    runs-on: arc-runner-set
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout code
         uses: actions/checkout@v4.1.1
diff --git a/scripts/lint.sh b/scripts/lint.sh
index 474cf70492e..eb9af19e000 100755
--- a/scripts/lint.sh
+++ b/scripts/lint.sh
@@ -3,33 +3,14 @@ set -e
 set -x
 set -o pipefail
 
-# Parse command line arguments
-while [[ $# -gt 0 ]]; do
-  key="$1"
-  case $key in
-    --changed-files)
-      CHANGED_FILES="$2"
-      shift # past argument
-      shift # past value
-      ;;
-    *)
-      # unknown option
-      shift
-      ;;
-  esac
-done
-
 # Default source directories
-SRC="src/zenml tests examples"
-SRC_NO_TESTS="src/zenml tests/harness"
-TESTS_EXAMPLES="tests examples"
+SRC=${1:-"src/zenml tests examples"}
+SRC_NO_TESTS=${1:-"src/zenml tests/harness"}
+TESTS_EXAMPLES=${1:-"tests examples"}
 
 export ZENML_DEBUG=1
 export ZENML_ANALYTICS_OPT_IN=false
-
-# Run ruff check on SRC_NO_TESTS
 ruff check $SRC_NO_TESTS
-
 # TODO: Fix docstrings in tests and examples and remove the `--extend-ignore D` flag
 ruff check $TESTS_EXAMPLES --extend-ignore D --extend-exclude "*.ipynb"
 
@@ -55,28 +36,7 @@ fi
 # autoflake replacement: checks for unused imports and variables
 ruff check $SRC --select F401,F841 --exclude "__init__.py" --exclude "*.ipynb" --isolated
 
-ruff format $SRC --check
-
-# check type annotations with mypy
-# If CHANGED_FILES is not set, check all files in SRC_NO_TESTS
-if [ -z "$CHANGED_FILES" ]; then
-    mypy $SRC_NO_TESTS
-else
-    # Convert the space-separated list to an array
-    IFS=' ' read -ra CHANGED_FILES_ARRAY <<< "$CHANGED_FILES"
-    
-    MYPY_FILES=()
-
-    for file in "${CHANGED_FILES_ARRAY[@]}"; do
-        if [[ $file == src/zenml* || $file == tests/harness* ]]; then
-            MYPY_FILES+=("$file")
-        fi
-    done
+ruff format $SRC  --check
 
-    # Run mypy only if there are files to check
-    if [ ${#MYPY_FILES[@]} -gt 0 ]; then
-        mypy "${MYPY_FILES[@]}"
-    else
-        echo "No files to check with mypy."
-    fi
-fi
\ No newline at end of file
+# check type annotations
+mypy $SRC_NO_TESTS
\ No newline at end of file
diff --git a/tests/harness/cfg/deployments.yaml b/tests/harness/cfg/deployments.yaml
index 36379970289..1ee09598f82 100644
--- a/tests/harness/cfg/deployments.yaml
+++ b/tests/harness/cfg/deployments.yaml
@@ -111,3 +111,14 @@ deployments:
     database: mariadb
     capabilities:
       server: true
+  - name: github-actions-server
+    description: >-
+      Local ZenML server and MariaDB both running in docker with docker-compose.
+    server: external
+    database: external
+    capabilities:
+      server: true
+    config:
+      url: http://127.0.0.1:8080
+      username: default
+      password: ''

From a712dcfd814269b0aee4046ef1a762f66433c7af Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Mon, 14 Oct 2024 01:12:34 +0100
Subject: [PATCH 14/60] format

---
 .github/workflows/integration-test-fast.yml | 40 ++++++++++-----------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/integration-test-fast.yml b/.github/workflows/integration-test-fast.yml
index 4ffab6c6b00..0851f44f47c 100644
--- a/.github/workflows/integration-test-fast.yml
+++ b/.github/workflows/integration-test-fast.yml
@@ -73,6 +73,26 @@ on:
         default: 30
 jobs:
   integration-tests-fast:
+    services:
+      mysql:
+        image: mysql:5.7
+        env:
+          MYSQL_ROOT_PASSWORD: zenml
+          MYSQL_DATABASE: zenml
+        ports: [3306:3306]
+        options: >-
+          --health-cmd="mysqladmin ping"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=3
+      zenml-server:
+        image: 715803424590.dkr.ecr.eu-central-1.amazonaws.com/zenml-public-pipelines:${{ github.sha }}
+        env:
+          ZENML_STORE_URL: mysql://root:zenml@mysql:3306/zenml
+          ZENML_SERVER_DEPLOYMENT_TYPE: docker
+          ZENML_SERVER_AUTO_ACTIVATE: 'True'
+          ZENML_SERVER_AUTO_CREATE_DEFAULT_USER: 'True'
+        ports: [8080:8080]
     name: integration-tests-fast
     runs-on: ${{ inputs.os }}
     strategy:
@@ -100,26 +120,6 @@ jobs:
     defaults:
       run:
         shell: bash
-    services:
-      mysql:
-        image: mysql:5.7
-        env:
-          MYSQL_ROOT_PASSWORD: zenml
-          MYSQL_DATABASE: zenml
-        ports: [3306:3306]
-        options: >-
-          --health-cmd="mysqladmin ping"
-          --health-interval=10s
-          --health-timeout=5s
-          --health-retries=3
-      zenml-server:
-        image: 715803424590.dkr.ecr.eu-central-1.amazonaws.com/zenml-public-pipelines:${{ github.sha }}
-        env:
-          ZENML_STORE_URL: mysql://root:zenml@mysql:3306/zenml
-          ZENML_SERVER_DEPLOYMENT_TYPE: docker
-          ZENML_SERVER_AUTO_ACTIVATE: 'True'
-          ZENML_SERVER_AUTO_CREATE_DEFAULT_USER: 'True'
-        ports: [8080:8080]
     steps:
       - name: Maximize space for Docker
         uses: easimon/maximize-build-space@v10

From 449fe79351d40a7921bce8bf5046068d0ec0178c Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Mon, 14 Oct 2024 01:20:14 +0100
Subject: [PATCH 15/60] Refactor port mappings in integration-test-fast.yml

---
 .github/workflows/integration-test-fast.yml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/integration-test-fast.yml b/.github/workflows/integration-test-fast.yml
index 0851f44f47c..ff4e8987436 100644
--- a/.github/workflows/integration-test-fast.yml
+++ b/.github/workflows/integration-test-fast.yml
@@ -79,7 +79,8 @@ jobs:
         env:
           MYSQL_ROOT_PASSWORD: zenml
           MYSQL_DATABASE: zenml
-        ports: [3306:3306]
+        ports: 
+          - 3306:3306
         options: >-
           --health-cmd="mysqladmin ping"
           --health-interval=10s
@@ -92,9 +93,10 @@ jobs:
           ZENML_SERVER_DEPLOYMENT_TYPE: docker
           ZENML_SERVER_AUTO_ACTIVATE: 'True'
           ZENML_SERVER_AUTO_CREATE_DEFAULT_USER: 'True'
-        ports: [8080:8080]
+        ports: 
+          - 8080:8080
     name: integration-tests-fast
-    runs-on: ${{ inputs.os }}
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:

From ea38e5ffa78e6ae2ab5bb009ca1ce8e364e9bda9 Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Mon, 14 Oct 2024 01:22:25 +0100
Subject: [PATCH 16/60] Refactor Dockerfile path in ci-fast.yml

---
 .github/workflows/ci-fast.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml
index 5d91c8dc146..c47b1227f34 100644
--- a/.github/workflows/ci-fast.yml
+++ b/.github/workflows/ci-fast.yml
@@ -130,7 +130,7 @@ jobs:
           context: .
           push: true
           tags: 715803424590.dkr.ecr.eu-central-1.amazonaws.com/zenml-public-pipelines:${{ github.sha }}
-          file: ./Dockerfile  # Adjust this path if your Dockerfile is located elsewhere
+          file: ./docker/zenml-server-dev.Dockerfile  # Adjust this path if your Dockerfile is located elsewhere
   custom-arc-runner-set-integration-test-server:
     needs: build-and-push
     if: github.event.pull_request.draft == false

From e6e729dc371c9e95c665906048f0e44a36e61a26 Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Mon, 14 Oct 2024 01:43:57 +0100
Subject: [PATCH 17/60] Refactor Dockerfile path in ci-fast.yml and remove
 unnecessary job dependency

---
 .github/workflows/ci-fast.yml               | 14 -------
 .github/workflows/integration-test-fast.yml | 44 ++++++++++-----------
 .github/workflows/unit-test.yml             |  2 +-
 3 files changed, 23 insertions(+), 37 deletions(-)

diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml
index c47b1227f34..4af797d6e4d 100644
--- a/.github/workflows/ci-fast.yml
+++ b/.github/workflows/ci-fast.yml
@@ -94,20 +94,6 @@ jobs:
       python-version: ${{ matrix.python-version }}
       os: ${{ matrix.os }}
     secrets: inherit
-  custom-arc-runner-set-integration-test-default:
-    if: github.event.pull_request.draft == false
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: ['3.12']
-        test_environment: [default]
-      fail-fast: false
-    uses: ./.github/workflows/integration-test-fast.yml
-    with:
-      os: ${{ matrix.os }}
-      python-version: ${{ matrix.python-version }}
-      test_environment: ${{ matrix.test_environment }}
-    secrets: inherit
   build-and-push:
     runs-on: ubuntu-latest
     permissions:
diff --git a/.github/workflows/integration-test-fast.yml b/.github/workflows/integration-test-fast.yml
index ff4e8987436..c2f32b5e370 100644
--- a/.github/workflows/integration-test-fast.yml
+++ b/.github/workflows/integration-test-fast.yml
@@ -73,28 +73,6 @@ on:
         default: 30
 jobs:
   integration-tests-fast:
-    services:
-      mysql:
-        image: mysql:5.7
-        env:
-          MYSQL_ROOT_PASSWORD: zenml
-          MYSQL_DATABASE: zenml
-        ports: 
-          - 3306:3306
-        options: >-
-          --health-cmd="mysqladmin ping"
-          --health-interval=10s
-          --health-timeout=5s
-          --health-retries=3
-      zenml-server:
-        image: 715803424590.dkr.ecr.eu-central-1.amazonaws.com/zenml-public-pipelines:${{ github.sha }}
-        env:
-          ZENML_STORE_URL: mysql://root:zenml@mysql:3306/zenml
-          ZENML_SERVER_DEPLOYMENT_TYPE: docker
-          ZENML_SERVER_AUTO_ACTIVATE: 'True'
-          ZENML_SERVER_AUTO_CREATE_DEFAULT_USER: 'True'
-        ports: 
-          - 8080:8080
     name: integration-tests-fast
     runs-on: ubuntu-latest
     strategy:
@@ -276,3 +254,25 @@ jobs:
           zenml integration list
           uv pip list
           uv pip check || true
+    services:
+      mysql:
+        image: mysql:5.7
+        env:
+          MYSQL_ROOT_PASSWORD: zenml
+          MYSQL_DATABASE: zenml
+        ports: 
+          - 3306:3306
+        options: >-
+          --health-cmd="mysqladmin ping"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=3
+      zenml-server:
+        image: 715803424590.dkr.ecr.eu-central-1.amazonaws.com/zenml-public-pipelines:${{ github.sha }}
+        env:
+          ZENML_STORE_URL: mysql://root:zenml@mysql:3306/zenml
+          ZENML_SERVER_DEPLOYMENT_TYPE: docker
+          ZENML_SERVER_AUTO_ACTIVATE: 'True'
+          ZENML_SERVER_AUTO_CREATE_DEFAULT_USER: 'True'
+        ports: 
+          - 8080:8080
\ No newline at end of file
diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
index 840d2f79e11..89e008458d7 100644
--- a/.github/workflows/unit-test.yml
+++ b/.github/workflows/unit-test.yml
@@ -109,7 +109,7 @@ jobs:
           OS: ${{ inputs.os }}
           CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
         run: |
-          bash scripts/lint.sh --changed-files "$CHANGED_FILES"
+          bash scripts/lint.sh
       - name: Run unit tests
         run: |
           bash scripts/test-coverage-xml.sh unit

From 21378cf5a326de244d2e3b4a70a032d966d15a6d Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Mon, 14 Oct 2024 01:54:19 +0100
Subject: [PATCH 18/60] Refactor Dockerfile path in ci-fast.yml and update
 GitHub Container Registry credentials

---
 .github/workflows/ci-fast.yml               | 22 ++++++++++-----------
 .github/workflows/integration-test-fast.yml | 10 +++++++---
 2 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml
index 4af797d6e4d..ac7bfaea020 100644
--- a/.github/workflows/ci-fast.yml
+++ b/.github/workflows/ci-fast.yml
@@ -98,25 +98,23 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: read
-      id-token: write
+      packages: write  # Needed for pushing to GHCR
     steps:
       - uses: actions/checkout@v4.1.1
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v1
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v1
         with:
-          role-to-assume: arn:aws:iam::715803424590:role/gh-action-role-zenml-quickstart-ecr
-          aws-region: eu-central-1
-      - name: Login to Amazon ECR
-        id: login-ecr
-        run: |
-          aws ecr get-login-password --region eu-central-1 | docker login --username AWS --password-stdin 715803424590.dkr.ecr.eu-central-1.amazonaws.com
-      - name: Build and push ZenML server image to AWS ECR
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and push ZenML server image to GHCR
         uses: docker/build-push-action@v2
         with:
           context: .
           push: true
-          tags: 715803424590.dkr.ecr.eu-central-1.amazonaws.com/zenml-public-pipelines:${{ github.sha }}
-          file: ./docker/zenml-server-dev.Dockerfile  # Adjust this path if your Dockerfile is located elsewhere
+          tags: ghcr.io/${{ github.repository_owner }}/zenml-server-github-actions-:${{
+            github.sha }}
+          file: ./docker/zenml-server-dev.Dockerfile
   custom-arc-runner-set-integration-test-server:
     needs: build-and-push
     if: github.event.pull_request.draft == false
diff --git a/.github/workflows/integration-test-fast.yml b/.github/workflows/integration-test-fast.yml
index c2f32b5e370..78d31749588 100644
--- a/.github/workflows/integration-test-fast.yml
+++ b/.github/workflows/integration-test-fast.yml
@@ -260,7 +260,7 @@ jobs:
         env:
           MYSQL_ROOT_PASSWORD: zenml
           MYSQL_DATABASE: zenml
-        ports: 
+        ports:
           - 3306:3306
         options: >-
           --health-cmd="mysqladmin ping"
@@ -268,11 +268,15 @@ jobs:
           --health-timeout=5s
           --health-retries=3
       zenml-server:
-        image: 715803424590.dkr.ecr.eu-central-1.amazonaws.com/zenml-public-pipelines:${{ github.sha }}
+        image: ghcr.io/${{ github.repository_owner }}/zenml-server-github-actions-:${{
+          github.sha }}
+        credentials:
+          username: ${{ github.actor }}
+          password: ${{ secrets.github_token }}
         env:
           ZENML_STORE_URL: mysql://root:zenml@mysql:3306/zenml
           ZENML_SERVER_DEPLOYMENT_TYPE: docker
           ZENML_SERVER_AUTO_ACTIVATE: 'True'
           ZENML_SERVER_AUTO_CREATE_DEFAULT_USER: 'True'
-        ports: 
+        ports:
           - 8080:8080
\ No newline at end of file

From 1d29c2b2a8127e3d748f5f418ad1b8e3567e0919 Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Mon, 14 Oct 2024 01:57:18 +0100
Subject: [PATCH 19/60] Refactor Dockerfile path in ci-fast.yml and update
 GitHub Container Registry credentials

---
 .github/workflows/ci-fast.yml               | 2 +-
 .github/workflows/integration-test-fast.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml
index ac7bfaea020..60823618c57 100644
--- a/.github/workflows/ci-fast.yml
+++ b/.github/workflows/ci-fast.yml
@@ -112,7 +112,7 @@ jobs:
         with:
           context: .
           push: true
-          tags: ghcr.io/${{ github.repository_owner }}/zenml-server-github-actions-:${{
+          tags: ghcr.io/${{ github.repository_owner }}/zenml-server-github-actions:${{
             github.sha }}
           file: ./docker/zenml-server-dev.Dockerfile
   custom-arc-runner-set-integration-test-server:
diff --git a/.github/workflows/integration-test-fast.yml b/.github/workflows/integration-test-fast.yml
index 78d31749588..5fbfd89d524 100644
--- a/.github/workflows/integration-test-fast.yml
+++ b/.github/workflows/integration-test-fast.yml
@@ -268,7 +268,7 @@ jobs:
           --health-timeout=5s
           --health-retries=3
       zenml-server:
-        image: ghcr.io/${{ github.repository_owner }}/zenml-server-github-actions-:${{
+        image: ghcr.io/${{ github.repository_owner }}/zenml-server-github-actions:${{
           github.sha }}
         credentials:
           username: ${{ github.actor }}

From f6b4d69eff3b75f8712cb9f5ced9a4ececbb7b31 Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Mon, 14 Oct 2024 02:13:17 +0100
Subject: [PATCH 20/60] Refactor Dockerfile path in ci-fast.yml and add
 github-actions-server-docker-orchestrator environment

---
 .github/workflows/ci-fast.yml                 |  19 +-
 .../integration-test-fast-services.yml        | 281 ++++++++++++++++++
 .github/workflows/integration-test-fast.yml   |  31 +-
 tests/harness/cfg/environments.yaml           |  11 +
 4 files changed, 312 insertions(+), 30 deletions(-)
 create mode 100644 .github/workflows/integration-test-fast-services.yml

diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml
index 60823618c57..208c2632481 100644
--- a/.github/workflows/ci-fast.yml
+++ b/.github/workflows/ci-fast.yml
@@ -94,6 +94,21 @@ jobs:
       python-version: ${{ matrix.python-version }}
       os: ${{ matrix.os }}
     secrets: inherit
+  custom-arc-runner-set-integration-test:
+    needs: build-and-push
+    if: github.event.pull_request.draft == false
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ['3.12']
+        test_environment: [default]
+      fail-fast: false
+    uses: ./.github/workflows/integration-test-fast.yml
+    with:
+      os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }}
+      test_environment: ${{ matrix.test_environment }}
+    secrets: inherit
   build-and-push:
     runs-on: ubuntu-latest
     permissions:
@@ -122,9 +137,9 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         python-version: ['3.12']
-        test_environment: [github-actions-server]
+        test_environment: [github-actions-server-docker-orchestrator]
       fail-fast: false
-    uses: ./.github/workflows/integration-test-fast.yml
+    uses: ./.github/workflows/integration-test-fast-services.yml
     with:
       os: ${{ matrix.os }}
       python-version: ${{ matrix.python-version }}
diff --git a/.github/workflows/integration-test-fast-services.yml b/.github/workflows/integration-test-fast-services.yml
new file mode 100644
index 00000000000..bfe40500123
--- /dev/null
+++ b/.github/workflows/integration-test-fast-services.yml
@@ -0,0 +1,281 @@
+---
+name: Integration Tests (Fast CI)
+on:
+  workflow_call:
+    inputs:
+      os:
+        description: OS
+        type: string
+        required: true
+      python-version:
+        description: Python version
+        type: string
+        required: true
+      test_environment:
+        description: The test environment
+        type: string
+        required: true
+      enable_tmate:
+        description: Enable tmate session for debugging
+        type: string
+        required: false
+        default: never
+      tmate_timeout:
+        description: Timeout for tmate session (minutes)
+        type: number
+        required: false
+        default: 30
+  workflow_dispatch:
+    inputs:
+      os:
+        description: OS
+        type: choice
+        options: [ubuntu-latest, macos-13, windows-latest, arc-runner-set]
+        required: false
+        default: ubuntu-latest
+      python-version:
+        description: Python version
+        type: choice
+        options: ['3.9', '3.10', '3.11', '3.12']
+        required: false
+        default: '3.9'
+      test_environment:
+        description: The test environment
+        type: choice
+        options:
+          # Default ZenML deployments
+          - default
+          - default-docker-orchestrator
+          - default-airflow-orchestrator
+          # Local ZenML server deployments
+          - local-server
+          - local-server-docker-orchestrator
+          - local-server-airflow-orchestrator
+          # Local ZenML docker-compose server deployments
+          - docker-server-mysql
+          - docker-server-mariadb
+          - docker-server-docker-orchestrator-mysql
+          - docker-server-docker-orchestrator-mariadb
+          - docker-server-airflow-orchestrator-mysql
+          - docker-server-airflow-orchestrator-mariadb
+          - github-actions-server-docker-orchestrator
+        required: false
+        default: default
+      enable_tmate:
+        description: Enable tmate session for debugging
+        type: choice
+        options: [no, on-failure, always, before-tests]
+        required: false
+        default: 'no'
+      tmate_timeout:
+        description: Timeout for tmate session (minutes)
+        type: number
+        required: false
+        default: 30
+jobs:
+  integration-tests-fast:
+    name: integration-tests-fast
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1, 2, 3]
+    env:
+      ZENML_DEBUG: 1
+      ZENML_ANALYTICS_OPT_IN: false
+      PYTHONIOENCODING: utf-8
+      UV_HTTP_TIMEOUT: 600
+      # on MAC OS, we need to set this environment variable
+      # to fix problems with the fork() calls (see this thread
+      # for more information: http://sealiesoftware.com/blog/archive/2017/6/5/Objective-C_and_fork_in_macOS_1013.html)
+      OBJC_DISABLE_INITIALIZE_FORK_SAFETY: 'YES'
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_US_EAST_1_ENV_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_US_EAST_1_ENV_SECRET_ACCESS_KEY }}
+      AWS_US_EAST_1_SERVER_URL: ${{ secrets.AWS_US_EAST_1_SERVER_URL }}
+      AWS_US_EAST_1_SERVER_USERNAME: ${{ secrets.AWS_US_EAST_1_SERVER_USERNAME }}
+      AWS_US_EAST_1_SERVER_PASSWORD: ${{ secrets.AWS_US_EAST_1_SERVER_PASSWORD }}
+      GCP_US_EAST4_SERVER_URL: ${{ secrets.GCP_US_EAST4_SERVER_URL }}
+      GCP_US_EAST4_SERVER_USERNAME: ${{ secrets.GCP_US_EAST4_SERVER_USERNAME }}
+      GCP_US_EAST4_SERVER_PASSWORD: ${{ secrets.GCP_US_EAST4_SERVER_PASSWORD }}
+    if: ${{ ! startsWith(github.event.head_commit.message, 'GitBook:') }}
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Maximize space for Docker
+        uses: easimon/maximize-build-space@v10
+        with:
+          root-reserve-mb: 20000
+          swap-size-mb: 1024
+          remove-dotnet: 'true'
+          remove-android: 'true'
+          remove-haskell: 'true'
+          build-mount-path: /var/lib/docker/
+        if: inputs.os == 'ubuntu-latest' && (contains(inputs.test_environment, 'docker')
+          || contains(inputs.test_environment, 'kubeflow') || contains(inputs.test_environment,
+          'airflow') || contains(inputs.test_environment, 'kubernetes'))
+      - name: Reload Docker
+        run: sudo systemctl restart docker
+        if: inputs.os == 'ubuntu-latest' && (contains(inputs.test_environment, 'docker')
+          || contains(inputs.test_environment, 'kubeflow') || contains(inputs.test_environment,
+          'airflow') || contains(inputs.test_environment, 'kubernetes'))
+      - uses: actions/checkout@v4.1.1
+      - name: Restore uv cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/uv
+          key: |
+            uv-${{ runner.os }}-${{ inputs.python-version }}-${{ hashFiles('src/zenml/integrations/*/__init__.py') }}
+          restore-keys: |
+            uv-${{ runner.os }}-${{ inputs.python-version }}-${{ hashFiles('src/zenml/integrations/*/__init__.py') }}
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ secrets.AWS_US_EAST_1_ENV_ROLE_ARN }}
+          aws-region: us-east-1
+        if: contains(inputs.test_environment, 'aws')
+      - name: Configure GCP credentials
+        uses: google-github-actions/auth@v2
+        with:
+          credentials_json: ${{ secrets.GCP_US_EAST4_ENV_CREDENTIALS }}
+        if: contains(inputs.test_environment, 'gcp')
+      - name: Set up gcloud SDK
+        uses: google-github-actions/setup-gcloud@v1
+        with:
+          install_components: gke-gcloud-auth-plugin
+        if: contains(inputs.test_environment, 'gcp')
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+        if: github.event.pull_request.head.repo.fork == false && (contains(inputs.test_environment,
+          'docker') || contains(inputs.test_environment, 'kubeflow') || contains(inputs.test_environment,
+          'airflow') || contains(inputs.test_environment, 'kubernetes'))
+      - name: Setup environment
+        uses: ./.github/actions/setup_environment
+        with:
+          cache_version: ${{ secrets.GH_ACTIONS_CACHE_KEY }}
+          python-version: ${{ inputs.python-version }}
+          os: ${{ inputs.os }}
+          runners_cache_access_key_id: ${{ secrets.RUNNERS_CACHE_ACCESS_KEY_ID }}
+          runners_cache_secret_access_key: ${{ secrets.RUNNERS_CACHE_SECRET_ACCESS_KEY }}
+          discord_webhook: ${{ secrets.DISCORD_WEBHOOK }}
+      - name: Install docker-compose for non-default environments
+        if: inputs.test_environment != 'default'
+        run: |
+          pip install uv
+          # see https://github.com/docker/docker-py/issues/3256 for why we need to pin requests
+          # docker-compose is deprecated and doesn't work with newer versions of docker
+          uv pip install --system "pyyaml==5.3.1" "requests<2.32.0" "docker==6.1.3" docker-compose
+      - name: Install Linux System Dependencies
+        if: (inputs.os == 'ubuntu-latest' || inputs.os == 'arc-runner-set')
+        run: sudo apt install graphviz
+      - name: Install MacOS System Dependencies
+        if: runner.os=='macOS'
+        run: brew install graphviz
+      - name: Install Windows System Dependencies
+        if: runner.os=='Windows'
+        run: choco install graphviz
+      - name: Unbreak python in github actions
+        if: runner.os=='macOS'
+        # github actions overwrites brew's python. Force it to reassert itself, by
+        # running in a separate step.
+        # Workaround GitHub Actions Python issues
+        # see https://github.com/Homebrew/homebrew-core/issues/165793#issuecomment-1989441193
+        run: |
+          find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete
+          sudo rm -rf /Library/Frameworks/Python.framework/
+          brew install --force python3 && brew unlink python3 && brew unlink python3 && brew link --overwrite python3
+      - name: Install Docker and Colima on MacOS
+        if: runner.os=='macOS'
+        run: |
+          export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1
+          brew update
+          brew install docker colima
+          brew reinstall --force qemu
+
+          # We need to mount the /private/tmp/zenml-test/ folder because
+          # this folder is also mounted in the Docker containers that are
+          # started by local ZenML orchestrators.
+          colima start --mount /private/tmp/zenml-test/:w
+
+          # This is required for the Docker Python SDK to work
+          sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock
+      - name: Install kubectl on Linux
+        run: |
+          curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+          sudo install -o root -g 0 -m 0755 kubectl /usr/local/bin/kubectl
+        if: (inputs.os == 'ubuntu-latest' || inputs.os == 'arc-runner-set')
+      - name: Install kubectl on MacOS
+        run: |
+          curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/darwin/amd64/kubectl"
+          sudo install -o root -g 0 -m 0755 kubectl /usr/local/bin/kubectl
+        if: runner.os=='macOS'
+      - name: Install K3D
+        run: |
+          curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash
+        if: runner.os!='Windows' && contains(inputs.test_environment, 'kubeflow')
+      - name: Login to Amazon ECR
+        id: login-ecr
+        run: |
+          aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 715803424590.dkr.ecr.us-east-1.amazonaws.com
+        if: contains(inputs.test_environment, 'aws')
+      - name: Login to Amazon EKS
+        id: login-eks
+        run: |
+          aws eks --region us-east-1 update-kubeconfig --name zenml-ci-cluster --alias zenml-ci-aws-us-east-1
+        if: contains(inputs.test_environment, 'aws')
+      - name: Login to Google ECR
+        run: |
+          gcloud auth configure-docker --project zenml-ci
+        if: contains(inputs.test_environment, 'gcp')
+      - name: Login to Google GKE
+        uses: google-github-actions/get-gke-credentials@v2
+        with:
+          cluster_name: zenml-ci-cluster
+          location: us-east4
+          project_id: zenml-ci
+        if: contains(inputs.test_environment, 'gcp')
+      - name: Setup tmate session before tests
+        if: ${{ inputs.enable_tmate == 'before-tests' }}
+        uses: mxschmitt/action-tmate@v3.17
+        timeout-minutes: ${{ inputs.tmate_timeout }}
+      - name: Sharded Integration Tests (Ubuntu) - Fast CI
+        # Ubuntu integration tests run as 6 shards
+        if: runner.os != 'macOS' && runner.os != 'Windows'
+        run: |
+          bash scripts/test-coverage-xml.sh integration ${{ inputs.test_environment }} 3 ${{ matrix.shard }}
+      - name: Setup tmate session after tests
+        if: ${{ inputs.enable_tmate == 'always' || (inputs.enable_tmate == 'on-failure' && failure()) }}
+        uses: mxschmitt/action-tmate@v3.17
+        timeout-minutes: ${{ inputs.tmate_timeout }}
+      - name: Verify Python Env unaffected
+        run: |-
+          zenml integration list
+          uv pip list
+          uv pip check || true
+    services:
+      mysql:
+        image: mysql:5.7
+        env:
+          MYSQL_ROOT_PASSWORD: zenml
+          MYSQL_DATABASE: zenml
+        ports: [3306:3306]
+        options: >-
+          --health-cmd="mysqladmin ping"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=3
+      zenml-server:
+        image: ghcr.io/${{ github.repository_owner }}/zenml-server-github-actions:${{
+          github.sha }}
+        credentials:
+          username: ${{ github.actor }}
+          password: ${{ secrets.github_token }}
+        env:
+          ZENML_STORE_URL: mysql://root:zenml@mysql:3306/zenml
+          ZENML_SERVER_DEPLOYMENT_TYPE: docker
+          ZENML_SERVER_AUTO_ACTIVATE: 'True'
+          ZENML_SERVER_AUTO_CREATE_DEFAULT_USER: 'True'
+        ports: [8080:8080]
diff --git a/.github/workflows/integration-test-fast.yml b/.github/workflows/integration-test-fast.yml
index 5fbfd89d524..835af58fafc 100644
--- a/.github/workflows/integration-test-fast.yml
+++ b/.github/workflows/integration-test-fast.yml
@@ -58,6 +58,7 @@ on:
           - docker-server-docker-orchestrator-mariadb
           - docker-server-airflow-orchestrator-mysql
           - docker-server-airflow-orchestrator-mariadb
+          - github-actions-server-docker-orchestrator
         required: false
         default: default
       enable_tmate:
@@ -78,7 +79,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        shard: [1, 2, 3, 4, 5, 6]
+        shard: [1, 2, 3]
     env:
       ZENML_DEBUG: 1
       ZENML_ANALYTICS_OPT_IN: false
@@ -244,7 +245,7 @@ jobs:
         # Ubuntu integration tests run as 6 shards
         if: runner.os != 'macOS' && runner.os != 'Windows'
         run: |
-          bash scripts/test-coverage-xml.sh integration ${{ inputs.test_environment }} 6 ${{ matrix.shard }}
+          bash scripts/test-coverage-xml.sh integration ${{ inputs.test_environment }} 3 ${{ matrix.shard }}
       - name: Setup tmate session after tests
         if: ${{ inputs.enable_tmate == 'always' || (inputs.enable_tmate == 'on-failure' && failure()) }}
         uses: mxschmitt/action-tmate@v3.17
@@ -254,29 +255,3 @@ jobs:
           zenml integration list
           uv pip list
           uv pip check || true
-    services:
-      mysql:
-        image: mysql:5.7
-        env:
-          MYSQL_ROOT_PASSWORD: zenml
-          MYSQL_DATABASE: zenml
-        ports:
-          - 3306:3306
-        options: >-
-          --health-cmd="mysqladmin ping"
-          --health-interval=10s
-          --health-timeout=5s
-          --health-retries=3
-      zenml-server:
-        image: ghcr.io/${{ github.repository_owner }}/zenml-server-github-actions:${{
-          github.sha }}
-        credentials:
-          username: ${{ github.actor }}
-          password: ${{ secrets.github_token }}
-        env:
-          ZENML_STORE_URL: mysql://root:zenml@mysql:3306/zenml
-          ZENML_SERVER_DEPLOYMENT_TYPE: docker
-          ZENML_SERVER_AUTO_ACTIVATE: 'True'
-          ZENML_SERVER_AUTO_CREATE_DEFAULT_USER: 'True'
-        ports:
-          - 8080:8080
\ No newline at end of file
diff --git a/tests/harness/cfg/environments.yaml b/tests/harness/cfg/environments.yaml
index 6451953c5f9..a323871587b 100644
--- a/tests/harness/cfg/environments.yaml
+++ b/tests/harness/cfg/environments.yaml
@@ -188,3 +188,14 @@ environments:
       - mlflow-local-registry
       - mlflow-local-deployer
     mandatory_requirements: [airflow-local]
+  - name: github-actions-server-docker-orchestrator
+    description: >-
+      Server using GitHub Actions services with docker orchestrator and all
+      local components.
+    deployment: github-actions-server
+    requirements:
+      - data-validators
+      - mlflow-local-tracker
+      - mlflow-local-registry
+      - mlflow-local-deployer
+    mandatory_requirements: [docker-local]

From 6a55f93cc049e3cca6d278d3d407e21241ce72e1 Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Mon, 14 Oct 2024 02:22:42 +0100
Subject: [PATCH 21/60] Refactor Dockerfile path in ci-fast.yml and update port
 mappings

---
 .github/workflows/integration-test-fast-services.yml | 6 ++++--
 scripts/format.sh                                    | 2 +-
 scripts/lint.sh                                      | 2 +-
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/integration-test-fast-services.yml b/.github/workflows/integration-test-fast-services.yml
index bfe40500123..f84a84af732 100644
--- a/.github/workflows/integration-test-fast-services.yml
+++ b/.github/workflows/integration-test-fast-services.yml
@@ -261,7 +261,8 @@ jobs:
         env:
           MYSQL_ROOT_PASSWORD: zenml
           MYSQL_DATABASE: zenml
-        ports: [3306:3306]
+        ports: 
+          - 3306:3306
         options: >-
           --health-cmd="mysqladmin ping"
           --health-interval=10s
@@ -278,4 +279,5 @@ jobs:
           ZENML_SERVER_DEPLOYMENT_TYPE: docker
           ZENML_SERVER_AUTO_ACTIVATE: 'True'
           ZENML_SERVER_AUTO_CREATE_DEFAULT_USER: 'True'
-        ports: [8080:8080]
+        ports: 
+          - 8080:8080
diff --git a/scripts/format.sh b/scripts/format.sh
index 77e166b781c..aee50b7867f 100755
--- a/scripts/format.sh
+++ b/scripts/format.sh
@@ -64,7 +64,7 @@ ruff format $SRC
 
 # standardizes / formats CI yaml files
 if [ "$SKIP_YAMLFIX" = false ]; then
-    yamlfix .github tests --exclude "dependabot.yml"
+    yamlfix .github tests --exclude "dependabot.yml" --exclude "integration-test-fast-services.yml"
 fi
 
 set +x
diff --git a/scripts/lint.sh b/scripts/lint.sh
index eb9af19e000..fffb6398af3 100755
--- a/scripts/lint.sh
+++ b/scripts/lint.sh
@@ -30,7 +30,7 @@ fi
 
 # checks for yaml formatting errors
 if [ "$SKIP_YAMLFIX" = false ]; then
-    yamlfix --check .github tests --exclude "dependabot.yml"
+    yamlfix --check .github tests --exclude "dependabot.yml" --exclude "integration-test-fast-services.yml"
 fi
 
 # autoflake replacement: checks for unused imports and variables

From de88ae429360dd156a81b22541a8b1aacf625c0b Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Mon, 14 Oct 2024 09:51:34 +0100
Subject: [PATCH 22/60] Refactor Dockerfile path in ci-fast.yml and update port
 mappings

---
 .github/workflows/integration-test-fast-services.yml | 9 +++++++--
 scripts/format.sh                                    | 2 +-
 scripts/lint.sh                                      | 2 +-
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/integration-test-fast-services.yml b/.github/workflows/integration-test-fast-services.yml
index f84a84af732..2a932af06b7 100644
--- a/.github/workflows/integration-test-fast-services.yml
+++ b/.github/workflows/integration-test-fast-services.yml
@@ -261,7 +261,7 @@ jobs:
         env:
           MYSQL_ROOT_PASSWORD: zenml
           MYSQL_DATABASE: zenml
-        ports: 
+        ports:
           - 3306:3306
         options: >-
           --health-cmd="mysqladmin ping"
@@ -279,5 +279,10 @@ jobs:
           ZENML_SERVER_DEPLOYMENT_TYPE: docker
           ZENML_SERVER_AUTO_ACTIVATE: 'True'
           ZENML_SERVER_AUTO_CREATE_DEFAULT_USER: 'True'
-        ports: 
+        ports:
           - 8080:8080
+        options: >-
+          --health-cmd="curl -f http://localhost:8080/health"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=3
diff --git a/scripts/format.sh b/scripts/format.sh
index aee50b7867f..c26ba6f22a6 100755
--- a/scripts/format.sh
+++ b/scripts/format.sh
@@ -64,7 +64,7 @@ ruff format $SRC
 
 # standardizes / formats CI yaml files
 if [ "$SKIP_YAMLFIX" = false ]; then
-    yamlfix .github tests --exclude "dependabot.yml" --exclude "integration-test-fast-services.yml"
+    yamlfix .github tests --exclude="dependabot.yml" --exclude="workflows/integration-test-fast-services.yml"
 fi
 
 set +x
diff --git a/scripts/lint.sh b/scripts/lint.sh
index fffb6398af3..cffa55692db 100755
--- a/scripts/lint.sh
+++ b/scripts/lint.sh
@@ -30,7 +30,7 @@ fi
 
 # checks for yaml formatting errors
 if [ "$SKIP_YAMLFIX" = false ]; then
-    yamlfix --check .github tests --exclude "dependabot.yml" --exclude "integration-test-fast-services.yml"
+    yamlfix .github tests --exclude="dependabot.yml" --exclude="workflows/integration-test-fast-services.yml"
 fi
 
 # autoflake replacement: checks for unused imports and variables

From 8a44266f20f23ae1c95167f901d2f638f1f39bb9 Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Mon, 14 Oct 2024 10:07:13 +0100
Subject: [PATCH 23/60] Refactor Dockerfile path in ci-fast.yml and update port
 mappings

---
 .../integration-test-fast-services.yml        | 19 +------------------
 tests/harness/cfg/deployments.yaml            |  2 +-
 2 files changed, 2 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/integration-test-fast-services.yml b/.github/workflows/integration-test-fast-services.yml
index 2a932af06b7..84f9ee59532 100644
--- a/.github/workflows/integration-test-fast-services.yml
+++ b/.github/workflows/integration-test-fast-services.yml
@@ -102,23 +102,6 @@ jobs:
       run:
         shell: bash
     steps:
-      - name: Maximize space for Docker
-        uses: easimon/maximize-build-space@v10
-        with:
-          root-reserve-mb: 20000
-          swap-size-mb: 1024
-          remove-dotnet: 'true'
-          remove-android: 'true'
-          remove-haskell: 'true'
-          build-mount-path: /var/lib/docker/
-        if: inputs.os == 'ubuntu-latest' && (contains(inputs.test_environment, 'docker')
-          || contains(inputs.test_environment, 'kubeflow') || contains(inputs.test_environment,
-          'airflow') || contains(inputs.test_environment, 'kubernetes'))
-      - name: Reload Docker
-        run: sudo systemctl restart docker
-        if: inputs.os == 'ubuntu-latest' && (contains(inputs.test_environment, 'docker')
-          || contains(inputs.test_environment, 'kubeflow') || contains(inputs.test_environment,
-          'airflow') || contains(inputs.test_environment, 'kubernetes'))
       - uses: actions/checkout@v4.1.1
       - name: Restore uv cache
         uses: actions/cache@v4
@@ -282,7 +265,7 @@ jobs:
         ports:
           - 8080:8080
         options: >-
-          --health-cmd="curl -f http://localhost:8080/health"
+          --health-cmd="curl -f http://127.0.0.1:8080/health"
           --health-interval=10s
           --health-timeout=5s
           --health-retries=3
diff --git a/tests/harness/cfg/deployments.yaml b/tests/harness/cfg/deployments.yaml
index 1ee09598f82..6974314f010 100644
--- a/tests/harness/cfg/deployments.yaml
+++ b/tests/harness/cfg/deployments.yaml
@@ -119,6 +119,6 @@ deployments:
     capabilities:
       server: true
     config:
-      url: http://127.0.0.1:8080
+      url: http://127.0.0.1:8080/
       username: default
       password: ''

From 1d0bc0b22458e5fd3ada319a25f0867ad36391fd Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Mon, 14 Oct 2024 22:49:50 +0100
Subject: [PATCH 24/60] Refactor Dockerfile path in ci-fast.yml and add
 integration-test-slow-services workflow

---
 .github/workflows/ci-slow.yml                 |  32 ++-
 .../integration-test-slow-services.yml        | 260 ++++++++++++++++++
 scripts/format.sh                             |   2 +-
 scripts/lint.sh                               |   2 +-
 4 files changed, 283 insertions(+), 13 deletions(-)
 create mode 100644 .github/workflows/integration-test-slow-services.yml

diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml
index b02fc73d6f0..053b87c385f 100644
--- a/.github/workflows/ci-slow.yml
+++ b/.github/workflows/ci-slow.yml
@@ -279,20 +279,15 @@ jobs:
     needs: run-slow-ci-label-is-set
     strategy:
       matrix:
-        os: [arc-runner-set]
-        python-version: ['3.9', '3.10', '3.11', '3.12']
-        test_environment:
-          - default
-          - docker-server-docker-orchestrator-mysql
-          - docker-server-docker-orchestrator-mariadb
+        os: [ubuntu-latest]
+        python-version: ['3.9', '3.10', '3.11']
+        test_environment: [default, docker-server-docker-orchestrator-mariadb]
         exclude:
           # docker is time-consuming to run, so we only run it on 3.9
-          - test_environment: docker-server-docker-orchestrator-mysql
-            python-version: '3.10'
-          - test_environment: docker-server-docker-orchestrator-mysql
-            python-version: '3.11'
-          - test_environment: docker-server-docker-orchestrator-mysql
+          - test_environment: default
             python-version: '3.12'
+          - test_environment: default
+            python-version: '3.10'
           - test_environment: docker-server-docker-orchestrator-mariadb
             python-version: '3.10'
           - test_environment: docker-server-docker-orchestrator-mariadb
@@ -306,3 +301,18 @@ jobs:
       python-version: ${{ matrix.python-version }}
       test_environment: ${{ matrix.test_environment }}
     secrets: inherit
+  custom-ubuntu-integration-test-gh-actions-server-docker-orchestrator:
+    if: github.event.pull_request.draft == false
+    needs: run-slow-ci-label-is-set
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ['3.9', '3.10', '3.11']
+        test_environment: [github-actions-server-docker-orchestrator]
+      fail-fast: false
+    uses: ./.github/workflows/integration-test-slow-services.yml
+    with:
+      os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }}
+      test_environment: ${{ matrix.test_environment }}
+    secrets: inherit
diff --git a/.github/workflows/integration-test-slow-services.yml b/.github/workflows/integration-test-slow-services.yml
new file mode 100644
index 00000000000..9a761034920
--- /dev/null
+++ b/.github/workflows/integration-test-slow-services.yml
@@ -0,0 +1,260 @@
+---
+name: Integration Tests (Slow CI)
+on:
+  workflow_call:
+    inputs:
+      os:
+        description: OS
+        type: string
+        required: true
+      python-version:
+        description: Python version
+        type: string
+        required: true
+      test_environment:
+        description: The test environment
+        type: string
+        required: true
+      enable_tmate:
+        description: Enable tmate session for debugging
+        type: string
+        required: false
+        default: never
+      tmate_timeout:
+        description: Timeout for tmate session (minutes)
+        type: number
+        required: false
+        default: 30
+  workflow_dispatch:
+    inputs:
+      os:
+        description: OS
+        type: choice
+        options: [ubuntu-latest, macos-13, windows-latest, arc-runner-set]
+        required: false
+        default: ubuntu-latest
+      python-version:
+        description: Python version
+        type: choice
+        options: ['3.9', '3.10', '3.11', '3.12']
+        required: false
+        default: '3.9'
+      test_environment:
+        description: The test environment
+        type: choice
+        options:
+          # Default ZenML deployments
+          - default
+          - default-docker-orchestrator
+          - default-airflow-orchestrator
+          # Local ZenML server deployments
+          - local-server
+          - local-server-docker-orchestrator
+          - local-server-airflow-orchestrator
+          # Local ZenML docker-compose server deployments
+          - docker-server-mysql
+          - docker-server-mariadb
+          - docker-server-docker-orchestrator-mysql
+          - docker-server-docker-orchestrator-mariadb
+          - docker-server-airflow-orchestrator-mysql
+          - docker-server-airflow-orchestrator-mariadb
+        required: false
+        default: default
+      enable_tmate:
+        description: Enable tmate session for debugging
+        type: choice
+        options: [no, on-failure, always, before-tests]
+        required: false
+        default: 'no'
+      tmate_timeout:
+        description: Timeout for tmate session (minutes)
+        type: number
+        required: false
+        default: 30
+jobs:
+  integration-tests-slow:
+    name: integration-tests-slow
+    runs-on: ${{ inputs.os }}
+    strategy:
+      fail-fast: false
+    env:
+      ZENML_DEBUG: 1
+      ZENML_ANALYTICS_OPT_IN: false
+      PYTHONIOENCODING: utf-8
+      UV_HTTP_TIMEOUT: 600
+      # on MAC OS, we need to set this environment variable
+      # to fix problems with the fork() calls (see this thread
+      # for more information: http://sealiesoftware.com/blog/archive/2017/6/5/Objective-C_and_fork_in_macOS_1013.html)
+      OBJC_DISABLE_INITIALIZE_FORK_SAFETY: 'YES'
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_US_EAST_1_ENV_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_US_EAST_1_ENV_SECRET_ACCESS_KEY }}
+      AWS_US_EAST_1_SERVER_URL: ${{ secrets.AWS_US_EAST_1_SERVER_URL }}
+      AWS_US_EAST_1_SERVER_USERNAME: ${{ secrets.AWS_US_EAST_1_SERVER_USERNAME }}
+      AWS_US_EAST_1_SERVER_PASSWORD: ${{ secrets.AWS_US_EAST_1_SERVER_PASSWORD }}
+      GCP_US_EAST4_SERVER_URL: ${{ secrets.GCP_US_EAST4_SERVER_URL }}
+      GCP_US_EAST4_SERVER_USERNAME: ${{ secrets.GCP_US_EAST4_SERVER_USERNAME }}
+      GCP_US_EAST4_SERVER_PASSWORD: ${{ secrets.GCP_US_EAST4_SERVER_PASSWORD }}
+    # TODO: add Windows testing for Python 3.11 and 3.12 back in
+    # TODO: add macos testing back in
+    if: ${{ ! startsWith(github.event.head_commit.message, 'GitBook:') && ! (inputs.os == 'windows-latest' && inputs.python-version == '3.11') && ! (inputs.os == 'windows-latest' && inputs.python-version == '3.12') && ! (inputs.os == 'macos-13' || inputs.os == 'macos-latest')  }}
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+        if: github.event.pull_request.head.repo.fork == false && (contains(inputs.test_environment,
+          'docker') || contains(inputs.test_environment, 'kubeflow') || contains(inputs.test_environment,
+          'airflow') || contains(inputs.test_environment, 'kubernetes'))
+      - uses: actions/checkout@v4.1.1
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ secrets.AWS_US_EAST_1_ENV_ROLE_ARN }}
+          aws-region: us-east-1
+        if: contains(inputs.test_environment, 'aws')
+      - name: Configure GCP credentials
+        uses: google-github-actions/auth@v2
+        with:
+          credentials_json: ${{ secrets.GCP_US_EAST4_ENV_CREDENTIALS }}
+        if: contains(inputs.test_environment, 'gcp')
+      - name: Set up gcloud SDK
+        uses: google-github-actions/setup-gcloud@v1
+        with:
+          install_components: gke-gcloud-auth-plugin
+        if: contains(inputs.test_environment, 'gcp')
+      - name: Setup environment
+        uses: ./.github/actions/setup_environment
+        with:
+          cache_version: ${{ secrets.GH_ACTIONS_CACHE_KEY }}
+          python-version: ${{ inputs.python-version }}
+          os: ${{ inputs.os }}
+          runners_cache_access_key_id: ${{ secrets.RUNNERS_CACHE_ACCESS_KEY_ID }}
+          runners_cache_secret_access_key: ${{ secrets.RUNNERS_CACHE_SECRET_ACCESS_KEY }}
+          discord_webhook: ${{ secrets.DISCORD_WEBHOOK }}
+      - name: Install docker-compose for non-default environments
+        if: inputs.test_environment != 'default'
+        run: |
+          pip install uv
+          # see https://github.com/docker/docker-py/issues/3256 for why we need to pin requests
+          # docker-compose is deprecated and doesn't work with newer versions of docker
+          uv pip install --system "pyyaml==5.3.1" "requests<2.32.0" "docker==6.1.3" docker-compose
+      - name: Install MacOS System Dependencies
+        if: runner.os=='macOS'
+        run: brew install libomp
+      - name: Unbreak Python in GHA for 3.9-3.10
+        if: runner.os=='macOS' && inputs.python-version != '3.11'
+        # github actions overwrites brew's python. Force it to reassert itself, by
+        # running in a separate step.
+        # Workaround GitHub Actions Python issues
+        # see https://github.com/Homebrew/homebrew-core/issues/165793#issuecomment-1989441193
+        run: |
+          find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete
+          sudo rm -rf /Library/Frameworks/Python.framework/
+          brew install --force python3 && brew unlink python3 && brew unlink python3 && brew link --overwrite python3
+      - name: Unbreak Python in GHA for 3.11
+        if: runner.os=='macOS' && inputs.python-version == '3.11'
+        run: |
+          # Unlink and re-link to prevent errors when github mac runner images
+          # https://github.com/actions/setup-python/issues/577
+          brew list -1 | grep python | while read formula; do brew unlink $formula; brew link --overwrite $formula; done
+      - name: Install Docker and Colima on MacOS
+        if: runner.os=='macOS'
+        run: |
+          export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1
+          brew update
+          brew install docker colima
+          brew reinstall --force qemu
+
+          # We need to mount the /private/tmp/zenml-test/ folder because
+          # this folder is also mounted in the Docker containers that are
+          # started by local ZenML orchestrators.
+          colima start --mount /private/tmp/zenml-test/:w
+
+          # This is required for the Docker Python SDK to work
+          sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock
+      - name: Install kubectl on Linux
+        run: |
+          curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+          sudo install -o root -g 0 -m 0755 kubectl /usr/local/bin/kubectl
+        if: (inputs.os == 'ubuntu-latest' || inputs.os == 'arc-runner-set')
+      - name: Install kubectl on MacOS
+        run: |
+          curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/darwin/amd64/kubectl"
+          sudo install -o root -g 0 -m 0755 kubectl /usr/local/bin/kubectl
+        if: runner.os=='macOS'
+      - name: Install K3D
+        run: |
+          curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash
+        if: runner.os!='Windows' && contains(inputs.test_environment, 'kubeflow')
+      - name: Login to Amazon ECR
+        id: login-ecr
+        run: |
+          aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 715803424590.dkr.ecr.us-east-1.amazonaws.com
+        if: contains(inputs.test_environment, 'aws')
+      - name: Login to Amazon EKS
+        id: login-eks
+        run: |
+          aws eks --region us-east-1 update-kubeconfig --name zenml-ci-cluster --alias zenml-ci-aws-us-east-1
+        if: contains(inputs.test_environment, 'aws')
+      - name: Login to Google ECR
+        run: |
+          gcloud auth configure-docker --project zenml-ci
+        if: contains(inputs.test_environment, 'gcp')
+      - name: Login to Google GKE
+        uses: google-github-actions/get-gke-credentials@v2
+        with:
+          cluster_name: zenml-ci-cluster
+          location: us-east4
+          project_id: zenml-ci
+        if: contains(inputs.test_environment, 'gcp')
+      - name: Setup tmate session before tests
+        if: ${{ inputs.enable_tmate == 'before-tests' }}
+        uses: mxschmitt/action-tmate@v3.17
+        timeout-minutes: ${{ inputs.tmate_timeout }}
+      - name: Integration Tests - Slow CI
+        run: |
+          bash scripts/test-coverage-xml.sh integration ${{ inputs.test_environment }}
+      - name: Setup tmate session after tests
+        if: ${{ inputs.enable_tmate == 'always' || (inputs.enable_tmate == 'on-failure' && failure()) }}
+        uses: mxschmitt/action-tmate@v3.17
+        timeout-minutes: ${{ inputs.tmate_timeout }}
+      - name: Verify Python Env unaffected
+        run: |-
+          zenml integration list
+          uv pip list
+          uv pip check || true
+    services:
+      mysql:
+        image: mysql:5.7
+        env:
+          MYSQL_ROOT_PASSWORD: zenml
+          MYSQL_DATABASE: zenml
+        ports: 
+          - 3306:3306
+        options: >-
+          --health-cmd="mysqladmin ping"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=3
+      zenml-server:
+        image: ghcr.io/${{ github.repository_owner }}/zenml-server-github-actions:${{
+          github.sha }}
+        credentials:
+          username: ${{ github.actor }}
+          password: ${{ secrets.github_token }}
+        env:
+          ZENML_STORE_URL: mysql://root:zenml@mysql:3306/zenml
+          ZENML_SERVER_DEPLOYMENT_TYPE: docker
+          ZENML_SERVER_AUTO_ACTIVATE: 'True'
+          ZENML_SERVER_AUTO_CREATE_DEFAULT_USER: 'True'
+        ports: 
+          - 8080:8080
+        options: >-
+          --health-cmd="curl -f http://127.0.0.1:8080/health"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=3
diff --git a/scripts/format.sh b/scripts/format.sh
index c26ba6f22a6..f6eb07dab99 100755
--- a/scripts/format.sh
+++ b/scripts/format.sh
@@ -64,7 +64,7 @@ ruff format $SRC
 
 # standardizes / formats CI yaml files
 if [ "$SKIP_YAMLFIX" = false ]; then
-    yamlfix .github tests --exclude="dependabot.yml" --exclude="workflows/integration-test-fast-services.yml"
+    yamlfix .github tests --exclude="dependabot.yml" --exclude="workflows/integration-test-fast-services.yml" --exclude="workflows/integration-test-slow-services.yml"
 fi
 
 set +x
diff --git a/scripts/lint.sh b/scripts/lint.sh
index cffa55692db..d6dca93b4fd 100755
--- a/scripts/lint.sh
+++ b/scripts/lint.sh
@@ -30,7 +30,7 @@ fi
 
 # checks for yaml formatting errors
 if [ "$SKIP_YAMLFIX" = false ]; then
-    yamlfix .github tests --exclude="dependabot.yml" --exclude="workflows/integration-test-fast-services.yml"
+    yamlfix .github tests --exclude="dependabot.yml" --exclude="workflows/integration-test-fast-services.yml" --exclude="workflows/integration-test-slow-services.yml"
 fi
 
 # autoflake replacement: checks for unused imports and variables

From 8646b5e0a935e93e47fe2beff14fb074fa66dc79 Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Mon, 14 Oct 2024 22:51:11 +0100
Subject: [PATCH 25/60] Refactor Dockerfile path in ci-slow.yml and update
 runs-on to ubuntu-latest

---
 .github/workflows/ci-slow.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml
index 053b87c385f..a04589a9c81 100644
--- a/.github/workflows/ci-slow.yml
+++ b/.github/workflows/ci-slow.yml
@@ -62,7 +62,7 @@ jobs:
     env:
       ZENML_ANALYTICS_OPT_IN: false
       ZENML_DEBUG: true
-    runs-on: arc-runner-set
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4.1.1
         with:
@@ -85,7 +85,7 @@ jobs:
     env:
       ZENML_ANALYTICS_OPT_IN: false
       ZENML_DEBUG: true
-    runs-on: arc-runner-set
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4.1.1
         with:
@@ -104,7 +104,7 @@ jobs:
         run: bash scripts/test-migrations.sh mysql random
   sqlite-db-migration-testing-full:
     needs: run-slow-ci-label-is-set
-    runs-on: arc-runner-set
+    runs-on: ubuntu-latest
     env:
       ZENML_ANALYTICS_OPT_IN: false
       ZENML_DEBUG: true
@@ -126,7 +126,7 @@ jobs:
     env:
       ZENML_ANALYTICS_OPT_IN: false
       ZENML_DEBUG: true
-    runs-on: arc-runner-set
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4.1.1
         with:
@@ -198,8 +198,8 @@ jobs:
     needs: run-slow-ci-label-is-set
     strategy:
       matrix:
-        os: [arc-runner-set]
-        python-version: ['3.9', '3.10', '3.11', '3.12']
+        os: [ubuntu-latest]
+        python-version: ['3.9', '3.10', '3.11']
       fail-fast: false
     uses: ./.github/workflows/unit-test.yml
     with:

From e0ce573a2abb92eb161cfa9a4e7406fb4980a18e Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Mon, 14 Oct 2024 23:10:46 +0100
Subject: [PATCH 26/60] Refactor ci-slow.yml to include custom-ubuntu-unit-test
 in the workflow dependencies

---
 .github/workflows/ci-slow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml
index a04589a9c81..d2789251504 100644
--- a/.github/workflows/ci-slow.yml
+++ b/.github/workflows/ci-slow.yml
@@ -303,7 +303,7 @@ jobs:
     secrets: inherit
   custom-ubuntu-integration-test-gh-actions-server-docker-orchestrator:
     if: github.event.pull_request.draft == false
-    needs: run-slow-ci-label-is-set
+    needs: [run-slow-ci-label-is-set, custom-ubuntu-unit-test]
     strategy:
       matrix:
         os: [ubuntu-latest]

From 4ba01db1400f24ca1dec6828ef8228384d36a004 Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Mon, 14 Oct 2024 23:24:41 +0100
Subject: [PATCH 27/60] Refactor ci-slow.yml to include custom-ubuntu-unit-test
 in the workflow dependencies

---
 .github/workflows/ci-fast.yml   |  17 ++++-
 .github/workflows/ci-slow.yml   |  48 +++++++++++--
 .github/workflows/linting.yml   | 120 ++++++++++++++++++++++++++++++++
 .github/workflows/unit-test.yml |   6 --
 4 files changed, 177 insertions(+), 14 deletions(-)
 create mode 100644 .github/workflows/linting.yml

diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml
index 208c2632481..4ee6d1cd89d 100644
--- a/.github/workflows/ci-fast.yml
+++ b/.github/workflows/ci-fast.yml
@@ -82,7 +82,20 @@ jobs:
       python-version: '3.9'
       os: ubuntu-latest
     secrets: inherit
+  linting:
+    if: github.event.pull_request.draft == false
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ['3.12']
+      fail-fast: false
+    uses: ./.github/workflows/linting.yml
+    with:
+      python-version: ${{ matrix.python-version }}
+      os: ${{ matrix.os }}
+    secrets: inherit
   custom-ubuntu-setup-and-unit-test:
+    needs: linting
     if: github.event.pull_request.draft == false
     strategy:
       matrix:
@@ -95,7 +108,7 @@ jobs:
       os: ${{ matrix.os }}
     secrets: inherit
   custom-arc-runner-set-integration-test:
-    needs: build-and-push
+    needs: [build-and-push, linting]
     if: github.event.pull_request.draft == false
     strategy:
       matrix:
@@ -131,7 +144,7 @@ jobs:
             github.sha }}
           file: ./docker/zenml-server-dev.Dockerfile
   custom-arc-runner-set-integration-test-server:
-    needs: build-and-push
+    needs: [build-and-push, linting]
     if: github.event.pull_request.draft == false
     strategy:
       matrix:
diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml
index d2789251504..17c53d84331 100644
--- a/.github/workflows/ci-slow.yml
+++ b/.github/workflows/ci-slow.yml
@@ -193,9 +193,21 @@ jobs:
           bash scripts/check-alembic-branches.sh
       - name: Install latest dashboard (test gitignore)
         run: bash scripts/install-dashboard.sh
+  ubuntu-linting:
+    if: github.event.pull_request.draft == false
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ['3.9', '3.10', '3.11']
+      fail-fast: false
+    uses: ./.github/workflows/linting.yml
+    with:
+      python-version: ${{ matrix.python-version }}
+      os: ${{ matrix.os }}
+    secrets: inherit
   custom-ubuntu-unit-test:
     if: github.event.pull_request.draft == false
-    needs: run-slow-ci-label-is-set
+    needs: [run-slow-ci-label-is-set, ubuntu-linting]
     strategy:
       matrix:
         os: [ubuntu-latest]
@@ -206,9 +218,21 @@ jobs:
       python-version: ${{ matrix.python-version }}
       os: ${{ matrix.os }}
     secrets: inherit
+  windows-linting:
+    if: github.event.pull_request.draft == false
+    strategy:
+      matrix:
+        os: [windows-latest]
+        python-version: ['3.9', '3.10', '3.11']
+      fail-fast: false
+    uses: ./.github/workflows/linting.yml
+    with:
+      python-version: ${{ matrix.python-version }}
+      os: ${{ matrix.os }}
+    secrets: inherit
   windows-unit-test:
     if: github.event.pull_request.draft == false
-    needs: run-slow-ci-label-is-set
+    needs: [run-slow-ci-label-is-set, windows-linting]
     strategy:
       matrix:
         os: [windows-latest]
@@ -219,9 +243,21 @@ jobs:
       python-version: ${{ matrix.python-version }}
       os: ${{ matrix.os }}
     secrets: inherit
+  macos-linting:
+    if: github.event.pull_request.draft == false
+    strategy:
+      matrix:
+        os: [macos-latest]
+        python-version: ['3.9', '3.10', '3.11']
+      fail-fast: false
+    uses: ./.github/workflows/linting.yml
+    with:
+      python-version: ${{ matrix.python-version }}
+      os: ${{ matrix.os }}
+    secrets: inherit
   macos-unit-test:
     if: github.event.pull_request.draft == false
-    needs: run-slow-ci-label-is-set
+    needs: [run-slow-ci-label-is-set, macos-linting]
     strategy:
       matrix:
         os: [macos-latest]
@@ -240,7 +276,7 @@ jobs:
     secrets: inherit
   windows-integration-test:
     if: github.event.pull_request.draft == false
-    needs: run-slow-ci-label-is-set
+    needs: [run-slow-ci-label-is-set, windows-unit-test]
     strategy:
       matrix:
         os: [windows-latest]
@@ -255,7 +291,7 @@ jobs:
     secrets: inherit
   macos-integration-test:
     if: github.event.pull_request.draft == false
-    needs: run-slow-ci-label-is-set
+    needs: [run-slow-ci-label-is-set, macos-unit-test]
     strategy:
       matrix:
         os: [macos-13]
@@ -276,7 +312,7 @@ jobs:
     secrets: inherit
   custom-ubuntu-integration-test:
     if: github.event.pull_request.draft == false
-    needs: run-slow-ci-label-is-set
+    needs: [run-slow-ci-label-is-set, custom-ubuntu-unit-test]
     strategy:
       matrix:
         os: [ubuntu-latest]
diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml
new file mode 100644
index 00000000000..61d84cdd840
--- /dev/null
+++ b/.github/workflows/linting.yml
@@ -0,0 +1,120 @@
+---
+name: Setup Python Environment, Lint and Unit Test
+on:
+  workflow_call:
+    inputs:
+      os:
+        description: OS
+        type: string
+        required: true
+      python-version:
+        description: Python version
+        type: string
+        required: true
+      enable_tmate:
+        description: Enable tmate session for debugging
+        type: string
+        required: false
+        default: never
+      install_integrations:
+        description: Install ZenML integrations
+        type: string
+        required: false
+        default: 'yes'
+      git-ref:
+        description: Git branch or ref
+        type: string
+        required: false
+        default: ''
+  workflow_dispatch:
+    inputs:
+      os:
+        description: OS
+        type: choice
+        options: [ubuntu-latest, macos-13, windows-latest, arc-runner-set]
+        required: false
+        default: ubuntu-latest
+      python-version:
+        description: Python version
+        type: choice
+        options: ['3.9', '3.10', '3.11', '3.12']
+        required: false
+        default: '3.9'
+      enable_tmate:
+        description: Enable tmate session for debugging
+        type: choice
+        options: [no, on-failure, always, before-tests]
+        required: false
+        default: 'no'
+      git-ref:
+        description: Git branch or ref
+        type: string
+        required: false
+        default: ''
+jobs:
+  unit-test:
+    name: unit-test
+    runs-on: ${{ inputs.os }}
+    env:
+      ZENML_DEBUG: 1
+      ZENML_ANALYTICS_OPT_IN: false
+      PYTHONIOENCODING: utf-8
+      UV_HTTP_TIMEOUT: 600
+      OBJC_DISABLE_INITIALIZE_FORK_SAFETY: 'YES'
+    if: ${{ ! startsWith(github.event.head_commit.message, 'GitBook:') && ! (inputs.os == 'windows-latest' && inputs.python-version == '3.11') && ! (inputs.os == 'windows-latest' && inputs.python-version == '3.12') }}
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4.1.1
+        with:
+          repository: ${{ github.repository }}
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0  # Fetch all history for all branches and tags
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v42
+        with:
+          files: |
+            src/**
+            tests/**
+            examples/**
+      - name: Restore uv cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/uv
+          key: |
+            uv-${{ runner.os }}-${{ inputs.python-version }}-${{ hashFiles('src/zenml/integrations/*/__init__.py') }}
+          restore-keys: |
+            uv-${{ runner.os }}-${{ inputs.python-version }}-${{ hashFiles('src/zenml/integrations/*/__init__.py') }}
+      - name: Install libomp
+        if: ${{ inputs.os == 'macos-latest' || inputs.os == 'macos-13' }}
+        run: brew install libomp
+      - name: Setup environment
+        uses: ./.github/actions/setup_environment
+        with:
+          cache_version: ${{ secrets.GH_ACTIONS_CACHE_KEY }}
+          python-version: ${{ inputs.python-version }}
+          os: ${{ inputs.os }}
+          install_integrations: ${{ inputs.install_integrations }}
+          runners_cache_access_key_id: ${{ secrets.RUNNERS_CACHE_ACCESS_KEY_ID }}
+          runners_cache_secret_access_key: ${{ secrets.RUNNERS_CACHE_SECRET_ACCESS_KEY }}
+          discord_webhook: ${{ secrets.DISCORD_WEBHOOK }}
+      - name: Setup tmate session before tests
+        if: ${{ inputs.enable_tmate == 'before-tests' }}
+        uses: mxschmitt/action-tmate@v3.17
+      - name: Lint check
+        env:
+          OS: ${{ inputs.os }}
+          CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
+        run: |
+          bash scripts/lint.sh
+      - name: Setup tmate session after tests
+        if: ${{ inputs.enable_tmate == 'always' || (inputs.enable_tmate == 'on-failure' && failure()) }}
+        uses: mxschmitt/action-tmate@v3.17
+      - name: Verify Python Env unaffected
+        run: |-
+          zenml integration list
+          uv pip list
+          uv pip check || true
diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
index 89e008458d7..8c1843ac693 100644
--- a/.github/workflows/unit-test.yml
+++ b/.github/workflows/unit-test.yml
@@ -104,12 +104,6 @@ jobs:
       - name: Setup tmate session before tests
         if: ${{ inputs.enable_tmate == 'before-tests' }}
         uses: mxschmitt/action-tmate@v3.17
-      - name: Lint check
-        env:
-          OS: ${{ inputs.os }}
-          CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
-        run: |
-          bash scripts/lint.sh
       - name: Run unit tests
         run: |
           bash scripts/test-coverage-xml.sh unit

From 732117e808777005079a6c325f1be8d4f52f1c46 Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Mon, 14 Oct 2024 23:26:41 +0100
Subject: [PATCH 28/60] Refactor ci-slow.yml to include custom-ubuntu-unit-test
 in the workflow dependencies

---
 .github/workflows/ci-slow.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml
index 17c53d84331..8c77abd5a8c 100644
--- a/.github/workflows/ci-slow.yml
+++ b/.github/workflows/ci-slow.yml
@@ -194,6 +194,7 @@ jobs:
       - name: Install latest dashboard (test gitignore)
         run: bash scripts/install-dashboard.sh
   ubuntu-linting:
+    needs: run-slow-ci-label-is-set
     if: github.event.pull_request.draft == false
     strategy:
       matrix:
@@ -219,6 +220,7 @@ jobs:
       os: ${{ matrix.os }}
     secrets: inherit
   windows-linting:
+    needs: run-slow-ci-label-is-set
     if: github.event.pull_request.draft == false
     strategy:
       matrix:
@@ -244,6 +246,7 @@ jobs:
       os: ${{ matrix.os }}
     secrets: inherit
   macos-linting:
+    needs: run-slow-ci-label-is-set
     if: github.event.pull_request.draft == false
     strategy:
       matrix:

From ff9fd7390fd9607ab7acecf03d4e83d73a1b9ec0 Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Sat, 19 Oct 2024 19:16:18 +0100
Subject: [PATCH 29/60] Refactor ci-slow.yml to update Python versions in the
 workflow dependencies

---
 .github/workflows/ci-fast.yml                        | 8 ++++----
 .github/workflows/ci-slow.yml                        | 4 ++--
 .github/workflows/integration-test-fast-services.yml | 6 +++---
 .github/workflows/integration-test-fast.yml          | 6 +++---
 .github/workflows/integration-test-slow-services.yml | 2 +-
 .github/workflows/integration-test-slow.yml          | 2 +-
 .github/workflows/linting.yml                        | 2 +-
 .github/workflows/templates-test.yml                 | 2 +-
 .github/workflows/unit-test.yml                      | 2 +-
 .github/workflows/update-templates-to-examples.yml   | 2 +-
 10 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml
index 4ee6d1cd89d..44d43875398 100644
--- a/.github/workflows/ci-fast.yml
+++ b/.github/workflows/ci-fast.yml
@@ -87,7 +87,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: ['3.12']
+        python-version: ['3.11']
       fail-fast: false
     uses: ./.github/workflows/linting.yml
     with:
@@ -100,7 +100,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: ['3.12']
+        python-version: ['3.11']
       fail-fast: false
     uses: ./.github/workflows/unit-test.yml
     with:
@@ -113,7 +113,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: ['3.12']
+        python-version: ['3.11']
         test_environment: [default]
       fail-fast: false
     uses: ./.github/workflows/integration-test-fast.yml
@@ -149,7 +149,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: ['3.12']
+        python-version: ['3.11']
         test_environment: [github-actions-server-docker-orchestrator]
       fail-fast: false
     uses: ./.github/workflows/integration-test-fast-services.yml
diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml
index 8c77abd5a8c..c9209342e69 100644
--- a/.github/workflows/ci-slow.yml
+++ b/.github/workflows/ci-slow.yml
@@ -319,7 +319,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: ['3.9', '3.10', '3.11']
+        python-version: ['3.9', '3.10', '3.12']
         test_environment: [default, docker-server-docker-orchestrator-mariadb]
         exclude:
           # docker is time-consuming to run, so we only run it on 3.9
@@ -330,7 +330,7 @@ jobs:
           - test_environment: docker-server-docker-orchestrator-mariadb
             python-version: '3.10'
           - test_environment: docker-server-docker-orchestrator-mariadb
-            python-version: '3.11'
+            python-version: '3.12'
           - test_environment: docker-server-docker-orchestrator-mariadb
             python-version: '3.12'
       fail-fast: false
diff --git a/.github/workflows/integration-test-fast-services.yml b/.github/workflows/integration-test-fast-services.yml
index 84f9ee59532..fb50a7fa0f8 100644
--- a/.github/workflows/integration-test-fast-services.yml
+++ b/.github/workflows/integration-test-fast-services.yml
@@ -38,7 +38,7 @@ on:
         type: choice
         options: ['3.9', '3.10', '3.11', '3.12']
         required: false
-        default: '3.9'
+        default: '3.11'
       test_environment:
         description: The test environment
         type: choice
@@ -79,7 +79,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        shard: [1, 2, 3]
+        shard: [1, 2, 3, 4, 5, 6]
     env:
       ZENML_DEBUG: 1
       ZENML_ANALYTICS_OPT_IN: false
@@ -228,7 +228,7 @@ jobs:
         # Ubuntu integration tests run as 6 shards
         if: runner.os != 'macOS' && runner.os != 'Windows'
         run: |
-          bash scripts/test-coverage-xml.sh integration ${{ inputs.test_environment }} 3 ${{ matrix.shard }}
+          bash scripts/test-coverage-xml.sh integration ${{ inputs.test_environment }} 6 ${{ matrix.shard }}
       - name: Setup tmate session after tests
         if: ${{ inputs.enable_tmate == 'always' || (inputs.enable_tmate == 'on-failure' && failure()) }}
         uses: mxschmitt/action-tmate@v3.17
diff --git a/.github/workflows/integration-test-fast.yml b/.github/workflows/integration-test-fast.yml
index 835af58fafc..823c5f27542 100644
--- a/.github/workflows/integration-test-fast.yml
+++ b/.github/workflows/integration-test-fast.yml
@@ -38,7 +38,7 @@ on:
         type: choice
         options: ['3.9', '3.10', '3.11', '3.12']
         required: false
-        default: '3.9'
+        default: '3.11'
       test_environment:
         description: The test environment
         type: choice
@@ -79,7 +79,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        shard: [1, 2, 3]
+        shard: [1, 2, 3, 4, 5, 6]
     env:
       ZENML_DEBUG: 1
       ZENML_ANALYTICS_OPT_IN: false
@@ -245,7 +245,7 @@ jobs:
         # Ubuntu integration tests run as 6 shards
         if: runner.os != 'macOS' && runner.os != 'Windows'
         run: |
-          bash scripts/test-coverage-xml.sh integration ${{ inputs.test_environment }} 3 ${{ matrix.shard }}
+          bash scripts/test-coverage-xml.sh integration ${{ inputs.test_environment }} 6 ${{ matrix.shard }}
       - name: Setup tmate session after tests
         if: ${{ inputs.enable_tmate == 'always' || (inputs.enable_tmate == 'on-failure' && failure()) }}
         uses: mxschmitt/action-tmate@v3.17
diff --git a/.github/workflows/integration-test-slow-services.yml b/.github/workflows/integration-test-slow-services.yml
index 9a761034920..84955b6b552 100644
--- a/.github/workflows/integration-test-slow-services.yml
+++ b/.github/workflows/integration-test-slow-services.yml
@@ -38,7 +38,7 @@ on:
         type: choice
         options: ['3.9', '3.10', '3.11', '3.12']
         required: false
-        default: '3.9'
+        default: '3.11'
       test_environment:
         description: The test environment
         type: choice
diff --git a/.github/workflows/integration-test-slow.yml b/.github/workflows/integration-test-slow.yml
index 368037de9f0..d74f7306804 100644
--- a/.github/workflows/integration-test-slow.yml
+++ b/.github/workflows/integration-test-slow.yml
@@ -38,7 +38,7 @@ on:
         type: choice
         options: ['3.9', '3.10', '3.11', '3.12']
         required: false
-        default: '3.9'
+        default: '3.11'
       test_environment:
         description: The test environment
         type: choice
diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml
index 61d84cdd840..ca8440124b6 100644
--- a/.github/workflows/linting.yml
+++ b/.github/workflows/linting.yml
@@ -39,7 +39,7 @@ on:
         type: choice
         options: ['3.9', '3.10', '3.11', '3.12']
         required: false
-        default: '3.9'
+        default: '3.11'
       enable_tmate:
         description: Enable tmate session for debugging
         type: choice
diff --git a/.github/workflows/templates-test.yml b/.github/workflows/templates-test.yml
index e2fb1d7ba79..d76a85a731d 100644
--- a/.github/workflows/templates-test.yml
+++ b/.github/workflows/templates-test.yml
@@ -24,7 +24,7 @@ on:
         type: choice
         options: ['3.9', '3.10', '3.11', '3.12']
         required: false
-        default: '3.9'
+        default: '3.11'
 jobs:
   all-template-tests:
     name: all-template-tests
diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
index 8c1843ac693..345d2d44698 100644
--- a/.github/workflows/unit-test.yml
+++ b/.github/workflows/unit-test.yml
@@ -39,7 +39,7 @@ on:
         type: choice
         options: ['3.9', '3.10', '3.11', '3.12']
         required: false
-        default: '3.9'
+        default: '3.11'
       enable_tmate:
         description: Enable tmate session for debugging
         type: choice
diff --git a/.github/workflows/update-templates-to-examples.yml b/.github/workflows/update-templates-to-examples.yml
index 9b55d1c9928..09862316dc7 100644
--- a/.github/workflows/update-templates-to-examples.yml
+++ b/.github/workflows/update-templates-to-examples.yml
@@ -24,7 +24,7 @@ on:
         type: choice
         options: ['3.9', '3.10', '3.11', '3.12']
         required: false
-        default: '3.9'
+        default: '3.11'
 jobs:
   update-e2e-batch-template-to-examples:
     name: update-e2e-batch-template-to-examples

From c9e1b65a085842e8f3d962d3930ad69d20da5e30 Mon Sep 17 00:00:00 2001
From: GitHub Actions <actions@github.com>
Date: Sat, 19 Oct 2024 19:37:18 +0000
Subject: [PATCH 30/60] Auto-update of LLM Finetuning template

---
 docs/book/how-to/handle-data-artifacts/datasets.md        | 5 ++++-
 docs/book/how-to/handle-data-artifacts/manage-big-data.md | 5 ++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/docs/book/how-to/handle-data-artifacts/datasets.md b/docs/book/how-to/handle-data-artifacts/datasets.md
index 32fc8ef2b99..3f3b3c19723 100644
--- a/docs/book/how-to/handle-data-artifacts/datasets.md
+++ b/docs/book/how-to/handle-data-artifacts/datasets.md
@@ -237,4 +237,7 @@ def analyze_data(data: pd.DataFrame) -> pd.DataFrame:
 
 By following these practices, you can create ZenML pipelines that efficiently handle complex data flows and multiple data sources while remaining adaptable to changing requirements. This approach allows you to leverage the power of custom Dataset classes throughout your machine learning workflows, ensuring consistency and flexibility as your projects evolve.
 
-For strategies on scaling your data processing as your datasets grow larger, refer to [scaling strategies for big data](manage-big-data.md).
\ No newline at end of file
+For strategies on scaling your data processing as your datasets grow larger, refer to [scaling strategies for big data](manage-big-data.md).<!-- For scarf -->
+<figure><img alt="ZenML Scarf" referrerpolicy="no-referrer-when-downgrade" src="https://static.scarf.sh/a.png?x-pxid=f0b4f458-0a54-4fcd-aa95-d5ee424815bc" /></figure>
+
+
diff --git a/docs/book/how-to/handle-data-artifacts/manage-big-data.md b/docs/book/how-to/handle-data-artifacts/manage-big-data.md
index f81091f436b..470548efc26 100644
--- a/docs/book/how-to/handle-data-artifacts/manage-big-data.md
+++ b/docs/book/how-to/handle-data-artifacts/manage-big-data.md
@@ -306,4 +306,7 @@ When selecting a scaling strategy, consider:
 
 Remember, it's often best to start simple and scale up as needed. ZenML's flexible architecture allows you to evolve your data processing strategies as your project grows.
 
-By implementing these scaling strategies, you can extend your ZenML pipelines to handle datasets of any size, ensuring that your machine learning workflows remain efficient and manageable as your projects scale. For more information on creating custom Dataset classes and managing complex data flows, refer back to [custom dataset classes](datasets.md).
\ No newline at end of file
+By implementing these scaling strategies, you can extend your ZenML pipelines to handle datasets of any size, ensuring that your machine learning workflows remain efficient and manageable as your projects scale. For more information on creating custom Dataset classes and managing complex data flows, refer back to [custom dataset classes](datasets.md).<!-- For scarf -->
+<figure><img alt="ZenML Scarf" referrerpolicy="no-referrer-when-downgrade" src="https://static.scarf.sh/a.png?x-pxid=f0b4f458-0a54-4fcd-aa95-d5ee424815bc" /></figure>
+
+

From c15a1c89a9249bdab057e3951feb6af26863b730 Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Sun, 20 Oct 2024 01:05:23 +0100
Subject: [PATCH 31/60] Refactor MLFlow deployment service and steps

---
 src/zenml/integrations/mlflow/services/mlflow_deployment.py | 4 ++--
 src/zenml/integrations/mlflow/steps/mlflow_deployer.py      | 2 +-
 src/zenml/integrations/mlflow/steps/mlflow_registry.py      | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/zenml/integrations/mlflow/services/mlflow_deployment.py b/src/zenml/integrations/mlflow/services/mlflow_deployment.py
index 38969f4e8de..d010ec1af77 100644
--- a/src/zenml/integrations/mlflow/services/mlflow_deployment.py
+++ b/src/zenml/integrations/mlflow/services/mlflow_deployment.py
@@ -225,7 +225,7 @@ def run(self) -> None:
             # to run the deploy the model on the local running environment
             if int(mlflow_version[0]) >= 2:
                 backend_kwargs["env_manager"] = "local"
-            backend = PyFuncBackend(  # type: ignore[no-untyped-call]
+            backend = PyFuncBackend(
                 config={},
                 no_conda=True,
                 workers=self.config.workers,
@@ -240,7 +240,7 @@ def run(self) -> None:
                     "stack."
                 )
             experiment_tracker.configure_mlflow()
-            backend.serve(  # type: ignore[no-untyped-call]
+            backend.serve(
                 model_uri=self.config.model_uri,
                 port=self.endpoint.status.port,
                 host="localhost",
diff --git a/src/zenml/integrations/mlflow/steps/mlflow_deployer.py b/src/zenml/integrations/mlflow/steps/mlflow_deployer.py
index 7eea8be4a11..f3075e9b47c 100644
--- a/src/zenml/integrations/mlflow/steps/mlflow_deployer.py
+++ b/src/zenml/integrations/mlflow/steps/mlflow_deployer.py
@@ -114,7 +114,7 @@ def mlflow_model_deployer_step(
     # Fetch the model URI from the MLflow artifact store
     model_uri = ""
     if mlflow_run_id and client.list_artifacts(mlflow_run_id, model_name):
-        model_uri = artifact_utils.get_artifact_uri(  # type: ignore[no-untyped-call]
+        model_uri = artifact_utils.get_artifact_uri(
             run_id=mlflow_run_id, artifact_path=model_name
         )
 
diff --git a/src/zenml/integrations/mlflow/steps/mlflow_registry.py b/src/zenml/integrations/mlflow/steps/mlflow_registry.py
index 87a9aaabe1c..093341f39a9 100644
--- a/src/zenml/integrations/mlflow/steps/mlflow_registry.py
+++ b/src/zenml/integrations/mlflow/steps/mlflow_registry.py
@@ -124,7 +124,7 @@ def mlflow_register_model_step(
     if not model_source_uri and client.list_artifacts(
         mlflow_run_id, trained_model_name
     ):
-        model_source_uri = artifact_utils.get_artifact_uri(  # type: ignore[no-untyped-call]
+        model_source_uri = artifact_utils.get_artifact_uri(
             run_id=mlflow_run_id, artifact_path=trained_model_name
         )
     if not model_source_uri:

From 40ac09a88094994d1b6e921d631f6a4a9a6edd31 Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Thu, 31 Oct 2024 12:38:24 +0100
Subject: [PATCH 32/60] Update lint script to check YAML formatting instead of
 fixing it

---
 scripts/lint.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/lint.sh b/scripts/lint.sh
index d6dca93b4fd..2d507e5020f 100755
--- a/scripts/lint.sh
+++ b/scripts/lint.sh
@@ -30,7 +30,7 @@ fi
 
 # checks for yaml formatting errors
 if [ "$SKIP_YAMLFIX" = false ]; then
-    yamlfix .github tests --exclude="dependabot.yml" --exclude="workflows/integration-test-fast-services.yml" --exclude="workflows/integration-test-slow-services.yml"
+    yamlfix --check .github tests --exclude="dependabot.yml" --exclude="workflows/integration-test-fast-services.yml" --exclude="workflows/integration-test-slow-services.yml"
 fi
 
 # autoflake replacement: checks for unused imports and variables

From 50ab43215dcfdce3c7efec38933e6ce9dad5de28 Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Thu, 31 Oct 2024 12:38:49 +0100
Subject: [PATCH 33/60] Refactor workflow YAML descriptions and simplify branch
 specification

---
 .github/workflows/release_finalize.yml | 4 ++--
 .github/workflows/release_prepare.yml  | 3 +--
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/release_finalize.yml b/.github/workflows/release_finalize.yml
index 9f486782a7a..d6ca4cc0681 100644
--- a/.github/workflows/release_finalize.yml
+++ b/.github/workflows/release_finalize.yml
@@ -4,11 +4,11 @@ on:
   workflow_dispatch:
     inputs:
       latest_version:
-        description: "The latest version of ZenML"
+        description: The latest version of ZenML
         required: true
         type: string
       new_version:
-        description: "The new version of ZenML"
+        description: The new version of ZenML
         required: true
         type: string
 env:
diff --git a/.github/workflows/release_prepare.yml b/.github/workflows/release_prepare.yml
index 1d60f6c3b0c..d8cbb5c89ee 100644
--- a/.github/workflows/release_prepare.yml
+++ b/.github/workflows/release_prepare.yml
@@ -2,8 +2,7 @@
 name: release-prepare
 on:
   push:
-    branches:
-      - "misc/prepare-release-*"
+    branches: [misc/prepare-release-*]
 env:
   ZENML_ANALYTICS_OPT_IN: false
   GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

From 00dbbb969a73698317f6d5be5e22e45b2d7670eb Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Fri, 1 Nov 2024 21:40:14 +0100
Subject: [PATCH 34/60] Update CI workflows to include additional test
 environments and comment out unused jobs

---
 .github/workflows/ci-fast.yml | 32 ++++++++++++++++----------------
 .github/workflows/ci-slow.yml | 30 +++++++++++++++---------------
 2 files changed, 31 insertions(+), 31 deletions(-)

diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml
index 44d43875398..86f4e83e159 100644
--- a/.github/workflows/ci-fast.yml
+++ b/.github/workflows/ci-fast.yml
@@ -114,7 +114,7 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         python-version: ['3.11']
-        test_environment: [default]
+        test_environment: [default, docker-server-docker-orchestrator-mysql]
       fail-fast: false
     uses: ./.github/workflows/integration-test-fast.yml
     with:
@@ -143,18 +143,18 @@ jobs:
           tags: ghcr.io/${{ github.repository_owner }}/zenml-server-github-actions:${{
             github.sha }}
           file: ./docker/zenml-server-dev.Dockerfile
-  custom-arc-runner-set-integration-test-server:
-    needs: [build-and-push, linting]
-    if: github.event.pull_request.draft == false
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: ['3.11']
-        test_environment: [github-actions-server-docker-orchestrator]
-      fail-fast: false
-    uses: ./.github/workflows/integration-test-fast-services.yml
-    with:
-      os: ${{ matrix.os }}
-      python-version: ${{ matrix.python-version }}
-      test_environment: ${{ matrix.test_environment }}
-    secrets: inherit
+# custom-arc-runner-set-integration-test-server:
+#   needs: [build-and-push, linting]
+#   if: github.event.pull_request.draft == false
+#   strategy:
+#     matrix:
+#       os: [ubuntu-latest]
+#       python-version: ['3.11']
+#       test_environment: [github-actions-server-docker-orchestrator]
+#     fail-fast: false
+#   uses: ./.github/workflows/integration-test-fast-services.yml
+#   with:
+#     os: ${{ matrix.os }}
+#     python-version: ${{ matrix.python-version }}
+#     test_environment: ${{ matrix.test_environment }}
+#   secrets: inherit
diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml
index af2ebd1c7ff..5a9a0688ae6 100644
--- a/.github/workflows/ci-slow.yml
+++ b/.github/workflows/ci-slow.yml
@@ -327,18 +327,18 @@ jobs:
       python-version: ${{ matrix.python-version }}
       test_environment: ${{ matrix.test_environment }}
     secrets: inherit
-  custom-ubuntu-integration-test-gh-actions-server-docker-orchestrator:
-    if: github.event.pull_request.draft == false
-    needs: [run-slow-ci-label-is-set, custom-ubuntu-unit-test]
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: ['3.9', '3.10', '3.11']
-        test_environment: [github-actions-server-docker-orchestrator]
-      fail-fast: false
-    uses: ./.github/workflows/integration-test-slow-services.yml
-    with:
-      os: ${{ matrix.os }}
-      python-version: ${{ matrix.python-version }}
-      test_environment: ${{ matrix.test_environment }}
-    secrets: inherit
+# custom-ubuntu-integration-test-gh-actions-server-docker-orchestrator:
+#   if: github.event.pull_request.draft == false
+#   needs: [run-slow-ci-label-is-set, custom-ubuntu-unit-test]
+#   strategy:
+#     matrix:
+#       os: [ubuntu-latest]
+#       python-version: ['3.9', '3.10', '3.11']
+#       test_environment: [github-actions-server-docker-orchestrator]
+#     fail-fast: false
+#   uses: ./.github/workflows/integration-test-slow-services.yml
+#   with:
+#     os: ${{ matrix.os }}
+#     python-version: ${{ matrix.python-version }}
+#     test_environment: ${{ matrix.test_environment }}
+#   secrets: inherit

From dfbb7b1748b50406cab35fad3008277df77fb146 Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Sat, 2 Nov 2024 00:35:11 +0100
Subject: [PATCH 35/60] Remove unnecessary comments and images from data
 artifact management documentation

---
 .../data-artifact-management/handle-data-artifacts/datasets.md | 3 +--
 .../handle-data-artifacts/manage-big-data.md                   | 3 +--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/docs/book/how-to/data-artifact-management/handle-data-artifacts/datasets.md b/docs/book/how-to/data-artifact-management/handle-data-artifacts/datasets.md
index 3f3b3c19723..502f21f3806 100644
--- a/docs/book/how-to/data-artifact-management/handle-data-artifacts/datasets.md
+++ b/docs/book/how-to/data-artifact-management/handle-data-artifacts/datasets.md
@@ -237,7 +237,6 @@ def analyze_data(data: pd.DataFrame) -> pd.DataFrame:
 
 By following these practices, you can create ZenML pipelines that efficiently handle complex data flows and multiple data sources while remaining adaptable to changing requirements. This approach allows you to leverage the power of custom Dataset classes throughout your machine learning workflows, ensuring consistency and flexibility as your projects evolve.
 
-For strategies on scaling your data processing as your datasets grow larger, refer to [scaling strategies for big data](manage-big-data.md).<!-- For scarf -->
-<figure><img alt="ZenML Scarf" referrerpolicy="no-referrer-when-downgrade" src="https://static.scarf.sh/a.png?x-pxid=f0b4f458-0a54-4fcd-aa95-d5ee424815bc" /></figure>
+For strategies on scaling your data processing as your datasets grow larger, refer to [scaling strategies for big data](manage-big-data.md).
 
 
diff --git a/docs/book/how-to/data-artifact-management/handle-data-artifacts/manage-big-data.md b/docs/book/how-to/data-artifact-management/handle-data-artifacts/manage-big-data.md
index 470548efc26..46fb93027f0 100644
--- a/docs/book/how-to/data-artifact-management/handle-data-artifacts/manage-big-data.md
+++ b/docs/book/how-to/data-artifact-management/handle-data-artifacts/manage-big-data.md
@@ -306,7 +306,6 @@ When selecting a scaling strategy, consider:
 
 Remember, it's often best to start simple and scale up as needed. ZenML's flexible architecture allows you to evolve your data processing strategies as your project grows.
 
-By implementing these scaling strategies, you can extend your ZenML pipelines to handle datasets of any size, ensuring that your machine learning workflows remain efficient and manageable as your projects scale. For more information on creating custom Dataset classes and managing complex data flows, refer back to [custom dataset classes](datasets.md).<!-- For scarf -->
-<figure><img alt="ZenML Scarf" referrerpolicy="no-referrer-when-downgrade" src="https://static.scarf.sh/a.png?x-pxid=f0b4f458-0a54-4fcd-aa95-d5ee424815bc" /></figure>
+By implementing these scaling strategies, you can extend your ZenML pipelines to handle datasets of any size, ensuring that your machine learning workflows remain efficient and manageable as your projects scale. For more information on creating custom Dataset classes and managing complex data flows, refer back to [custom dataset classes](datasets.md).
 
 

From ccddde776e6b20aee586e8529e2b21a93376db6a Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Sat, 2 Nov 2024 00:36:35 +0100
Subject: [PATCH 36/60] Fix type hinting for artifact URI retrieval in MLflow
 integration

---
 docs/book/component-guide/model-deployers/mlflow.md         | 2 +-
 src/zenml/integrations/mlflow/__init__.py                   | 2 +-
 src/zenml/integrations/mlflow/services/mlflow_deployment.py | 4 ++--
 src/zenml/integrations/mlflow/steps/mlflow_deployer.py      | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/book/component-guide/model-deployers/mlflow.md b/docs/book/component-guide/model-deployers/mlflow.md
index dfcfd38a72f..3ba84c69022 100644
--- a/docs/book/component-guide/model-deployers/mlflow.md
+++ b/docs/book/component-guide/model-deployers/mlflow.md
@@ -98,7 +98,7 @@ def deploy_model() -> Optional[MLFlowDeploymentService]:
     experiment_tracker.configure_mlflow()
     client = MlflowClient()
     model_name = "model" # set the model name that was logged
-    model_uri = artifact_utils.get_artifact_uri(
+    model_uri = artifact_utils.get_artifact_uri(  # type: ignore[no-untyped-call]
         run_id=mlflow_run_id, artifact_path=model_name
     )
     mlflow_deployment_config = MLFlowDeploymentConfig(
diff --git a/src/zenml/integrations/mlflow/__init__.py b/src/zenml/integrations/mlflow/__init__.py
index 017b769e4ae..fcd9c1f0545 100644
--- a/src/zenml/integrations/mlflow/__init__.py
+++ b/src/zenml/integrations/mlflow/__init__.py
@@ -58,7 +58,7 @@ def get_requirements(cls, target_os: Optional[str] = None) -> List[str]:
         from zenml.integrations.pandas import PandasIntegration
 
         reqs = [
-            "mlflow>=2.1.1,<2.17.0",
+            "mlflow>=2.1.1,<3",
             # TODO: remove this requirement once rapidjson is fixed
             "python-rapidjson<1.15",
             # When you do:
diff --git a/src/zenml/integrations/mlflow/services/mlflow_deployment.py b/src/zenml/integrations/mlflow/services/mlflow_deployment.py
index d010ec1af77..38969f4e8de 100644
--- a/src/zenml/integrations/mlflow/services/mlflow_deployment.py
+++ b/src/zenml/integrations/mlflow/services/mlflow_deployment.py
@@ -225,7 +225,7 @@ def run(self) -> None:
             # to run the deploy the model on the local running environment
             if int(mlflow_version[0]) >= 2:
                 backend_kwargs["env_manager"] = "local"
-            backend = PyFuncBackend(
+            backend = PyFuncBackend(  # type: ignore[no-untyped-call]
                 config={},
                 no_conda=True,
                 workers=self.config.workers,
@@ -240,7 +240,7 @@ def run(self) -> None:
                     "stack."
                 )
             experiment_tracker.configure_mlflow()
-            backend.serve(
+            backend.serve(  # type: ignore[no-untyped-call]
                 model_uri=self.config.model_uri,
                 port=self.endpoint.status.port,
                 host="localhost",
diff --git a/src/zenml/integrations/mlflow/steps/mlflow_deployer.py b/src/zenml/integrations/mlflow/steps/mlflow_deployer.py
index f3075e9b47c..7eea8be4a11 100644
--- a/src/zenml/integrations/mlflow/steps/mlflow_deployer.py
+++ b/src/zenml/integrations/mlflow/steps/mlflow_deployer.py
@@ -114,7 +114,7 @@ def mlflow_model_deployer_step(
     # Fetch the model URI from the MLflow artifact store
     model_uri = ""
     if mlflow_run_id and client.list_artifacts(mlflow_run_id, model_name):
-        model_uri = artifact_utils.get_artifact_uri(
+        model_uri = artifact_utils.get_artifact_uri(  # type: ignore[no-untyped-call]
             run_id=mlflow_run_id, artifact_path=model_name
         )
 

From dafd35043e465bab764ed4d4389755aaf8ce9b3d Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Sat, 2 Nov 2024 01:20:43 +0100
Subject: [PATCH 37/60] updatte

---
 docs/book/component-guide/model-deployers/mlflow.md           | 2 +-
 .../handle-data-artifacts/datasets.md                         | 4 +---
 .../handle-data-artifacts/manage-big-data.md                  | 4 +---
 src/zenml/integrations/mlflow/steps/mlflow_registry.py        | 2 +-
 4 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/docs/book/component-guide/model-deployers/mlflow.md b/docs/book/component-guide/model-deployers/mlflow.md
index 3ba84c69022..dfcfd38a72f 100644
--- a/docs/book/component-guide/model-deployers/mlflow.md
+++ b/docs/book/component-guide/model-deployers/mlflow.md
@@ -98,7 +98,7 @@ def deploy_model() -> Optional[MLFlowDeploymentService]:
     experiment_tracker.configure_mlflow()
     client = MlflowClient()
     model_name = "model" # set the model name that was logged
-    model_uri = artifact_utils.get_artifact_uri(  # type: ignore[no-untyped-call]
+    model_uri = artifact_utils.get_artifact_uri(
         run_id=mlflow_run_id, artifact_path=model_name
     )
     mlflow_deployment_config = MLFlowDeploymentConfig(
diff --git a/docs/book/how-to/data-artifact-management/handle-data-artifacts/datasets.md b/docs/book/how-to/data-artifact-management/handle-data-artifacts/datasets.md
index 502f21f3806..32fc8ef2b99 100644
--- a/docs/book/how-to/data-artifact-management/handle-data-artifacts/datasets.md
+++ b/docs/book/how-to/data-artifact-management/handle-data-artifacts/datasets.md
@@ -237,6 +237,4 @@ def analyze_data(data: pd.DataFrame) -> pd.DataFrame:
 
 By following these practices, you can create ZenML pipelines that efficiently handle complex data flows and multiple data sources while remaining adaptable to changing requirements. This approach allows you to leverage the power of custom Dataset classes throughout your machine learning workflows, ensuring consistency and flexibility as your projects evolve.
 
-For strategies on scaling your data processing as your datasets grow larger, refer to [scaling strategies for big data](manage-big-data.md).
-
-
+For strategies on scaling your data processing as your datasets grow larger, refer to [scaling strategies for big data](manage-big-data.md).
\ No newline at end of file
diff --git a/docs/book/how-to/data-artifact-management/handle-data-artifacts/manage-big-data.md b/docs/book/how-to/data-artifact-management/handle-data-artifacts/manage-big-data.md
index 46fb93027f0..f81091f436b 100644
--- a/docs/book/how-to/data-artifact-management/handle-data-artifacts/manage-big-data.md
+++ b/docs/book/how-to/data-artifact-management/handle-data-artifacts/manage-big-data.md
@@ -306,6 +306,4 @@ When selecting a scaling strategy, consider:
 
 Remember, it's often best to start simple and scale up as needed. ZenML's flexible architecture allows you to evolve your data processing strategies as your project grows.
 
-By implementing these scaling strategies, you can extend your ZenML pipelines to handle datasets of any size, ensuring that your machine learning workflows remain efficient and manageable as your projects scale. For more information on creating custom Dataset classes and managing complex data flows, refer back to [custom dataset classes](datasets.md).
-
-
+By implementing these scaling strategies, you can extend your ZenML pipelines to handle datasets of any size, ensuring that your machine learning workflows remain efficient and manageable as your projects scale. For more information on creating custom Dataset classes and managing complex data flows, refer back to [custom dataset classes](datasets.md).
\ No newline at end of file
diff --git a/src/zenml/integrations/mlflow/steps/mlflow_registry.py b/src/zenml/integrations/mlflow/steps/mlflow_registry.py
index 093341f39a9..87a9aaabe1c 100644
--- a/src/zenml/integrations/mlflow/steps/mlflow_registry.py
+++ b/src/zenml/integrations/mlflow/steps/mlflow_registry.py
@@ -124,7 +124,7 @@ def mlflow_register_model_step(
     if not model_source_uri and client.list_artifacts(
         mlflow_run_id, trained_model_name
     ):
-        model_source_uri = artifact_utils.get_artifact_uri(
+        model_source_uri = artifact_utils.get_artifact_uri(  # type: ignore[no-untyped-call]
             run_id=mlflow_run_id, artifact_path=trained_model_name
         )
     if not model_source_uri:

From df6c41e2ee1aee3a701411eaa646a52e1cfa35ac Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Tue, 5 Nov 2024 12:43:38 +0100
Subject: [PATCH 38/60] Update GitHub Actions to use actions/checkout@v4.2.2
 and actions/setup-python@v5.3.0

---
 .github/actions/setup_environment/action.yml  |  2 +-
 .github/workflows/ci-fast.yml                 | 22 +++++++++----------
 .github/workflows/ci-slow.yml                 | 22 +++++++++----------
 .github/workflows/codeql.yml                  |  2 +-
 .github/workflows/generate-test-duration.yml  |  2 +-
 .github/workflows/image-optimiser.yml         |  2 +-
 .../integration-test-fast-services.yml        |  2 +-
 .github/workflows/integration-test-fast.yml   |  2 +-
 .../integration-test-slow-services.yml        |  2 +-
 .github/workflows/integration-test-slow.yml   |  2 +-
 .github/workflows/linting.yml                 |  2 +-
 .github/workflows/publish_api_docs.yml        |  4 ++--
 .github/workflows/publish_docker_image.yml    |  2 +-
 .github/workflows/publish_helm_chart.yml      |  2 +-
 .github/workflows/publish_stack_templates.yml |  2 +-
 .github/workflows/publish_to_pypi.yml         |  4 ++--
 .github/workflows/publish_to_pypi_nightly.yml |  4 ++--
 .github/workflows/release.yml                 | 10 ++++-----
 .github/workflows/release_finalize.yml        | 10 ++++-----
 .github/workflows/release_prepare.yml         | 12 +++++-----
 .github/workflows/spellcheck.yml              |  2 +-
 .github/workflows/trivy-zenml-core.yml        |  2 +-
 .github/workflows/trivy-zenserver.yml         |  2 +-
 .github/workflows/unit-test.yml               |  2 +-
 .../update-templates-to-examples.yml          |  8 +++----
 25 files changed, 64 insertions(+), 64 deletions(-)

diff --git a/.github/actions/setup_environment/action.yml b/.github/actions/setup_environment/action.yml
index d5b70818292..29bd5f7bc98 100644
--- a/.github/actions/setup_environment/action.yml
+++ b/.github/actions/setup_environment/action.yml
@@ -30,7 +30,7 @@ runs:
   using: composite
   steps:
     - name: Set up Python
-      uses: actions/setup-python@v5.0.0
+      uses: actions/setup-python@v5.3.0
       with:
         python-version: ${{ inputs.python-version }}
     - name: Delete error-causing bash
diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml
index 86f4e83e159..562dc04ea22 100644
--- a/.github/workflows/ci-fast.yml
+++ b/.github/workflows/ci-fast.yml
@@ -19,9 +19,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
       - name: Set up Python
-        uses: actions/setup-python@v5.0.0
+        uses: actions/setup-python@v5.3.0
         with:
           python-version: '3.10'
       - name: Install darglint using uv
@@ -40,11 +40,11 @@ jobs:
     if: github.event.pull_request.draft == false || github.event_name == 'workflow_dispatch'
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
         with:
           fetch-depth: 0
       - name: Set up Python 3.9
-        uses: actions/setup-python@v5.0.0
+        uses: actions/setup-python@v5.3.0
         with:
           python-version: '3.9'
       - name: Test migrations across versions
@@ -54,9 +54,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
       - name: Spelling checker
-        uses: crate-ci/typos@v1.17.0
+        uses: crate-ci/typos@v1.27.0
         with:
           files: .
           config: ./.typos.toml
@@ -65,9 +65,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
       - name: Set up Python 3.11
-        uses: actions/setup-python@v5.0.0
+        uses: actions/setup-python@v5.3.0
         with:
           python-version: '3.11'
       - name: Test API docs buildable
@@ -128,15 +128,15 @@ jobs:
       contents: read
       packages: write  # Needed for pushing to GHCR
     steps:
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.2.2
       - name: Log in to GitHub Container Registry
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3.3.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Build and push ZenML server image to GHCR
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v6.9.0
         with:
           context: .
           push: true
diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml
index 5a9a0688ae6..a113a093030 100644
--- a/.github/workflows/ci-slow.yml
+++ b/.github/workflows/ci-slow.yml
@@ -23,7 +23,7 @@ jobs:
       # With dynamic approach dev can set label and rerun this flow to make it running.
       - name: Get PR labels
         id: pr-labels
-        uses: actions/github-script@v5
+        uses: actions/github-script@v7.0.1
         with:
           script: |
             const prNumber = ${{ github.event.pull_request.number }};
@@ -47,11 +47,11 @@ jobs:
       ZENML_DEBUG: true
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.2.2
         with:
           fetch-depth: 0
       - name: Set up Python 3.9
-        uses: actions/setup-python@v5.0.0
+        uses: actions/setup-python@v5.3.0
         with:
           python-version: '3.9'
       - name: Login to Docker Hub
@@ -70,11 +70,11 @@ jobs:
       ZENML_DEBUG: true
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.2.2
         with:
           fetch-depth: 0
       - name: Set up Python 3.9
-        uses: actions/setup-python@v5.0.0
+        uses: actions/setup-python@v5.3.0
         with:
           python-version: '3.9'
       - name: Login to Docker Hub
@@ -94,11 +94,11 @@ jobs:
     if: github.event.pull_request.draft == false
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
         with:
           fetch-depth: 0
       - name: Set up Python 3.9
-        uses: actions/setup-python@v5.0.0
+        uses: actions/setup-python@v5.3.0
         with:
           python-version: '3.9'
       - name: Test migrations across versions
@@ -111,11 +111,11 @@ jobs:
       ZENML_DEBUG: true
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.2.2
         with:
           fetch-depth: 0
       - name: Set up Python 3.9
-        uses: actions/setup-python@v5.0.0
+        uses: actions/setup-python@v5.3.0
         with:
           python-version: '3.9'
       - name: Login to Docker Hub
@@ -131,9 +131,9 @@ jobs:
     needs: run-slow-ci-label-is-set
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.2.2
       - name: Set up Python
-        uses: actions/setup-python@v5.0.0
+        uses: actions/setup-python@v5.3.0
         with:
           python-version: '3.11'
       - name: Install uv
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
index 1decfe4f6cf..c52b8fb71cd 100644
--- a/.github/workflows/codeql.yml
+++ b/.github/workflows/codeql.yml
@@ -23,7 +23,7 @@ jobs:
         language: [python]
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v3
         with:
diff --git a/.github/workflows/generate-test-duration.yml b/.github/workflows/generate-test-duration.yml
index a4c441c1bf9..de6847e361a 100644
--- a/.github/workflows/generate-test-duration.yml
+++ b/.github/workflows/generate-test-duration.yml
@@ -23,7 +23,7 @@ jobs:
       run:
         shell: bash
     steps:
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.2.2
         with:
           ref: develop
       - name: Setup environment
diff --git a/.github/workflows/image-optimiser.yml b/.github/workflows/image-optimiser.yml
index c64ca19ddc3..a7a5c560d8e 100644
--- a/.github/workflows/image-optimiser.yml
+++ b/.github/workflows/image-optimiser.yml
@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Repo
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
       - name: Compress Images
         uses: calibreapp/image-actions@1.1.0
         with:
diff --git a/.github/workflows/integration-test-fast-services.yml b/.github/workflows/integration-test-fast-services.yml
index fb50a7fa0f8..a42ad367735 100644
--- a/.github/workflows/integration-test-fast-services.yml
+++ b/.github/workflows/integration-test-fast-services.yml
@@ -102,7 +102,7 @@ jobs:
       run:
         shell: bash
     steps:
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.2.2
       - name: Restore uv cache
         uses: actions/cache@v4
         with:
diff --git a/.github/workflows/integration-test-fast.yml b/.github/workflows/integration-test-fast.yml
index 823c5f27542..74a6679ae44 100644
--- a/.github/workflows/integration-test-fast.yml
+++ b/.github/workflows/integration-test-fast.yml
@@ -119,7 +119,7 @@ jobs:
         if: inputs.os == 'ubuntu-latest' && (contains(inputs.test_environment, 'docker')
           || contains(inputs.test_environment, 'kubeflow') || contains(inputs.test_environment,
           'airflow') || contains(inputs.test_environment, 'kubernetes'))
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.2.2
       - name: Restore uv cache
         uses: actions/cache@v4
         with:
diff --git a/.github/workflows/integration-test-slow-services.yml b/.github/workflows/integration-test-slow-services.yml
index 84955b6b552..bb4c377b431 100644
--- a/.github/workflows/integration-test-slow-services.yml
+++ b/.github/workflows/integration-test-slow-services.yml
@@ -109,7 +109,7 @@ jobs:
         if: github.event.pull_request.head.repo.fork == false && (contains(inputs.test_environment,
           'docker') || contains(inputs.test_environment, 'kubeflow') || contains(inputs.test_environment,
           'airflow') || contains(inputs.test_environment, 'kubernetes'))
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.2.2
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:
diff --git a/.github/workflows/integration-test-slow.yml b/.github/workflows/integration-test-slow.yml
index d74f7306804..f9376844ef9 100644
--- a/.github/workflows/integration-test-slow.yml
+++ b/.github/workflows/integration-test-slow.yml
@@ -126,7 +126,7 @@ jobs:
         if: github.event.pull_request.head.repo.fork == false && (contains(inputs.test_environment,
           'docker') || contains(inputs.test_environment, 'kubeflow') || contains(inputs.test_environment,
           'airflow') || contains(inputs.test_environment, 'kubernetes'))
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.2.2
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:
diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml
index ca8440124b6..2acb10f8932 100644
--- a/.github/workflows/linting.yml
+++ b/.github/workflows/linting.yml
@@ -67,7 +67,7 @@ jobs:
         shell: bash
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
         with:
           repository: ${{ github.repository }}
           ref: ${{ github.event.pull_request.head.sha }}
diff --git a/.github/workflows/publish_api_docs.yml b/.github/workflows/publish_api_docs.yml
index 5eca8442997..ae18d2a3bb7 100644
--- a/.github/workflows/publish_api_docs.yml
+++ b/.github/workflows/publish_api_docs.yml
@@ -13,7 +13,7 @@ jobs:
       ZENML_ANALYTICS_OPT_IN: false
       PYTHONIOENCODING: utf-8
     steps:
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.2.2
         with:
           fetch-depth: 0  # fetch all commits/branches including gh-pages
       - name: Get the version from the github branch name
@@ -27,7 +27,7 @@ jobs:
       - run: npm install
       - run: npm install html-minifier -g
       - name: Set up Python 3.11
-        uses: actions/setup-python@v5.0.0
+        uses: actions/setup-python@v5.3.0
         with:
           python-version: '3.11'
       - name: Setup git user
diff --git a/.github/workflows/publish_docker_image.yml b/.github/workflows/publish_docker_image.yml
index ef88763135f..81ef19a2ba7 100644
--- a/.github/workflows/publish_docker_image.yml
+++ b/.github/workflows/publish_docker_image.yml
@@ -25,7 +25,7 @@ jobs:
       ZENML_ANALYTICS_OPT_IN: false
       PYTHONIOENCODING: utf-8
     steps:
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.2.2
         with:
           ref: ${{ inputs.zenml_nightly && 'develop' || github.ref }}
       - name: Determine version
diff --git a/.github/workflows/publish_helm_chart.yml b/.github/workflows/publish_helm_chart.yml
index 5d7a17d2e6f..c7a2fed50e4 100644
--- a/.github/workflows/publish_helm_chart.yml
+++ b/.github/workflows/publish_helm_chart.yml
@@ -17,7 +17,7 @@ jobs:
       PYTHONIOENCODING: utf-8
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
 
       # The following sed command replaces the version number in Chart.yaml with the tag version.
       # It replaces the line that starts with "version: " with "version: <tag version>"
diff --git a/.github/workflows/publish_stack_templates.yml b/.github/workflows/publish_stack_templates.yml
index 6068f8c91ec..804e6e74a8a 100644
--- a/.github/workflows/publish_stack_templates.yml
+++ b/.github/workflows/publish_stack_templates.yml
@@ -12,7 +12,7 @@ jobs:
       id-token: write
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
       # Setup AWS CLI
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v2
diff --git a/.github/workflows/publish_to_pypi.yml b/.github/workflows/publish_to_pypi.yml
index b04864fdb60..6bf1f02c090 100644
--- a/.github/workflows/publish_to_pypi.yml
+++ b/.github/workflows/publish_to_pypi.yml
@@ -15,12 +15,12 @@ jobs:
       ZENML_ANALYTICS_OPT_IN: false
       PYTHONIOENCODING: utf-8
     steps:
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.2.2
       - name: Get the version from the github tag ref
         id: get_version
         run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
       - name: Set up Python
-        uses: actions/setup-python@v5.0.0
+        uses: actions/setup-python@v5.3.0
         with:
           python-version: '3.9'
       - name: Install Poetry
diff --git a/.github/workflows/publish_to_pypi_nightly.yml b/.github/workflows/publish_to_pypi_nightly.yml
index d61f8835fba..4487b61879d 100644
--- a/.github/workflows/publish_to_pypi_nightly.yml
+++ b/.github/workflows/publish_to_pypi_nightly.yml
@@ -17,11 +17,11 @@ jobs:
       ZENML_ANALYTICS_OPT_IN: false
       PYTHONIOENCODING: utf-8
     steps:
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.2.2
         with:
           ref: develop
       - name: Set up Python
-        uses: actions/setup-python@v5.0.0
+        uses: actions/setup-python@v5.3.0
         with:
           python-version: '3.9'
       - name: Install Poetry
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index a04394bc03f..fe14b836714 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -17,11 +17,11 @@ jobs:
       ZENML_DEBUG: true
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
         with:
           fetch-depth: 0
       - name: Set up Python 3.9
-        uses: actions/setup-python@v5.0.0
+        uses: actions/setup-python@v5.3.0
         with:
           python-version: '3.9'
       - name: Login to Docker Hub
@@ -38,11 +38,11 @@ jobs:
       ZENML_DEBUG: true
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
         with:
           fetch-depth: 0
       - name: Set up Python 3.9
-        uses: actions/setup-python@v5.0.0
+        uses: actions/setup-python@v5.3.0
         with:
           python-version: '3.9'
       - name: Test migrations across versions
@@ -54,7 +54,7 @@ jobs:
       ZENML_DEBUG: true
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
         with:
           fetch-depth: 0
       - name: Set up Python 3.9
diff --git a/.github/workflows/release_finalize.yml b/.github/workflows/release_finalize.yml
index d6ca4cc0681..c76b0226f7c 100644
--- a/.github/workflows/release_finalize.yml
+++ b/.github/workflows/release_finalize.yml
@@ -26,7 +26,7 @@ jobs:
           git config --global user.name "ZenML GmbH"
       # Check out develop
       - name: Checkout code
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
         with:
           ref: develop
       # Create the release branch
@@ -46,7 +46,7 @@ jobs:
           git config --global user.name "ZenML GmbH"
       # Check out the previous release branch
       - name: Checkout code
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
         with:
           ref: release/${{ github.event.inputs.latest_version }}
       # Create the docs update PR
@@ -66,7 +66,7 @@ jobs:
           git config --global user.name "ZenML GmbH"
       # Check out develop
       - name: Checkout code
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
         with:
           ref: develop
       # Create the migration test version if necessary
@@ -80,7 +80,7 @@ jobs:
     steps:
       # Check out develop
       - name: Checkout code
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
         with:
           ref: develop
       # Setting up the Python
@@ -115,7 +115,7 @@ jobs:
           git config --global user.name "ZenML GmbH"
       # Check out legacy docs branch
       - name: Checkout code
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
         with:
           ref: docs/legacy-docs-page
       # Append new version to the legacy docs table
diff --git a/.github/workflows/release_prepare.yml b/.github/workflows/release_prepare.yml
index d8cbb5c89ee..d11725da083 100644
--- a/.github/workflows/release_prepare.yml
+++ b/.github/workflows/release_prepare.yml
@@ -24,7 +24,7 @@ jobs:
       # Check out main to get the old version
       - name: Checkout code
         id: checkout-code
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
         with:
           ref: main
       # Configure Git
@@ -46,7 +46,7 @@ jobs:
     steps:
       # Check out the code
       - name: Checkout code
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
       # Configure Git
       - name: Configure git
         shell: bash
@@ -60,7 +60,7 @@ jobs:
           scripts/validate-new-version.sh ${{ needs.fetch-versions.outputs.new_version }}
       # Set up Python
       - name: Set up Python
-        uses: actions/setup-python@v5.0.0
+        uses: actions/setup-python@v5.3.0
         with:
           python-version: '3.12'
       # Install ZenML
@@ -141,7 +141,7 @@ jobs:
     steps:
       # Check out the prepare-release branch
       - name: Checkout code
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
       # Sign in to Google
       - uses: google-github-actions/setup-gcloud@v0
         with:
@@ -182,7 +182,7 @@ jobs:
     steps:
       # Check out the code
       - name: Checkout code
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
       # Setting up Python
       - name: Set up Python
         uses: actions/setup-python@v2
@@ -220,7 +220,7 @@ jobs:
     steps:
       # Check out the code
       - name: Checkout code
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
       # Setting up Python
       - name: Set up Python
         uses: actions/setup-python@v2
diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml
index 288a2a1f0df..c0ddb508744 100644
--- a/.github/workflows/spellcheck.yml
+++ b/.github/workflows/spellcheck.yml
@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
       - name: Spelling checker
         uses: crate-ci/typos@v1.17.0
         with:
diff --git a/.github/workflows/trivy-zenml-core.yml b/.github/workflows/trivy-zenml-core.yml
index 0df6b004151..f64fa0c80bd 100644
--- a/.github/workflows/trivy-zenml-core.yml
+++ b/.github/workflows/trivy-zenml-core.yml
@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
       - name: zenml-Trivy vulnerability scanner
         uses: aquasecurity/trivy-action@0.19.0
         with:
diff --git a/.github/workflows/trivy-zenserver.yml b/.github/workflows/trivy-zenserver.yml
index 16054b36e51..0601e5c3413 100644
--- a/.github/workflows/trivy-zenserver.yml
+++ b/.github/workflows/trivy-zenserver.yml
@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
       - name: zenserver-Trivy vulnerability scanner
         uses: aquasecurity/trivy-action@0.19.0
         with:
diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
index 345d2d44698..ce6878f84de 100644
--- a/.github/workflows/unit-test.yml
+++ b/.github/workflows/unit-test.yml
@@ -67,7 +67,7 @@ jobs:
         shell: bash
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
         with:
           repository: ${{ github.repository }}
           ref: ${{ github.event.pull_request.head.sha }}
diff --git a/.github/workflows/update-templates-to-examples.yml b/.github/workflows/update-templates-to-examples.yml
index 09862316dc7..e15a35665fe 100644
--- a/.github/workflows/update-templates-to-examples.yml
+++ b/.github/workflows/update-templates-to-examples.yml
@@ -57,7 +57,7 @@ jobs:
             Breaking changes affecting templates have been introduced. To mitigate this issue,\
             please make the code in zenml-io/template-e2e-batch compatible with new version of\
             ZenML core, release it and update release tag in zenml.cli.base.ZENML_PROJECT_TEMPLATES"
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.2.2
         with:
           ref: ${{ github.event.pull_request.head.ref }}
       - name: Check-out fresh E2E template
@@ -129,7 +129,7 @@ jobs:
             Breaking changes affecting templates have been introduced. To mitigate this issue,\
             please make the code in zenml-io/template-nlp compatible with new version of\
             ZenML core, release it and update release tag in zenml.cli.base.ZENML_PROJECT_TEMPLATES"
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.2.2
         with:
           ref: ${{ github.event.pull_request.head.ref }}
       - name: Check-out fresh NLP template
@@ -201,7 +201,7 @@ jobs:
             Breaking changes affecting templates have been introduced. To mitigate this issue,\
             please make the code in zenml-io/template-starter compatible with new version of\
             ZenML core, release it and update release tag in zenml.cli.base.ZENML_PROJECT_TEMPLATES"
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.2.2
         with:
           ref: ${{ github.event.pull_request.head.ref }}
       - name: Check-out fresh Starter template
@@ -272,7 +272,7 @@ jobs:
             Breaking changes affecting templates have been introduced. To mitigate this issue,\
             please make the code in zenml-io/template-llm-finetuning compatible with new version of\
             ZenML core, release it and update release tag in zenml.cli.base.ZENML_PROJECT_TEMPLATES"
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.2.2
         with:
           ref: ${{ github.event.pull_request.head.ref }}
       - name: Check-out fresh LLM Finetuning template

From 0fb93ed16248818c08abc4dbf18270246205138d Mon Sep 17 00:00:00 2001
From: Safoine El Khabich <34200873+safoinme@users.noreply.github.com>
Date: Tue, 5 Nov 2024 12:44:41 +0100
Subject: [PATCH 39/60] Update .github/workflows/ci-fast.yml

Co-authored-by: Alex Strick van Linschoten <strickvl@users.noreply.github.com>
---
 .github/workflows/ci-fast.yml | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml
index 562dc04ea22..383302ab464 100644
--- a/.github/workflows/ci-fast.yml
+++ b/.github/workflows/ci-fast.yml
@@ -143,18 +143,3 @@ jobs:
           tags: ghcr.io/${{ github.repository_owner }}/zenml-server-github-actions:${{
             github.sha }}
           file: ./docker/zenml-server-dev.Dockerfile
-# custom-arc-runner-set-integration-test-server:
-#   needs: [build-and-push, linting]
-#   if: github.event.pull_request.draft == false
-#   strategy:
-#     matrix:
-#       os: [ubuntu-latest]
-#       python-version: ['3.11']
-#       test_environment: [github-actions-server-docker-orchestrator]
-#     fail-fast: false
-#   uses: ./.github/workflows/integration-test-fast-services.yml
-#   with:
-#     os: ${{ matrix.os }}
-#     python-version: ${{ matrix.python-version }}
-#     test_environment: ${{ matrix.test_environment }}
-#   secrets: inherit

From 85745c54f428c594efa2d3dc1090be8a07d58014 Mon Sep 17 00:00:00 2001
From: Safoine El Khabich <34200873+safoinme@users.noreply.github.com>
Date: Tue, 5 Nov 2024 12:44:49 +0100
Subject: [PATCH 40/60] Update .github/workflows/ci-slow.yml

Co-authored-by: Alex Strick van Linschoten <strickvl@users.noreply.github.com>
---
 .github/workflows/ci-slow.yml | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml
index a113a093030..399c05cf99f 100644
--- a/.github/workflows/ci-slow.yml
+++ b/.github/workflows/ci-slow.yml
@@ -327,18 +327,3 @@ jobs:
       python-version: ${{ matrix.python-version }}
       test_environment: ${{ matrix.test_environment }}
     secrets: inherit
-# custom-ubuntu-integration-test-gh-actions-server-docker-orchestrator:
-#   if: github.event.pull_request.draft == false
-#   needs: [run-slow-ci-label-is-set, custom-ubuntu-unit-test]
-#   strategy:
-#     matrix:
-#       os: [ubuntu-latest]
-#       python-version: ['3.9', '3.10', '3.11']
-#       test_environment: [github-actions-server-docker-orchestrator]
-#     fail-fast: false
-#   uses: ./.github/workflows/integration-test-slow-services.yml
-#   with:
-#     os: ${{ matrix.os }}
-#     python-version: ${{ matrix.python-version }}
-#     test_environment: ${{ matrix.test_environment }}
-#   secrets: inherit

From 44d4f2c2694ba93f9a08926310c1c96d42544131 Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Tue, 5 Nov 2024 13:03:35 +0100
Subject: [PATCH 41/60] Fix typo in deprecation validator variable name in
 test_deprecation_utils.py

---
 tests/unit/utils/test_deprecation_utils.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/unit/utils/test_deprecation_utils.py b/tests/unit/utils/test_deprecation_utils.py
index 97dd809bb47..c44ddaa4180 100644
--- a/tests/unit/utils/test_deprecation_utils.py
+++ b/tests/unit/utils/test_deprecation_utils.py
@@ -30,7 +30,7 @@ class Model(BaseModel):
         old: Optional[str] = None
         new: Optional[str] = None
 
-        _deprecatation_validator = (
+        _deprecation_validator = (
             deprecation_utils.deprecate_pydantic_attributes(
                 "deprecated", ("old", "new")
             )
@@ -56,7 +56,7 @@ class Model(BaseModel):
     class InvalidAttributeNameModel(BaseModel):
         deprecated: Optional[str] = None
 
-        _deprecatation_validator = (
+        _deprecation_validator = (
             deprecation_utils.deprecate_pydantic_attributes("not_an_attribute")
         )
 
@@ -66,7 +66,7 @@ class InvalidAttributeNameModel(BaseModel):
     class DeprecateRequiredAttributeModel(BaseModel):
         deprecated: str
 
-        _deprecatation_validator = (
+        _deprecation_validator = (
             deprecation_utils.deprecate_pydantic_attributes("deprecated")
         )
 

From a38e6a016d28fd6c75c79b68d56122da0233a2a4 Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Tue, 5 Nov 2024 13:36:21 +0100
Subject: [PATCH 42/60] Fix formatting issue in migration guide for ZenML
 server

---
 .../migration-guide/migration-zero-twenty.md                    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/book/how-to/advanced-topics/manage-zenml-server/migration-guide/migration-zero-twenty.md b/docs/book/how-to/advanced-topics/manage-zenml-server/migration-guide/migration-zero-twenty.md
index 964c8d3bfc9..ae2dbcbaec6 100644
--- a/docs/book/how-to/advanced-topics/manage-zenml-server/migration-guide/migration-zero-twenty.md
+++ b/docs/book/how-to/advanced-topics/manage-zenml-server/migration-guide/migration-zero-twenty.md
@@ -221,7 +221,7 @@ Running with active project: 'default' (global)
 ┃ ACTIVE │ STACK NAME           │ STACK ID             │ SHARED │ OWNER   │ ORCHESTRATOR          │ ARTIFACT_STORE    │ CONTAINER_REGISTRY   │ SECRETS_MANAGER       │ MODEL_DEPLOYER      │ EXPERIMENT_TRACKER   ┃
 ┠────────┼──────────────────────┼──────────────────────┼────────┼─────────┼───────────────────────┼───────────────────┼──────────────────────┼───────────────────────┼─────────────────────┼──────────────────────┨
 ┃        │ zenbytes_aws_kubeflo │ 9fe90f0b-2a79-47d9-8 │        │ default │ zenbytes_eks_orchestr │ zenbytes_s3_store │ zenbytes_ecr_registr │ zenbytes_aws_secret_m │ zenbytes_eks_seldon │                      ┃
-┃        │ w_stack              │ f80-04e45ff02cdb     │        │         │ ator                  │                   │ y                    │ anager                │                     │                      ┃
+┃        │ w_stack              │ f80-04e45ff02cdb     │        │         │ ator                  │                   │ y                    │ manager                │                     │                      ┃
 ┠────────┼──────────────────────┼──────────────────────┼────────┼─────────┼───────────────────────┼───────────────────┼──────────────────────┼───────────────────────┼─────────────────────┼──────────────────────┨
 ┃   👉   │ default              │ 7a587e0c-30fd-402f-a │        │ default │ default               │ default           │                      │                       │                     │                      ┃
 ┃        │                      │ 3a8-03651fe1458f     │        │         │                       │                   │                      │                       │                     │                      ┃

From a13395dcf98d201c04da00e7e6d4d043aabb8086 Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Thu, 7 Nov 2024 09:47:09 +0100
Subject: [PATCH 43/60] Update CI workflows to use ubuntu-latest and adjust
 Python versions

---
 .github/workflows/ci-fast.yml                 |  2 +-
 .github/workflows/ci-slow.yml                 | 34 ++++++++-----------
 .github/workflows/generate-test-duration.yml  |  4 +--
 .../integration-test-fast-services.yml        |  4 +--
 .github/workflows/integration-test-fast.yml   |  4 +--
 .../integration-test-slow-services.yml        |  2 +-
 .github/workflows/integration-test-slow.yml   |  2 +-
 .github/workflows/linting.yml                 |  2 +-
 .github/workflows/release.yml                 | 12 +++----
 .github/workflows/unit-test.yml               |  2 +-
 .../update-templates-to-examples.yml          |  2 +-
 scripts/format.sh                             |  2 +-
 12 files changed, 34 insertions(+), 38 deletions(-)

diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml
index 383302ab464..578e99b47bc 100644
--- a/.github/workflows/ci-fast.yml
+++ b/.github/workflows/ci-fast.yml
@@ -107,7 +107,7 @@ jobs:
       python-version: ${{ matrix.python-version }}
       os: ${{ matrix.os }}
     secrets: inherit
-  custom-arc-runner-set-integration-test:
+  ubuntu-latest-integration-test:
     needs: [build-and-push, linting]
     if: github.event.pull_request.draft == false
     strategy:
diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml
index 399c05cf99f..7aab5a8cfd0 100644
--- a/.github/workflows/ci-slow.yml
+++ b/.github/workflows/ci-slow.yml
@@ -177,7 +177,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: ['3.9', '3.10', '3.11']
+        python-version: ['3.9', '3.10', '3.12']
       fail-fast: false
     uses: ./.github/workflows/linting.yml
     with:
@@ -190,7 +190,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: ['3.9', '3.10', '3.11']
+        python-version: ['3.9', '3.10', '3.12']
       fail-fast: false
     uses: ./.github/workflows/unit-test.yml
     with:
@@ -203,7 +203,7 @@ jobs:
     strategy:
       matrix:
         os: [windows-latest]
-        python-version: ['3.9', '3.10', '3.11']
+        python-version: ['3.9', '3.10', '3.12']
       fail-fast: false
     uses: ./.github/workflows/linting.yml
     with:
@@ -291,35 +291,31 @@ jobs:
       python-version: ${{ matrix.python-version }}
       test_environment: ${{ matrix.test_environment }}
     secrets: inherit
-  custom-ubuntu-integration-test:
+  ubuntu-latest-integration-test:
     if: github.event.pull_request.draft == false
     needs: [run-slow-ci-label-is-set, custom-ubuntu-unit-test]
     strategy:
       matrix:
         os: [ubuntu-latest]
         python-version: ['3.9', '3.10', '3.12']
-        test_environment: [default, docker-server-docker-orchestrator-mariadb]
+        test_environment:
+          - default
+          - docker-server-docker-orchestrator-mysql
+          - docker-server-docker-orchestrator-mariadb
         exclude:
           # docker is time-consuming to run, so we only run it on 3.9
-          - test_environment: default
-            python-version: '3.12'
-          - test_environment: default
+          - test_environment: docker-server-docker-orchestrator-mysql
+            python-version: '3.9'
+          - test_environment: docker-server-docker-orchestrator-mysql
             python-version: '3.10'
+          - test_environment: docker-server-docker-orchestrator-mysql
+            python-version: '3.12'
           - test_environment: docker-server-docker-orchestrator-mariadb
-            python-version: '3.10'
+            python-version: '3.9'
           - test_environment: docker-server-docker-orchestrator-mariadb
-            python-version: '3.12'
+            python-version: '3.10'
           - test_environment: docker-server-docker-orchestrator-mariadb
             python-version: '3.12'
-          # IMPORTANT: Since we are using the following combinations
-          # in our `ci-fast` workflow, this combination has been
-          # excluded from the `ci-slow` workflow.
-          - os: arc-runner-set
-            test_environment: default
-            python-version: '3.10'
-          - os: arc-runner-set
-            test_environment: docker-server-docker-orchestrator-mysql
-            python-version: '3.10'
       fail-fast: false
     uses: ./.github/workflows/integration-test-slow.yml
     with:
diff --git a/.github/workflows/generate-test-duration.yml b/.github/workflows/generate-test-duration.yml
index de6847e361a..36d220836ab 100644
--- a/.github/workflows/generate-test-duration.yml
+++ b/.github/workflows/generate-test-duration.yml
@@ -7,7 +7,7 @@ on:
 jobs:
   generate-test-duration-file:
     name: Generate test duration file
-    runs-on: arc-runner-set
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
     env:
@@ -31,7 +31,7 @@ jobs:
         with:
           cache_version: ${{ secrets.GH_ACTIONS_CACHE_KEY }}
           python-version: '3.10'
-          os: arc-runner-set
+          os: ubuntu-latest
           runners_cache_access_key_id: ${{ secrets.RUNNERS_CACHE_ACCESS_KEY_ID }}
           runners_cache_secret_access_key: ${{ secrets.RUNNERS_CACHE_SECRET_ACCESS_KEY }}
           discord_webhook: ${{ secrets.DISCORD_WEBHOOK }}
diff --git a/.github/workflows/integration-test-fast-services.yml b/.github/workflows/integration-test-fast-services.yml
index a42ad367735..63a0b7a3b99 100644
--- a/.github/workflows/integration-test-fast-services.yml
+++ b/.github/workflows/integration-test-fast-services.yml
@@ -30,7 +30,7 @@ on:
       os:
         description: OS
         type: choice
-        options: [ubuntu-latest, macos-13, windows-latest, arc-runner-set]
+        options: [ubuntu-latest, macos-13, windows-latest]
         required: false
         default: ubuntu-latest
       python-version:
@@ -75,7 +75,7 @@ on:
 jobs:
   integration-tests-fast:
     name: integration-tests-fast
-    runs-on: ubuntu-latest
+    runs-on: ${{ inputs.os }}
     strategy:
       fail-fast: false
       matrix:
diff --git a/.github/workflows/integration-test-fast.yml b/.github/workflows/integration-test-fast.yml
index 74a6679ae44..cc2c661d18c 100644
--- a/.github/workflows/integration-test-fast.yml
+++ b/.github/workflows/integration-test-fast.yml
@@ -30,7 +30,7 @@ on:
       os:
         description: OS
         type: choice
-        options: [ubuntu-latest, macos-13, windows-latest, arc-runner-set]
+        options: [ubuntu-latest, macos-13, windows-latest]
         required: false
         default: ubuntu-latest
       python-version:
@@ -75,7 +75,7 @@ on:
 jobs:
   integration-tests-fast:
     name: integration-tests-fast
-    runs-on: ubuntu-latest
+    runs-on: ${{ inputs.os }}
     strategy:
       fail-fast: false
       matrix:
diff --git a/.github/workflows/integration-test-slow-services.yml b/.github/workflows/integration-test-slow-services.yml
index bb4c377b431..76fc95e223c 100644
--- a/.github/workflows/integration-test-slow-services.yml
+++ b/.github/workflows/integration-test-slow-services.yml
@@ -30,7 +30,7 @@ on:
       os:
         description: OS
         type: choice
-        options: [ubuntu-latest, macos-13, windows-latest, arc-runner-set]
+        options: [ubuntu-latest, macos-13, windows-latest]
         required: false
         default: ubuntu-latest
       python-version:
diff --git a/.github/workflows/integration-test-slow.yml b/.github/workflows/integration-test-slow.yml
index f9376844ef9..94d5f2e2fb5 100644
--- a/.github/workflows/integration-test-slow.yml
+++ b/.github/workflows/integration-test-slow.yml
@@ -30,7 +30,7 @@ on:
       os:
         description: OS
         type: choice
-        options: [ubuntu-latest, macos-13, windows-latest, arc-runner-set]
+        options: [ubuntu-latest, macos-13, windows-latest]
         required: false
         default: ubuntu-latest
       python-version:
diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml
index 2acb10f8932..b702c37fcad 100644
--- a/.github/workflows/linting.yml
+++ b/.github/workflows/linting.yml
@@ -31,7 +31,7 @@ on:
       os:
         description: OS
         type: choice
-        options: [ubuntu-latest, macos-13, windows-latest, arc-runner-set]
+        options: [ubuntu-latest, macos-13, windows-latest]
         required: false
         default: ubuntu-latest
       python-version:
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index fe14b836714..84fce611f94 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -7,11 +7,11 @@ jobs:
   setup-and-test:
     uses: ./.github/workflows/unit-test.yml
     with:
-      os: arc-runner-set
+      os: ubuntu-latest
       python-version: '3.9'
     secrets: inherit
   mysql-db-migration-testing:
-    runs-on: arc-runner-set
+    runs-on: ubuntu-latest
     env:
       ZENML_ANALYTICS_OPT_IN: false
       ZENML_DEBUG: true
@@ -32,7 +32,7 @@ jobs:
       - name: Test migrations across versions
         run: bash scripts/test-migrations.sh mysql
   sqlite-db-migration-testing:
-    runs-on: arc-runner-set
+    runs-on: ubuntu-latest
     env:
       ZENML_ANALYTICS_OPT_IN: false
       ZENML_DEBUG: true
@@ -48,7 +48,7 @@ jobs:
       - name: Test migrations across versions
         run: bash scripts/test-migrations.sh sqlite
   mariadb-db-migration-testing:
-    runs-on: arc-runner-set
+    runs-on: ubuntu-latest
     env:
       ZENML_ANALYTICS_OPT_IN: false
       ZENML_DEBUG: true
@@ -78,7 +78,7 @@ jobs:
     uses: ./.github/workflows/publish_to_pypi.yml
     secrets: inherit
   wait-for-package-release:
-    runs-on: arc-runner-set
+    runs-on: ubuntu-latest
     needs: publish-python-package
     steps:
       - name: Sleep for 4 minutes
@@ -95,7 +95,7 @@ jobs:
     uses: ./.github/workflows/publish_helm_chart.yml
     secrets: inherit
   wait-for-package-release-again:
-    runs-on: arc-runner-set
+    runs-on: ubuntu-latest
     needs: publish-helm-chart
     steps:
       - name: Sleep for 4 minutes
diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
index ce6878f84de..9ec7fbfc305 100644
--- a/.github/workflows/unit-test.yml
+++ b/.github/workflows/unit-test.yml
@@ -31,7 +31,7 @@ on:
       os:
         description: OS
         type: choice
-        options: [ubuntu-latest, macos-13, windows-latest, arc-runner-set]
+        options: [ubuntu-latest, macos-13, windows-latest]
         required: false
         default: ubuntu-latest
       python-version:
diff --git a/.github/workflows/update-templates-to-examples.yml b/.github/workflows/update-templates-to-examples.yml
index e15a35665fe..fa3aff51924 100644
--- a/.github/workflows/update-templates-to-examples.yml
+++ b/.github/workflows/update-templates-to-examples.yml
@@ -16,7 +16,7 @@ on:
       os:
         description: OS
         type: choice
-        options: [ubuntu-latest, macos-13, windows-latest, arc-runner-set]
+        options: [ubuntu-latest, macos-13, windows-latest]
         required: false
         default: ubuntu-latest
       python-version:
diff --git a/scripts/format.sh b/scripts/format.sh
index f37c1fa2839..52616b13c8b 100755
--- a/scripts/format.sh
+++ b/scripts/format.sh
@@ -64,5 +64,5 @@ ruff format $SRC
 
 # standardizes / formats CI yaml files
 if [ "$SKIP_YAMLFIX" = false ]; then
-    yamlfix .github tests --exclude="dependabot.yml" --exclude="workflows/integration-test-fast-services.yml" --exclude="workflows/integration-test-slow-services.yml"
+        yamlfix .github tests -e "dependabot.yml" -e "workflows/release_prepare.yml" -e "workflows/release_finalize.yml"
 fi

From 666650cb8c9d093a1ece471b336e7fd5f4df8a06 Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Thu, 7 Nov 2024 09:47:12 +0100
Subject: [PATCH 44/60] Fix formatting in integration-test-slow-services.yml
 for ports section

---
 .github/workflows/integration-test-slow-services.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/integration-test-slow-services.yml b/.github/workflows/integration-test-slow-services.yml
index 76fc95e223c..f6ae3725bce 100644
--- a/.github/workflows/integration-test-slow-services.yml
+++ b/.github/workflows/integration-test-slow-services.yml
@@ -251,7 +251,7 @@ jobs:
           ZENML_SERVER_DEPLOYMENT_TYPE: docker
           ZENML_SERVER_AUTO_ACTIVATE: 'True'
           ZENML_SERVER_AUTO_CREATE_DEFAULT_USER: 'True'
-        ports: 
+        ports:   
           - 8080:8080
         options: >-
           --health-cmd="curl -f http://127.0.0.1:8080/health"

From d633f2f8197f4009e403844979f257d07553b7d1 Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Thu, 7 Nov 2024 09:49:45 +0100
Subject: [PATCH 45/60] Rename CI workflow jobs for clarity and consistency

---
 .github/workflows/ci-fast.yml | 2 +-
 .github/workflows/ci-slow.yml | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml
index 578e99b47bc..5bca2b58766 100644
--- a/.github/workflows/ci-fast.yml
+++ b/.github/workflows/ci-fast.yml
@@ -94,7 +94,7 @@ jobs:
       python-version: ${{ matrix.python-version }}
       os: ${{ matrix.os }}
     secrets: inherit
-  custom-ubuntu-setup-and-unit-test:
+  ubuntu-setup-and-unit-test:
     needs: linting
     if: github.event.pull_request.draft == false
     strategy:
diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml
index 7aab5a8cfd0..871a99bd007 100644
--- a/.github/workflows/ci-slow.yml
+++ b/.github/workflows/ci-slow.yml
@@ -184,7 +184,7 @@ jobs:
       python-version: ${{ matrix.python-version }}
       os: ${{ matrix.os }}
     secrets: inherit
-  custom-ubuntu-unit-test:
+  ubuntu-unit-test:
     if: github.event.pull_request.draft == false
     needs: [run-slow-ci-label-is-set, ubuntu-linting]
     strategy:
@@ -293,7 +293,7 @@ jobs:
     secrets: inherit
   ubuntu-latest-integration-test:
     if: github.event.pull_request.draft == false
-    needs: [run-slow-ci-label-is-set, custom-ubuntu-unit-test]
+    needs: [run-slow-ci-label-is-set, ubuntu-unit-test]
     strategy:
       matrix:
         os: [ubuntu-latest]

From 1d2dbd4da6c675f1afa4c4fcc957dc8f36a2c6e8 Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Thu, 7 Nov 2024 09:54:16 +0100
Subject: [PATCH 46/60] Fix formatting in release_prepare.yml for branch
 specification

---
 .github/workflows/release_prepare.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/release_prepare.yml b/.github/workflows/release_prepare.yml
index d11725da083..f4ab9ba9959 100644
--- a/.github/workflows/release_prepare.yml
+++ b/.github/workflows/release_prepare.yml
@@ -2,7 +2,8 @@
 name: release-prepare
 on:
   push:
-    branches: [misc/prepare-release-*]
+    branches:
+      - 'misc/prepare-release-*'
 env:
   ZENML_ANALYTICS_OPT_IN: false
   GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

From bc310033ace26e337c74814388fbbb269e6a4003 Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Thu, 7 Nov 2024 09:54:32 +0100
Subject: [PATCH 47/60] Add integration test workflows to YAML formatting
 script

---
 scripts/format.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/format.sh b/scripts/format.sh
index 52616b13c8b..9c052505e09 100755
--- a/scripts/format.sh
+++ b/scripts/format.sh
@@ -64,5 +64,5 @@ ruff format $SRC
 
 # standardizes / formats CI yaml files
 if [ "$SKIP_YAMLFIX" = false ]; then
-        yamlfix .github tests -e "dependabot.yml" -e "workflows/release_prepare.yml" -e "workflows/release_finalize.yml"
+        yamlfix .github tests -e "dependabot.yml" -e "workflows/release_prepare.yml" -e "workflows/release_finalize.yml" -e "workflows/integration-test-fast-services.yml" -e "workflows/integration-test-slow-services.yml"
 fi

From 8a92624ce7a702eccb08064d4e1812b4dccc2eed Mon Sep 17 00:00:00 2001
From: GitHub Actions <actions@github.com>
Date: Thu, 7 Nov 2024 08:56:57 +0000
Subject: [PATCH 48/60] Auto-update of LLM Finetuning template

---
 examples/llm_finetuning/.copier-answers.yml   | 2 +-
 examples/llm_finetuning/steps/log_metadata.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/llm_finetuning/.copier-answers.yml b/examples/llm_finetuning/.copier-answers.yml
index 09cb600a5fa..386863f54e8 100644
--- a/examples/llm_finetuning/.copier-answers.yml
+++ b/examples/llm_finetuning/.copier-answers.yml
@@ -1,5 +1,5 @@
 # Changes here will be overwritten by Copier
-_commit: 2024.09.24
+_commit: 2024.08.29-1-g7af7693
 _src_path: gh:zenml-io/template-llm-finetuning
 bf16: true
 cuda_version: cuda11.8
diff --git a/examples/llm_finetuning/steps/log_metadata.py b/examples/llm_finetuning/steps/log_metadata.py
index 14371b78b6e..645f98cc8ea 100644
--- a/examples/llm_finetuning/steps/log_metadata.py
+++ b/examples/llm_finetuning/steps/log_metadata.py
@@ -34,7 +34,7 @@ def log_metadata_from_step_artifact(
 
     context = get_step_context()
     metadata_dict: Dict[str, Any] = (
-        context.pipeline_run.steps[step_name].outputs[artifact_name].load()
+        context.pipeline_run.steps[step_name].outputs[artifact_name][0].load()
     )
 
     metadata = {artifact_name: metadata_dict}

From 02732a3b2bc0e0255fd1b67f97ca284618e2ec39 Mon Sep 17 00:00:00 2001
From: GitHub Actions <actions@github.com>
Date: Thu, 7 Nov 2024 08:57:18 +0000
Subject: [PATCH 49/60] Auto-update of Starter template

---
 examples/mlops_starter/.copier-answers.yml     | 2 +-
 examples/mlops_starter/quickstart.ipynb        | 4 ++--
 examples/mlops_starter/run.py                  | 4 ++--
 examples/mlops_starter/steps/model_promoter.py | 8 +++-----
 4 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/examples/mlops_starter/.copier-answers.yml b/examples/mlops_starter/.copier-answers.yml
index 8b1fb8187ed..e17f27ee551 100644
--- a/examples/mlops_starter/.copier-answers.yml
+++ b/examples/mlops_starter/.copier-answers.yml
@@ -1,5 +1,5 @@
 # Changes here will be overwritten by Copier
-_commit: 2024.09.24
+_commit: 2024.10.21
 _src_path: gh:zenml-io/template-starter
 email: info@zenml.io
 full_name: ZenML GmbH
diff --git a/examples/mlops_starter/quickstart.ipynb b/examples/mlops_starter/quickstart.ipynb
index df8c010b5ea..6fba7a0e8cc 100644
--- a/examples/mlops_starter/quickstart.ipynb
+++ b/examples/mlops_starter/quickstart.ipynb
@@ -994,8 +994,8 @@
     "@pipeline\n",
     "def inference(preprocess_pipeline_id: UUID):\n",
     "    \"\"\"Model batch inference pipeline\"\"\"\n",
-    "    # random_state = client.get_artifact_version(name_id_or_prefix=preprocess_pipeline_id).metadata[\"random_state\"].value\n",
-    "    # target = client.get_artifact_version(name_id_or_prefix=preprocess_pipeline_id).run_metadata['target'].value\n",
+    "    # random_state = client.get_artifact_version(name_id_or_prefix=preprocess_pipeline_id).metadata[\"random_state\"]\n",
+    "    # target = client.get_artifact_version(name_id_or_prefix=preprocess_pipeline_id).run_metadata['target']\n",
     "    random_state = 42\n",
     "    target = \"target\"\n",
     "\n",
diff --git a/examples/mlops_starter/run.py b/examples/mlops_starter/run.py
index d7b1a7f11b2..16a352588d6 100644
--- a/examples/mlops_starter/run.py
+++ b/examples/mlops_starter/run.py
@@ -239,8 +239,8 @@ def main(
         #  to get the random state and target column
         random_state = preprocess_pipeline_artifact.run_metadata[
             "random_state"
-        ].value
-        target = preprocess_pipeline_artifact.run_metadata["target"].value
+        ]
+        target = preprocess_pipeline_artifact.run_metadata["target"]
         run_args_inference["random_state"] = random_state
         run_args_inference["target"] = target
 
diff --git a/examples/mlops_starter/steps/model_promoter.py b/examples/mlops_starter/steps/model_promoter.py
index 52040638496..43d43ceac1f 100644
--- a/examples/mlops_starter/steps/model_promoter.py
+++ b/examples/mlops_starter/steps/model_promoter.py
@@ -58,11 +58,9 @@ def model_promoter(accuracy: float, stage: str = "production") -> bool:
         try:
             stage_model = client.get_model_version(current_model.name, stage)
             # We compare their metrics
-            prod_accuracy = (
-                stage_model.get_artifact("sklearn_classifier")
-                .run_metadata["test_accuracy"]
-                .value
-            )
+            prod_accuracy = stage_model.get_artifact(
+                "sklearn_classifier"
+            ).run_metadata["test_accuracy"]
             if float(accuracy) > float(prod_accuracy):
                 # If current model has better metrics, we promote it
                 is_promoted = True

From 8e7ca14a92c251ceeba700c0d9d69eb0feace16b Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Thu, 7 Nov 2024 10:02:57 +0100
Subject: [PATCH 50/60] Fix quotes in branch specification for
 release_prepare.yml

---
 .github/workflows/release_prepare.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/release_prepare.yml b/.github/workflows/release_prepare.yml
index f4ab9ba9959..b77a1b136e7 100644
--- a/.github/workflows/release_prepare.yml
+++ b/.github/workflows/release_prepare.yml
@@ -3,7 +3,7 @@ name: release-prepare
 on:
   push:
     branches:
-      - 'misc/prepare-release-*'
+      - "misc/prepare-release-*"
 env:
   ZENML_ANALYTICS_OPT_IN: false
   GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

From f9ea2e5b46a3671354dcda8a1d407d078ab96ea5 Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Thu, 7 Nov 2024 10:11:21 +0100
Subject: [PATCH 51/60] Standardize YAML file checks in format and lint scripts

---
 scripts/format.sh | 2 +-
 scripts/lint.sh   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/format.sh b/scripts/format.sh
index 9c052505e09..d3dd117f80b 100755
--- a/scripts/format.sh
+++ b/scripts/format.sh
@@ -64,5 +64,5 @@ ruff format $SRC
 
 # standardizes / formats CI yaml files
 if [ "$SKIP_YAMLFIX" = false ]; then
-        yamlfix .github tests -e "dependabot.yml" -e "workflows/release_prepare.yml" -e "workflows/release_finalize.yml" -e "workflows/integration-test-fast-services.yml" -e "workflows/integration-test-slow-services.yml"
+    yamlfix --check .github tests -e "dependabot.yml" -e "workflows/release_prepare.yml" -e "workflows/release_finalize.yml" -e "workflows/integration-test-fast-services.yml" -e "workflows/integration-test-slow-services.yml"
 fi
diff --git a/scripts/lint.sh b/scripts/lint.sh
index 2d507e5020f..c4c8b24a823 100755
--- a/scripts/lint.sh
+++ b/scripts/lint.sh
@@ -30,7 +30,7 @@ fi
 
 # checks for yaml formatting errors
 if [ "$SKIP_YAMLFIX" = false ]; then
-    yamlfix --check .github tests --exclude="dependabot.yml" --exclude="workflows/integration-test-fast-services.yml" --exclude="workflows/integration-test-slow-services.yml"
+    yamlfix --check .github tests -e "dependabot.yml" -e "workflows/release_prepare.yml" -e "workflows/release_finalize.yml" -e "workflows/integration-test-fast-services.yml" -e "workflows/integration-test-slow-services.yml"
 fi
 
 # autoflake replacement: checks for unused imports and variables

From 4b34e99a664efc68178b4ff3b504d587a153b235 Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Thu, 7 Nov 2024 10:19:17 +0100
Subject: [PATCH 52/60] Refactor CI workflow by removing build-and-push job and
 adjusting dependencies

---
 .github/workflows/ci-fast.yml | 44 +++++++++++++++++------------------
 1 file changed, 22 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml
index 5bca2b58766..d8aa9eabae7 100644
--- a/.github/workflows/ci-fast.yml
+++ b/.github/workflows/ci-fast.yml
@@ -108,7 +108,7 @@ jobs:
       os: ${{ matrix.os }}
     secrets: inherit
   ubuntu-latest-integration-test:
-    needs: [build-and-push, linting]
+    needs: [linting]
     if: github.event.pull_request.draft == false
     strategy:
       matrix:
@@ -122,24 +122,24 @@ jobs:
       python-version: ${{ matrix.python-version }}
       test_environment: ${{ matrix.test_environment }}
     secrets: inherit
-  build-and-push:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write  # Needed for pushing to GHCR
-    steps:
-      - uses: actions/checkout@v4.2.2
-      - name: Log in to GitHub Container Registry
-        uses: docker/login-action@v3.3.0
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Build and push ZenML server image to GHCR
-        uses: docker/build-push-action@v6.9.0
-        with:
-          context: .
-          push: true
-          tags: ghcr.io/${{ github.repository_owner }}/zenml-server-github-actions:${{
-            github.sha }}
-          file: ./docker/zenml-server-dev.Dockerfile
+# build-and-push:
+#   runs-on: ubuntu-latest
+#   permissions:
+#     contents: read
+#     packages: write  # Needed for pushing to GHCR
+#   steps:
+#     - uses: actions/checkout@v4.2.2
+#     - name: Log in to GitHub Container Registry
+#       uses: docker/login-action@v3.3.0
+#       with:
+#         registry: ghcr.io
+#         username: ${{ github.actor }}
+#         password: ${{ secrets.GITHUB_TOKEN }}
+#     - name: Build and push ZenML server image to GHCR
+#       uses: docker/build-push-action@v6.9.0
+#       with:
+#         context: .
+#         push: true
+#         tags: ghcr.io/${{ github.repository_owner }}/zenml-server-github-actions:${{
+#           github.sha }}
+#         file: ./docker/zenml-server-dev.Dockerfile

From 14f883c12d2e4e9ff19a2b9e5ef3c3a2a7df21f3 Mon Sep 17 00:00:00 2001
From: GitHub Actions <actions@github.com>
Date: Thu, 7 Nov 2024 09:20:33 +0000
Subject: [PATCH 53/60] Auto-update of E2E template

---
 examples/e2e/.copier-answers.yml                            | 2 +-
 examples/e2e/steps/deployment/deployment_deploy.py          | 2 +-
 examples/e2e/steps/hp_tuning/hp_tuning_select_best_model.py | 2 +-
 examples/e2e/steps/promotion/promote_with_metric_compare.py | 6 +++---
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/examples/e2e/.copier-answers.yml b/examples/e2e/.copier-answers.yml
index 74cc33d8594..b008b2c1e99 100644
--- a/examples/e2e/.copier-answers.yml
+++ b/examples/e2e/.copier-answers.yml
@@ -1,5 +1,5 @@
 # Changes here will be overwritten by Copier
-_commit: 2024.10.10
+_commit: 2024.10.21
 _src_path: gh:zenml-io/template-e2e-batch
 data_quality_checks: true
 email: info@zenml.io
diff --git a/examples/e2e/steps/deployment/deployment_deploy.py b/examples/e2e/steps/deployment/deployment_deploy.py
index 3fb0d879f3f..dad351e45be 100644
--- a/examples/e2e/steps/deployment/deployment_deploy.py
+++ b/examples/e2e/steps/deployment/deployment_deploy.py
@@ -67,7 +67,7 @@ def deployment_deploy() -> (
             registry_model_name=model.name,
             registry_model_version=model.run_metadata[
                 "model_registry_version"
-            ].value,
+            ],
             replace_existing=True,
         )
     else:
diff --git a/examples/e2e/steps/hp_tuning/hp_tuning_select_best_model.py b/examples/e2e/steps/hp_tuning/hp_tuning_select_best_model.py
index 7d5a6bc33ea..65e524ecd98 100644
--- a/examples/e2e/steps/hp_tuning/hp_tuning_select_best_model.py
+++ b/examples/e2e/steps/hp_tuning/hp_tuning_select_best_model.py
@@ -50,7 +50,7 @@ def hp_tuning_select_best_model(
         hp_output = model.get_data_artifact("hp_result")
         model_: ClassifierMixin = hp_output.load()
         # fetch metadata we attached earlier
-        metric = float(hp_output.run_metadata["metric"].value)
+        metric = float(hp_output.run_metadata["metric"])
         if best_model is None or best_metric < metric:
             best_model = model_
     ### YOUR CODE ENDS HERE ###
diff --git a/examples/e2e/steps/promotion/promote_with_metric_compare.py b/examples/e2e/steps/promotion/promote_with_metric_compare.py
index 038d219d32d..6bc580f47ba 100644
--- a/examples/e2e/steps/promotion/promote_with_metric_compare.py
+++ b/examples/e2e/steps/promotion/promote_with_metric_compare.py
@@ -92,14 +92,14 @@ def promote_with_metric_compare(
         # Promote in Model Registry
         latest_version_model_registry_number = latest_version.run_metadata[
             "model_registry_version"
-        ].value
+        ]
         if current_version_number is None:
             current_version_model_registry_number = (
                 latest_version_model_registry_number
             )
         else:
             current_version_model_registry_number = (
-                current_version.run_metadata["model_registry_version"].value
+                current_version.run_metadata["model_registry_version"]
             )
         promote_in_model_registry(
             latest_version=latest_version_model_registry_number,
@@ -111,7 +111,7 @@ def promote_with_metric_compare(
     else:
         promoted_version = current_version.run_metadata[
             "model_registry_version"
-        ].value
+        ]
 
     logger.info(
         f"Current model version in `{target_env}` is `{promoted_version}` registered in Model Registry"

From d4907e917c0325b942580a849a5826f93d175886 Mon Sep 17 00:00:00 2001
From: GitHub Actions <actions@github.com>
Date: Thu, 7 Nov 2024 09:24:07 +0000
Subject: [PATCH 54/60] Auto-update of NLP template

---
 examples/e2e_nlp/.copier-answers.yml                 |  2 +-
 examples/e2e_nlp/gradio/requirements.txt             |  2 +-
 .../e2e_nlp/steps/promotion/promote_get_metrics.py   | 12 ++++--------
 3 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/examples/e2e_nlp/.copier-answers.yml b/examples/e2e_nlp/.copier-answers.yml
index 3ca2ba198fe..e509aae2760 100644
--- a/examples/e2e_nlp/.copier-answers.yml
+++ b/examples/e2e_nlp/.copier-answers.yml
@@ -1,5 +1,5 @@
 # Changes here will be overwritten by Copier
-_commit: 2024.09.23
+_commit: 2024.10.21
 _src_path: gh:zenml-io/template-nlp
 accelerator: cpu
 cloud_of_choice: aws
diff --git a/examples/e2e_nlp/gradio/requirements.txt b/examples/e2e_nlp/gradio/requirements.txt
index 1bddfdfb85b..b53f1df9e62 100644
--- a/examples/e2e_nlp/gradio/requirements.txt
+++ b/examples/e2e_nlp/gradio/requirements.txt
@@ -9,4 +9,4 @@ pandas==1.5.3
 session_info==1.0.0
 scikit-learn==1.5.0
 transformers==4.28.1
-IPython==7.34.0
\ No newline at end of file
+IPython==8.10.0
\ No newline at end of file
diff --git a/examples/e2e_nlp/steps/promotion/promote_get_metrics.py b/examples/e2e_nlp/steps/promotion/promote_get_metrics.py
index 7f2951a5865..b24ac42245c 100644
--- a/examples/e2e_nlp/steps/promotion/promote_get_metrics.py
+++ b/examples/e2e_nlp/steps/promotion/promote_get_metrics.py
@@ -56,9 +56,7 @@ def promote_get_metrics() -> (
 
     # Get current model version metric in current run
     model = get_step_context().model
-    current_metrics = (
-        model.get_model_artifact("model").run_metadata["metrics"].value
-    )
+    current_metrics = model.get_model_artifact("model").run_metadata["metrics"]
     logger.info(f"Current model version metrics are {current_metrics}")
 
     # Get latest saved model version metric in target environment
@@ -72,11 +70,9 @@ def promote_get_metrics() -> (
     except KeyError:
         latest_version = None
     if latest_version:
-        latest_metrics = (
-            latest_version.get_model_artifact("model")
-            .run_metadata["metrics"]
-            .value
-        )
+        latest_metrics = latest_version.get_model_artifact(
+            "model"
+        ).run_metadata["metrics"]
         logger.info(f"Latest model version metrics are {latest_metrics}")
     else:
         logger.info("No currently promoted model version found.")

From 7edbaed6bafd21e5b1bf78d058c34ea4561a533e Mon Sep 17 00:00:00 2001
From: GitHub Actions <actions@github.com>
Date: Thu, 7 Nov 2024 15:24:00 +0000
Subject: [PATCH 55/60] Auto-update of LLM Finetuning template

---
 examples/llm_finetuning/.copier-answers.yml   | 2 +-
 examples/llm_finetuning/steps/log_metadata.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/llm_finetuning/.copier-answers.yml b/examples/llm_finetuning/.copier-answers.yml
index dd85e236760..4004897928b 100644
--- a/examples/llm_finetuning/.copier-answers.yml
+++ b/examples/llm_finetuning/.copier-answers.yml
@@ -1,5 +1,5 @@
 # Changes here will be overwritten by Copier
-_commit: 2024.09.24-1-g378145b
+_commit: 2024.10.30
 _src_path: gh:zenml-io/template-llm-finetuning
 bf16: true
 cuda_version: cuda11.8
diff --git a/examples/llm_finetuning/steps/log_metadata.py b/examples/llm_finetuning/steps/log_metadata.py
index 645f98cc8ea..14371b78b6e 100644
--- a/examples/llm_finetuning/steps/log_metadata.py
+++ b/examples/llm_finetuning/steps/log_metadata.py
@@ -34,7 +34,7 @@ def log_metadata_from_step_artifact(
 
     context = get_step_context()
     metadata_dict: Dict[str, Any] = (
-        context.pipeline_run.steps[step_name].outputs[artifact_name][0].load()
+        context.pipeline_run.steps[step_name].outputs[artifact_name].load()
     )
 
     metadata = {artifact_name: metadata_dict}

From 39aeb8a1112aaf1ac735959fa0a699541d4b55b7 Mon Sep 17 00:00:00 2001
From: GitHub Actions <actions@github.com>
Date: Thu, 7 Nov 2024 15:24:39 +0000
Subject: [PATCH 56/60] Auto-update of Starter template

---
 examples/mlops_starter/.copier-answers.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/mlops_starter/.copier-answers.yml b/examples/mlops_starter/.copier-answers.yml
index 21ba51bc459..fd6b937c7c9 100644
--- a/examples/mlops_starter/.copier-answers.yml
+++ b/examples/mlops_starter/.copier-answers.yml
@@ -1,5 +1,5 @@
 # Changes here will be overwritten by Copier
-_commit: 2024.09.24-3-g2c1a682
+_commit: 2024.10.30
 _src_path: gh:zenml-io/template-starter
 email: info@zenml.io
 full_name: ZenML GmbH

From de69c0edb1cd0bc686a9f2172e11240e16c3ddd8 Mon Sep 17 00:00:00 2001
From: GitHub Actions <actions@github.com>
Date: Thu, 7 Nov 2024 15:31:01 +0000
Subject: [PATCH 57/60] Auto-update of E2E template

---
 examples/e2e/.copier-answers.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/e2e/.copier-answers.yml b/examples/e2e/.copier-answers.yml
index 04c970cb9a4..cd687be59df 100644
--- a/examples/e2e/.copier-answers.yml
+++ b/examples/e2e/.copier-answers.yml
@@ -1,5 +1,5 @@
 # Changes here will be overwritten by Copier
-_commit: 2024.10.10-5-g6edd133
+_commit: 2024.10.30
 _src_path: gh:zenml-io/template-e2e-batch
 data_quality_checks: true
 email: info@zenml.io

From af5af8dc3bc2137626be1a95268d37123d376a9d Mon Sep 17 00:00:00 2001
From: GitHub Actions <actions@github.com>
Date: Thu, 7 Nov 2024 15:33:39 +0000
Subject: [PATCH 58/60] Auto-update of NLP template

---
 examples/e2e_nlp/.copier-answers.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/e2e_nlp/.copier-answers.yml b/examples/e2e_nlp/.copier-answers.yml
index a78ae8bac68..e13858e7da1 100644
--- a/examples/e2e_nlp/.copier-answers.yml
+++ b/examples/e2e_nlp/.copier-answers.yml
@@ -1,5 +1,5 @@
 # Changes here will be overwritten by Copier
-_commit: 2024.10.21-3-ge37d83a
+_commit: 2024.10.30
 _src_path: gh:zenml-io/template-nlp
 accelerator: cpu
 cloud_of_choice: aws

From 19f8928cd95b2d2b8eb6dd21e6cba2cc28515282 Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Fri, 8 Nov 2024 10:46:15 +0100
Subject: [PATCH 59/60] Update Python version matrix in CI workflows to include
 3.11

---
 .github/workflows/ci-fast.yml | 29 ++++++++---------------------
 .github/workflows/ci-slow.yml |  8 ++++----
 2 files changed, 12 insertions(+), 25 deletions(-)

diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml
index d8aa9eabae7..e0c26841e9e 100644
--- a/.github/workflows/ci-fast.yml
+++ b/.github/workflows/ci-fast.yml
@@ -99,6 +99,10 @@ jobs:
     if: github.event.pull_request.draft == false
     strategy:
       matrix:
+        # IMPORTANT: Since we are using the combination of `arc-runner-set`
+        # and `3.10` in our `ci-fast` workflow, this combination has been
+        # excluded from the `ci-slow` workflow. If you change the configuration
+        # here, please adjust the configuration of `ci-slow` accordingly.
         os: [ubuntu-latest]
         python-version: ['3.11']
       fail-fast: false
@@ -112,6 +116,10 @@ jobs:
     if: github.event.pull_request.draft == false
     strategy:
       matrix:
+        # IMPORTANT: Since we are using the combination of `arc-runner-set`
+        # and `3.10` in our `ci-fast` workflow, this combination has been
+        # excluded from the `ci-slow` workflow. If you change the configuration
+        # here, please adjust the configuration of `ci-slow` accordingly.
         os: [ubuntu-latest]
         python-version: ['3.11']
         test_environment: [default, docker-server-docker-orchestrator-mysql]
@@ -122,24 +130,3 @@ jobs:
       python-version: ${{ matrix.python-version }}
       test_environment: ${{ matrix.test_environment }}
     secrets: inherit
-# build-and-push:
-#   runs-on: ubuntu-latest
-#   permissions:
-#     contents: read
-#     packages: write  # Needed for pushing to GHCR
-#   steps:
-#     - uses: actions/checkout@v4.2.2
-#     - name: Log in to GitHub Container Registry
-#       uses: docker/login-action@v3.3.0
-#       with:
-#         registry: ghcr.io
-#         username: ${{ github.actor }}
-#         password: ${{ secrets.GITHUB_TOKEN }}
-#     - name: Build and push ZenML server image to GHCR
-#       uses: docker/build-push-action@v6.9.0
-#       with:
-#         context: .
-#         push: true
-#         tags: ghcr.io/${{ github.repository_owner }}/zenml-server-github-actions:${{
-#           github.sha }}
-#         file: ./docker/zenml-server-dev.Dockerfile
diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml
index 871a99bd007..a3413d09e6b 100644
--- a/.github/workflows/ci-slow.yml
+++ b/.github/workflows/ci-slow.yml
@@ -177,7 +177,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: ['3.9', '3.10', '3.12']
+        python-version: ['3.9', '3.10', '3.11', '3.12']
       fail-fast: false
     uses: ./.github/workflows/linting.yml
     with:
@@ -190,7 +190,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: ['3.9', '3.10', '3.12']
+        python-version: ['3.9', '3.10', '3.11', '3.12']
       fail-fast: false
     uses: ./.github/workflows/unit-test.yml
     with:
@@ -203,7 +203,7 @@ jobs:
     strategy:
       matrix:
         os: [windows-latest]
-        python-version: ['3.9', '3.10', '3.12']
+        python-version: ['3.9', '3.10', '3.11', '3.12']
       fail-fast: false
     uses: ./.github/workflows/linting.yml
     with:
@@ -229,7 +229,7 @@ jobs:
     strategy:
       matrix:
         os: [macos-latest]
-        python-version: ['3.9', '3.10', '3.11']
+        python-version: ['3.9', '3.10', '3.11', '3.12']
       fail-fast: false
     uses: ./.github/workflows/linting.yml
     with:

From b7069bf0cc5ad005ff9d041440d248b84e85e13d Mon Sep 17 00:00:00 2001
From: Safoine El khabich <safoine.ext@zenml.io>
Date: Fri, 8 Nov 2024 13:22:39 +0100
Subject: [PATCH 60/60] Remove Python 3.11 from CI workflow version matrix

---
 .github/workflows/ci-slow.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml
index a3413d09e6b..59a367ad559 100644
--- a/.github/workflows/ci-slow.yml
+++ b/.github/workflows/ci-slow.yml
@@ -177,7 +177,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: ['3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.9', '3.10', '3.12']
       fail-fast: false
     uses: ./.github/workflows/linting.yml
     with:
@@ -190,7 +190,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: ['3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.9', '3.10', '3.12']
       fail-fast: false
     uses: ./.github/workflows/unit-test.yml
     with: