From 91db66a47070e800c08eec7d371393b5f2403972 Mon Sep 17 00:00:00 2001 From: Andrei Vishniakov <31008759+avishniakov@users.noreply.github.com> Date: Mon, 7 Oct 2024 09:51:03 +0200 Subject: [PATCH 1/4] remove outdated warning --- docs/book/component-guide/orchestrators/skypilot-vm.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/book/component-guide/orchestrators/skypilot-vm.md b/docs/book/component-guide/orchestrators/skypilot-vm.md index 61d489223a5..5875ba14518 100644 --- a/docs/book/component-guide/orchestrators/skypilot-vm.md +++ b/docs/book/component-guide/orchestrators/skypilot-vm.md @@ -10,10 +10,6 @@ The SkyPilot VM Orchestrator is an integration provided by ZenML that allows you This component is only meant to be used within the context of a [remote ZenML deployment scenario](../../getting-started/deploying-zenml/README.md). Usage with a local ZenML deployment may lead to unexpected behavior! {% endhint %} -{% hint style="warning" %} -SkyPilot VM Orchestrator is currently supported only for Python 3.8 and 3.9. -{% endhint %} - ## When to use it You should use the SkyPilot VM Orchestrator if: From b7a00d715f725fa07491e67bebfda41de408f90a Mon Sep 17 00:00:00 2001 From: Andrei Vishniakov <31008759+avishniakov@users.noreply.github.com> Date: Mon, 7 Oct 2024 10:09:36 +0200 Subject: [PATCH 2/4] remove 3.8 traces --- .github/workflows/ci-fast.yml | 2 +- .github/workflows/ci-slow.yml | 28 ++++++++----------- .github/workflows/integration-test-fast.yml | 4 +-- .github/workflows/integration-test-slow.yml | 6 ++-- .github/workflows/nightly_build.yml | 2 +- .github/workflows/publish_to_pypi.yml | 2 +- .github/workflows/publish_to_pypi_nightly.yml | 2 +- .github/workflows/release.yml | 4 +-- .github/workflows/templates-test.yml | 4 +-- .github/workflows/unit-test.yml | 8 ++---- .../update-templates-to-examples.yml | 4 +-- README.md | 2 +- docs/book/getting-started/installation.md | 2 +- pyproject.toml | 11 ++------ release-cloudbuild.yaml | 23 --------------- src/zenml/cli/integration.py | 9 ------ .../integrations/huggingface/__init__.py | 6 +--- src/zenml/integrations/tensorflow/__init__.py | 9 ------ tests/harness/utils.py | 3 -- 19 files changed, 35 insertions(+), 96 deletions(-) diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml index 23ed8009c4e..27f5863f175 100644 --- a/.github/workflows/ci-fast.yml +++ b/.github/workflows/ci-fast.yml @@ -79,7 +79,7 @@ jobs: == false uses: ./.github/workflows/update-templates-to-examples.yml with: - python-version: '3.8' + python-version: '3.11' os: ubuntu-latest secrets: inherit custom-ubuntu-setup-and-unit-test: diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml index 592c8466dd3..72b2a0e21a6 100644 --- a/.github/workflows/ci-slow.yml +++ b/.github/workflows/ci-slow.yml @@ -152,7 +152,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5.0.0 with: - python-version: '3.8' + python-version: '3.11' - name: Install uv run: | curl -LsSf https://astral.sh/uv/install.sh | sh @@ -199,7 +199,7 @@ jobs: strategy: matrix: os: [arc-runner-set] - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12'] fail-fast: false uses: ./.github/workflows/unit-test.yml with: @@ -212,7 +212,7 @@ jobs: strategy: matrix: os: [windows-latest] - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12'] fail-fast: false uses: ./.github/workflows/unit-test.yml with: @@ -225,14 +225,12 @@ jobs: strategy: matrix: os: [macos-latest] - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] - # Python 3.8 and 3.9 is on macos-13 but not macos-latest (macos-14-arm64) + python-version: ['3.9', '3.10', '3.11', '3.12'] + # Python 3.9 is on macos-13 but not macos-latest (macos-14-arm64) # https://github.com/actions/setup-python/issues/696#issuecomment-1637587760 exclude: - - {python-version: '3.8', os: macos-latest} - {python-version: '3.9', os: macos-latest} include: - - {python-version: '3.8', os: macos-13} - {python-version: '3.9', os: macos-13} fail-fast: false uses: ./.github/workflows/unit-test.yml @@ -246,7 +244,7 @@ jobs: strategy: matrix: os: [windows-latest] - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12'] test_environment: [default] fail-fast: false uses: ./.github/workflows/integration-test-slow.yml @@ -261,14 +259,12 @@ jobs: strategy: matrix: os: [macos-13] - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] - # Python 3.8 and 3.9 is on macos-13 but not macos-latest (macos-14-arm64) + python-version: ['3.9', '3.10', '3.11', '3.12'] + # Python 3.9 is on macos-13 but not macos-latest (macos-14-arm64) # https://github.com/actions/setup-python/issues/696#issuecomment-1637587760 exclude: - - {python-version: '3.8', os: macos-latest} - {python-version: '3.9', os: macos-latest} include: - - {python-version: '3.8', os: macos-13} - {python-version: '3.9', os: macos-13} test_environment: [default] fail-fast: false @@ -284,25 +280,25 @@ jobs: strategy: matrix: os: [arc-runner-set] - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12'] test_environment: - default - docker-server-docker-orchestrator-mysql - docker-server-docker-orchestrator-mariadb exclude: - # docker is time-consuming to run, so we only run it on 3.8 + # docker is time-consuming to run, so we only run it on 3.11 - test_environment: docker-server-docker-orchestrator-mysql python-version: '3.9' - test_environment: docker-server-docker-orchestrator-mysql python-version: '3.10' - test_environment: docker-server-docker-orchestrator-mysql - python-version: '3.11' + python-version: '3.12' - test_environment: docker-server-docker-orchestrator-mariadb python-version: '3.9' - test_environment: docker-server-docker-orchestrator-mariadb python-version: '3.10' - test_environment: docker-server-docker-orchestrator-mariadb - python-version: '3.11' + python-version: '3.12' fail-fast: false uses: ./.github/workflows/integration-test-slow.yml with: diff --git a/.github/workflows/integration-test-fast.yml b/.github/workflows/integration-test-fast.yml index 82d1c260aee..8762e276a37 100644 --- a/.github/workflows/integration-test-fast.yml +++ b/.github/workflows/integration-test-fast.yml @@ -36,9 +36,9 @@ on: python-version: description: Python version type: choice - options: ['3.8', '3.9', '3.10', '3.11', '3.12'] + options: ['3.9', '3.10', '3.11', '3.12'] required: false - default: '3.8' + default: '3.11' test_environment: description: The test environment type: choice diff --git a/.github/workflows/integration-test-slow.yml b/.github/workflows/integration-test-slow.yml index 002898be544..d74f7306804 100644 --- a/.github/workflows/integration-test-slow.yml +++ b/.github/workflows/integration-test-slow.yml @@ -36,9 +36,9 @@ on: python-version: description: Python version type: choice - options: ['3.8', '3.9', '3.10', '3.11', '3.12'] + options: ['3.9', '3.10', '3.11', '3.12'] required: false - default: '3.8' + default: '3.11' test_environment: description: The test environment type: choice @@ -162,7 +162,7 @@ jobs: - name: Install MacOS System Dependencies if: runner.os=='macOS' run: brew install libomp - - name: Unbreak Python in GHA for 3.8-3.10 + - name: Unbreak Python in GHA for 3.9-3.10 if: runner.os=='macOS' && inputs.python-version != '3.11' # github actions overwrites brew's python. Force it to reassert itself, by # running in a separate step. diff --git a/.github/workflows/nightly_build.yml b/.github/workflows/nightly_build.yml index a9d18a8cd95..6a8d2c04ce0 100644 --- a/.github/workflows/nightly_build.yml +++ b/.github/workflows/nightly_build.yml @@ -10,7 +10,7 @@ jobs: uses: ./.github/workflows/unit-test.yml with: os: ubuntu-latest - python-version: '3.8' + python-version: '3.11' git-ref: develop secrets: inherit if: github.repository == 'zenml-io/zenml' diff --git a/.github/workflows/publish_to_pypi.yml b/.github/workflows/publish_to_pypi.yml index 4d2c0ed03ca..c2a81e42ee3 100644 --- a/.github/workflows/publish_to_pypi.yml +++ b/.github/workflows/publish_to_pypi.yml @@ -22,7 +22,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5.0.0 with: - python-version: '3.8' + python-version: '3.11' - name: Install Poetry uses: snok/install-poetry@v1.3.4 with: diff --git a/.github/workflows/publish_to_pypi_nightly.yml b/.github/workflows/publish_to_pypi_nightly.yml index a82e1e9db79..67deaac282b 100644 --- a/.github/workflows/publish_to_pypi_nightly.yml +++ b/.github/workflows/publish_to_pypi_nightly.yml @@ -23,7 +23,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5.0.0 with: - python-version: '3.8' + python-version: '3.11' - name: Install Poetry uses: snok/install-poetry@v1.3.4 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 598a7600de7..1b7881e64e3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,7 +8,7 @@ jobs: uses: ./.github/workflows/unit-test.yml with: os: arc-runner-set - python-version: '3.8' + python-version: '3.11' secrets: inherit mlstacks-compatibility-check: needs: setup-and-test @@ -19,7 +19,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5.0.0 with: - python-version: '3.8' + python-version: '3.11' - name: Install uv run: | curl -LsSf https://astral.sh/uv/install.sh | sh diff --git a/.github/workflows/templates-test.yml b/.github/workflows/templates-test.yml index e476b5170c1..d76a85a731d 100644 --- a/.github/workflows/templates-test.yml +++ b/.github/workflows/templates-test.yml @@ -22,9 +22,9 @@ on: python-version: description: Python version type: choice - options: ['3.8', '3.9', '3.10', '3.11', '3.12'] + options: ['3.9', '3.10', '3.11', '3.12'] required: false - default: '3.8' + default: '3.11' jobs: all-template-tests: name: all-template-tests diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index fae07ae41ce..31eb80f71b7 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -37,9 +37,9 @@ on: python-version: description: Python version type: choice - options: ['3.8', '3.9', '3.10', '3.11', '3.12'] + options: ['3.9', '3.10', '3.11', '3.12'] required: false - default: '3.8' + default: '3.11' enable_tmate: description: Enable tmate session for debugging type: choice @@ -105,10 +105,6 @@ jobs: ZENML_ANALYTICS_OPT_IN: false ZENML_DEBUG: true - # - name: Upload coverage - # # only do it for python 3.8, we don't need to do it for every version - # if: ${{ inputs.os == 'arc-runner-set' && inputs.python-version == '3.8' }} - # uses: codecov/codecov-action@v2 - name: Setup tmate session after tests if: ${{ inputs.enable_tmate == 'always' || (inputs.enable_tmate == 'on-failure' && failure()) }} uses: mxschmitt/action-tmate@v3.17 diff --git a/.github/workflows/update-templates-to-examples.yml b/.github/workflows/update-templates-to-examples.yml index 7a9accb4293..0473babc442 100644 --- a/.github/workflows/update-templates-to-examples.yml +++ b/.github/workflows/update-templates-to-examples.yml @@ -22,9 +22,9 @@ on: python-version: description: Python version type: choice - options: ['3.8', '3.9', '3.10', '3.11', '3.12'] + options: ['3.9', '3.10', '3.11', '3.12'] required: false - default: '3.8' + default: '3.11' jobs: update-e2e-batch-template-to-examples: name: update-e2e-batch-template-to-examples diff --git a/README.md b/README.md index 740e0d4a14d..373e86b4577 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ Thank you for your support! 🌟 ## 🤸 Quickstart [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/zenml-io/zenml/blob/main/examples/quickstart/quickstart.ipynb) -[Install ZenML](https://docs.zenml.io/getting-started/installation) via [PyPI](https://pypi.org/project/zenml/). Python 3.8 - 3.11 is required: +[Install ZenML](https://docs.zenml.io/getting-started/installation) via [PyPI](https://pypi.org/project/zenml/). Python 3.9 - 3.12 is required: ```bash pip install "zenml[server]" notebook diff --git a/docs/book/getting-started/installation.md b/docs/book/getting-started/installation.md index 307bf84fb39..a631c6157a2 100644 --- a/docs/book/getting-started/installation.md +++ b/docs/book/getting-started/installation.md @@ -11,7 +11,7 @@ pip install zenml ``` {% hint style="warning" %} -Note that ZenML currently supports **Python 3.8, 3.9, 3.10, and 3.11**. Please make sure that you are using a supported Python version. +Note that ZenML currently supports **Python 3.9, 3.10, 3.11 and 3.12**. Please make sure that you are using a supported Python version. {% endhint %} ## Install with the dashboard diff --git a/pyproject.toml b/pyproject.toml index 6922df1d452..d5250d0521e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,6 @@ classifiers = [ "Intended Audience :: System Administrators", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", @@ -55,7 +54,7 @@ psutil = ">=5.0.0" pydantic = "~2.8" pydantic-settings = "*" pymysql = { version = "~1.1.0,>=1.1.1" } -python = ">=3.8,<3.13" +python = ">=3.9,<3.13" python-dateutil = "^2.8.1" pyyaml = ">=6.0.1" rich = { extras = ["jupyter"], version = ">=12.0.0" } @@ -342,8 +341,8 @@ exclude = [ ] src = ["src", "test"] -# use Python 3.8 as the minimum version for autofixing -target-version = "py38" +# use Python 3.9 as the minimum version for autofixing +target-version = "py39" [tool.ruff.format] @@ -402,10 +401,6 @@ strict = true namespace_packages = true show_error_codes = true -# temporary fix for python 3.8 https://github.com/apache/airflow/discussions/19006 -# remove once the issue is solved in airflow -exclude = "airflow/" - [[tool.mypy.overrides]] module = ["airflow.*"] follow_imports = "skip" diff --git a/release-cloudbuild.yaml b/release-cloudbuild.yaml index bbaccad7728..86e04ab8751 100644 --- a/release-cloudbuild.yaml +++ b/release-cloudbuild.yaml @@ -1,26 +1,4 @@ steps: - # build client base image - python 3.8 - - name: gcr.io/cloud-builders/docker - args: - - '-c' - - | - docker build \ - --build-arg ZENML_VERSION=$TAG_NAME \ - --build-arg PYTHON_VERSION=3.8 \ - --target client \ - -f docker/base.Dockerfile . \ - -t $$USERNAME/zenml:$TAG_NAME-py3.8 - - # use latest tags only for official releases - if [[ $TAG_NAME =~ ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]; then - docker tag $$USERNAME/zenml:$TAG_NAME-py3.8 $$USERNAME/zenml:py3.8 - fi - id: build-base-3.8 - waitFor: ['-'] - entrypoint: bash - secretEnv: - - USERNAME - # build client base image - python 3.9 - name: gcr.io/cloud-builders/docker args: @@ -152,7 +130,6 @@ steps: id: push-base waitFor: - docker-login - - build-base-3.8 - build-base-3.9 - build-base-3.10 - build-base-3.11 diff --git a/src/zenml/cli/integration.py b/src/zenml/cli/integration.py index a9f51cf32a9..622cc712483 100644 --- a/src/zenml/cli/integration.py +++ b/src/zenml/cli/integration.py @@ -282,15 +282,6 @@ def install( else: integration_set = set(integrations) - # TODO: remove once python 3.8 is deprecated - if sys.version_info.minor == 8 and "tensorflow" in integration_set: - warning( - "Python 3.8 with TensorFlow is not fully compatible with " - "Pydantic 2 requirements. Consider upgrading to a " - "higher Python version if you would like to use the " - "Tensorflow integration." - ) - if sys.version_info.minor == 12 and "tensorflow" in integration_set: warning( "The TensorFlow integration is not yet compatible with Python " diff --git a/src/zenml/integrations/huggingface/__init__.py b/src/zenml/integrations/huggingface/__init__.py index f84128dd2c5..c415d791881 100644 --- a/src/zenml/integrations/huggingface/__init__.py +++ b/src/zenml/integrations/huggingface/__init__.py @@ -59,11 +59,7 @@ def get_requirements(cls, target_os: Optional[str] = None) -> List[str]: "fsspec<=2023.12.0", ] - # In python 3.8 higher transformers version lead to other packages breaking - if sys.version_info.minor > 8: - requirements += ["transformers"] - else: - requirements += ["transformers<=4.31"] + requirements += ["transformers"] # Add the pandas integration requirements from zenml.integrations.pandas import PandasIntegration diff --git a/src/zenml/integrations/tensorflow/__init__.py b/src/zenml/integrations/tensorflow/__init__.py index 22d9e6440d7..dfeb1c12da1 100644 --- a/src/zenml/integrations/tensorflow/__init__.py +++ b/src/zenml/integrations/tensorflow/__init__.py @@ -40,15 +40,6 @@ def activate(cls) -> None: from zenml.integrations.tensorflow import materializers # noqa - if sys.version_info.minor == 8: - logger.warning( - "Python 3.8 with TensorFlow is not fully " - "compatible with Pydantic 2 requirements. " - "Consider upgrading to a higher Python " - "version if you would like to use the " - "Tensorflow integration." - ) - @classmethod def get_requirements(cls, target_os: Optional[str] = None) -> List[str]: """Defines platform specific requirements for the integration. diff --git a/tests/harness/utils.py b/tests/harness/utils.py index 6ec7e38fadb..aa31d4fae77 100644 --- a/tests/harness/utils.py +++ b/tests/harness/utils.py @@ -56,9 +56,6 @@ def cleanup_folder(path: str) -> None: "Skipping deletion of temp dir at teardown, due to " "Windows Permission error" ) - # TODO[HIGH]: Implement fixture cleanup for Windows where - # shutil.rmtree fails on files that are in use on python 3.7 and - # 3.8 else: shutil.rmtree(path) From 9d0566f2356397789e0e57020d07732124a4f2e6 Mon Sep 17 00:00:00 2001 From: Andrei Vishniakov <31008759+avishniakov@users.noreply.github.com> Date: Mon, 7 Oct 2024 10:14:02 +0200 Subject: [PATCH 3/4] lint --- .github/workflows/unit-test.yml | 1 - .../zen_stores/test_secrets_store.py | 44 ++++++++++--------- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 31eb80f71b7..03752da2c37 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -104,7 +104,6 @@ jobs: env: ZENML_ANALYTICS_OPT_IN: false ZENML_DEBUG: true - - name: Setup tmate session after tests if: ${{ inputs.enable_tmate == 'always' || (inputs.enable_tmate == 'on-failure' && failure()) }} uses: mxschmitt/action-tmate@v3.17 diff --git a/tests/integration/functional/zen_stores/test_secrets_store.py b/tests/integration/functional/zen_stores/test_secrets_store.py index bc3d79af393..29b7d42be02 100644 --- a/tests/integration/functional/zen_stores/test_secrets_store.py +++ b/tests/integration/functional/zen_stores/test_secrets_store.py @@ -1114,15 +1114,16 @@ def test_list_secrets_filter(): aria_secret_name = sample_name("arias-whiskers") axl_secret_name = sample_name("axls-whiskers") - with SecretContext( - secret_name=aria_secret_name - ) as secret_one, SecretContext( - secret_name=aria_secret_name, scope=SecretScope.USER - ) as secret_two, SecretContext( - secret_name=axl_secret_name - ) as secret_three, SecretContext( - secret_name=axl_secret_name, scope=SecretScope.USER - ) as secret_four: + with ( + SecretContext(secret_name=aria_secret_name) as secret_one, + SecretContext( + secret_name=aria_secret_name, scope=SecretScope.USER + ) as secret_two, + SecretContext(secret_name=axl_secret_name) as secret_three, + SecretContext( + secret_name=axl_secret_name, scope=SecretScope.USER + ) as secret_four, + ): all_secrets = store.list_secrets(SecretFilter()).items assert len(all_secrets) >= 4 assert set( @@ -1263,17 +1264,20 @@ def test_list_secrets_pagination_and_sorting(): suffix = sample_name("") - with SecretContext( - secret_name=f"arias-whiskers-{suffix}" - ) as secret_one, SecretContext( - secret_name=f"arias-spots-{suffix}", - scope=SecretScope.USER, - ) as secret_two, SecretContext( - secret_name=f"axls-whiskers-{suffix}", - ) as secret_three, SecretContext( - secret_name=f"axls-spots-{suffix}", - scope=SecretScope.USER, - ) as secret_four: + with ( + SecretContext(secret_name=f"arias-whiskers-{suffix}") as secret_one, + SecretContext( + secret_name=f"arias-spots-{suffix}", + scope=SecretScope.USER, + ) as secret_two, + SecretContext( + secret_name=f"axls-whiskers-{suffix}", + ) as secret_three, + SecretContext( + secret_name=f"axls-spots-{suffix}", + scope=SecretScope.USER, + ) as secret_four, + ): secrets = store.list_secrets( SecretFilter( name=f"endswith:{suffix}", From bcd42190b6512d377fc23c03bfa191615237f2fd Mon Sep 17 00:00:00 2001 From: Andrei Vishniakov <31008759+avishniakov@users.noreply.github.com> Date: Mon, 7 Oct 2024 10:46:03 +0200 Subject: [PATCH 4/4] revert 3.8 removal --- .github/workflows/ci-fast.yml | 2 +- .github/workflows/ci-slow.yml | 28 +++++++----- .github/workflows/integration-test-fast.yml | 4 +- .github/workflows/integration-test-slow.yml | 6 +-- .github/workflows/nightly_build.yml | 2 +- .github/workflows/publish_to_pypi.yml | 2 +- .github/workflows/publish_to_pypi_nightly.yml | 2 +- .github/workflows/release.yml | 4 +- .github/workflows/templates-test.yml | 4 +- .github/workflows/unit-test.yml | 9 +++- .../update-templates-to-examples.yml | 4 +- README.md | 2 +- docs/book/getting-started/installation.md | 2 +- pyproject.toml | 11 +++-- release-cloudbuild.yaml | 23 ++++++++++ src/zenml/cli/integration.py | 9 ++++ .../integrations/huggingface/__init__.py | 6 ++- src/zenml/integrations/tensorflow/__init__.py | 9 ++++ tests/harness/utils.py | 3 ++ .../zen_stores/test_secrets_store.py | 44 +++++++++---------- 20 files changed, 119 insertions(+), 57 deletions(-) diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml index 27f5863f175..23ed8009c4e 100644 --- a/.github/workflows/ci-fast.yml +++ b/.github/workflows/ci-fast.yml @@ -79,7 +79,7 @@ jobs: == false uses: ./.github/workflows/update-templates-to-examples.yml with: - python-version: '3.11' + python-version: '3.8' os: ubuntu-latest secrets: inherit custom-ubuntu-setup-and-unit-test: diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml index 72b2a0e21a6..a9e994f9b8a 100644 --- a/.github/workflows/ci-slow.yml +++ b/.github/workflows/ci-slow.yml @@ -152,7 +152,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5.0.0 with: - python-version: '3.11' + python-version: '3.8' - name: Install uv run: | curl -LsSf https://astral.sh/uv/install.sh | sh @@ -199,7 +199,7 @@ jobs: strategy: matrix: os: [arc-runner-set] - python-version: ['3.9', '3.10', '3.11', '3.12'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] fail-fast: false uses: ./.github/workflows/unit-test.yml with: @@ -212,7 +212,7 @@ jobs: strategy: matrix: os: [windows-latest] - python-version: ['3.9', '3.10', '3.11', '3.12'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] fail-fast: false uses: ./.github/workflows/unit-test.yml with: @@ -225,12 +225,14 @@ jobs: strategy: matrix: os: [macos-latest] - python-version: ['3.9', '3.10', '3.11', '3.12'] - # Python 3.9 is on macos-13 but not macos-latest (macos-14-arm64) + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + # Python 3.8 and 3.9 is on macos-13 but not macos-latest (macos-14-arm64) # https://github.com/actions/setup-python/issues/696#issuecomment-1637587760 exclude: + - {python-version: '3.8', os: macos-latest} - {python-version: '3.9', os: macos-latest} include: + - {python-version: '3.8', os: macos-13} - {python-version: '3.9', os: macos-13} fail-fast: false uses: ./.github/workflows/unit-test.yml @@ -244,7 +246,7 @@ jobs: strategy: matrix: os: [windows-latest] - python-version: ['3.9', '3.10', '3.11', '3.12'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] test_environment: [default] fail-fast: false uses: ./.github/workflows/integration-test-slow.yml @@ -259,12 +261,14 @@ jobs: strategy: matrix: os: [macos-13] - python-version: ['3.9', '3.10', '3.11', '3.12'] - # Python 3.9 is on macos-13 but not macos-latest (macos-14-arm64) + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + # Python 3.8 and 3.9 is on macos-13 but not macos-latest (macos-14-arm64) # https://github.com/actions/setup-python/issues/696#issuecomment-1637587760 exclude: + - {python-version: '3.8', os: macos-latest} - {python-version: '3.9', os: macos-latest} include: + - {python-version: '3.8', os: macos-13} - {python-version: '3.9', os: macos-13} test_environment: [default] fail-fast: false @@ -280,23 +284,27 @@ jobs: strategy: matrix: os: [arc-runner-set] - python-version: ['3.9', '3.10', '3.11', '3.12'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] test_environment: - default - docker-server-docker-orchestrator-mysql - docker-server-docker-orchestrator-mariadb exclude: - # docker is time-consuming to run, so we only run it on 3.11 + # docker is time-consuming to run, so we only run it on 3.8 - test_environment: docker-server-docker-orchestrator-mysql python-version: '3.9' - test_environment: docker-server-docker-orchestrator-mysql python-version: '3.10' + - test_environment: docker-server-docker-orchestrator-mysql + python-version: '3.11' - test_environment: docker-server-docker-orchestrator-mysql python-version: '3.12' - test_environment: docker-server-docker-orchestrator-mariadb python-version: '3.9' - test_environment: docker-server-docker-orchestrator-mariadb python-version: '3.10' + - test_environment: docker-server-docker-orchestrator-mariadb + python-version: '3.11' - test_environment: docker-server-docker-orchestrator-mariadb python-version: '3.12' fail-fast: false diff --git a/.github/workflows/integration-test-fast.yml b/.github/workflows/integration-test-fast.yml index 8762e276a37..82d1c260aee 100644 --- a/.github/workflows/integration-test-fast.yml +++ b/.github/workflows/integration-test-fast.yml @@ -36,9 +36,9 @@ on: python-version: description: Python version type: choice - options: ['3.9', '3.10', '3.11', '3.12'] + options: ['3.8', '3.9', '3.10', '3.11', '3.12'] required: false - default: '3.11' + default: '3.8' test_environment: description: The test environment type: choice diff --git a/.github/workflows/integration-test-slow.yml b/.github/workflows/integration-test-slow.yml index d74f7306804..002898be544 100644 --- a/.github/workflows/integration-test-slow.yml +++ b/.github/workflows/integration-test-slow.yml @@ -36,9 +36,9 @@ on: python-version: description: Python version type: choice - options: ['3.9', '3.10', '3.11', '3.12'] + options: ['3.8', '3.9', '3.10', '3.11', '3.12'] required: false - default: '3.11' + default: '3.8' test_environment: description: The test environment type: choice @@ -162,7 +162,7 @@ jobs: - name: Install MacOS System Dependencies if: runner.os=='macOS' run: brew install libomp - - name: Unbreak Python in GHA for 3.9-3.10 + - name: Unbreak Python in GHA for 3.8-3.10 if: runner.os=='macOS' && inputs.python-version != '3.11' # github actions overwrites brew's python. Force it to reassert itself, by # running in a separate step. diff --git a/.github/workflows/nightly_build.yml b/.github/workflows/nightly_build.yml index 6a8d2c04ce0..a9d18a8cd95 100644 --- a/.github/workflows/nightly_build.yml +++ b/.github/workflows/nightly_build.yml @@ -10,7 +10,7 @@ jobs: uses: ./.github/workflows/unit-test.yml with: os: ubuntu-latest - python-version: '3.11' + python-version: '3.8' git-ref: develop secrets: inherit if: github.repository == 'zenml-io/zenml' diff --git a/.github/workflows/publish_to_pypi.yml b/.github/workflows/publish_to_pypi.yml index c2a81e42ee3..4d2c0ed03ca 100644 --- a/.github/workflows/publish_to_pypi.yml +++ b/.github/workflows/publish_to_pypi.yml @@ -22,7 +22,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5.0.0 with: - python-version: '3.11' + python-version: '3.8' - name: Install Poetry uses: snok/install-poetry@v1.3.4 with: diff --git a/.github/workflows/publish_to_pypi_nightly.yml b/.github/workflows/publish_to_pypi_nightly.yml index 67deaac282b..a82e1e9db79 100644 --- a/.github/workflows/publish_to_pypi_nightly.yml +++ b/.github/workflows/publish_to_pypi_nightly.yml @@ -23,7 +23,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5.0.0 with: - python-version: '3.11' + python-version: '3.8' - name: Install Poetry uses: snok/install-poetry@v1.3.4 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1b7881e64e3..598a7600de7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,7 +8,7 @@ jobs: uses: ./.github/workflows/unit-test.yml with: os: arc-runner-set - python-version: '3.11' + python-version: '3.8' secrets: inherit mlstacks-compatibility-check: needs: setup-and-test @@ -19,7 +19,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5.0.0 with: - python-version: '3.11' + python-version: '3.8' - name: Install uv run: | curl -LsSf https://astral.sh/uv/install.sh | sh diff --git a/.github/workflows/templates-test.yml b/.github/workflows/templates-test.yml index d76a85a731d..e476b5170c1 100644 --- a/.github/workflows/templates-test.yml +++ b/.github/workflows/templates-test.yml @@ -22,9 +22,9 @@ on: python-version: description: Python version type: choice - options: ['3.9', '3.10', '3.11', '3.12'] + options: ['3.8', '3.9', '3.10', '3.11', '3.12'] required: false - default: '3.11' + default: '3.8' jobs: all-template-tests: name: all-template-tests diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 03752da2c37..fae07ae41ce 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -37,9 +37,9 @@ on: python-version: description: Python version type: choice - options: ['3.9', '3.10', '3.11', '3.12'] + options: ['3.8', '3.9', '3.10', '3.11', '3.12'] required: false - default: '3.11' + default: '3.8' enable_tmate: description: Enable tmate session for debugging type: choice @@ -104,6 +104,11 @@ jobs: env: ZENML_ANALYTICS_OPT_IN: false ZENML_DEBUG: true + + # - name: Upload coverage + # # only do it for python 3.8, we don't need to do it for every version + # if: ${{ inputs.os == 'arc-runner-set' && inputs.python-version == '3.8' }} + # uses: codecov/codecov-action@v2 - name: Setup tmate session after tests if: ${{ inputs.enable_tmate == 'always' || (inputs.enable_tmate == 'on-failure' && failure()) }} uses: mxschmitt/action-tmate@v3.17 diff --git a/.github/workflows/update-templates-to-examples.yml b/.github/workflows/update-templates-to-examples.yml index 0473babc442..7a9accb4293 100644 --- a/.github/workflows/update-templates-to-examples.yml +++ b/.github/workflows/update-templates-to-examples.yml @@ -22,9 +22,9 @@ on: python-version: description: Python version type: choice - options: ['3.9', '3.10', '3.11', '3.12'] + options: ['3.8', '3.9', '3.10', '3.11', '3.12'] required: false - default: '3.11' + default: '3.8' jobs: update-e2e-batch-template-to-examples: name: update-e2e-batch-template-to-examples diff --git a/README.md b/README.md index 373e86b4577..c13d60c5abe 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ Thank you for your support! 🌟 ## 🤸 Quickstart [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/zenml-io/zenml/blob/main/examples/quickstart/quickstart.ipynb) -[Install ZenML](https://docs.zenml.io/getting-started/installation) via [PyPI](https://pypi.org/project/zenml/). Python 3.9 - 3.12 is required: +[Install ZenML](https://docs.zenml.io/getting-started/installation) via [PyPI](https://pypi.org/project/zenml/). Python 3.8 - 3.12 is required: ```bash pip install "zenml[server]" notebook diff --git a/docs/book/getting-started/installation.md b/docs/book/getting-started/installation.md index a631c6157a2..b1a598b73f3 100644 --- a/docs/book/getting-started/installation.md +++ b/docs/book/getting-started/installation.md @@ -11,7 +11,7 @@ pip install zenml ``` {% hint style="warning" %} -Note that ZenML currently supports **Python 3.9, 3.10, 3.11 and 3.12**. Please make sure that you are using a supported Python version. +Note that ZenML currently supports **Python 3.8, 3.9, 3.10, 3.11 and 3.12**. Please make sure that you are using a supported Python version. {% endhint %} ## Install with the dashboard diff --git a/pyproject.toml b/pyproject.toml index d5250d0521e..6922df1d452 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ classifiers = [ "Intended Audience :: System Administrators", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", @@ -54,7 +55,7 @@ psutil = ">=5.0.0" pydantic = "~2.8" pydantic-settings = "*" pymysql = { version = "~1.1.0,>=1.1.1" } -python = ">=3.9,<3.13" +python = ">=3.8,<3.13" python-dateutil = "^2.8.1" pyyaml = ">=6.0.1" rich = { extras = ["jupyter"], version = ">=12.0.0" } @@ -341,8 +342,8 @@ exclude = [ ] src = ["src", "test"] -# use Python 3.9 as the minimum version for autofixing -target-version = "py39" +# use Python 3.8 as the minimum version for autofixing +target-version = "py38" [tool.ruff.format] @@ -401,6 +402,10 @@ strict = true namespace_packages = true show_error_codes = true +# temporary fix for python 3.8 https://github.com/apache/airflow/discussions/19006 +# remove once the issue is solved in airflow +exclude = "airflow/" + [[tool.mypy.overrides]] module = ["airflow.*"] follow_imports = "skip" diff --git a/release-cloudbuild.yaml b/release-cloudbuild.yaml index 86e04ab8751..bbaccad7728 100644 --- a/release-cloudbuild.yaml +++ b/release-cloudbuild.yaml @@ -1,4 +1,26 @@ steps: + # build client base image - python 3.8 + - name: gcr.io/cloud-builders/docker + args: + - '-c' + - | + docker build \ + --build-arg ZENML_VERSION=$TAG_NAME \ + --build-arg PYTHON_VERSION=3.8 \ + --target client \ + -f docker/base.Dockerfile . \ + -t $$USERNAME/zenml:$TAG_NAME-py3.8 + + # use latest tags only for official releases + if [[ $TAG_NAME =~ ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]; then + docker tag $$USERNAME/zenml:$TAG_NAME-py3.8 $$USERNAME/zenml:py3.8 + fi + id: build-base-3.8 + waitFor: ['-'] + entrypoint: bash + secretEnv: + - USERNAME + # build client base image - python 3.9 - name: gcr.io/cloud-builders/docker args: @@ -130,6 +152,7 @@ steps: id: push-base waitFor: - docker-login + - build-base-3.8 - build-base-3.9 - build-base-3.10 - build-base-3.11 diff --git a/src/zenml/cli/integration.py b/src/zenml/cli/integration.py index 622cc712483..a9f51cf32a9 100644 --- a/src/zenml/cli/integration.py +++ b/src/zenml/cli/integration.py @@ -282,6 +282,15 @@ def install( else: integration_set = set(integrations) + # TODO: remove once python 3.8 is deprecated + if sys.version_info.minor == 8 and "tensorflow" in integration_set: + warning( + "Python 3.8 with TensorFlow is not fully compatible with " + "Pydantic 2 requirements. Consider upgrading to a " + "higher Python version if you would like to use the " + "Tensorflow integration." + ) + if sys.version_info.minor == 12 and "tensorflow" in integration_set: warning( "The TensorFlow integration is not yet compatible with Python " diff --git a/src/zenml/integrations/huggingface/__init__.py b/src/zenml/integrations/huggingface/__init__.py index c415d791881..f84128dd2c5 100644 --- a/src/zenml/integrations/huggingface/__init__.py +++ b/src/zenml/integrations/huggingface/__init__.py @@ -59,7 +59,11 @@ def get_requirements(cls, target_os: Optional[str] = None) -> List[str]: "fsspec<=2023.12.0", ] - requirements += ["transformers"] + # In python 3.8 higher transformers version lead to other packages breaking + if sys.version_info.minor > 8: + requirements += ["transformers"] + else: + requirements += ["transformers<=4.31"] # Add the pandas integration requirements from zenml.integrations.pandas import PandasIntegration diff --git a/src/zenml/integrations/tensorflow/__init__.py b/src/zenml/integrations/tensorflow/__init__.py index dfeb1c12da1..22d9e6440d7 100644 --- a/src/zenml/integrations/tensorflow/__init__.py +++ b/src/zenml/integrations/tensorflow/__init__.py @@ -40,6 +40,15 @@ def activate(cls) -> None: from zenml.integrations.tensorflow import materializers # noqa + if sys.version_info.minor == 8: + logger.warning( + "Python 3.8 with TensorFlow is not fully " + "compatible with Pydantic 2 requirements. " + "Consider upgrading to a higher Python " + "version if you would like to use the " + "Tensorflow integration." + ) + @classmethod def get_requirements(cls, target_os: Optional[str] = None) -> List[str]: """Defines platform specific requirements for the integration. diff --git a/tests/harness/utils.py b/tests/harness/utils.py index aa31d4fae77..6ec7e38fadb 100644 --- a/tests/harness/utils.py +++ b/tests/harness/utils.py @@ -56,6 +56,9 @@ def cleanup_folder(path: str) -> None: "Skipping deletion of temp dir at teardown, due to " "Windows Permission error" ) + # TODO[HIGH]: Implement fixture cleanup for Windows where + # shutil.rmtree fails on files that are in use on python 3.7 and + # 3.8 else: shutil.rmtree(path) diff --git a/tests/integration/functional/zen_stores/test_secrets_store.py b/tests/integration/functional/zen_stores/test_secrets_store.py index 29b7d42be02..bc3d79af393 100644 --- a/tests/integration/functional/zen_stores/test_secrets_store.py +++ b/tests/integration/functional/zen_stores/test_secrets_store.py @@ -1114,16 +1114,15 @@ def test_list_secrets_filter(): aria_secret_name = sample_name("arias-whiskers") axl_secret_name = sample_name("axls-whiskers") - with ( - SecretContext(secret_name=aria_secret_name) as secret_one, - SecretContext( - secret_name=aria_secret_name, scope=SecretScope.USER - ) as secret_two, - SecretContext(secret_name=axl_secret_name) as secret_three, - SecretContext( - secret_name=axl_secret_name, scope=SecretScope.USER - ) as secret_four, - ): + with SecretContext( + secret_name=aria_secret_name + ) as secret_one, SecretContext( + secret_name=aria_secret_name, scope=SecretScope.USER + ) as secret_two, SecretContext( + secret_name=axl_secret_name + ) as secret_three, SecretContext( + secret_name=axl_secret_name, scope=SecretScope.USER + ) as secret_four: all_secrets = store.list_secrets(SecretFilter()).items assert len(all_secrets) >= 4 assert set( @@ -1264,20 +1263,17 @@ def test_list_secrets_pagination_and_sorting(): suffix = sample_name("") - with ( - SecretContext(secret_name=f"arias-whiskers-{suffix}") as secret_one, - SecretContext( - secret_name=f"arias-spots-{suffix}", - scope=SecretScope.USER, - ) as secret_two, - SecretContext( - secret_name=f"axls-whiskers-{suffix}", - ) as secret_three, - SecretContext( - secret_name=f"axls-spots-{suffix}", - scope=SecretScope.USER, - ) as secret_four, - ): + with SecretContext( + secret_name=f"arias-whiskers-{suffix}" + ) as secret_one, SecretContext( + secret_name=f"arias-spots-{suffix}", + scope=SecretScope.USER, + ) as secret_two, SecretContext( + secret_name=f"axls-whiskers-{suffix}", + ) as secret_three, SecretContext( + secret_name=f"axls-spots-{suffix}", + scope=SecretScope.USER, + ) as secret_four: secrets = store.list_secrets( SecretFilter( name=f"endswith:{suffix}",