From 6ca20584319f52856e249f3588a6706239e2abd9 Mon Sep 17 00:00:00 2001 From: Robbe Sneyders Date: Mon, 19 Feb 2024 16:42:21 +0100 Subject: [PATCH] Update docker image versions in examples and tests --- data_explorer/Dockerfile | 2 +- docs/components/containerized_components.md | 2 +- docs/components/lightweight_components.md | 2 +- docs/guides/installation.md | 2 +- .../components/dummy_component/Dockerfile | 2 +- examples/sample_pipeline/pipeline.py | 2 +- scripts/build_base_image.sh | 2 +- tests/examples/example_component/Dockerfile | 2 +- tests/pipeline/test_compiler.py | 6 +++--- tests/pipeline/test_lightweight_component.py | 14 +++++++------- 10 files changed, 18 insertions(+), 18 deletions(-) diff --git a/data_explorer/Dockerfile b/data_explorer/Dockerfile index 46c1c5b2b..59ef59bdb 100644 --- a/data_explorer/Dockerfile +++ b/data_explorer/Dockerfile @@ -1,4 +1,4 @@ -FROM --platform=linux/amd64 python:3.8-slim +FROM --platform=linux/amd64 python:3.10-slim ENV PYTHONUNBUFFERED=1 ENV PYTHONIOENCODING=UTF-8 diff --git a/docs/components/containerized_components.md b/docs/components/containerized_components.md index 389ad357d..f0b8bd039 100644 --- a/docs/components/containerized_components.md +++ b/docs/components/containerized_components.md @@ -48,7 +48,7 @@ The `Dockerfile` defines how to build the component into a Docker image. An exam defined below. ```bash -FROM --platform=linux/amd64 python:3.8-slim +FROM --platform=linux/amd64 python:3.10-slim # install requirements COPY requirements.txt ./ diff --git a/docs/components/lightweight_components.md b/docs/components/lightweight_components.md index 67e6bb865..467107954 100644 --- a/docs/components/lightweight_components.md +++ b/docs/components/lightweight_components.md @@ -89,7 +89,7 @@ image instead of the default one. Make sure you install Fondant in the base imag in the `extra_requires` argument. ```python title="pipeline.py" -@lightweight_component(base_image="python:3.8-slim") +@lightweight_component(base_image="python:3.10-slim") ``` ## Optimizing loaded data diff --git a/docs/guides/installation.md b/docs/guides/installation.md index 82bbafb53..ed6c32b92 100644 --- a/docs/guides/installation.md +++ b/docs/guides/installation.md @@ -65,7 +65,7 @@ Check out the [guide](../components/publishing_components.md) on publishing comp ## Docker installation To execute pipelines locally, you must -have [Docker compose](https://docs.docker.com/compose/install/) and Python >=3.8 +have [Docker compose](https://docs.docker.com/compose/install/) and Python >=3.9 installed on your system. We only support Docker compose V2 and above. If you have an older version of Docker compose, please upgrade to the latest version as described in the [Docker compose migration doc](https://docs.docker.com/compose/migrate/). diff --git a/examples/sample_pipeline/components/dummy_component/Dockerfile b/examples/sample_pipeline/components/dummy_component/Dockerfile index e15bbb588..b103d06af 100644 --- a/examples/sample_pipeline/components/dummy_component/Dockerfile +++ b/examples/sample_pipeline/components/dummy_component/Dockerfile @@ -1,4 +1,4 @@ -FROM --platform=linux/amd64 python:3.8-slim +FROM --platform=linux/amd64 python:3.10-slim # System dependencies RUN apt-get update && \ diff --git a/examples/sample_pipeline/pipeline.py b/examples/sample_pipeline/pipeline.py index 28c01ceda..900b79e13 100644 --- a/examples/sample_pipeline/pipeline.py +++ b/examples/sample_pipeline/pipeline.py @@ -41,7 +41,7 @@ @lightweight_component( - base_image="python:3.8", + base_image="python:3.10", extra_requires=[ f"fondant[component]@git+https://github.com/ml6team/fondant@" f"{os.environ.get('FONDANT_VERSION', 'main')}", diff --git a/scripts/build_base_image.sh b/scripts/build_base_image.sh index fa15f16f2..dc8d56375 100755 --- a/scripts/build_base_image.sh +++ b/scripts/build_base_image.sh @@ -17,7 +17,7 @@ while [[ "$#" -gt 0 ]]; do case $1 in esac; shift; done # Supported Python versions -python_versions=("3.8" "3.9" "3.10" "3.11") +python_versions=("3.9" "3.10" "3.11") for tag in "${tags[@]}"; do for python_version in "${python_versions[@]}"; do diff --git a/tests/examples/example_component/Dockerfile b/tests/examples/example_component/Dockerfile index ee4ae333b..95ef9f1db 100644 --- a/tests/examples/example_component/Dockerfile +++ b/tests/examples/example_component/Dockerfile @@ -1 +1 @@ -FROM --platform=linux/amd64 python:3.8-slim as base \ No newline at end of file +FROM --platform=linux/amd64 python:3.10-slim as base \ No newline at end of file diff --git a/tests/pipeline/test_compiler.py b/tests/pipeline/test_compiler.py index 545b56d00..ceee63922 100644 --- a/tests/pipeline/test_compiler.py +++ b/tests/pipeline/test_compiler.py @@ -417,7 +417,7 @@ def test_kubeflow_component_spec_from_lightweight_component( ) @lightweight_component( - base_image="python:3.8-slim-buster", + base_image="python:3.10-slim-buster", extra_requires=["pandas", "dask"], produces={"x": pa.int32(), "y": pa.int32()}, ) @@ -442,7 +442,7 @@ def load(self) -> dd.DataFrame: compiler.compile(pipeline=pipeline, output_path=output_path) pipeline_configs = KubeflowPipelineConfigs.from_spec(output_path) assert pipeline_configs.component_configs["createdata"].image == ( - "python:3.8-slim-buster" + "python:3.10-slim-buster" ) assert pipeline_configs.component_configs["createdata"].command == [ "sh", @@ -793,7 +793,7 @@ def test_sagemaker_generate_script(tmp_path_factory): def test_sagemaker_generate_script_lightweight_component(tmp_path_factory): @lightweight_component( - base_image="python:3.8-slim-buster", + base_image="python:3.10-slim-buster", extra_requires=["pandas", "dask"], ) class CreateData(DaskLoadComponent): diff --git a/tests/pipeline/test_lightweight_component.py b/tests/pipeline/test_lightweight_component.py index bb4644538..d5a4ab3be 100644 --- a/tests/pipeline/test_lightweight_component.py +++ b/tests/pipeline/test_lightweight_component.py @@ -35,7 +35,7 @@ def load_pipeline(caplog): ) @lightweight_component( - base_image="python:3.8-slim-buster", + base_image="python:3.10-slim-buster", extra_requires=["pandas", "dask"], produces={"x": pa.int32(), "y": pa.int32(), "z": pa.int32()}, ) @@ -100,7 +100,7 @@ def test_lightweight_component_sdk(default_fondant_image, load_pipeline): assert operation_spec_dict == { "specification": { "name": "CreateData", - "image": "python:3.8-slim-buster", + "image": "python:3.10-slim-buster", "description": "lightweight component", "consumes": {"additionalProperties": True}, "produces": { @@ -159,7 +159,7 @@ def transform(self, dataframe: pd.DataFrame) -> pd.DataFrame: def test_consumes_mapping_all_fields(tmp_path_factory, load_pipeline): @lightweight_component( - base_image="python:3.8", + base_image="python:3.10", extra_requires=[ "fondant[component]@git+https://github.com/ml6team/fondant@main", ], @@ -194,7 +194,7 @@ def transform(self, dataframe: pd.DataFrame) -> pd.DataFrame: def test_consumes_mapping_specific_fields(tmp_path_factory, load_pipeline): @lightweight_component( - base_image="python:3.8", + base_image="python:3.10", extra_requires=[ "fondant[component]@git+https://github.com/ml6team/fondant@main", ], @@ -231,7 +231,7 @@ def transform(self, dataframe: pd.DataFrame) -> pd.DataFrame: def test_consumes_mapping_additional_fields(tmp_path_factory, load_pipeline): @lightweight_component( - base_image="python:3.8", + base_image="python:3.10", extra_requires=[ "fondant[component]@git+https://github.com/ml6team/fondant@main", ], @@ -268,7 +268,7 @@ def transform(self, dataframe: pd.DataFrame) -> pd.DataFrame: def test_produces_mapping_additional_fields(tmp_path_factory, load_pipeline): @lightweight_component( - base_image="python:3.8", + base_image="python:3.10", extra_requires=[ "fondant[component]@git+https://github.com/ml6team/fondant@main", ], @@ -322,7 +322,7 @@ def load(self) -> str: def test_valid_load_component(): @lightweight_component( - base_image="python:3.8-slim-buster", + base_image="python:3.10-slim-buster", ) class CreateData(DaskLoadComponent): def load(self) -> dd.DataFrame: