Add dask and docker as default dependencies (#893)

Fixes:
- #891

 Fixes (maybe needs more work):
- #877

README.md

@@ -114,12 +114,7 @@ pip install fondant
 ```
 
 Fondant also includes extra dependencies for specific runners, storage integrations and publishing 
-components to registries. 
-We can install the local runner to enable local pipeline execution:
-
-```
-pip install fondant[docker]
-```
+components to registries. The dependencies for the local runner (docker) is included by default.
 
 For more detailed installation options, check the [**installation page**](https://fondant.ai/en/latest/guides/installation/)on our documentation.
 

docs/guides/installation.md

@@ -51,13 +51,7 @@ Fondant also includes extra dependencies for specific runners, storage integrati
 
 ### Publishing components dependencies
 
-For publishing components to registries: 
-
-```bash
-pip install fondant[docker]
-```
-
-Check out the [guide](../components/publishing_components.md) on publishing components to registries.
+For publishing components to registries check out the [guide](../components/publishing_components.md) on publishing components to registries.
 
 ### Runner specific dependencies
 

examples/sample_pipeline/components/dummy_component/Dockerfile

@@ -11,7 +11,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 
 # Install fondant
 ARG FONDANT_VERSION=main
-RUN pip3 install fondant[component,aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN pip3 install fondant[aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component/src

images/Dockerfile

@@ -8,5 +8,5 @@ RUN apt-get update && \
 
 # Install Fondant
 ARG FONDANT_VERSION=main
-RUN pip3 install fondant[component,aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN pip3 install fondant[aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 

pyproject.toml

@@ -49,33 +49,32 @@ jsonschema = ">= 4.18"
 pyarrow = ">= 11.0.0"
 pyyaml = ">= 5.3.1"
 
-dask = { version = ">= 2023.4.1", extras = ["dataframe", "distributed", "diagnostics"], optional = true }
+dask = { version = ">= 2023.4.1", extras = ["dataframe", "distributed", "diagnostics"]}
+docker = ">= 6.1.3"
+
 dask-cuda = { version = ">=23.4.1", optional = true }
 
 gcsfs = { version = ">= 2023.10.0", optional = true }
 s3fs = { version = ">= 2023.4.0", optional = true }
 adlfs = { version = ">= 2023.4.0", optional = true }
 
-docker = {version = ">= 6.1.3", optional = true }
 kfp = { version = "2.6.0", optional = true, extras =["kubernetes"] }
 google-cloud-aiplatform =  { version = "1.34.0", optional = true}
 sagemaker = {version = ">= 2.197.0", optional = true}
 boto3 = {version = "1.28.64", optional = true}
 
 [tool.poetry.extras]
-component = ["dask"]
 gpu = ["dask-cuda"]
 
 aws = ["s3fs"]
 azure = ["adlfs"]
 gcp = ["gcsfs"]
 
-kfp = ["docker", "kfp"]
-vertex = ["docker", "kfp", "google-cloud-aiplatform"]
+kfp = ["kfp"]
+vertex = ["kfp", "google-cloud-aiplatform"]
 sagemaker = ["sagemaker", "boto3"]
-docker = ["docker"]
 
-all = ["dask", "dask-cuda", "s3fs", "adlfs", "gcsfs", "docker", "kfp", "google-cloud-aiplatform",
+all = ["dask-cuda", "s3fs", "adlfs", "gcsfs", "kfp", "google-cloud-aiplatform",
     "sagemaker", "boto3"]
 
 [tool.poetry.group.test]

src/fondant/build.py

@@ -1,11 +1,14 @@
 """Module holding implementation to build Fondant components, used by the `fondant build`
 command.
 """
+
 import logging
 import re
 import typing as t
 from pathlib import Path
 
+import docker
+
 from fondant.pipeline import ComponentOp
 
 logger = logging.getLogger(__name__)
@@ -22,17 +25,6 @@ def build_component(  # ruff: noqa: PLR0912, PLR0915
     pull: bool = False,
     target: t.Optional[str] = None,
 ) -> None:
-    try:
-        import docker
-    except ImportError:
-        msg = (
-            "You need to install `docker` to use the `fondant build` command, you can install "
-            "it with `pip install fondant[docker]`"
-        )
-        raise SystemExit(
-            msg,
-        )
-
     component_op = ComponentOp.from_component_yaml(component_dir)
     component_spec = component_op.component_spec
 

src/fondant/component/__init__.py

@@ -1,19 +1,6 @@
-try:
-    pass
-except ImportError:
-    msg = (
-        "You need to install fondant using the `component` extra to develop or run a component."
-        "You can install it with `pip install fondant[component]`"
-    )
-    raise SystemExit(
-        msg,
-    )
+# fmt: off
+from .component import (BaseComponent, Component, DaskLoadComponent,  # noqa
+                        DaskTransformComponent, DaskWriteComponent,
+                        PandasTransformComponent)
 
-from .component import (  # noqa
-    BaseComponent,
-    Component,
-    DaskLoadComponent,
-    DaskTransformComponent,
-    DaskWriteComponent,
-    PandasTransformComponent,
-)
+# fmt: on

src/fondant/components/caption_images/Dockerfile

@@ -12,7 +12,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=main
-RUN pip3 install fondant[component,aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN pip3 install fondant[aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component

src/fondant/components/chunk_text/Dockerfile

@@ -12,7 +12,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=main
-RUN pip3 install fondant[component,aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN pip3 install fondant[aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component

src/fondant/components/crop_images/Dockerfile

@@ -12,7 +12,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=main
-RUN pip3 install fondant[component,aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN pip3 install fondant[aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component/src

src/fondant/components/download_images/Dockerfile

@@ -12,7 +12,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=main
-RUN pip3 install fondant[component,aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN pip3 install fondant[aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component

src/fondant/components/embed_images/Dockerfile

@@ -12,7 +12,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=main
-RUN pip3 install fondant[component,gpu,aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN pip3 install fondant[gpu,aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component/src

src/fondant/components/embed_text/Dockerfile

@@ -12,7 +12,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=main
-RUN pip3 install fondant[component,aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN pip3 install fondant[aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component

src/fondant/components/extract_image_resolution/Dockerfile

@@ -12,7 +12,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=main
-RUN pip3 install fondant[component,aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN pip3 install fondant[aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component/src

src/fondant/components/filter_image_resolution/Dockerfile

@@ -12,7 +12,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=main
-RUN pip3 install fondant[component,aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN pip3 install fondant[aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component/src

src/fondant/components/filter_language/Dockerfile

@@ -12,7 +12,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=main
-RUN pip3 install fondant[component,aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN pip3 install fondant[aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component

src/fondant/components/filter_text_length/Dockerfile

@@ -12,7 +12,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=main
-RUN pip3 install fondant[component,aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN pip3 install fondant[aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component

src/fondant/components/generate_minhash/Dockerfile

@@ -12,7 +12,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=main
-RUN pip3 install fondant[component,aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN pip3 install fondant[aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component

src/fondant/components/index_aws_opensearch/Dockerfile

@@ -12,7 +12,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=main
-RUN pip3 install fondant[component,aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN pip3 install fondant[aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component

src/fondant/components/index_qdrant/Dockerfile

@@ -12,7 +12,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=main
-RUN pip3 install fondant[component,aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN pip3 install fondant[aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component

src/fondant/components/index_weaviate/Dockerfile

@@ -12,7 +12,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=main
-RUN pip3 install fondant[component,aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN pip3 install fondant[aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component

src/fondant/components/load_from_csv/Dockerfile

@@ -8,7 +8,7 @@ RUN apt-get update && \
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=main
-RUN pip3 install fondant[component,aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN pip3 install fondant[aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component

src/fondant/components/load_from_files/Dockerfile

@@ -12,7 +12,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=main
-RUN pip3 install fondant[component,aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN pip3 install fondant[aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component

src/fondant/components/load_from_hf_hub/Dockerfile

@@ -12,7 +12,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=main
-RUN pip3 install fondant[component,aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN pip3 install fondant[aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component/src

src/fondant/components/load_from_parquet/Dockerfile

@@ -12,7 +12,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=main
-RUN pip3 install fondant[component,aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN pip3 install fondant[aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component/src

src/fondant/components/load_from_pdf/Dockerfile

@@ -12,7 +12,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=main
-RUN pip3 install fondant[component,aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN pip3 install fondant[aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component

src/fondant/components/resize_images/Dockerfile

@@ -12,7 +12,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=main
-RUN pip3 install fondant[component,aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN pip3 install fondant[aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component/src

src/fondant/components/retrieve_laion_by_embedding/Dockerfile

@@ -12,7 +12,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=main
-RUN pip3 install fondant[component,aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN pip3 install fondant[aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component

src/fondant/components/retrieve_laion_by_prompt/Dockerfile

@@ -12,7 +12,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=main
-RUN pip3 install fondant[component,aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN pip3 install fondant[aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component

src/fondant/components/segment_images/Dockerfile

@@ -12,7 +12,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=main
-RUN pip3 install fondant[component,aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN pip3 install fondant[aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component/src

src/fondant/components/write_to_file/Dockerfile

@@ -12,7 +12,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=main
-RUN pip3 install fondant[component,aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN pip3 install fondant[aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component

src/fondant/components/write_to_hf_hub/Dockerfile

@@ -12,7 +12,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 # Install Fondant
 # This is split from other requirements to leverage caching
 ARG FONDANT_VERSION=main
-RUN pip3 install fondant[component,aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+RUN pip3 install fondant[aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder
 WORKDIR /component/src