Migrate to KfpV2

.github/workflows/pipeline.yaml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10']
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}

components/download_images/fondant_component.yaml

@@ -39,7 +39,7 @@ args:
   resize_only_if_bigger: 
     description: If True, resize only if image is bigger than image_size.
     type: bool
-    default: 'False'
+    default: False
   min_image_size:
     description: Minimum size of the images.
     type: int

docs/components/component_spec.md

@@ -124,8 +124,6 @@ The `args` section describes which arguments the component takes. Each argument
 `description` and a `type`, which should be one of the builtin Python types. Additionally, you can 
 set an optional `default` value for each argument.
 
-_Note:_ default iterable arguments such as `dict` and `list` have to be passed as a string 
-(e.g. `'{"foo":1, "bar":2}`, `'["foo","bar]'`)
 ```yaml
 args:
   custom_argument:

docs/pipeline.md

@@ -30,7 +30,8 @@ def build_pipeline():
             "batch_size": 2,  
             "max_new_tokens": 50,  
         },  
-        number_of_gpus=1,
+        number_of_accelerators=1,
+        accelerator_name="GPU",
         node_pool_label="node_pool",  
         node_pool_name="model-inference-pool",  
     )

examples/pipelines/controlnet-interior-design/pipeline.py

@@ -45,15 +45,17 @@
         "batch_size": 2,
         "max_new_tokens": 50,
     },
-    number_of_gpus=1,
+    number_of_accelerators=1,
+    accelerator_name="GPU",
 )
 segment_images_op = ComponentOp.from_registry(
     name="segment_images",
     arguments={
         "model_id": "openmmlab/upernet-convnext-small",
         "batch_size": 2,
     },
-    number_of_gpus=1,
+    number_of_accelerators=1,
+    accelerator_name="GPU",
 )
 
 write_to_hub_controlnet = ComponentOp(

examples/pipelines/datacomp/pipeline.py

@@ -41,64 +41,66 @@
         "dataset_name": "nielsr/datacomp-small-with-text-embeddings",
         "column_name_mapping": load_component_column_mapping,
         "index_column": "uid",
-        # "n_rows_to_load": 1000,
+        "n_rows_to_load": 1000,
+    },
+    node_pool_label="node_pool",
+    node_pool_name="n2-standard-64-pool",
+    cache=False,
+)
+download_images_op = ComponentOp.from_registry(
+    name="download_images",
+    arguments={
+        "retries": 2,
+        "min_image_size": 0,
+        "max_aspect_ratio": float("inf"),
+    },
+    node_pool_label="node_pool",
+    node_pool_name="n2-standard-64-pool",
+    input_partition_rows=1000,
+    cache=False,
+)
+detect_text_op = ComponentOp(
+    component_dir="components/detect_text",
+    arguments={
+        "batch_size": 2,
+    },
+    node_pool_label="node_pool",
+    node_pool_name="model-inference-mega-pool",
+    number_of_accelerators=1,
+    accelerator_name="GPU",
+    cache=False,
+)
+mask_images_op = ComponentOp(
+    component_dir="components/mask_images",
+    node_pool_label="node_pool",
+    node_pool_name="n2-standard-64-pool",
+    cache=False,
+)
+embed_images_op = ComponentOp.from_registry(
+    name="image_embedding",
+    arguments={
+        "batch_size": 2,
+    },
+    node_pool_label="node_pool",
+    node_pool_name="model-inference-mega-pool",
+    number_of_accelerators=1,
+    accelerator_name="GPU",
+    cache=False,
+)
+add_clip_score_op = ComponentOp(
+    component_dir="components/add_clip_score",
+    node_pool_label="node_pool",
+    node_pool_name="n2-standard-64-pool",
+    cache=False,
+)
+filter_clip_score_op = ComponentOp(
+    component_dir="components/filter_clip_score",
+    arguments={
+        "pct_threshold": 0.5,
     },
     node_pool_label="node_pool",
     node_pool_name="n2-standard-64-pool",
 )
-# download_images_op = ComponentOp.from_registry(
-#     name="download_images",
-#     arguments={
-#         "retries": 2,
-#         "min_image_size": 0,
-#         "max_aspect_ratio": float("inf"),
-#     },
-#     node_pool_label="node_pool",
-#     node_pool_name="n2-standard-64-pool",
-#     input_partition_rows=1000,
-#     cache=False,
-# )
-# detect_text_op = ComponentOp(
-#     component_dir="components/detect_text",
-#     arguments={
-#         "batch_size": 2,
-#     },
-#     node_pool_label="node_pool",
-#     node_pool_name="model-inference-mega-pool",
-#     number_of_gpus=1,
-#     cache=False,
-# )
-# mask_images_op = ComponentOp(
-#     component_dir="components/mask_images",
-#     node_pool_label="node_pool",
-#     node_pool_name="n2-standard-64-pool",
-#     cache=False,
-# )
-# embed_images_op = ComponentOp.from_registry(
-#     name="embed_images",
-#     arguments={
-#         "batch_size": 2,
-#     },
-#     node_pool_label="node_pool",
-#     node_pool_name="model-inference-mega-pool",
-#     number_of_gpus=1,
-#     cache=False,
-# )
-# add_clip_score_op = ComponentOp(
-#     component_dir="components/add_clip_score",
-#     node_pool_label="node_pool",
-#     node_pool_name="n2-standard-64-pool",
-#     cache=False,
-# )
-# filter_clip_score_op = ComponentOp(
-#     component_dir="components/filter_clip_score",
-#     arguments={
-#         "pct_threshold": 0.5,
-#     },
-#     node_pool_label="node_pool",
-#     node_pool_name="n2-standard-64-pool",
-# )
-
 
 # add ops to pipeline
 pipeline.add_op(load_from_hub_op)

examples/pipelines/finetune_stable_diffusion/pipeline.py

@@ -69,7 +69,8 @@
         "batch_size": 2,
         "max_new_tokens": 50,
     },
-    number_of_gpus=1,
+    number_of_accelerators=1,
+    accelerator_name="GPU",
 )
 
 write_to_hub = ComponentOp(
@@ -80,7 +81,8 @@
         "hf_token": "hf_token",
         "image_column_names": ["images_data"],
     },
-    number_of_gpus=1,
+    number_of_accelerators=1,
+    accelerator_name="GPU",
 )
 
 pipeline = Pipeline(

pyproject.toml

@@ -33,15 +33,14 @@ classifiers = [
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
-    "Programming Language :: Python :: 3.11",
     "Topic :: Software Development",
     "Topic :: Software Development :: Libraries",
     "Topic :: Software Development :: Libraries :: Python Modules",
     "Typing :: Typed",
 ]
 
 [tool.poetry.dependencies]
-python = ">= 3.8"
+python = ">= 3.8 < 3.11"
 dask = {extras = ["dataframe", "distributed", "diagnostics"], version = ">= 2023.4.1"}
 importlib-resources = { version = ">= 1.3", python = "<3.9" }
 jsonschema = ">= 4.18"
@@ -51,14 +50,16 @@ fsspec = { version = ">= 2023.4.0", optional = true}
 gcsfs = { version = ">= 2023.4.0", optional = true }
 s3fs = { version = ">= 2023.4.0", optional = true }
 adlfs = { version = ">= 2023.4.0", optional = true }
-kfp = { version = ">= 1.8.19, < 2", optional = true }
+kfp = { version = "2.0.1", optional = true, extras =["kubernetes"] }
 pandas = { version = ">= 1.3.5", optional = true }
+google-cloud-aiplatform =  { version = "1.32.0", optional = true}
 
 [tool.poetry.extras]
 aws = ["fsspec", "s3fs"]
 azure = ["fsspec", "adlfs"]
 gcp = ["fsspec", "gcsfs"]
 kfp = ["kfp"]
+vertex = ["kfp", "google-cloud-aiplatform"]
 
 [tool.poetry.group.test.dependencies]
 pre-commit = "^3.1.1"