Enable optional component arguments

fondant/component.py

@@ -93,7 +93,6 @@ def _add_and_parse_args(cls, spec: ComponentSpec):
         component_arguments = cls._get_component_arguments(spec)
 
         for arg in component_arguments.values():
-            # Input manifest is not required for loading component
             if arg.name in cls.optional_fondant_arguments():
                 input_required = False
                 default = None
@@ -106,7 +105,7 @@ def _add_and_parse_args(cls, spec: ComponentSpec):
 
             parser.add_argument(
                 f"--{arg.name}",
-                type=kubeflow2python_type[arg.type],  # type: ignore
+                type=kubeflow2python_type(arg.type),  # type: ignore
                 required=input_required,
                 default=default,
                 help=arg.description,

fondant/component_spec.py

@@ -17,14 +17,21 @@
 from fondant.schema import Field, KubeflowCommandArguments, Type
 
 # TODO: remove after upgrading to kfpv2
-kubeflow2python_type = {
+kubeflow_to_python_type_dict = {
     "String": str,
     "Integer": int,
     "Float": float,
     "Boolean": ast.literal_eval,
     "JsonObject": json.loads,
     "JsonArray": json.loads,
 }
+
+
+def kubeflow2python_type(type_: str) -> t.Any:
+    map_fn = kubeflow_to_python_type_dict[type_]
+    return lambda value: map_fn(value) if value != "None" else None  # type: ignore
+
+
 # TODO: Change after upgrading to kfp v2
 # :https://www.kubeflow.org/docs/components/pipelines/v2/data-types/parameters/
 python2kubeflow_type = {
@@ -47,7 +54,6 @@ class Argument:
         description: argument description
         type: the python argument type (str, int, ...)
         default: default value of the argument (defaults to None)
-
     """
 
     name: str
@@ -234,6 +240,7 @@ def from_fondant_component_spec(
                     "name": "component_spec",
                     "description": "The component specification as a dictionary",
                     "type": "JsonObject",
+                    "default": "None",
                 },
                 *(
                     {

fondant/schemas/component_spec.json

@@ -98,4 +98,4 @@
       }
     }
   }
-}
+}

tests/example_specs/component_specs/kubeflow_component.yaml

@@ -10,6 +10,7 @@ inputs:
 -   name: component_spec
     description: The component specification as a dictionary
     type: JsonObject
+    default: None
 -   name: storage_args
     description: Storage arguments
     type: String

tests/example_specs/components/arguments/component.yaml

@@ -15,10 +15,14 @@ args:
     description: default float argument
     type: float
     default: 3.14
-  bool_default_arg:
+  bool_false_default_arg:
     description: default bool argument
     type: bool
     default: 'False'
+  bool_true_default_arg:
+    description: default bool argument
+    type: bool
+    default: 'True'
   list_default_arg:
     description: default list argument
     type: list
@@ -27,7 +31,38 @@ args:
     description: default dict argument
     type: dict
     default: '{"foo":1, "bar":2}'
-  override_default_string_arg:
-    description: default argument that can be overriden
+  string_default_arg_none:
+    description: default string argument
+    type: str
+    default: None
+  integer_default_arg_none:
+    description: default integer argument
+    type: int
+    default: None
+  float_default_arg_none:
+    description: default float argument
+    type: float
+    default: None
+  bool_default_arg_none:
+    description: default bool argument
+    type: bool
+    default: None
+  list_default_arg_none:
+    description: default list argument
+    type: list
+    default: None
+  dict_default_arg_none:
+    description: default dict argument
+    type: dict
+    default: None
+  override_default_arg:
+    description: argument with default python value type that can be overriden
     type: str
     default: foo
+  override_default_none_arg:
+    description: argument with default None value type that can be overriden with a valid python type
+    type: float
+    default: None
+  override_default_arg_with_none:
+    description: argument with default python type that can be overriden with None
+    type: str

tests/example_specs/components/arguments/component_default_args.yaml

@@ -0,0 +1,69 @@
+name: Example component
+description: This is an example component
+image: example_component:latest
+
+args:
+  string_default_arg:
+    description: default string argument
+    type: str
+    default: foo
+  integer_default_arg:
+    description: default integer argument
+    type: int
+    default: 1
+  float_default_arg:
+    description: default float argument
+    type: float
+    default: 3.14
+  bool_false_default_arg:
+    description: default bool argument
+    type: bool
+    default: 'False'
+  bool_true_default_arg:
+    description: default bool argument
+    type: bool
+    default: 'True'
+  list_default_arg:
+    description: default list argument
+    type: list
+    default: '["foo", "bar"]'
+  dict_default_arg:
+    description: default dict argument
+    type: dict
+    default: '{"foo":1, "bar":2}'
+  string_default_arg_none:
+    description: default string argument
+    type: str
+    default: None
+  integer_default_arg_none:
+    description: default integer argument
+    type: int
+    default: None
+  float_default_arg_none:
+    description: default float argument
+    type: float
+    default: None
+  bool_default_arg_none:
+    description: default bool argument
+    type: bool
+    default: None
+  list_default_arg_none:
+    description: default list argument
+    type: list
+    default: None
+  dict_default_arg_none:
+    description: default dict argument
+    type: dict
+    default: None
+  override_default_arg:
+    description: argument with default python value type that can be overriden
+    type: str
+    default: foo
+  override_default_none_arg:
+    description: argument with default None value type that can be overriden with a valid python type
+    type: float
+    default: None
+  override_default_arg_with_none:
+    description: argument with default python type that can be overriden with None
+    type: str
+

tests/test_component.py

@@ -61,12 +61,16 @@ def test_component_arguments():
         str(components_path / "arguments/input_manifest.json"),
         "--metadata",
         "{}",
-        "--override_default_string_arg",
-        "bar",
         "--output_manifest_path",
         str(components_path / "arguments/output_manifest.json"),
         "--component_spec",
         yaml_file_to_json_string(components_path / "arguments/component.yaml"),
+        "--override_default_arg",
+        "bar",
+        "--override_default_none_arg",
+        "3.14",
+        "--override_default_arg_with_none",
+        "None",
     ]
 
     class MyComponent(Component):
@@ -83,10 +87,19 @@ def _process_dataset(self, manifest: Manifest) -> t.Union[None, dd.DataFrame]:
         "string_default_arg": "foo",
         "integer_default_arg": 1,
         "float_default_arg": 3.14,
-        "bool_default_arg": False,
+        "bool_false_default_arg": False,
+        "bool_true_default_arg": True,
         "list_default_arg": ["foo", "bar"],
         "dict_default_arg": {"foo": 1, "bar": 2},
-        "override_default_string_arg": "bar",
+        "string_default_arg_none": None,
+        "integer_default_arg_none": None,
+        "float_default_arg_none": None,
+        "bool_default_arg_none": None,
+        "list_default_arg_none": None,
+        "dict_default_arg_none": None,
+        "override_default_arg": "bar",
+        "override_default_none_arg": 3.14,
+        "override_default_arg_with_none": None,
     }