Skip to content

Commit

Permalink
fix(sdk): fix load_test (kubeflow#6978)
Browse files Browse the repository at this point in the history 
* fix(sdk): fix load_test

* concat placeholder

* release notes
  • Loading branch information
@ji-yaqi @abaland
ji-yaqi authored and abaland committed May 29, 2022
1 parent c29ef8a commit c2e1a46
Show file tree
Hide file tree
Showing 5 changed files with 118 additions and 129 deletions.
12 changes: 5 additions & 7 deletions sdk/python/kfp/v2/compiler_cli_tests/compiler_cli_tests.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,15 +107,13 @@ def test_pipeline_with_importer(self):
# def test_pipeline_with_ontology(self):
# self._test_compile_py_to_json('pipeline_with_ontology')

# TODO: re-enable the test, debug load_component_from_file error
# def test_pipeline_with_if_placeholder(self):
# self._test_compile_py_to_json('pipeline_with_if_placeholder')
def test_pipeline_with_if_placeholder(self):
self._test_compile_py_to_json('pipeline_with_if_placeholder')

# TODO: re-enable the test, debug load_component_from_file error
# def test_pipeline_with_concat_placeholder(self):
# self._test_compile_py_to_json('pipeline_with_concat_placeholder')
def test_pipeline_with_concat_placeholder(self):
self._test_compile_py_to_json('pipeline_with_concat_placeholder')

# TODO: re-enable the test, debug load_component_from_file error
# TODO: re-enable the test, debug add_node_selector_constraint error
# def test_pipeline_with_resource_spec(self):
# self._test_compile_py_to_json('pipeline_with_resource_spec')

Expand Down
94 changes: 45 additions & 49 deletions sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_concat_placeholder.json
View file
Original file line number Diff line number Diff line change
@@ -1,63 +1,59 @@
{
"pipelineSpec": {
"components": {
"comp-component-with-concat-placeholder": {
"executorLabel": "exec-component-with-concat-placeholder",
"inputDefinitions": {
"parameters": {
"input_prefix": {
"parameterType": "STRING"
}
"components": {
"comp-component-with-concat-placeholder": {
"executorLabel": "exec-component-with-concat-placeholder",
"inputDefinitions": {
"parameters": {
"input_prefix": {
"parameterType": "STRING"
}
}
}
},
"deploymentSpec": {
"executors": {
"exec-component-with-concat-placeholder": {
"container": {
"args": [
"--arg0",
"{{$.inputs.parameters['input_prefix']}}some value"
],
"image": "gcr.io/my-project/my-image"
}
}
},
"defaultPipelineRoot": "dummy_root",
"deploymentSpec": {
"executors": {
"exec-component-with-concat-placeholder": {
"container": {
"args": [
"--arg0",
"{{$.inputs.parameters['input_prefix']}}some value"
],
"image": "gcr.io/my-project/my-image"
}
}
},
"pipelineInfo": {
"name": "one-step-pipeline-with-concat-placeholder"
},
"root": {
"dag": {
"tasks": {
"component-with-concat-placeholder": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-component-with-concat-placeholder"
},
"inputs": {
"parameters": {
"input_prefix": {
"runtimeValue": {
"constant": "some prefix:"
}
}
},
"pipelineInfo": {
"name": "one-step-pipeline-with-concat-placeholder"
},
"root": {
"dag": {
"tasks": {
"component-with-concat-placeholder": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-component-with-concat-placeholder"
},
"inputs": {
"parameters": {
"input_prefix": {
"runtimeValue": {
"constant": "some prefix:"
}
}
},
"taskInfo": {
"name": "component-with-concat-placeholder"
}
},
"taskInfo": {
"name": "component-with-concat-placeholder"
}
}
}
},
"schemaVersion": "2.1.0",
"sdkVersion": "kfp-1.8.6"
}
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root"
}
"schemaVersion": "2.1.0",
"sdkVersion": "kfp-1.8.9"
}
134 changes: 65 additions & 69 deletions sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_if_placeholder.json
View file
Original file line number Diff line number Diff line change
@@ -1,84 +1,80 @@
{
"pipelineSpec": {
"components": {
"comp-component-with-optional-inputs": {
"executorLabel": "exec-component-with-optional-inputs",
"inputDefinitions": {
"parameters": {
"optional_input_1": {
"parameterType": "STRING"
},
"required_input": {
"parameterType": "STRING"
}
"components": {
"comp-component-with-optional-inputs": {
"executorLabel": "exec-component-with-optional-inputs",
"inputDefinitions": {
"parameters": {
"optional_input_1": {
"parameterType": "STRING"
},
"required_input": {
"parameterType": "STRING"
}
}
}
},
"deploymentSpec": {
"executors": {
"exec-component-with-optional-inputs": {
"container": {
"args": [
"--arg0",
"{{$.inputs.parameters['required_input']}}",
"--arg1",
"{{$.inputs.parameters['optional_input_1']}}",
"--arg3",
"default value"
],
"image": "gcr.io/my-project/my-image"
}
}
},
"defaultPipelineRoot": "dummy_root",
"deploymentSpec": {
"executors": {
"exec-component-with-optional-inputs": {
"container": {
"args": [
"--arg0",
"{{$.inputs.parameters['required_input']}}",
"--arg1",
"{{$.inputs.parameters['optional_input_1']}}",
"--arg3",
"default value"
],
"image": "gcr.io/my-project/my-image"
}
}
},
"pipelineInfo": {
"name": "one-step-pipeline-with-if-placeholder"
},
"root": {
"dag": {
"tasks": {
"component-with-optional-inputs": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-component-with-optional-inputs"
},
"inputs": {
"parameters": {
"optional_input_1": {
"componentInputParameter": "input1"
},
"required_input": {
"componentInputParameter": "input0"
}
}
},
"pipelineInfo": {
"name": "one-step-pipeline-with-if-placeholder"
},
"root": {
"dag": {
"tasks": {
"component-with-optional-inputs": {
"cachingOptions": {
"enableCache": true
},
"componentRef": {
"name": "comp-component-with-optional-inputs"
},
"inputs": {
"parameters": {
"optional_input_1": {
"componentInputParameter": "input1"
},
"required_input": {
"componentInputParameter": "input0"
}
},
"taskInfo": {
"name": "component-with-optional-inputs"
}
}
}
},
"inputDefinitions": {
"parameters": {
"input0": {
"parameterType": "STRING"
},
"input1": {
"parameterType": "STRING"
},
"input2": {
"parameterType": "STRING"
"taskInfo": {
"name": "component-with-optional-inputs"
}
}
}
},
"schemaVersion": "2.1.0",
"sdkVersion": "kfp-1.8.6"
"inputDefinitions": {
"parameters": {
"input0": {
"parameterType": "STRING"
},
"input1": {
"parameterType": "STRING"
},
"input2": {
"parameterType": "STRING"
}
}
}
},
"runtimeConfig": {
"gcsOutputDirectory": "dummy_root"
}
"schemaVersion": "2.1.0",
"sdkVersion": "kfp-1.8.9"
}
5 changes: 2 additions & 3 deletions sdk/python/kfp/v2/components/structures.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -373,8 +373,8 @@ def _check_valid_placeholder_reference(cls, valid_inputs: Sequence[str],
if arg.if_structure.input_name not in valid_inputs:
raise ValueError(
f'Argument "{arg}" references non-existing input.')
for placeholder in itertools.chain(arg.if_structure.then,
arg.if_structure.otherwise):
for placeholder in itertools.chain(arg.if_structure.then or [],
arg.if_structure.otherwise or []):
cls._check_valid_placeholder_reference(valid_inputs,
valid_outputs,
placeholder)
Expand Down Expand Up @@ -573,7 +573,6 @@ def load_from_component_yaml(cls, component_yaml: str) -> 'ComponentSpec':
Returns:
Component spec in the form of V2 ComponentSpec.
"""

json_component = yaml.safe_load(component_yaml)
try:
return ComponentSpec.parse_obj(json_component)
Expand Down
2 changes: 1 addition & 1 deletion sdk/python/kfp/v2/components/yaml_component.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ def load_component_from_file(file_path: str) -> base_component.BaseComponent:
file_path: A string containing path to the YAML file.
"""
with open(file_path, 'rb') as component_stream:
return load_component_from_text(component_stream)
return load_component_from_text(component_stream.read())


def load_component_from_url(url: str,
Expand Down

0 comments on commit c2e1a46

Please sign in to comment.