From a41998f85abdc353131d10660602026bd78513c7 Mon Sep 17 00:00:00 2001 From: connor-mccarthy Date: Wed, 4 May 2022 15:26:30 -0600 Subject: [PATCH] fix traversal func --- ...eline_with_importer_dynamic_variables.yaml | 47 ++++++++++--------- sdk/python/kfp/components/importer_node.py | 44 ++++++++--------- 2 files changed, 43 insertions(+), 48 deletions(-) diff --git a/sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_importer_dynamic_variables.yaml b/sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_importer_dynamic_variables.yaml index f17edb41cb8..c091c9ac70c 100644 --- a/sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_importer_dynamic_variables.yaml +++ b/sdk/python/kfp/compiler_cli_tests/test_data/pipeline_with_importer_dynamic_variables.yaml @@ -48,8 +48,8 @@ deploymentSpec: runtimeParameter: uri metadata: containerSpec: - imageUri: "{{$.inputs.parameters['metadata_1']}}" - name: "{{$.inputs.parameters['metadata_0']}}" + imageUri: '{{$.inputs.parameters[''metadata_2'']}}' + name: '{{$.inputs.parameters[''metadata_1'']}}' typeSchema: schemaTitle: system.Artifact schemaVersion: 0.0.1 @@ -58,36 +58,37 @@ deploymentSpec: artifactUri: constant: gs://ml-pipeline-playground/shakespeare1.txt metadata: - containerSpec: {} - name: "{{$.inputs.parameters['metadata_0']}}" + containerSpec: + imageUri: us-docker.pkg.dev/vertex-ai/prediction/tf2-gpu.2-5:latest + name: '{{$.inputs.parameters[''metadata_1'']}}' typeSchema: schemaTitle: system.Artifact schemaVersion: 0.0.1 exec-make-name: container: args: - - --executor_input - - "{{$}}" - - --function_to_execute - - make_name + - --executor_input + - '{{$}}' + - --function_to_execute + - make_name command: - - sh - - -c - - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ - \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-alpha.2'\ - \ && \"$0\" \"$@\"\n" - - sh - - -ec - - 'program_path=$(mktemp -d) + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-alpha.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) - printf "%s" "$0" > "$program_path/ephemeral_component.py" + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" - ' - - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ - \ *\n\ndef make_name(name: str) -> str:\n return name\n\n" + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef make_name(name: str) -> str:\n return name\n\n" image: python:3.7 pipelineInfo: name: pipeline-with-importer @@ -115,7 +116,7 @@ root: componentRef: name: comp-importer-2 dependentTasks: - - make-name + - make-name inputs: parameters: metadata_0: diff --git a/sdk/python/kfp/components/importer_node.py b/sdk/python/kfp/components/importer_node.py index 1769af0c768..809c1246656 100644 --- a/sdk/python/kfp/components/importer_node.py +++ b/sdk/python/kfp/components/importer_node.py @@ -57,32 +57,26 @@ def traverse_dict_and_create_metadata_inputs(d: Any) -> Dict: nonlocal metadata_inputs nonlocal _metadata_counter nonlocal call_inputs - res = {} - for k, v in d.items(): - if isinstance(k, pipeline_channel.PipelineParameterChannel): - res[k] = placeholders.input_parameter_placeholder( - METADATA_KEY_PREFIX + str(_metadata_counter)) - metadata_inputs[METADATA_KEY_PREFIX + - str(_metadata_counter)] = structures.InputSpec( - type='String') - call_inputs[METADATA_KEY_PREFIX + str(_metadata_counter)] = v - _metadata_counter += 1 - if isinstance(v, dict): - res[k] = traverse_dict_and_create_metadata_inputs(v) - elif isinstance(v, list): - res[k] = [ - traverse_dict_and_create_metadata_inputs(el) for el in v - ] - elif isinstance(v, pipeline_channel.PipelineParameterChannel): - res[k] = placeholders.input_parameter_placeholder( - METADATA_KEY_PREFIX + str(_metadata_counter)) - metadata_inputs[METADATA_KEY_PREFIX + - str(_metadata_counter)] = structures.InputSpec( - type='String') - call_inputs[METADATA_KEY_PREFIX + str(_metadata_counter)] = v - _metadata_counter += 1 - return res + if isinstance(d, pipeline_channel.PipelineParameterChannel): + metadata_inputs[METADATA_KEY_PREFIX + + str(_metadata_counter)] = structures.InputSpec( + type='String') + call_inputs[METADATA_KEY_PREFIX + str(_metadata_counter)] = d + _metadata_counter += 1 + return placeholders.input_parameter_placeholder( + METADATA_KEY_PREFIX + str(_metadata_counter)) + elif isinstance(d, dict): + return { + traverse_dict_and_create_metadata_inputs(k): + traverse_dict_and_create_metadata_inputs(v) + for k, v in d.items() + } + + elif isinstance(d, list): + return [traverse_dict_and_create_metadata_inputs(el) for el in d] + else: + return d metadata_with_placeholders = traverse_dict_and_create_metadata_inputs( metadata)