Skip to content

Commit

Permalink
feat(sdk): add load_component_from_* (#6822)
Browse files Browse the repository at this point in the history
* feat(sdk): add load_component_from_*

* address comments'
:

* update release notes
  • Loading branch information
ji-yaqi authored Oct 29, 2021
1 parent 2e94575 commit f1bb852
Show file tree
Hide file tree
Showing 4 changed files with 140 additions and 1 deletion.
1 change: 1 addition & 0 deletions sdk/RELEASE.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

* Support passing parameters in v2 using google.protobuf.Value [\#6804](https://github.com/kubeflow/pipelines/pull/6804).
* Implement experimental v2 `@component` component [\#6825](https://github.com/kubeflow/pipelines/pull/6825)
* Add load_component_from_* for v2 [\#6822](https://github.com/kubeflow/pipelines/pull/6822)

## Breaking Changes

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
name: component_1
inputs:
input1: {type: String}
outputs:
output1: {type: String}
implementation:
container:
image: alpine
commands:
- sh
- -c
- 'set -ex
echo "$0" > "$1"'
- {inputValue: input1}
- {outputPath: output1}
38 changes: 37 additions & 1 deletion sdk/python/kfp/v2/components/experimental/yaml_component.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,12 @@
# limitations under the License.
"""Functions for loading component from yaml."""

__all__ = [
'load_component_from_text',
'load_component_from_url',
'load_component_from_file',
]

from kfp.v2.components.experimental import base_component
from kfp.v2.components.experimental import structures

Expand All @@ -27,4 +33,34 @@ def execute(self, *args, **kwargs):
def load_component_from_text(text: str) -> base_component.BaseComponent:
"""Loads component from text."""
return YamlComponent(
component_spec=structures.ComponentSpec.load_from_component_yaml(text))
structures.ComponentSpec.load_from_component_yaml(text))

def load_component_from_file(file_path: str) -> base_component.BaseComponent:
"""Loads component from file.
Args:
file_path: A string containing path to the YAML file.
"""
with open(file_path, 'rb') as component_stream:
return load_component_from_text(component_stream)

def load_component_from_url(url: str, auth=None) -> base_component.BaseComponent:
"""Loads component from url.
Args:
url: A string containing path to the url containing YAML file.
auth: The authentication credentials necessary for url access.
"""

if url is None:
raise TypeError

if url.startswith('gs://'):
#Replacing the gs:// URI with https:// URI (works for public objects)
url = 'https://storage.googleapis.com/' + url[len('gs://'):]

import requests
resp = requests.get(url, auth=auth)
resp.raise_for_status()

return load_component_from_text(resp.content)
86 changes: 86 additions & 0 deletions sdk/python/kfp/v2/components/experimental/yaml_component_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
# Copyright 2021 The Kubeflow Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for kfp.v2.components.experimental.yaml_component."""

import requests
import unittest
import textwrap

from pathlib import Path
from unittest import mock

from kfp.v2.components.experimental import yaml_component
from kfp.v2.components.experimental import structures

SAMPLE_YAML = textwrap.dedent("""\
name: component_1
inputs:
input1: {type: String}
outputs:
output1: {type: String}
implementation:
container:
image: alpine
commands:
- sh
- -c
- 'set -ex
echo "$0" > "$1"'
- {inputValue: input1}
- {outputPath: output1}
""")

class YamlComponentTest(unittest.TestCase):

def test_load_component_from_text(self):
component = yaml_component.load_component_from_text(SAMPLE_YAML)
self.assertEqual(component.component_spec.name, 'component_1')
self.assertEqual(component.component_spec.outputs, {'output1': structures.OutputSpec(type='String')})
self.assertEqual(component._component_inputs, {'input1'})
self.assertEqual(component.name, 'component_1')
self.assertEqual(component.component_spec.implementation.container.image, 'alpine')

def test_load_component_from_file(self):
component_path = Path(
__file__).parent/'test_data'/'simple_yaml.yaml'
component = yaml_component.load_component_from_file(component_path)
self.assertEqual(component.component_spec.name, 'component_1')
self.assertEqual(component.component_spec.outputs, {'output1': structures.OutputSpec(type='String')})
self.assertEqual(component._component_inputs, {'input1'})
self.assertEqual(component.name, 'component_1')
self.assertEqual(component.component_spec.implementation.container.image, 'alpine')

def test_load_component_from_url(self):
component_url = 'https://raw.githubusercontent.com/some/repo/components/component_group/component.yaml'

def mock_response_factory(url, params=None, **kwargs):
if url == component_url:
response = requests.Response()
response.url = component_url
response.status_code = 200
response._content = SAMPLE_YAML
return response
raise RuntimeError('Unexpected URL "{}"'.format(url))

with mock.patch('requests.get', mock_response_factory):
component = yaml_component.load_component_from_url(component_url)
self.assertEqual(component.component_spec.name, 'component_1')
self.assertEqual(component.component_spec.outputs, {'output1': structures.OutputSpec(type='String')})
self.assertEqual(component._component_inputs, {'input1'})
self.assertEqual(component.name, 'component_1')
self.assertEqual(component.component_spec.implementation.container.image, 'alpine')

if __name__ == '__main__':
unittest.main()

0 comments on commit f1bb852

Please sign in to comment.