diff --git a/sdk/python/kfp/__init__.py b/sdk/python/kfp/__init__.py index a72b6c3e13c..33c61dae084 100644 --- a/sdk/python/kfp/__init__.py +++ b/sdk/python/kfp/__init__.py @@ -18,3 +18,7 @@ __version__ = '2.0.0b0' +TYPE_CHECK = True +# COMPILING_FOR_V2 is True when using kfp.v2.compiler or use (v1) kfp.compiler +# with V2_COMPATIBLE or V2_ENGINE mode +COMPILING_FOR_V2 = False diff --git a/sdk/python/kfp/deprecated/dsl/__init__.py b/sdk/python/kfp/deprecated/dsl/__init__.py index 705c4b2f801..5adc0414443 100644 --- a/sdk/python/kfp/deprecated/dsl/__init__.py +++ b/sdk/python/kfp/deprecated/dsl/__init__.py @@ -20,7 +20,7 @@ from ._pipeline_volume import PipelineVolume from ._volume_snapshot_op import VolumeSnapshotOp from ._ops_group import OpsGroup, ExitHandler, Condition, ParallelFor, SubGraph -from ._component import python_component, graph_component, component +from ._component import graph_component, component def importer(*args, **kwargs): diff --git a/sdk/python/kfp/deprecated/dsl/_component.py b/sdk/python/kfp/deprecated/dsl/_component.py index b490ad51f06..e104641a1d5 100644 --- a/sdk/python/kfp/deprecated/dsl/_component.py +++ b/sdk/python/kfp/deprecated/dsl/_component.py @@ -20,54 +20,54 @@ import kfp.deprecated as kfp -@deprecated( - version='0.2.6', - reason='This decorator does not seem to be used, so we deprecate it. ' - 'If you need this decorator, please create an issue at ' - 'https://github.com/kubeflow/pipelines/issues', -) -def python_component(name, - description=None, - base_image=None, - target_component_file: str = None): - """Decorator for Python component functions. - - This decorator adds the metadata to the function object itself. - - Args: - name: Human-readable name of the component - description: Optional. Description of the component - base_image: Optional. Docker container image to use as the base of the - component. Needs to have Python 3.5+ installed. - target_component_file: Optional. Local file to store the component - definition. The file can then be used for sharing. - - Returns: - The same function (with some metadata fields set). - - Example: - :: - - @dsl.python_component( - name='my awesome component', - description='Come, Let\'s play', - base_image='tensorflow/tensorflow:1.11.0-py3', - ) - def my_component(a: str, b: int) -> str: - ... - """ - - def _python_component(func): - func._component_human_name = name - if description: - func._component_description = description - if base_image: - func._component_base_image = base_image - if target_component_file: - func._component_target_component_file = target_component_file - return func - - return _python_component +# @deprecated( +# version='0.2.6', +# reason='This decorator does not seem to be used, so we deprecate it. ' +# 'If you need this decorator, please create an issue at ' +# 'https://github.com/kubeflow/pipelines/issues', +# ) +# def python_component(name, +# description=None, +# base_image=None, +# target_component_file: str = None): +# """Decorator for Python component functions. + +# This decorator adds the metadata to the function object itself. + +# Args: +# name: Human-readable name of the component +# description: Optional. Description of the component +# base_image: Optional. Docker container image to use as the base of the +# component. Needs to have Python 3.5+ installed. +# target_component_file: Optional. Local file to store the component +# definition. The file can then be used for sharing. + +# Returns: +# The same function (with some metadata fields set). + +# Example: +# :: + +# @dsl.python_component( +# name='my awesome component', +# description='Come, Let\'s play', +# base_image='tensorflow/tensorflow:1.11.0-py3', +# ) +# def my_component(a: str, b: int) -> str: +# ... +# """ + +# def _python_component(func): +# func._component_human_name = name +# if description: +# func._component_description = description +# if base_image: +# func._component_base_image = base_image +# if target_component_file: +# func._component_target_component_file = target_component_file +# return func + +# return _python_component def component(func): diff --git a/sdk/python/kfp/deprecated/dsl/_component_bridge.py b/sdk/python/kfp/deprecated/dsl/_component_bridge.py index 1243d627494..730d253c819 100644 --- a/sdk/python/kfp/deprecated/dsl/_component_bridge.py +++ b/sdk/python/kfp/deprecated/dsl/_component_bridge.py @@ -19,7 +19,6 @@ import pathlib from typing import Any, Mapping, Optional -# import kfp.deprecated as kfp from kfp.deprecated._config import COMPILING_FOR_V2 from kfp.deprecated.components import _structures, _data_passing from kfp.deprecated.components import _components diff --git a/sdk/python/tests/dsl/component_tests.py b/sdk/python/tests/dsl/component_tests.py index 8b135ce8640..0ee0c85f946 100644 --- a/sdk/python/tests/dsl/component_tests.py +++ b/sdk/python/tests/dsl/component_tests.py @@ -23,729 +23,7 @@ from kfp.deprecated.dsl import ContainerOp, Pipeline, PipelineParam from kfp.deprecated.components.structures import ComponentSpec, InputSpec, OutputSpec - -class TestPythonComponent(unittest.TestCase): - - def test_component_metadata(self): - """Test component decorator metadata.""" - - class MockContainerOp: - - def _set_metadata(self, component_meta): - self._metadata = component_meta - - @component - def componentA( - a: {'ArtifactA': { - 'file_type': 'csv' - }}, - b: Integer() = 12, - c: {'ArtifactB': { - 'path_type': 'file', - 'file_type': 'tsv' - }} = 'gs://hello/world' - ) -> { - 'model': Integer() - }: - return MockContainerOp() - - containerOp = componentA(1, 2, c=3) - - golden_meta = ComponentSpec(name='ComponentA', inputs=[], outputs=[]) - golden_meta.inputs.append( - InputSpec(name='a', type={'ArtifactA': { - 'file_type': 'csv' - }})) - golden_meta.inputs.append( - InputSpec( - name='b', - type={ - 'Integer': { - 'openapi_schema_validator': { - "type": "integer" - } - } - }, - default="12", - optional=True)) - golden_meta.inputs.append( - InputSpec( - name='c', - type={'ArtifactB': { - 'path_type': 'file', - 'file_type': 'tsv' - }}, - default='gs://hello/world', - optional=True)) - golden_meta.outputs.append( - OutputSpec( - name='model', - type={ - 'Integer': { - 'openapi_schema_validator': { - "type": "integer" - } - } - })) - - self.assertEqual(containerOp._metadata, golden_meta) - - def test_component_metadata_standard_type_annotation(self): - """Test component decorator metadata.""" - - class MockContainerOp: - - def _set_metadata(self, component_meta): - self._metadata = component_meta - - @component - def componentA(a: float, b: List[int], c: Optional[str] = None) -> None: - return MockContainerOp() - - containerOp = componentA('str_value', '[1,2,3]') - - golden_meta = ComponentSpec(name='ComponentA', inputs=[], outputs=None) - golden_meta.inputs.append(InputSpec(name='a', type='Float')) - golden_meta.inputs.append( - InputSpec( - name='b', - type='typing.List[int]' if sys.version_info >= - (3, 7) else 'List')) - golden_meta.inputs.append( - InputSpec(name='c', type='String', default=None, optional=True)) - - self.assertEqual(containerOp._metadata, golden_meta) - - def test_python_component_decorator(self): - # Deprecated - from kfp.deprecated.dsl import python_component - from kfp.deprecated.components import create_component_from_func - - expected_name = 'Sum component name' - expected_description = 'Sum component description' - expected_image = 'org/image' - - @python_component( - name=expected_name, - description=expected_description, - base_image=expected_image) - def add_two_numbers_decorated( - a: float, - b: float, - ) -> float: - """Returns sum of two arguments.""" - return a + b - - op = create_component_from_func(add_two_numbers_decorated) - - component_spec = op.component_spec - self.assertEqual(component_spec.name, expected_name) - self.assertEqual(component_spec.description.strip(), - expected_description.strip()) - self.assertEqual(component_spec.implementation.container.image, - expected_image) - - def test_type_check_with_same_representation(self): - """Test type check at the decorator.""" - kfp.TYPE_CHECK = True - - @component - def a_op(field_l: Integer()) -> { - 'field_m': GCSPath(), - 'field_n': { - 'customized_type': { - 'property_a': 'value_a', - 'property_b': 'value_b' - } - }, - 'field_o': 'GcsUri' - }: - return ContainerOp( - name='operator a', - image='gcr.io/ml-pipeline/component-a', - arguments=[ - '--field-l', - field_l, - ], - file_outputs={ - 'field_m': '/schema.txt', - 'field_n': '/feature.txt', - 'field_o': '/output.txt' - }) - - @component - def b_op( - field_x: { - 'customized_type': { - 'property_a': 'value_a', - 'property_b': 'value_b' - } - }, - field_y: 'GcsUri', # noqa: F821 TODO - field_z: GCSPath() - ) -> { - 'output_model_uri': 'GcsUri' - }: - return ContainerOp( - name='operator b', - image='gcr.io/ml-pipeline/component-b', - command=[ - 'python3', - field_x, - ], - arguments=[ - '--field-y', - field_y, - '--field-z', - field_z, - ], - file_outputs={ - 'output_model_uri': '/schema.txt', - }) - - a = a_op(field_l=12) - b = b_op( - field_x=a.outputs['field_n'], - field_y=a.outputs['field_o'], - field_z=a.outputs['field_m']) - - def test_type_check_with_different_represenation(self): - """Test type check at the decorator.""" - kfp.TYPE_CHECK = True - - @component - def a_op(field_l: Integer()) -> { - 'field_m': 'GCSPath', - 'field_n': { - 'customized_type': { - 'property_a': 'value_a', - 'property_b': 'value_b' - } - }, - 'field_o': 'Integer' - }: - return ContainerOp( - name='operator a', - image='gcr.io/ml-pipeline/component-b', - arguments=[ - '--field-l', - field_l, - ], - file_outputs={ - 'field_m': '/schema.txt', - 'field_n': '/feature.txt', - 'field_o': '/output.txt' - }) - - @component - def b_op( - field_x: { - 'customized_type': { - 'property_a': 'value_a', - 'property_b': 'value_b' - } - }, field_y: Integer(), field_z: GCSPath() - ) -> { - 'output_model_uri': 'GcsUri' - }: - return ContainerOp( - name='operator b', - image='gcr.io/ml-pipeline/component-a', - command=[ - 'python3', - field_x, - ], - arguments=[ - '--field-y', - field_y, - '--field-z', - field_z, - ], - file_outputs={ - 'output_model_uri': '/schema.txt', - }) - - a = a_op(field_l=12) - b = b_op( - field_x=a.outputs['field_n'], - field_y=a.outputs['field_o'], - field_z=a.outputs['field_m']) - - def test_type_check_with_inconsistent_types_property_value(self): - """Test type check at the decorator.""" - kfp.TYPE_CHECK = True - - @component - def a_op(field_l: Integer()) -> { - 'field_m': { - 'ArtifactB': { - 'path_type': 'file', - 'file_type': 'tsv' - } - }, - 'field_n': { - 'customized_type': { - 'property_a': 'value_a', - 'property_b': 'value_b' - } - }, - 'field_o': 'Integer' - }: - return ContainerOp( - name='operator a', - image='gcr.io/ml-pipeline/component-b', - arguments=[ - '--field-l', - field_l, - ], - file_outputs={ - 'field_m': '/schema.txt', - 'field_n': '/feature.txt', - 'field_o': '/output.txt' - }) - - @component - def b_op( - field_x: { - 'customized_type': { - 'property_a': 'value_a', - 'property_b': 'value_b' - } - }, field_y: Integer(), - field_z: {'ArtifactB': { - 'path_type': 'file', - 'file_type': 'csv' - }} - ) -> { - 'output_model_uri': 'GcsUri' - }: - return ContainerOp( - name='operator b', - image='gcr.io/ml-pipeline/component-a', - command=[ - 'python3', - field_x, - ], - arguments=[ - '--field-y', - field_y, - '--field-z', - field_z, - ], - file_outputs={ - 'output_model_uri': '/schema.txt', - }) - - with self.assertRaises(InconsistentTypeException): - a = a_op(field_l=12) - b = b_op( - field_x=a.outputs['field_n'], - field_y=a.outputs['field_o'], - field_z=a.outputs['field_m']) - - def test_type_check_with_inconsistent_types_type_name(self): - """Test type check at the decorator.""" - kfp.TYPE_CHECK = True - - @component - def a_op(field_l: Integer()) -> { - 'field_m': { - 'ArtifactB': { - 'path_type': 'file', - 'file_type': 'tsv' - } - }, - 'field_n': { - 'customized_type': { - 'property_a': 'value_a', - 'property_b': 'value_b' - } - }, - 'field_o': 'Integer' - }: - return ContainerOp( - name='operator a', - image='gcr.io/ml-pipeline/component-b', - arguments=[ - '--field-l', - field_l, - ], - file_outputs={ - 'field_m': '/schema.txt', - 'field_n': '/feature.txt', - 'field_o': '/output.txt' - }) - - @component - def b_op( - field_x: { - 'customized_type_a': { - 'property_a': 'value_a', - 'property_b': 'value_b' - } - }, field_y: Integer(), - field_z: {'ArtifactB': { - 'path_type': 'file', - 'file_type': 'tsv' - }} - ) -> { - 'output_model_uri': 'GcsUri' - }: - return ContainerOp( - name='operator b', - image='gcr.io/ml-pipeline/component-a', - command=[ - 'python3', - field_x, - ], - arguments=[ - '--field-y', - field_y, - '--field-z', - field_z, - ], - file_outputs={ - 'output_model_uri': '/schema.txt', - }) - - with self.assertRaises(InconsistentTypeException): - a = a_op(field_l=12) - b = b_op( - field_x=a.outputs['field_n'], - field_y=a.outputs['field_o'], - field_z=a.outputs['field_m']) - - def test_type_check_without_types(self): - """Test type check at the decorator.""" - kfp.TYPE_CHECK = True - - @component - def a_op(field_l: Integer()) -> { - 'field_m': { - 'ArtifactB': { - 'path_type': 'file', - 'file_type': 'tsv' - } - }, - 'field_n': { - 'customized_type': { - 'property_a': 'value_a', - 'property_b': 'value_b' - } - } - }: - return ContainerOp( - name='operator a', - image='gcr.io/ml-pipeline/component-b', - arguments=[ - '--field-l', - field_l, - ], - file_outputs={ - 'field_m': '/schema.txt', - 'field_n': '/feature.txt', - 'field_o': '/output.txt' - }) - - @component - def b_op( - field_x, field_y: Integer(), - field_z: {'ArtifactB': { - 'path_type': 'file', - 'file_type': 'tsv' - }} - ) -> { - 'output_model_uri': 'GcsUri' - }: - return ContainerOp( - name='operator b', - image='gcr.io/ml-pipeline/component-a', - command=[ - 'python3', - field_x, - ], - arguments=[ - '--field-y', - field_y, - '--field-z', - field_z, - ], - file_outputs={ - 'output_model_uri': '/schema.txt', - }) - - a = a_op(field_l=12) - b = b_op( - field_x=a.outputs['field_n'], - field_y=a.outputs['field_o'], - field_z=a.outputs['field_m']) - - def test_type_check_nonnamed_inputs(self): - """Test type check at the decorator.""" - kfp.TYPE_CHECK = True - - @component - def a_op(field_l: Integer()) -> { - 'field_m': { - 'ArtifactB': { - 'path_type': 'file', - 'file_type': 'tsv' - } - }, - 'field_n': { - 'customized_type': { - 'property_a': 'value_a', - 'property_b': 'value_b' - } - } - }: - return ContainerOp( - name='operator a', - image='gcr.io/ml-pipeline/component-b', - arguments=[ - '--field-l', - field_l, - ], - file_outputs={ - 'field_m': '/schema.txt', - 'field_n': '/feature.txt', - 'field_o': '/output.txt' - }) - - @component - def b_op( - field_x, field_y: Integer(), - field_z: {'ArtifactB': { - 'path_type': 'file', - 'file_type': 'tsv' - }} - ) -> { - 'output_model_uri': 'GcsUri' - }: - return ContainerOp( - name='operator b', - image='gcr.io/ml-pipeline/component-a', - command=[ - 'python3', - field_x, - ], - arguments=[ - '--field-y', - field_y, - '--field-z', - field_z, - ], - file_outputs={ - 'output_model_uri': '/schema.txt', - }) - - a = a_op(field_l=12) - b = b_op( - a.outputs['field_n'], - field_z=a.outputs['field_m'], - field_y=a.outputs['field_o']) - - def test_type_check_with_inconsistent_types_disabled(self): - """Test type check at the decorator.""" - kfp.TYPE_CHECK = False - - @component - def a_op(field_l: Integer()) -> { - 'field_m': { - 'ArtifactB': { - 'path_type': 'file', - 'file_type': 'tsv' - } - }, - 'field_n': { - 'customized_type': { - 'property_a': 'value_a', - 'property_b': 'value_b' - } - }, - 'field_o': 'Integer' - }: - return ContainerOp( - name='operator a', - image='gcr.io/ml-pipeline/component-b', - arguments=[ - '--field-l', - field_l, - ], - file_outputs={ - 'field_m': '/schema.txt', - 'field_n': '/feature.txt', - 'field_o': '/output.txt' - }) - - @component - def b_op( - field_x: { - 'customized_type_a': { - 'property_a': 'value_a', - 'property_b': 'value_b' - } - }, field_y: Integer(), - field_z: {'ArtifactB': { - 'path_type': 'file', - 'file_type': 'tsv' - }} - ) -> { - 'output_model_uri': 'GcsUri' - }: - return ContainerOp( - name='operator b', - image='gcr.io/ml-pipeline/component-a', - command=[ - 'python3', - field_x, - ], - arguments=[ - '--field-y', - field_y, - '--field-z', - field_z, - ], - file_outputs={ - 'output_model_uri': '/schema.txt', - }) - - a = a_op(field_l=12) - b = b_op( - field_x=a.outputs['field_n'], - field_y=a.outputs['field_o'], - field_z=a.outputs['field_m']) - - def test_type_check_with_openapi_schema(self): - """Test type check at the decorator.""" - kfp.TYPE_CHECK = True - - @component - def a_op(field_l: Integer()) -> { - 'field_m': 'GCSPath', - 'field_n': { - 'customized_type': { - 'openapi_schema_validator': - '{"type": "string", "pattern": "^gs://.*$"}' - } - }, - 'field_o': 'Integer' - }: - return ContainerOp( - name='operator a', - image='gcr.io/ml-pipeline/component-b', - arguments=[ - '--field-l', - field_l, - ], - file_outputs={ - 'field_m': '/schema.txt', - 'field_n': '/feature.txt', - 'field_o': '/output.txt' - }) - - @component - def b_op( - field_x: { - 'customized_type': { - 'openapi_schema_validator': - '{"type": "string", "pattern": "^gs://.*$"}' - } - }, field_y: Integer(), field_z: GCSPath() - ) -> { - 'output_model_uri': 'GcsUri' - }: - return ContainerOp( - name='operator b', - image='gcr.io/ml-pipeline/component-a', - command=[ - 'python3', - field_x, - ], - arguments=[ - '--field-y', - field_y, - '--field-z', - field_z, - ], - file_outputs={ - 'output_model_uri': '/schema.txt', - }) - - a = a_op(field_l=12) - b = b_op( - field_x=a.outputs['field_n'], - field_y=a.outputs['field_o'], - field_z=a.outputs['field_m']) - - def test_type_check_with_ignore_type(self): - """Test type check at the decorator.""" - kfp.TYPE_CHECK = True - - @component - def a_op(field_l: Integer()) -> { - 'field_m': 'GCSPath', - 'field_n': { - 'customized_type': { - 'openapi_schema_validator': - '{"type": "string", "pattern": "^gs://.*$"}' - } - }, - 'field_o': 'Integer' - }: - return ContainerOp( - name='operator a', - image='gcr.io/ml-pipeline/component-b', - arguments=[ - '--field-l', - field_l, - ], - file_outputs={ - 'field_m': '/schema.txt', - 'field_n': '/feature.txt', - 'field_o': '/output.txt' - }) - - @component - def b_op( - field_x: { - 'customized_type': { - 'openapi_schema_validator': - '{"type": "string", "pattern": "^gcs://.*$"}' - } - }, field_y: Integer(), field_z: GCSPath() - ) -> { - 'output_model_uri': 'GcsUri' - }: - return ContainerOp( - name='operator b', - image='gcr.io/ml-pipeline/component-a', - command=[ - 'python3', - field_x, - ], - arguments=[ - '--field-y', - field_y, - '--field-z', - field_z, - ], - file_outputs={ - 'output_model_uri': '/schema.txt', - }) - - a = a_op(field_l=12) - with self.assertRaises(InconsistentTypeException): - b = b_op( - field_x=a.outputs['field_n'], - field_y=a.outputs['field_o'], - field_z=a.outputs['field_m']) - b = b_op( - field_x=a.outputs['field_n'].ignore_type(), - field_y=a.outputs['field_o'], - field_z=a.outputs['field_m']) - - +@unittest.skip("deprecated") class TestGraphComponent(unittest.TestCase): def test_graphcomponent_basic(self):