diff --git a/docs/conf.py b/docs/conf.py index 4fb3cebb314..f00b5f3ed0c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,7 +58,16 @@ 'sphinx_click', 'm2r2', 'sphinx_immaterial', + 'autodocsumm', ] +autodoc_member_order = 'bysource' +autodoc_default_options = { + 'members': True, + 'imported-members': True, + 'undoc-members': True, + 'show-inheritance': False, + 'autosummary': True, +} html_theme = 'sphinx_immaterial' html_title = 'KFP SDK API Reference' @@ -79,7 +88,7 @@ 'edit_uri': 'blob/master/docs', 'globaltoc_collapse': - True, + False, 'features': [ 'navigation.expand', # "navigation.tabs", diff --git a/docs/index.rst b/docs/index.rst index bdeeed634e2..fcf778a8927 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,11 +3,8 @@ Kubeflow Pipelines SDK API Reference .. mdinclude:: ../sdk/python/README.md -.. maxdepth and titlesonly settings ensure TOC items don't expand recursively, bloating the TOC (only applies if :hidden: is removed) .. toctree:: :caption: Contents - :maxdepth: 1 - :titlesonly: :hidden: Home diff --git a/docs/requirements.txt b/docs/requirements.txt index de221453bba..0e1274a496e 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,4 @@ +autodocsumm==0.2.9 sdk/python sphinx==5.0.2 sphinx-click==4.3.0 diff --git a/docs/source/client.rst b/docs/source/client.rst index 5a7b852adb2..36711caeeb0 100644 --- a/docs/source/client.rst +++ b/docs/source/client.rst @@ -2,7 +2,3 @@ kfp.client ========================== .. automodule:: kfp.client - :members: - :undoc-members: - :show-inheritance: - :noindex: diff --git a/docs/source/compiler.rst b/docs/source/compiler.rst index 2e89b6a7b73..1c5ce9118ed 100644 --- a/docs/source/compiler.rst +++ b/docs/source/compiler.rst @@ -1,8 +1,5 @@ kfp.compiler ========================== +.. glossary:: .. automodule:: kfp.compiler - :members: - :undoc-members: - :show-inheritance: - :noindex: diff --git a/docs/source/components.rst b/docs/source/components.rst index 43b93b56c17..13f01ae7a45 100644 --- a/docs/source/components.rst +++ b/docs/source/components.rst @@ -2,7 +2,3 @@ kfp.components ========================== .. automodule:: kfp.components - :members: - :undoc-members: - :show-inheritance: - :noindex: diff --git a/docs/source/dsl.rst b/docs/source/dsl.rst index 05fad48d561..5ffcd59309f 100644 --- a/docs/source/dsl.rst +++ b/docs/source/dsl.rst @@ -2,10 +2,3 @@ kfp.dsl ========================== .. automodule:: kfp.dsl - :members: - :undoc-members: - :show-inheritance: - :noindex: - - .. autodata:: Input - .. autodata:: Output diff --git a/docs/source/kfp.rst b/docs/source/kfp.rst index a9b79a6c770..01df01fbc9d 100644 --- a/docs/source/kfp.rst +++ b/docs/source/kfp.rst @@ -6,6 +6,6 @@ API Reference dsl compiler - client components + client registry diff --git a/docs/source/registry.rst b/docs/source/registry.rst index 8a9b966dc46..e69fab9a965 100644 --- a/docs/source/registry.rst +++ b/docs/source/registry.rst @@ -2,7 +2,3 @@ kfp.registry ========================== .. automodule:: kfp.registry - :members: - :undoc-members: - :show-inheritance: - :noindex: diff --git a/sdk/python/kfp/components/__init__.py b/sdk/python/kfp/components/__init__.py index 02ee9300c21..3f183b62675 100644 --- a/sdk/python/kfp/components/__init__.py +++ b/sdk/python/kfp/components/__init__.py @@ -15,9 +15,9 @@ # limitations under the License. __all__ = [ - 'load_component_from_text', 'load_component_from_file', 'load_component_from_url', + 'load_component_from_text', 'PythonComponent', 'BaseComponent', 'ContainerComponent', diff --git a/sdk/python/kfp/components/executor_test.py b/sdk/python/kfp/components/executor_test.py index 303c050ee5d..ea5c9e6149d 100644 --- a/sdk/python/kfp/components/executor_test.py +++ b/sdk/python/kfp/components/executor_test.py @@ -27,10 +27,10 @@ from kfp.components.types.artifact_types import Dataset from kfp.components.types.artifact_types import Metrics from kfp.components.types.artifact_types import Model -from kfp.components.types.type_annotations import Input from kfp.components.types.type_annotations import InputPath -from kfp.components.types.type_annotations import Output from kfp.components.types.type_annotations import OutputPath +from kfp.dsl import Input +from kfp.dsl import Output class ExecutorTest(unittest.TestCase): diff --git a/sdk/python/kfp/components/types/custom_artifact_types_test.py b/sdk/python/kfp/components/types/custom_artifact_types_test.py index c4de629df45..ed856db23b0 100644 --- a/sdk/python/kfp/components/types/custom_artifact_types_test.py +++ b/sdk/python/kfp/components/types/custom_artifact_types_test.py @@ -28,10 +28,10 @@ from kfp.components.types import custom_artifact_types from kfp.components.types.artifact_types import Artifact from kfp.components.types.artifact_types import Dataset -from kfp.components.types.type_annotations import Input from kfp.components.types.type_annotations import InputPath -from kfp.components.types.type_annotations import Output from kfp.components.types.type_annotations import OutputPath +from kfp.dsl import Input +from kfp.dsl import Output Alias = Artifact artifact_types_alias = artifact_types diff --git a/sdk/python/kfp/components/types/type_annotations.py b/sdk/python/kfp/components/types/type_annotations.py index 74ed3170786..595526074d2 100644 --- a/sdk/python/kfp/components/types/type_annotations.py +++ b/sdk/python/kfp/components/types/type_annotations.py @@ -22,13 +22,6 @@ from kfp.components.types import type_annotations from kfp.components.types import type_utils -try: - from typing import Annotated -except ImportError: - from typing_extensions import Annotated - -T = TypeVar('T') - class OutputPath: """Type annotation used in component definitions for indicating a parameter @@ -119,67 +112,14 @@ def construct_type_for_inputpath_or_outputpath( return type_ -class InputAnnotation(): +class InputAnnotation: """Marker type for input artifacts.""" -class OutputAnnotation(): +class OutputAnnotation: """Marker type for output artifacts.""" -Input = Annotated[T, InputAnnotation] -Input.__doc__ = """Type generic used to represent an input artifact of type ``T``, where ``T`` is an artifact class. - -Use ``Input[Artifact]`` or ``Output[Artifact]`` to indicate whether the enclosed artifact is a component input or output. - -Args: - T: The type of the input artifact. - -Example: - :: - - @dsl.component - def artifact_producer(model: Output[Artifact]): - with open(model.path, 'w') as f: - f.write('my model') - - @dsl.component - def artifact_consumer(model: Input[Artifact]): - print(model) - - @dsl.pipeline(name='my-pipeline') - def my_pipeline(): - producer_task = artifact_producer() - artifact_consumer(model=producer_task.output) -""" - -Output = Annotated[T, OutputAnnotation] -Output.__doc__ = """A type generic used to represent an output artifact of type ``T``, where ``T`` is an artifact class. The argument typed with this annotation is provided at runtime by the executing backend and does not need to be passed as an input by the pipeline author (see example). - -Use ``Input[Artifact]`` or ``Output[Artifact]`` to indicate whether the enclosed artifact is a component input or output. - -Args: - T: The type of the output artifact. - -Example: - :: - - @dsl.component - def artifact_producer(model: Output[Artifact]): - with open(model.path, 'w') as f: - f.write('my model') - - @dsl.component - def artifact_consumer(model: Input[Artifact]): - print(model) - - @dsl.pipeline(name='my-pipeline') - def my_pipeline(): - producer_task = artifact_producer() - artifact_consumer(model=producer_task.output) -""" - - def is_artifact_annotation(typ) -> bool: if not hasattr(typ, '__metadata__'): return False @@ -207,6 +147,8 @@ def is_output_artifact(typ) -> bool: def get_io_artifact_class(typ): + from kfp.dsl import Input + from kfp.dsl import Output if not is_artifact_annotation(typ): return None if typ == Input or typ == Output: @@ -222,6 +164,9 @@ def get_io_artifact_annotation(typ): return typ.__metadata__[0] +T = TypeVar('T') + + def maybe_strip_optional_from_annotation(annotation: T) -> T: """Strips 'Optional' from 'Optional[]' if applicable. diff --git a/sdk/python/kfp/components/types/type_annotations_test.py b/sdk/python/kfp/components/types/type_annotations_test.py index abd5b1680e3..f311a610361 100644 --- a/sdk/python/kfp/components/types/type_annotations_test.py +++ b/sdk/python/kfp/components/types/type_annotations_test.py @@ -20,12 +20,12 @@ from kfp.components.types import artifact_types from kfp.components.types import type_annotations from kfp.components.types.artifact_types import Model -from kfp.components.types.type_annotations import Input from kfp.components.types.type_annotations import InputAnnotation from kfp.components.types.type_annotations import InputPath -from kfp.components.types.type_annotations import Output from kfp.components.types.type_annotations import OutputAnnotation from kfp.components.types.type_annotations import OutputPath +from kfp.dsl import Input +from kfp.dsl import Output class AnnotationsTest(parameterized.TestCase): diff --git a/sdk/python/kfp/dsl/__init__.py b/sdk/python/kfp/dsl/__init__.py index 2ebcc081e18..895308593db 100644 --- a/sdk/python/kfp/dsl/__init__.py +++ b/sdk/python/kfp/dsl/__init__.py @@ -17,35 +17,42 @@ __all__ = [ 'component', 'container_component', - 'ContainerSpec', - 'importer', 'pipeline', - 'PipelineTask', - 'PipelineTaskFinalStatus', + 'importer', + 'ContainerSpec', 'Condition', 'ExitHandler', 'ParallelFor', - 'Artifact', - 'ClassificationMetrics', - 'Dataset', - 'HTML', - 'Markdown', - 'Metrics', - 'Model', - 'SlicedClassificationMetrics', 'Input', 'Output', 'InputPath', 'OutputPath', + 'IfPresentPlaceholder', + 'ConcatPlaceholder', + 'PipelineTaskFinalStatus', 'PIPELINE_JOB_NAME_PLACEHOLDER', 'PIPELINE_JOB_RESOURCE_NAME_PLACEHOLDER', 'PIPELINE_JOB_ID_PLACEHOLDER', 'PIPELINE_TASK_NAME_PLACEHOLDER', 'PIPELINE_TASK_ID_PLACEHOLDER', - 'IfPresentPlaceholder', - 'ConcatPlaceholder', + 'Artifact', + 'ClassificationMetrics', + 'Dataset', + 'HTML', + 'Markdown', + 'Metrics', + 'Model', + 'SlicedClassificationMetrics', + 'PipelineTask', ] +try: + from typing import Annotated +except ImportError: + from typing_extensions import Annotated + +from typing import TypeVar + from kfp.components.component_decorator import component from kfp.components.container_component_decorator import container_component from kfp.components.importer_node import importer @@ -66,11 +73,13 @@ from kfp.components.types.artifact_types import Metrics from kfp.components.types.artifact_types import Model from kfp.components.types.artifact_types import SlicedClassificationMetrics -from kfp.components.types.type_annotations import Input +from kfp.components.types.type_annotations import InputAnnotation from kfp.components.types.type_annotations import InputPath -from kfp.components.types.type_annotations import Output +from kfp.components.types.type_annotations import OutputAnnotation from kfp.components.types.type_annotations import OutputPath +# hack: constants and custom type generics have to be defined here to be captured by autodoc and autodocsumm used in ./docs/conf.py + PIPELINE_JOB_NAME_PLACEHOLDER = '{{$.pipeline_job_name}}' """A placeholder used to obtain a pipeline job name within a task at pipeline runtime. @@ -140,3 +149,55 @@ def my_pipeline(): value=dsl.PIPELINE_TASK_ID_PLACEHOLDER, ) """ +T = TypeVar('T') +Input = Annotated[T, InputAnnotation] +"""Type generic used to represent an input artifact of type ``T``, where ``T`` is an artifact class. + +Use ``Input[Artifact]`` or ``Output[Artifact]`` to indicate whether the enclosed artifact is a component input or output. + +Args: + T: The type of the input artifact. + +Example: + :: + + @dsl.component + def artifact_producer(model: Output[Artifact]): + with open(model.path, 'w') as f: + f.write('my model') + + @dsl.component + def artifact_consumer(model: Input[Artifact]): + print(model) + + @dsl.pipeline(name='my-pipeline') + def my_pipeline(): + producer_task = artifact_producer() + artifact_consumer(model=producer_task.output) +""" + +Output = Annotated[T, OutputAnnotation] +"""A type generic used to represent an output artifact of type ``T``, where ``T`` is an artifact class. The argument typed with this annotation is provided at runtime by the executing backend and does not need to be passed as an input by the pipeline author (see example). + +Use ``Input[Artifact]`` or ``Output[Artifact]`` to indicate whether the enclosed artifact is a component input or output. + +Args: + T: The type of the output artifact. + +Example: + :: + + @dsl.component + def artifact_producer(model: Output[Artifact]): + with open(model.path, 'w') as f: + f.write('my model') + + @dsl.component + def artifact_consumer(model: Input[Artifact]): + print(model) + + @dsl.pipeline(name='my-pipeline') + def my_pipeline(): + producer_task = artifact_producer() + artifact_consumer(model=producer_task.output) +""" diff --git a/sdk/python/kfp/registry/__init__.py b/sdk/python/kfp/registry/__init__.py index 401f00ff533..997d3e66b62 100644 --- a/sdk/python/kfp/registry/__init__.py +++ b/sdk/python/kfp/registry/__init__.py @@ -15,8 +15,8 @@ # limitations under the License. __all__ = [ - 'ApiAuth', 'RegistryClient', + 'ApiAuth', ] from kfp.registry.registry_client import ApiAuth from kfp.registry.registry_client import RegistryClient