From 0a2c3f6db14afe0630181089df4d8b01ed0958a8 Mon Sep 17 00:00:00 2001 From: connor-mccarthy Date: Thu, 6 Oct 2022 12:59:00 -0600 Subject: [PATCH 1/7] use autodoc default options --- docs/conf.py | 7 +++++++ docs/source/client.rst | 4 ---- docs/source/compiler.rst | 5 +---- docs/source/components.rst | 4 ---- docs/source/dsl.rst | 8 ++------ docs/source/kfp.rst | 2 +- docs/source/registry.rst | 4 ---- 7 files changed, 11 insertions(+), 23 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 4fb3cebb314..59abcc8006b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -59,6 +59,13 @@ 'm2r2', 'sphinx_immaterial', ] +autodoc_member_order = 'bysource' +autodoc_default_options = { + 'members': True, + 'imported-members': True, + 'undoc-members': True, + 'show-inheritance': False, +} html_theme = 'sphinx_immaterial' html_title = 'KFP SDK API Reference' diff --git a/docs/source/client.rst b/docs/source/client.rst index 5a7b852adb2..36711caeeb0 100644 --- a/docs/source/client.rst +++ b/docs/source/client.rst @@ -2,7 +2,3 @@ kfp.client ========================== .. automodule:: kfp.client - :members: - :undoc-members: - :show-inheritance: - :noindex: diff --git a/docs/source/compiler.rst b/docs/source/compiler.rst index 2e89b6a7b73..1c5ce9118ed 100644 --- a/docs/source/compiler.rst +++ b/docs/source/compiler.rst @@ -1,8 +1,5 @@ kfp.compiler ========================== +.. glossary:: .. automodule:: kfp.compiler - :members: - :undoc-members: - :show-inheritance: - :noindex: diff --git a/docs/source/components.rst b/docs/source/components.rst index 43b93b56c17..13f01ae7a45 100644 --- a/docs/source/components.rst +++ b/docs/source/components.rst @@ -2,7 +2,3 @@ kfp.components ========================== .. automodule:: kfp.components - :members: - :undoc-members: - :show-inheritance: - :noindex: diff --git a/docs/source/dsl.rst b/docs/source/dsl.rst index 05fad48d561..6191fea6ad3 100644 --- a/docs/source/dsl.rst +++ b/docs/source/dsl.rst @@ -2,10 +2,6 @@ kfp.dsl ========================== .. automodule:: kfp.dsl - :members: - :undoc-members: - :show-inheritance: - :noindex: - .. autodata:: Input - .. autodata:: Output + .. autodata:: Input + .. autodata:: Output diff --git a/docs/source/kfp.rst b/docs/source/kfp.rst index a9b79a6c770..01df01fbc9d 100644 --- a/docs/source/kfp.rst +++ b/docs/source/kfp.rst @@ -6,6 +6,6 @@ API Reference dsl compiler - client components + client registry diff --git a/docs/source/registry.rst b/docs/source/registry.rst index 8a9b966dc46..e69fab9a965 100644 --- a/docs/source/registry.rst +++ b/docs/source/registry.rst @@ -2,7 +2,3 @@ kfp.registry ========================== .. automodule:: kfp.registry - :members: - :undoc-members: - :show-inheritance: - :noindex: From 155eaa2cb30eb93614a8d42aacde47f8692fc806 Mon Sep 17 00:00:00 2001 From: connor-mccarthy Date: Thu, 6 Oct 2022 13:00:11 -0600 Subject: [PATCH 2/7] sort symbol order by typical usage patterns --- sdk/python/kfp/components/__init__.py | 2 +- sdk/python/kfp/dsl/__init__.py | 28 +++++++++++++-------------- sdk/python/kfp/registry/__init__.py | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/sdk/python/kfp/components/__init__.py b/sdk/python/kfp/components/__init__.py index 02ee9300c21..3f183b62675 100644 --- a/sdk/python/kfp/components/__init__.py +++ b/sdk/python/kfp/components/__init__.py @@ -15,9 +15,9 @@ # limitations under the License. __all__ = [ - 'load_component_from_text', 'load_component_from_file', 'load_component_from_url', + 'load_component_from_text', 'PythonComponent', 'BaseComponent', 'ContainerComponent', diff --git a/sdk/python/kfp/dsl/__init__.py b/sdk/python/kfp/dsl/__init__.py index 2ebcc081e18..66335e18f6d 100644 --- a/sdk/python/kfp/dsl/__init__.py +++ b/sdk/python/kfp/dsl/__init__.py @@ -17,33 +17,33 @@ __all__ = [ 'component', 'container_component', - 'ContainerSpec', - 'importer', 'pipeline', - 'PipelineTask', - 'PipelineTaskFinalStatus', + 'importer', + 'ContainerSpec', 'Condition', 'ExitHandler', 'ParallelFor', - 'Artifact', - 'ClassificationMetrics', - 'Dataset', - 'HTML', - 'Markdown', - 'Metrics', - 'Model', - 'SlicedClassificationMetrics', 'Input', 'Output', 'InputPath', 'OutputPath', + 'IfPresentPlaceholder', + 'ConcatPlaceholder', + 'PipelineTaskFinalStatus', 'PIPELINE_JOB_NAME_PLACEHOLDER', 'PIPELINE_JOB_RESOURCE_NAME_PLACEHOLDER', 'PIPELINE_JOB_ID_PLACEHOLDER', 'PIPELINE_TASK_NAME_PLACEHOLDER', 'PIPELINE_TASK_ID_PLACEHOLDER', - 'IfPresentPlaceholder', - 'ConcatPlaceholder', + 'Artifact', + 'ClassificationMetrics', + 'Dataset', + 'HTML', + 'Markdown', + 'Metrics', + 'Model', + 'SlicedClassificationMetrics', + 'PipelineTask', ] from kfp.components.component_decorator import component diff --git a/sdk/python/kfp/registry/__init__.py b/sdk/python/kfp/registry/__init__.py index 401f00ff533..997d3e66b62 100644 --- a/sdk/python/kfp/registry/__init__.py +++ b/sdk/python/kfp/registry/__init__.py @@ -15,8 +15,8 @@ # limitations under the License. __all__ = [ - 'ApiAuth', 'RegistryClient', + 'ApiAuth', ] from kfp.registry.registry_client import ApiAuth from kfp.registry.registry_client import RegistryClient From 17573506369e2cb5951853c7a2c55f44a8193ee5 Mon Sep 17 00:00:00 2001 From: connor-mccarthy Date: Thu, 6 Oct 2022 13:00:50 -0600 Subject: [PATCH 3/7] include summary of submodule symbols --- docs/conf.py | 2 ++ docs/requirements.txt | 1 + 2 files changed, 3 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index 59abcc8006b..fb69f790dd0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,6 +58,7 @@ 'sphinx_click', 'm2r2', 'sphinx_immaterial', + 'autodocsumm', ] autodoc_member_order = 'bysource' autodoc_default_options = { @@ -65,6 +66,7 @@ 'imported-members': True, 'undoc-members': True, 'show-inheritance': False, + 'autosummary': True, } html_theme = 'sphinx_immaterial' diff --git a/docs/requirements.txt b/docs/requirements.txt index de221453bba..0e1274a496e 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,4 @@ +autodocsumm==0.2.9 sdk/python sphinx==5.0.2 sphinx-click==4.3.0 From 21787efe6cd2c9fe438ddfcbc5a821ba40c41744 Mon Sep 17 00:00:00 2001 From: connor-mccarthy Date: Thu, 6 Oct 2022 13:13:29 -0600 Subject: [PATCH 4/7] expose Input[] and Output[] type generics --- docs/source/dsl.rst | 3 - .../kfp/components/types/type_annotations.py | 63 +----------------- sdk/python/kfp/dsl/__init__.py | 65 ++++++++++++++++++- 3 files changed, 66 insertions(+), 65 deletions(-) diff --git a/docs/source/dsl.rst b/docs/source/dsl.rst index 6191fea6ad3..5ffcd59309f 100644 --- a/docs/source/dsl.rst +++ b/docs/source/dsl.rst @@ -2,6 +2,3 @@ kfp.dsl ========================== .. automodule:: kfp.dsl - - .. autodata:: Input - .. autodata:: Output diff --git a/sdk/python/kfp/components/types/type_annotations.py b/sdk/python/kfp/components/types/type_annotations.py index 74ed3170786..c12c0224178 100644 --- a/sdk/python/kfp/components/types/type_annotations.py +++ b/sdk/python/kfp/components/types/type_annotations.py @@ -22,13 +22,6 @@ from kfp.components.types import type_annotations from kfp.components.types import type_utils -try: - from typing import Annotated -except ImportError: - from typing_extensions import Annotated - -T = TypeVar('T') - class OutputPath: """Type annotation used in component definitions for indicating a parameter @@ -127,59 +120,6 @@ class OutputAnnotation(): """Marker type for output artifacts.""" -Input = Annotated[T, InputAnnotation] -Input.__doc__ = """Type generic used to represent an input artifact of type ``T``, where ``T`` is an artifact class. - -Use ``Input[Artifact]`` or ``Output[Artifact]`` to indicate whether the enclosed artifact is a component input or output. - -Args: - T: The type of the input artifact. - -Example: - :: - - @dsl.component - def artifact_producer(model: Output[Artifact]): - with open(model.path, 'w') as f: - f.write('my model') - - @dsl.component - def artifact_consumer(model: Input[Artifact]): - print(model) - - @dsl.pipeline(name='my-pipeline') - def my_pipeline(): - producer_task = artifact_producer() - artifact_consumer(model=producer_task.output) -""" - -Output = Annotated[T, OutputAnnotation] -Output.__doc__ = """A type generic used to represent an output artifact of type ``T``, where ``T`` is an artifact class. The argument typed with this annotation is provided at runtime by the executing backend and does not need to be passed as an input by the pipeline author (see example). - -Use ``Input[Artifact]`` or ``Output[Artifact]`` to indicate whether the enclosed artifact is a component input or output. - -Args: - T: The type of the output artifact. - -Example: - :: - - @dsl.component - def artifact_producer(model: Output[Artifact]): - with open(model.path, 'w') as f: - f.write('my model') - - @dsl.component - def artifact_consumer(model: Input[Artifact]): - print(model) - - @dsl.pipeline(name='my-pipeline') - def my_pipeline(): - producer_task = artifact_producer() - artifact_consumer(model=producer_task.output) -""" - - def is_artifact_annotation(typ) -> bool: if not hasattr(typ, '__metadata__'): return False @@ -222,6 +162,9 @@ def get_io_artifact_annotation(typ): return typ.__metadata__[0] +T = TypeVar('T') + + def maybe_strip_optional_from_annotation(annotation: T) -> T: """Strips 'Optional' from 'Optional[]' if applicable. diff --git a/sdk/python/kfp/dsl/__init__.py b/sdk/python/kfp/dsl/__init__.py index 66335e18f6d..895308593db 100644 --- a/sdk/python/kfp/dsl/__init__.py +++ b/sdk/python/kfp/dsl/__init__.py @@ -46,6 +46,13 @@ 'PipelineTask', ] +try: + from typing import Annotated +except ImportError: + from typing_extensions import Annotated + +from typing import TypeVar + from kfp.components.component_decorator import component from kfp.components.container_component_decorator import container_component from kfp.components.importer_node import importer @@ -66,11 +73,13 @@ from kfp.components.types.artifact_types import Metrics from kfp.components.types.artifact_types import Model from kfp.components.types.artifact_types import SlicedClassificationMetrics -from kfp.components.types.type_annotations import Input +from kfp.components.types.type_annotations import InputAnnotation from kfp.components.types.type_annotations import InputPath -from kfp.components.types.type_annotations import Output +from kfp.components.types.type_annotations import OutputAnnotation from kfp.components.types.type_annotations import OutputPath +# hack: constants and custom type generics have to be defined here to be captured by autodoc and autodocsumm used in ./docs/conf.py + PIPELINE_JOB_NAME_PLACEHOLDER = '{{$.pipeline_job_name}}' """A placeholder used to obtain a pipeline job name within a task at pipeline runtime. @@ -140,3 +149,55 @@ def my_pipeline(): value=dsl.PIPELINE_TASK_ID_PLACEHOLDER, ) """ +T = TypeVar('T') +Input = Annotated[T, InputAnnotation] +"""Type generic used to represent an input artifact of type ``T``, where ``T`` is an artifact class. + +Use ``Input[Artifact]`` or ``Output[Artifact]`` to indicate whether the enclosed artifact is a component input or output. + +Args: + T: The type of the input artifact. + +Example: + :: + + @dsl.component + def artifact_producer(model: Output[Artifact]): + with open(model.path, 'w') as f: + f.write('my model') + + @dsl.component + def artifact_consumer(model: Input[Artifact]): + print(model) + + @dsl.pipeline(name='my-pipeline') + def my_pipeline(): + producer_task = artifact_producer() + artifact_consumer(model=producer_task.output) +""" + +Output = Annotated[T, OutputAnnotation] +"""A type generic used to represent an output artifact of type ``T``, where ``T`` is an artifact class. The argument typed with this annotation is provided at runtime by the executing backend and does not need to be passed as an input by the pipeline author (see example). + +Use ``Input[Artifact]`` or ``Output[Artifact]`` to indicate whether the enclosed artifact is a component input or output. + +Args: + T: The type of the output artifact. + +Example: + :: + + @dsl.component + def artifact_producer(model: Output[Artifact]): + with open(model.path, 'w') as f: + f.write('my model') + + @dsl.component + def artifact_consumer(model: Input[Artifact]): + print(model) + + @dsl.pipeline(name='my-pipeline') + def my_pipeline(): + producer_task = artifact_producer() + artifact_consumer(model=producer_task.output) +""" From 403ae4928c3b6a2b68f01ea319b51fdac191aff1 Mon Sep 17 00:00:00 2001 From: connor-mccarthy Date: Thu, 6 Oct 2022 13:14:10 -0600 Subject: [PATCH 5/7] include tables of contents on API reference docs pages --- docs/index.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index bdeeed634e2..fcf778a8927 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,11 +3,8 @@ Kubeflow Pipelines SDK API Reference .. mdinclude:: ../sdk/python/README.md -.. maxdepth and titlesonly settings ensure TOC items don't expand recursively, bloating the TOC (only applies if :hidden: is removed) .. toctree:: :caption: Contents - :maxdepth: 1 - :titlesonly: :hidden: Home From 621fca23b04d857b6272cbb79720e32e828c63bf Mon Sep 17 00:00:00 2001 From: connor-mccarthy Date: Thu, 6 Oct 2022 13:19:06 -0600 Subject: [PATCH 6/7] expand global TOC for discoverability --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index fb69f790dd0..f00b5f3ed0c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -88,7 +88,7 @@ 'edit_uri': 'blob/master/docs', 'globaltoc_collapse': - True, + False, 'features': [ 'navigation.expand', # "navigation.tabs", From 279963244b158e7f7bb2d542af3f46fab7e8bf19 Mon Sep 17 00:00:00 2001 From: connor-mccarthy Date: Thu, 6 Oct 2022 13:45:38 -0600 Subject: [PATCH 7/7] fix missing reference / circular import problem --- sdk/python/kfp/components/executor_test.py | 4 ++-- .../kfp/components/types/custom_artifact_types_test.py | 4 ++-- sdk/python/kfp/components/types/type_annotations.py | 6 ++++-- sdk/python/kfp/components/types/type_annotations_test.py | 4 ++-- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/sdk/python/kfp/components/executor_test.py b/sdk/python/kfp/components/executor_test.py index 303c050ee5d..ea5c9e6149d 100644 --- a/sdk/python/kfp/components/executor_test.py +++ b/sdk/python/kfp/components/executor_test.py @@ -27,10 +27,10 @@ from kfp.components.types.artifact_types import Dataset from kfp.components.types.artifact_types import Metrics from kfp.components.types.artifact_types import Model -from kfp.components.types.type_annotations import Input from kfp.components.types.type_annotations import InputPath -from kfp.components.types.type_annotations import Output from kfp.components.types.type_annotations import OutputPath +from kfp.dsl import Input +from kfp.dsl import Output class ExecutorTest(unittest.TestCase): diff --git a/sdk/python/kfp/components/types/custom_artifact_types_test.py b/sdk/python/kfp/components/types/custom_artifact_types_test.py index c4de629df45..ed856db23b0 100644 --- a/sdk/python/kfp/components/types/custom_artifact_types_test.py +++ b/sdk/python/kfp/components/types/custom_artifact_types_test.py @@ -28,10 +28,10 @@ from kfp.components.types import custom_artifact_types from kfp.components.types.artifact_types import Artifact from kfp.components.types.artifact_types import Dataset -from kfp.components.types.type_annotations import Input from kfp.components.types.type_annotations import InputPath -from kfp.components.types.type_annotations import Output from kfp.components.types.type_annotations import OutputPath +from kfp.dsl import Input +from kfp.dsl import Output Alias = Artifact artifact_types_alias = artifact_types diff --git a/sdk/python/kfp/components/types/type_annotations.py b/sdk/python/kfp/components/types/type_annotations.py index c12c0224178..595526074d2 100644 --- a/sdk/python/kfp/components/types/type_annotations.py +++ b/sdk/python/kfp/components/types/type_annotations.py @@ -112,11 +112,11 @@ def construct_type_for_inputpath_or_outputpath( return type_ -class InputAnnotation(): +class InputAnnotation: """Marker type for input artifacts.""" -class OutputAnnotation(): +class OutputAnnotation: """Marker type for output artifacts.""" @@ -147,6 +147,8 @@ def is_output_artifact(typ) -> bool: def get_io_artifact_class(typ): + from kfp.dsl import Input + from kfp.dsl import Output if not is_artifact_annotation(typ): return None if typ == Input or typ == Output: diff --git a/sdk/python/kfp/components/types/type_annotations_test.py b/sdk/python/kfp/components/types/type_annotations_test.py index abd5b1680e3..f311a610361 100644 --- a/sdk/python/kfp/components/types/type_annotations_test.py +++ b/sdk/python/kfp/components/types/type_annotations_test.py @@ -20,12 +20,12 @@ from kfp.components.types import artifact_types from kfp.components.types import type_annotations from kfp.components.types.artifact_types import Model -from kfp.components.types.type_annotations import Input from kfp.components.types.type_annotations import InputAnnotation from kfp.components.types.type_annotations import InputPath -from kfp.components.types.type_annotations import Output from kfp.components.types.type_annotations import OutputAnnotation from kfp.components.types.type_annotations import OutputPath +from kfp.dsl import Input +from kfp.dsl import Output class AnnotationsTest(parameterized.TestCase):