chore(sdk): improve KFP SDK reference documentation (kubeflow#8337)

* use autodoc default options

* sort symbol order by typical usage patterns

* include summary of submodule symbols

* expose Input[] and Output[] type generics

* include tables of contents on API reference docs pages

* expand global TOC for discoverability

* fix missing reference / circular import problem

docs/conf.py

@@ -58,7 +58,16 @@
     'sphinx_click',
     'm2r2',
     'sphinx_immaterial',
+    'autodocsumm',
 ]
+autodoc_member_order = 'bysource'
+autodoc_default_options = {
+    'members': True,
+    'imported-members': True,
+    'undoc-members': True,
+    'show-inheritance': False,
+    'autosummary': True,
+}
 
 html_theme = 'sphinx_immaterial'
 html_title = 'KFP SDK API Reference'
@@ -79,7 +88,7 @@
     'edit_uri':
         'blob/master/docs',
     'globaltoc_collapse':
-        True,
+        False,
     'features': [
         'navigation.expand',
         # "navigation.tabs",

docs/index.rst

@@ -3,11 +3,8 @@ Kubeflow Pipelines SDK API Reference
 
 .. mdinclude:: ../sdk/python/README.md
 
-.. maxdepth and titlesonly settings ensure TOC items don't expand recursively, bloating the TOC (only applies if :hidden: is removed)
 .. toctree::
    :caption: Contents
-   :maxdepth: 1
-   :titlesonly:
    :hidden:
 
    Home <self>

docs/requirements.txt

@@ -1,3 +1,4 @@
+autodocsumm==0.2.9
 sdk/python
 sphinx==5.0.2
 sphinx-click==4.3.0

docs/source/client.rst

@@ -2,7 +2,3 @@ kfp.client
 ==========================
 
 .. automodule:: kfp.client
-   :members:
-   :undoc-members:
-   :show-inheritance:
-   :noindex:

docs/source/compiler.rst

@@ -1,8 +1,5 @@
 kfp.compiler
 ==========================
 
+.. glossary::
 .. automodule:: kfp.compiler
-   :members:
-   :undoc-members:
-   :show-inheritance:
-   :noindex:

docs/source/components.rst

@@ -2,7 +2,3 @@ kfp.components
 ==========================
 
 .. automodule:: kfp.components
-   :members:
-   :undoc-members:
-   :show-inheritance:
-   :noindex:

docs/source/dsl.rst

@@ -2,10 +2,3 @@ kfp.dsl
 ==========================
 
 .. automodule:: kfp.dsl
-   :members:
-   :undoc-members:
-   :show-inheritance:
-   :noindex:
-
-   .. autodata:: Input
-   .. autodata:: Output

docs/source/kfp.rst

@@ -6,6 +6,6 @@ API Reference
 
    dsl
    compiler
-   client
    components
+   client
    registry

docs/source/registry.rst

@@ -2,7 +2,3 @@ kfp.registry
 ==========================
 
 .. automodule:: kfp.registry
-   :members:
-   :undoc-members:
-   :show-inheritance:
-   :noindex:

sdk/python/kfp/components/__init__.py

@@ -15,9 +15,9 @@
 # limitations under the License.
 
 __all__ = [
-    'load_component_from_text',
     'load_component_from_file',
     'load_component_from_url',
+    'load_component_from_text',
     'PythonComponent',
     'BaseComponent',
     'ContainerComponent',

sdk/python/kfp/components/executor_test.py

@@ -27,10 +27,10 @@
 from kfp.components.types.artifact_types import Dataset
 from kfp.components.types.artifact_types import Metrics
 from kfp.components.types.artifact_types import Model
-from kfp.components.types.type_annotations import Input
 from kfp.components.types.type_annotations import InputPath
-from kfp.components.types.type_annotations import Output
 from kfp.components.types.type_annotations import OutputPath
+from kfp.dsl import Input
+from kfp.dsl import Output
 
 
 class ExecutorTest(unittest.TestCase):

sdk/python/kfp/components/types/custom_artifact_types_test.py

@@ -28,10 +28,10 @@
 from kfp.components.types import custom_artifact_types
 from kfp.components.types.artifact_types import Artifact
 from kfp.components.types.artifact_types import Dataset
-from kfp.components.types.type_annotations import Input
 from kfp.components.types.type_annotations import InputPath
-from kfp.components.types.type_annotations import Output
 from kfp.components.types.type_annotations import OutputPath
+from kfp.dsl import Input
+from kfp.dsl import Output
 
 Alias = Artifact
 artifact_types_alias = artifact_types

sdk/python/kfp/components/types/type_annotations.py

@@ -22,13 +22,6 @@
 from kfp.components.types import type_annotations
 from kfp.components.types import type_utils
 
-try:
-    from typing import Annotated
-except ImportError:
-    from typing_extensions import Annotated
-
-T = TypeVar('T')
-
 
 class OutputPath:
     """Type annotation used in component definitions for indicating a parameter
@@ -119,67 +112,14 @@ def construct_type_for_inputpath_or_outputpath(
         return type_
 
 
-class InputAnnotation():
+class InputAnnotation:
     """Marker type for input artifacts."""
 
 
-class OutputAnnotation():
+class OutputAnnotation:
     """Marker type for output artifacts."""
 
 
-Input = Annotated[T, InputAnnotation]
-Input.__doc__ = """Type generic used to represent an input artifact of type ``T``, where ``T`` is an artifact class.
-
-Use ``Input[Artifact]`` or ``Output[Artifact]`` to indicate whether the enclosed artifact is a component input or output.
-
-Args:
-    T: The type of the input artifact.
-
-Example:
-  ::
-
-    @dsl.component
-    def artifact_producer(model: Output[Artifact]):
-        with open(model.path, 'w') as f:
-            f.write('my model')
-
-    @dsl.component
-    def artifact_consumer(model: Input[Artifact]):
-        print(model)
-
-    @dsl.pipeline(name='my-pipeline')
-    def my_pipeline():
-        producer_task = artifact_producer()
-        artifact_consumer(model=producer_task.output)
-"""
-
-Output = Annotated[T, OutputAnnotation]
-Output.__doc__ = """A type generic used to represent an output artifact of type ``T``, where ``T`` is an artifact class. The argument typed with this annotation is provided at runtime by the executing backend and does not need to be passed as an input by the pipeline author (see example).
-
-Use ``Input[Artifact]`` or ``Output[Artifact]`` to indicate whether the enclosed artifact is a component input or output.
-
-Args:
-    T: The type of the output artifact.
-
-Example:
-  ::
-
-    @dsl.component
-    def artifact_producer(model: Output[Artifact]):
-        with open(model.path, 'w') as f:
-            f.write('my model')
-
-    @dsl.component
-    def artifact_consumer(model: Input[Artifact]):
-        print(model)
-
-    @dsl.pipeline(name='my-pipeline')
-    def my_pipeline():
-        producer_task = artifact_producer()
-        artifact_consumer(model=producer_task.output)
-"""
-
-
 def is_artifact_annotation(typ) -> bool:
     if not hasattr(typ, '__metadata__'):
         return False
@@ -207,6 +147,8 @@ def is_output_artifact(typ) -> bool:
 
 
 def get_io_artifact_class(typ):
+    from kfp.dsl import Input
+    from kfp.dsl import Output
     if not is_artifact_annotation(typ):
         return None
     if typ == Input or typ == Output:
@@ -222,6 +164,9 @@ def get_io_artifact_annotation(typ):
     return typ.__metadata__[0]
 
 
+T = TypeVar('T')
+
+
 def maybe_strip_optional_from_annotation(annotation: T) -> T:
     """Strips 'Optional' from 'Optional[<type>]' if applicable.
 

sdk/python/kfp/components/types/type_annotations_test.py

@@ -20,12 +20,12 @@
 from kfp.components.types import artifact_types
 from kfp.components.types import type_annotations
 from kfp.components.types.artifact_types import Model
-from kfp.components.types.type_annotations import Input
 from kfp.components.types.type_annotations import InputAnnotation
 from kfp.components.types.type_annotations import InputPath
-from kfp.components.types.type_annotations import Output
 from kfp.components.types.type_annotations import OutputAnnotation
 from kfp.components.types.type_annotations import OutputPath
+from kfp.dsl import Input
+from kfp.dsl import Output
 
 
 class AnnotationsTest(parameterized.TestCase):

sdk/python/kfp/dsl/__init__.py

@@ -17,35 +17,42 @@
 __all__ = [
     'component',
     'container_component',
-    'ContainerSpec',
-    'importer',
     'pipeline',
-    'PipelineTask',
-    'PipelineTaskFinalStatus',
+    'importer',
+    'ContainerSpec',
     'Condition',
     'ExitHandler',
     'ParallelFor',
-    'Artifact',
-    'ClassificationMetrics',
-    'Dataset',
-    'HTML',
-    'Markdown',
-    'Metrics',
-    'Model',
-    'SlicedClassificationMetrics',
     'Input',
     'Output',
     'InputPath',
     'OutputPath',
+    'IfPresentPlaceholder',
+    'ConcatPlaceholder',
+    'PipelineTaskFinalStatus',
     'PIPELINE_JOB_NAME_PLACEHOLDER',
     'PIPELINE_JOB_RESOURCE_NAME_PLACEHOLDER',
     'PIPELINE_JOB_ID_PLACEHOLDER',
     'PIPELINE_TASK_NAME_PLACEHOLDER',
     'PIPELINE_TASK_ID_PLACEHOLDER',
-    'IfPresentPlaceholder',
-    'ConcatPlaceholder',
+    'Artifact',
+    'ClassificationMetrics',
+    'Dataset',
+    'HTML',
+    'Markdown',
+    'Metrics',
+    'Model',
+    'SlicedClassificationMetrics',
+    'PipelineTask',
 ]
 
+try:
+    from typing import Annotated
+except ImportError:
+    from typing_extensions import Annotated
+
+from typing import TypeVar
+
 from kfp.components.component_decorator import component
 from kfp.components.container_component_decorator import container_component
 from kfp.components.importer_node import importer
@@ -66,11 +73,13 @@
 from kfp.components.types.artifact_types import Metrics
 from kfp.components.types.artifact_types import Model
 from kfp.components.types.artifact_types import SlicedClassificationMetrics
-from kfp.components.types.type_annotations import Input
+from kfp.components.types.type_annotations import InputAnnotation
 from kfp.components.types.type_annotations import InputPath
-from kfp.components.types.type_annotations import Output
+from kfp.components.types.type_annotations import OutputAnnotation
 from kfp.components.types.type_annotations import OutputPath
 
+# hack: constants and custom type generics have to be defined here to be captured by autodoc and autodocsumm used in ./docs/conf.py
+
 PIPELINE_JOB_NAME_PLACEHOLDER = '{{$.pipeline_job_name}}'
 """A placeholder used to obtain a pipeline job name within a task at pipeline runtime.
 
@@ -140,3 +149,55 @@ def my_pipeline():
                 value=dsl.PIPELINE_TASK_ID_PLACEHOLDER,
             )
 """
+T = TypeVar('T')
+Input = Annotated[T, InputAnnotation]
+"""Type generic used to represent an input artifact of type ``T``, where ``T`` is an artifact class.
+
+Use ``Input[Artifact]`` or ``Output[Artifact]`` to indicate whether the enclosed artifact is a component input or output.
+
+Args:
+    T: The type of the input artifact.
+
+Example:
+  ::
+
+    @dsl.component
+    def artifact_producer(model: Output[Artifact]):
+        with open(model.path, 'w') as f:
+            f.write('my model')
+
+    @dsl.component
+    def artifact_consumer(model: Input[Artifact]):
+        print(model)
+
+    @dsl.pipeline(name='my-pipeline')
+    def my_pipeline():
+        producer_task = artifact_producer()
+        artifact_consumer(model=producer_task.output)
+"""
+
+Output = Annotated[T, OutputAnnotation]
+"""A type generic used to represent an output artifact of type ``T``, where ``T`` is an artifact class. The argument typed with this annotation is provided at runtime by the executing backend and does not need to be passed as an input by the pipeline author (see example).
+
+Use ``Input[Artifact]`` or ``Output[Artifact]`` to indicate whether the enclosed artifact is a component input or output.
+
+Args:
+    T: The type of the output artifact.
+
+Example:
+  ::
+
+    @dsl.component
+    def artifact_producer(model: Output[Artifact]):
+        with open(model.path, 'w') as f:
+            f.write('my model')
+
+    @dsl.component
+    def artifact_consumer(model: Input[Artifact]):
+        print(model)
+
+    @dsl.pipeline(name='my-pipeline')
+    def my_pipeline():
+        producer_task = artifact_producer()
+        artifact_consumer(model=producer_task.output)
+"""

sdk/python/kfp/registry/__init__.py

@@ -15,8 +15,8 @@
 # limitations under the License.
 
 __all__ = [
-    'ApiAuth',
     'RegistryClient',
+    'ApiAuth',
 ]
 from kfp.registry.registry_client import ApiAuth
 from kfp.registry.registry_client import RegistryClient