Add Usage Examples to public APIs (#521)

Signed-off-by: wild-endeavor <[email protected]>

docs/source/_templates/custom.rst

@@ -0,0 +1,37 @@
+{{ fullname | escape | underline}}
+
+.. currentmodule:: {{ module }}
+
+{% if objtype == 'class' %}
+
+.. autoclass:: {{ objname }}
+
+   {% block methods %}
+   {% if methods %}
+
+   .. rubric:: {{ _('Methods') }}
+   {% for item in methods %}
+   .. automethod:: {{ item }}
+   {%- endfor %}
+   {% endif %}
+   {% endblock %}
+
+   {% block attributes %}
+   {% if attributes %}
+
+   .. rubric:: {{ _('Attributes') }}
+   {% for item in attributes %}
+   .. autoattribute:: {{ item }}
+   {%- endfor %}
+
+   {% endif %}
+   {% endblock %}
+
+
+{% endif %}
+
+{% if objtype == 'function' %}
+
+.. autofunction:: {{ objname }}
+
+{% endif %}

docs/source/conf.py

@@ -27,17 +27,16 @@
 # The full version, including alpha/beta/rc tags
 release = "0.16.0b9"
 
-
 # -- General configuration ---------------------------------------------------
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
+    "sphinx.ext.napoleon",
     "sphinx.ext.autodoc",
     "sphinx.ext.autosummary",
     "sphinx.ext.autosectionlabel",
-    "sphinx.ext.napoleon",
     "sphinx.ext.todo",
     "sphinx.ext.viewcode",
     "sphinx.ext.doctest",
@@ -83,7 +82,6 @@
 # This pattern also affects html_static_path and html_extra_path .
 exclude_patterns = []
 
-
 # -- Options for HTML output -------------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
@@ -120,7 +118,7 @@
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ["_static"]
+html_static_path = []
 
 # Custom sidebar templates, must be a dictionary that maps document names
 # to template names.
@@ -134,13 +132,11 @@
     "home_page": "https://docs.flyte.org",
 }
 
-
 # -- Options for HTMLHelp output ---------------------------------------------
 
 # Output file base name for HTML help builder.
 htmlhelp_basename = "flytekitdoc"
 
-
 # -- Options for LaTeX output ------------------------------------------------
 
 latex_elements = {
@@ -165,14 +161,12 @@
     (master_doc, "flytekit.tex", "Flytekit Documentation", "Flyte", "manual"),
 ]
 
-
 # -- Options for manual page output ------------------------------------------
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [(master_doc, "flytekit", "Flytekit Documentation", [author], 1)]
 
-
 # -- Options for Texinfo output ----------------------------------------------
 
 # Grouping the document tree into Texinfo files. List of tuples
@@ -190,13 +184,27 @@
     ),
 ]
 
-
 # -- Extension configuration -------------------------------------------------
 # intersphinx configuration
 intersphinx_mapping = {
     "python": ("https://docs.python.org/3", None),
     "flytectl": ("https://flytectl.readthedocs.io/en/latest/", None),
+    "idl": ("https://flyteidl.readthedocs.io/en/latest/", None),
     # "flytectl": ("/Users/yourusername/go/src/github.com/flyteorg/flytectl/docs/build/html", None),
     "cookbook": ("https://flytecookbook.readthedocs.io/en/latest/", None),
     "flyte": ("https://flyte.readthedocs.io/en/latest/", None),
 }
+
+inheritance_graph_attrs = {
+    "resolution": 300.0,
+}
+
+inheritance_node_attrs = {
+    "bgcolor": "aliceblue",
+}
+
+inheritance_edge_attrs = {
+    "color": "darkgray",
+}
+
+autoclass_content = "both"

docs/source/design/authoring.rst

@@ -41,19 +41,23 @@ Please see the documentation on each of the classes for details.
 .. autoclass:: flytekit.core.base_task.PythonTask
    :noindex:
 
-.. autoclass:: flytekit.core.python_function_task.PythonAutoContainerTask
+.. autoclass:: flytekit.core.python_auto_container.PythonAutoContainerTask
    :noindex:
 
 .. autoclass:: flytekit.core.python_function_task.PythonFunctionTask
    :noindex:
 
 Workflows
 =========
-There is currently only one :py:class:`Workflow <flytekit.core.workflow.Workflow>` class.
+There are two workflow classes, which both inherit from the :py:class:`WorkflowBase <flytekit.core.workflow.WorkflowBase>` class.
 
 .. autoclass:: flytekit.core.workflow.PythonFunctionWorkflow
    :noindex:
 
+.. autoclass:: flytekit.core.workflow.ImperativeWorkflow
+   :noindex:
+
+
 Launch Plan
 ===========
 There is also only one :py:class:`LaunchPlan <flytekit.core.launch_plan.LaunchPlan>` class.

docs/source/design/clis.rst

@@ -14,6 +14,10 @@ The client code is located in ``flytekit/clients`` and there are two.
 * Similar to the :ref:`design-models` files, but a bit more complex, the ``raw`` one is basically a wrapper around the Protobuf generated code, with some handling for authentication in place, and acts as a mechanism for autocompletion and comments.
 * The ``friendly`` client uses the ``raw`` client, adds handling of things like pagination, and is structurally more aligned with the functionality and call pattern of the CLI itself.
 
+.. autoclass:: flytekit.clients.friendly.SynchronousFlyteClient
+
+.. autoclass:: flytekit.clients.raw.RawSynchronousFlyteClient
+
 ***********************
 Command Line Interfaces
 ***********************

docs/source/tasks.extend.rst

@@ -1,11 +1,15 @@
 #############################
-Build your custom task types
+Build Custom Task Types
 #############################
 
-
+These modules are useful to extend the base task types.
 
 .. automodule:: flytekit.core.base_task
-    :members:
+   :no-members:
+   :no-inherited-members:
+   :no-special-members:
 
 .. automodule:: flytekit.core.python_function_task
-    :members:
+   :no-members:
+   :no-inherited-members:
+   :no-special-members:

flytekit/__init__.py

@@ -10,19 +10,21 @@
 Basic Authoring
 ===============
 
-These are the essentials needed to get started writing tasks and workflows. The elements here correspond well with :std:ref:`Basic <cookbook:sphx_glr_auto_core_flyte_basics>` section of the cookbook.
+These are the essentials needed to get started writing tasks and workflows. The elements here correspond well with :std:ref:`Basics <cookbook:sphx_glr_auto_core_flyte_basics>` section of the user guide.
 
 .. autosummary::
    :nosignatures:
+   :template: custom.rst
    :toctree: generated/
 
-   task - This is the basic decorator to use to turn a properly type-annotated function into a Flyte task.
-   workflow - This will turn a properly type-annotated function into a Flyte workflow.
-   kwtypes - Helper function that makes it slightly easier to declare certain functions.
-   current_context - This function returns an ExecutionParameters object.
-   ExecutionParameters - This object gives the user at Task run time useful information about the execution environment.
-   FlyteContext - This is an flytekit-internal context that is sometimes useful for end-users to have access to.
-
+   task
+   workflow
+   kwtypes
+   current_context
+   ExecutionParameters
+   FlyteContext
+   map_task
+   ~core.workflow.ImperativeWorkflow
 
 Running Locally
 ------------------
@@ -37,6 +39,7 @@
 
 .. autosummary::
    :nosignatures:
+   :template: custom.rst
    :toctree: generated/
 
    conditional
@@ -47,6 +50,7 @@
 
 .. autosummary::
    :nosignatures:
+   :template: custom.rst
    :toctree: generated/
 
    TaskMetadata - Wrapper object that allows users to specify Task
@@ -60,22 +64,30 @@
 
 .. autosummary::
    :nosignatures:
+   :template: custom.rst
    :toctree: generated/
 
    dynamic
 
-Scheduling and Notifications
+Scheduling
 ============================
 
-See the :py:mod:`Notifications Module <flytekit.core.notification>` and
-:py:mod:`Schedules Module <flytekit.core.schedule>` for more information.
-
 .. autosummary::
    :nosignatures:
+   :template: custom.rst
    :toctree: generated/
 
    CronSchedule
    FixedRate
+
+Notifications
+============================
+
+.. autosummary::
+   :nosignatures:
+   :template: custom.rst
+   :toctree: generated/
+
    Email
    PagerDuty
    Slack
@@ -85,6 +97,7 @@
 
 .. autosummary::
    :nosignatures:
+   :template: custom.rst
    :toctree: generated/
 
    get_reference_entity
@@ -99,6 +112,7 @@
 
 .. autosummary::
    :nosignatures:
+   :template: custom.rst
    :toctree: generated/
 
    SQLTask
@@ -112,22 +126,12 @@
 
 .. autosummary::
    :nosignatures:
+   :template: custom.rst
    :toctree: generated/
 
    Secret
    SecurityContext
 
-Core Modules
-=============
-
-.. autosummary::
-   :nosignatures:
-   :toctree: generated/
-
-   core.dynamic_workflow_task
-   core.notification
-   core.schedule
-
 """
 
 
@@ -161,7 +165,7 @@ def current_context() -> ExecutionParameters:
 
     Usage
 
-    .. code-block::
+    .. code-block:: python
 
         flytekit.current_context().logging.info(...)
 

flytekit/clients/friendly.py

@@ -28,6 +28,17 @@
 
 
 class SynchronousFlyteClient(_RawSynchronousFlyteClient):
+    """
+    This is a low-level client that users can use to make direct gRPC service calls to the control plane. See the
+    :std:doc:`service spec <idl:protos/docs/service/index>`. This is more user-friendly interface than the
+    :py:class:`raw client <flytekit.clients.raw.RawSynchronousFlyteClient>` so users should try to use this class
+    first. Create a client by ::
+
+        SynchronousFlyteClient("your.domain:port", insecure=True)
+        # insecure should be True if your flyteadmin deployment doesn't have SSL enabled
+
+    """
+
     @property
     def raw(self):
         """

flytekit/common/tasks/sdk_runnable.py

@@ -91,12 +91,19 @@ def check_group_key(group: str, key: str):
 # TODO: Clean up working dir name
 class ExecutionParameters(object):
     """
-    This is the context object that is accessible to every @task method. It can be accessed using
+    This is a run-time user-centric context object that is accessible to every @task method. It can be accessed using
 
     .. code-block:: python
 
         flytekit.current_context()
 
+    This object provides the following
+    * a statsd handler
+    * a logging handler
+    * the execution ID as an :py:class:`flytekit.models.core.identifier.WorkflowExecutionIdentifier` object
+    * a working directory for the user to write arbitrary files to
+
+    Please do not confuse this object with the :py:class:`flytekit.FlyteContext` object.
     """
 
     @dataclass(init=False)

flytekit/common/translator.py

@@ -360,7 +360,6 @@ def get_serializable(
       the parent entity this function is called with.
     :param settings: used to pick up project/domain/name - to be deprecated.
     :param entity: The local flyte entity to try to convert (along with its dependencies)
-    :param fast: For tasks only, fast serialization produces a different command.
     :return: The resulting control plane entity, in addition to being added to the mutable entity_mapping parameter
       is also returned.
     """