User Documentation Proposal (#1200)

Signed-off-by: Kevin Su <[email protected]>

flytekit/__init__.py

@@ -190,6 +190,7 @@
 from flytekit.models.common import Annotations, AuthRole, Labels
 from flytekit.models.core.execution import WorkflowExecutionPhase
 from flytekit.models.core.types import BlobType
+from flytekit.models.documentation import Description, Documentation, SourceCode
 from flytekit.models.literals import Blob, BlobMetadata, Literal, Scalar
 from flytekit.models.types import LiteralType
 from flytekit.types import directory, file, numpy, schema

flytekit/configuration/__init__.py

@@ -463,7 +463,7 @@ class GCSConfig(object):
     gsutil_parallelism: bool = False
 
     @classmethod
-    def auto(self, config_file: typing.Union[str, ConfigFile] = None) -> GCSConfig:
+    def auto(cls, config_file: typing.Union[str, ConfigFile] = None) -> GCSConfig:
         config_file = get_config_file(config_file)
         kwargs = {}
         kwargs = set_if_exists(kwargs, "gsutil_parallelism", _internal.GCP.GSUTIL_PARALLELISM.read(config_file))
@@ -647,6 +647,7 @@ class SerializationSettings(object):
     domain: typing.Optional[str] = None
     version: typing.Optional[str] = None
     env: Optional[Dict[str, str]] = None
+    git_repo: Optional[str] = None
     python_interpreter: str = DEFAULT_RUNTIME_PYTHON_INTERPRETER
     flytekit_virtualenv_root: Optional[str] = None
     fast_serialization_settings: Optional[FastSerializationSettings] = None
@@ -719,6 +720,7 @@ def new_builder(self) -> Builder:
             version=self.version,
             image_config=self.image_config,
             env=self.env.copy() if self.env else None,
+            git_repo=self.git_repo,
             flytekit_virtualenv_root=self.flytekit_virtualenv_root,
             python_interpreter=self.python_interpreter,
             fast_serialization_settings=self.fast_serialization_settings,
@@ -768,6 +770,7 @@ class Builder(object):
         version: str
         image_config: ImageConfig
         env: Optional[Dict[str, str]] = None
+        git_repo: Optional[str] = None
         flytekit_virtualenv_root: Optional[str] = None
         python_interpreter: Optional[str] = None
         fast_serialization_settings: Optional[FastSerializationSettings] = None
@@ -783,6 +786,7 @@ def build(self) -> SerializationSettings:
                 version=self.version,
                 image_config=self.image_config,
                 env=self.env,
+                git_repo=self.git_repo,
                 flytekit_virtualenv_root=self.flytekit_virtualenv_root,
                 python_interpreter=self.python_interpreter,
                 fast_serialization_settings=self.fast_serialization_settings,

flytekit/configuration/file.py

@@ -224,7 +224,7 @@ def legacy_config(self) -> _configparser.ConfigParser:
         return self._legacy_config
 
     @property
-    def yaml_config(self) -> typing.Dict[str, Any]:
+    def yaml_config(self) -> typing.Dict[str, typing.Any]:
         return self._yaml_config
 
 

flytekit/core/base_task.py

@@ -45,6 +45,7 @@
 from flytekit.models import literals as _literal_models
 from flytekit.models import task as _task_model
 from flytekit.models.core import workflow as _workflow_model
+from flytekit.models.documentation import Description, Documentation
 from flytekit.models.interface import Variable
 from flytekit.models.security import SecurityContext
 
@@ -156,6 +157,7 @@ def __init__(
         metadata: Optional[TaskMetadata] = None,
         task_type_version=0,
         security_ctx: Optional[SecurityContext] = None,
+        docs: Optional[Documentation] = None,
         **kwargs,
     ):
         self._task_type = task_type
@@ -164,6 +166,7 @@ def __init__(
         self._metadata = metadata if metadata else TaskMetadata()
         self._task_type_version = task_type_version
         self._security_ctx = security_ctx
+        self._docs = docs
 
         FlyteEntities.entities.append(self)
 
@@ -195,6 +198,10 @@ def task_type_version(self) -> int:
     def security_context(self) -> SecurityContext:
         return self._security_ctx
 
+    @property
+    def docs(self) -> Documentation:
+        return self._docs
+
     def get_type_for_input_var(self, k: str, v: Any) -> type:
         """
         Returns the python native type for the given input variable
@@ -390,6 +397,17 @@ def __init__(
         self._environment = environment if environment else {}
         self._task_config = task_config
         self._disable_deck = disable_deck
+        if self._python_interface.docstring:
+            if self.docs is None:
+                self._docs = Documentation(
+                    short_description=self._python_interface.docstring.short_description,
+                    long_description=Description(value=self._python_interface.docstring.long_description),
+                )
+            else:
+                if self._python_interface.docstring.short_description:
+                    self._docs.short_description = self._python_interface.docstring.short_description
+                if self._python_interface.docstring.long_description:
+                    self._docs.long_description = Description(value=self._python_interface.docstring.long_description)
 
     # TODO lets call this interface and the other as flyte_interface?
     @property

flytekit/core/interface.py

@@ -207,7 +207,6 @@ def transform_interface_to_typed_interface(
     """
     if interface is None:
         return None
-
     if interface.docstring is None:
         input_descriptions = output_descriptions = {}
     else:

flytekit/core/task.py

@@ -7,6 +7,7 @@
 from flytekit.core.python_function_task import PythonFunctionTask
 from flytekit.core.reference_entity import ReferenceEntity, TaskReference
 from flytekit.core.resources import Resources
+from flytekit.models.documentation import Documentation
 from flytekit.models.security import Secret
 
 
@@ -89,6 +90,7 @@ def task(
     secret_requests: Optional[List[Secret]] = None,
     execution_mode: Optional[PythonFunctionTask.ExecutionBehavior] = PythonFunctionTask.ExecutionBehavior.DEFAULT,
     task_resolver: Optional[TaskResolverMixin] = None,
+    docs: Optional[Documentation] = None,
     disable_deck: bool = True,
 ) -> Union[Callable, PythonFunctionTask]:
     """
@@ -179,6 +181,7 @@ def foo2():
     :param execution_mode: This is mainly for internal use. Please ignore. It is filled in automatically.
     :param task_resolver: Provide a custom task resolver.
     :param disable_deck: If true, this task will not output deck html file
+    :param docs: Documentation about this task
     """
 
     def wrapper(fn) -> PythonFunctionTask:
@@ -204,6 +207,7 @@ def wrapper(fn) -> PythonFunctionTask:
             execution_mode=execution_mode,
             task_resolver=task_resolver,
             disable_deck=disable_deck,
+            docs=docs,
         )
         update_wrapper(task_instance, fn)
         return task_instance

flytekit/core/workflow.py

@@ -39,6 +39,7 @@
 from flytekit.models import interface as _interface_models
 from flytekit.models import literals as _literal_models
 from flytekit.models.core import workflow as _workflow_model
+from flytekit.models.documentation import Description, Documentation
 
 GLOBAL_START_NODE = Node(
     id=_common_constants.GLOBAL_INPUT_NODE_ID,
@@ -168,6 +169,7 @@ def __init__(
         workflow_metadata: WorkflowMetadata,
         workflow_metadata_defaults: WorkflowMetadataDefaults,
         python_interface: Interface,
+        docs: Optional[Documentation] = None,
         **kwargs,
     ):
         self._name = name
@@ -179,13 +181,31 @@ def __init__(
         self._unbound_inputs = set()
         self._nodes = []
         self._output_bindings: List[_literal_models.Binding] = []
+        self._docs = docs
+
+        if self._python_interface.docstring:
+            if self.docs is None:
+                self._docs = Documentation(
+                    short_description=self._python_interface.docstring.short_description,
+                    long_description=Description(value=self._python_interface.docstring.long_description),
+                )
+            else:
+                if self._python_interface.docstring.short_description:
+                    self._docs.short_description = self._python_interface.docstring.short_description
+                if self._python_interface.docstring.long_description:
+                    self._docs = Description(value=self._python_interface.docstring.long_description)
+
         FlyteEntities.entities.append(self)
         super().__init__(**kwargs)
 
     @property
     def name(self) -> str:
         return self._name
 
+    @property
+    def docs(self):
+        return self._docs
+
     @property
     def short_name(self) -> str:
         return extract_obj_name(self._name)
@@ -571,7 +591,8 @@ def __init__(
         workflow_function: Callable,
         metadata: Optional[WorkflowMetadata],
         default_metadata: Optional[WorkflowMetadataDefaults],
-        docstring: Docstring = None,
+        docstring: Optional[Docstring] = None,
+        docs: Optional[Documentation] = None,
     ):
         name, _, _, _ = extract_task_module(workflow_function)
         self._workflow_function = workflow_function
@@ -586,6 +607,7 @@ def __init__(
             workflow_metadata=metadata,
             workflow_metadata_defaults=default_metadata,
             python_interface=native_interface,
+            docs=docs,
         )
 
     @property
@@ -690,6 +712,7 @@ def workflow(
     _workflow_function=None,
     failure_policy: Optional[WorkflowFailurePolicy] = None,
     interruptible: bool = False,
+    docs: Optional[Documentation] = None,
 ):
     """
     This decorator declares a function to be a Flyte workflow. Workflows are declarative entities that construct a DAG
@@ -718,6 +741,7 @@ def workflow(
     :param _workflow_function: This argument is implicitly passed and represents the decorated function.
     :param failure_policy: Use the options in flytekit.WorkflowFailurePolicy
     :param interruptible: Whether or not tasks launched from this workflow are by default interruptible
+    :param docs: Description entity for the workflow
     """
 
     def wrapper(fn):
@@ -730,6 +754,7 @@ def wrapper(fn):
             metadata=workflow_metadata,
             default_metadata=workflow_metadata_defaults,
             docstring=Docstring(callable_=fn),
+            docs=docs,
         )
         workflow_instance.compile()
         update_wrapper(workflow_instance, fn)

flytekit/models/admin/workflow.py

@@ -1,20 +1,29 @@
+import typing
+
 from flyteidl.admin import workflow_pb2 as _admin_workflow
 
 from flytekit.models import common as _common
 from flytekit.models.core import compiler as _compiler_models
 from flytekit.models.core import identifier as _identifier
 from flytekit.models.core import workflow as _core_workflow
+from flytekit.models.documentation import Documentation
 
 
 class WorkflowSpec(_common.FlyteIdlEntity):
-    def __init__(self, template, sub_workflows):
+    def __init__(
+        self,
+        template: _core_workflow.WorkflowTemplate,
+        sub_workflows: typing.List[_core_workflow.WorkflowTemplate],
+        docs: typing.Optional[Documentation] = None,
+    ):
         """
         This object fully encapsulates the specification of a workflow
         :param flytekit.models.core.workflow.WorkflowTemplate template:
         :param list[flytekit.models.core.workflow.WorkflowTemplate] sub_workflows:
         """
         self._template = template
         self._sub_workflows = sub_workflows
+        self._docs = docs
 
     @property
     def template(self):
@@ -30,13 +39,21 @@ def sub_workflows(self):
         """
         return self._sub_workflows
 
+    @property
+    def docs(self):
+        """
+        :rtype: Description entity for the workflow
+        """
+        return self._docs
+
     def to_flyte_idl(self):
         """
         :rtype: flyteidl.admin.workflow_pb2.WorkflowSpec
         """
         return _admin_workflow.WorkflowSpec(
             template=self._template.to_flyte_idl(),
             sub_workflows=[s.to_flyte_idl() for s in self._sub_workflows],
+            description=self._docs.to_flyte_idl() if self._docs else None,
         )
 
     @classmethod
@@ -48,6 +65,7 @@ def from_flyte_idl(cls, pb2_object):
         return cls(
             _core_workflow.WorkflowTemplate.from_flyte_idl(pb2_object.template),
             [_core_workflow.WorkflowTemplate.from_flyte_idl(s) for s in pb2_object.sub_workflows],
+            Documentation.from_flyte_idl(pb2_object.description) if pb2_object.description else None,
         )
 
 

flytekit/models/documentation.py

@@ -0,0 +1,93 @@
+from dataclasses import dataclass
+from enum import Enum
+from typing import Optional
+
+from flyteidl.admin import description_entity_pb2
+
+from flytekit.models import common as _common_models
+
+
+@dataclass
+class Description(_common_models.FlyteIdlEntity):
+    """
+    Full user description with formatting preserved. This can be rendered
+    by clients, such as the console or command line tools with in-tact
+    formatting.
+    """
+
+    class DescriptionFormat(Enum):
+        UNKNOWN = 0
+        MARKDOWN = 1
+        HTML = 2
+        RST = 3
+
+    value: Optional[str] = None
+    uri: Optional[str] = None
+    icon_link: Optional[str] = None
+    format: DescriptionFormat = DescriptionFormat.RST
+
+    def to_flyte_idl(self):
+        return description_entity_pb2.Description(
+            value=self.value if self.value else None,
+            uri=self.uri if self.uri else None,
+            format=self.format.value,
+            icon_link=self.icon_link,
+        )
+
+    @classmethod
+    def from_flyte_idl(cls, pb2_object: description_entity_pb2.Description) -> "Description":
+        return cls(
+            value=pb2_object.value if pb2_object.value else None,
+            uri=pb2_object.uri if pb2_object.uri else None,
+            format=Description.DescriptionFormat(pb2_object.format),
+            icon_link=pb2_object.icon_link if pb2_object.icon_link else None,
+        )
+
+
+@dataclass
+class SourceCode(_common_models.FlyteIdlEntity):
+    """
+    Link to source code used to define this task or workflow.
+    """
+
+    link: Optional[str] = None
+
+    def to_flyte_idl(self):
+        return description_entity_pb2.SourceCode(link=self.link)
+
+    @classmethod
+    def from_flyte_idl(cls, pb2_object: description_entity_pb2.SourceCode) -> "SourceCode":
+        return cls(link=pb2_object.link) if pb2_object.link else None
+
+
+@dataclass
+class Documentation(_common_models.FlyteIdlEntity):
+    """
+    DescriptionEntity contains detailed description for the task/workflow/launch plan.
+    Documentation could provide insight into the algorithms, business use case, etc.
+    Args:
+        short_description (str): One-liner overview of the entity.
+        long_description (Optional[Description]): Full user description with formatting preserved.
+        source_code (Optional[SourceCode]): link to source code used to define this entity
+    """
+
+    short_description: Optional[str] = None
+    long_description: Optional[Description] = None
+    source_code: Optional[SourceCode] = None
+
+    def to_flyte_idl(self):
+        return description_entity_pb2.DescriptionEntity(
+            short_description=self.short_description,
+            long_description=self.long_description.to_flyte_idl() if self.long_description else None,
+            source_code=self.source_code.to_flyte_idl() if self.source_code else None,
+        )
+
+    @classmethod
+    def from_flyte_idl(cls, pb2_object: description_entity_pb2.DescriptionEntity) -> "Documentation":
+        return cls(
+            short_description=pb2_object.short_description,
+            long_description=Description.from_flyte_idl(pb2_object.long_description)
+            if pb2_object.long_description
+            else None,
+            source_code=SourceCode.from_flyte_idl(pb2_object.source_code) if pb2_object.source_code else None,
+        )