From 7e7018a09851f35fd099f150c296eb7580cfd67f Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Sat, 25 Jun 2022 04:17:23 +0800 Subject: [PATCH 01/38] Doc hub Signed-off-by: Kevin Su --- flytekit/configuration/__init__.py | 2 +- flytekit/configuration/file.py | 2 +- flytekit/core/base_task.py | 7 +++ flytekit/core/interface.py | 2 +- flytekit/core/task.py | 6 +- flytekit/models/documentation.py | 95 ++++++++++++++++++++++++++++++ flytekit/models/task.py | 21 ++++++- flytekit/tools/translator.py | 2 +- 8 files changed, 129 insertions(+), 8 deletions(-) create mode 100644 flytekit/models/documentation.py diff --git a/flytekit/configuration/__init__.py b/flytekit/configuration/__init__.py index 188f6ebeee..4a055c2524 100644 --- a/flytekit/configuration/__init__.py +++ b/flytekit/configuration/__init__.py @@ -467,7 +467,7 @@ class GCSConfig(object): gsutil_parallelism: bool = False @classmethod - def auto(self, config_file: typing.Union[str, ConfigFile] = None) -> GCSConfig: + def auto(cls, config_file: typing.Union[str, ConfigFile] = None) -> GCSConfig: config_file = get_config_file(config_file) kwargs = {} kwargs = set_if_exists(kwargs, "gsutil_parallelism", _internal.GCP.GSUTIL_PARALLELISM.read(config_file)) diff --git a/flytekit/configuration/file.py b/flytekit/configuration/file.py index b329a16112..2bb2d123ae 100644 --- a/flytekit/configuration/file.py +++ b/flytekit/configuration/file.py @@ -221,7 +221,7 @@ def legacy_config(self) -> _configparser.ConfigParser: return self._legacy_config @property - def yaml_config(self) -> typing.Dict[str, Any]: + def yaml_config(self) -> typing.Dict[str, typing.Any]: return self._yaml_config diff --git a/flytekit/core/base_task.py b/flytekit/core/base_task.py index cc3319e359..c31eae2d51 100644 --- a/flytekit/core/base_task.py +++ b/flytekit/core/base_task.py @@ -46,6 +46,7 @@ from flytekit.models import literals as _literal_models from flytekit.models import task as _task_model from flytekit.models.core import workflow as _workflow_model +from flytekit.models.documentation import Documentation from flytekit.models.interface import Variable from flytekit.models.security import SecurityContext @@ -157,6 +158,7 @@ def __init__( metadata: Optional[TaskMetadata] = None, task_type_version=0, security_ctx: Optional[SecurityContext] = None, + docs: Optional[Documentation] = None, **kwargs, ): self._task_type = task_type @@ -165,6 +167,7 @@ def __init__( self._metadata = metadata if metadata else TaskMetadata() self._task_type_version = task_type_version self._security_ctx = security_ctx + self._docs = docs FlyteEntities.entities.append(self) @@ -196,6 +199,10 @@ def task_type_version(self) -> int: def security_context(self) -> SecurityContext: return self._security_ctx + @property + def docs(self) -> Documentation: + return self._docs + def get_type_for_input_var(self, k: str, v: Any) -> type: """ Returns the python native type for the given input variable diff --git a/flytekit/core/interface.py b/flytekit/core/interface.py index fed0793e4a..bb80f4fae0 100644 --- a/flytekit/core/interface.py +++ b/flytekit/core/interface.py @@ -306,7 +306,7 @@ def transform_function_to_interface(fn: typing.Callable, docstring: Optional[Doc def transform_variable_map( variable_map: Dict[str, type], - descriptions: Dict[str, str] = {}, + descriptions: Dict[str, str], ) -> Dict[str, _interface_models.Variable]: """ Given a map of str (names of inputs for instance) to their Python native types, return a map of the name to a diff --git a/flytekit/core/task.py b/flytekit/core/task.py index d210aaa2ed..a0777ea312 100644 --- a/flytekit/core/task.py +++ b/flytekit/core/task.py @@ -7,6 +7,7 @@ from flytekit.core.python_function_task import PythonFunctionTask from flytekit.core.reference_entity import ReferenceEntity, TaskReference from flytekit.core.resources import Resources +from flytekit.models.documentation import Documentation from flytekit.models.security import Secret @@ -89,7 +90,8 @@ def task( secret_requests: Optional[List[Secret]] = None, execution_mode: Optional[PythonFunctionTask.ExecutionBehavior] = PythonFunctionTask.ExecutionBehavior.DEFAULT, task_resolver: Optional[TaskResolverMixin] = None, - disable_deck: bool = False, + disable_deck: Optional[bool] = False, + docs: Optional[Documentation] = None, ) -> Union[Callable, PythonFunctionTask]: """ This is the core decorator to use for any task type in flytekit. @@ -179,6 +181,7 @@ def foo2(): :param execution_mode: This is mainly for internal use. Please ignore. It is filled in automatically. :param task_resolver: Provide a custom task resolver. :param disable_deck: If true, this task will not output deck html file + :param docs: Documentation about this task """ def wrapper(fn) -> PythonFunctionTask: @@ -204,6 +207,7 @@ def wrapper(fn) -> PythonFunctionTask: execution_mode=execution_mode, task_resolver=task_resolver, disable_deck=disable_deck, + docs=docs, ) update_wrapper(task_instance, fn) return task_instance diff --git a/flytekit/models/documentation.py b/flytekit/models/documentation.py new file mode 100644 index 0000000000..0917a3c5b6 --- /dev/null +++ b/flytekit/models/documentation.py @@ -0,0 +1,95 @@ +from dataclasses import dataclass +from enum import Enum +from typing import Dict, List, Optional + +from flyteidl.admin import entity_description_pb2 as _entity_description_pb2 + +from flytekit.models import common as _common_models + + +@dataclass +class LongDescription(_common_models.FlyteIdlEntity): + class DescriptionFormat(Enum): + UNKNOWN = 0 + MARKDOWN = 1 + HTML = 2 + RST = 3 + + values: Optional[str] = "" + uri: Optional[str] = "" + icon_link: Optional[str] = "" + format: DescriptionFormat = DescriptionFormat.RST + + @classmethod + def from_flyte_idl(cls, pb2_object: _entity_description_pb2.LongDescription) -> "LongDescription": + return cls( + values=pb2_object.values, + uri=pb2_object.uri, + format=LongDescription.DescriptionFormat(pb2_object.long_format), + icon_link=pb2_object.icon_link, + ) + + def to_flyte_idl(self): + return _entity_description_pb2.LongDescription( + values=self.values, uri=self.uri, long_format=self.format.value, icon_link=self.icon_link + ) + + +@dataclass +class SourceCode(_common_models.FlyteIdlEntity): + file: Optional[str] = None + line_number: Optional[int] = None + repo: Optional[str] = None + branch: Optional[str] = None + link: Optional[str] = None + language: Optional[str] = None + + @classmethod + def from_flyte_idl(cls, pb2_object: _entity_description_pb2.SourceCode) -> "SourceCode": + return cls( + file=pb2_object.file, + line_number=pb2_object.line_number, + repo=pb2_object.repo, + branch=pb2_object.branch, + link=pb2_object.link, + language=pb2_object.language, + ) + + def to_flyte_idl(self): + return _entity_description_pb2.SourceCode( + file=self.file, + line_number=self.line_number, + repo=self.repo, + branch=self.branch, + link=self.link, + language=self.language, + ) + + +@dataclass +class Documentation(_common_models.FlyteIdlEntity): + + short_description: str + long_description: LongDescription + source_code: Optional[SourceCode] = None + tags: Optional[List[str]] = None + labels: Optional[Dict[str, str]] = None + + @classmethod + def from_flyte_idl(cls, pb2_object: _entity_description_pb2.EntityDescription) -> "Documentation": + return cls( + short_description=pb2_object.short_description, + long_description=LongDescription.from_flyte_idl(pb2_object.long_description), + source_code=SourceCode.from_flyte_idl(pb2_object.source_code), + tags=pb2_object.tags, + labels=pb2_object.labels, + ) + + def to_flyte_idl(self): + return _entity_description_pb2.EntityDescription( + short_description=self.short_description, + long_description=self.long_description.to_flyte_idl() if self.long_description else None, + tags=self.tags, + labels=self.labels, + source=self.source_code.to_flyte_idl() if self.source_code else None, + ) diff --git a/flytekit/models/task.py b/flytekit/models/task.py index f2ff5efd89..521983cfc7 100644 --- a/flytekit/models/task.py +++ b/flytekit/models/task.py @@ -13,6 +13,7 @@ from flytekit.models import literals as _literals from flytekit.models import security as _sec from flytekit.models.core import identifier as _identifier +from flytekit.models.documentation import Documentation class Resources(_common.FlyteIdlEntity): @@ -480,11 +481,12 @@ def from_flyte_idl(cls, pb2_object): class TaskSpec(_common.FlyteIdlEntity): - def __init__(self, template): + def __init__(self, template, entity_description): """ :param TaskTemplate template: """ self._template = template + self._entity_description = entity_description @property def template(self): @@ -493,11 +495,21 @@ def template(self): """ return self._template + @property + def entity_description(self): + """ + :rtype: Documentation + """ + return self._entity_description + def to_flyte_idl(self): """ :rtype: flyteidl.admin.tasks_pb2.TaskSpec """ - return _admin_task.TaskSpec(template=self.template.to_flyte_idl()) + return _admin_task.TaskSpec( + template=self.template.to_flyte_idl(), + entity_description=self.entity_description.to_flyte_idl() if self._entity_description else None, + ) @classmethod def from_flyte_idl(cls, pb2_object): @@ -505,7 +517,10 @@ def from_flyte_idl(cls, pb2_object): :param flyteidl.admin.tasks_pb2.TaskSpec pb2_object: :rtype: TaskSpec """ - return cls(TaskTemplate.from_flyte_idl(pb2_object.template)) + return cls( + TaskTemplate.from_flyte_idl(pb2_object.template), + Documentation.from_flyte_idl(pb2_object.entity_description), + ) class Task(_common.FlyteIdlEntity): diff --git a/flytekit/tools/translator.py b/flytekit/tools/translator.py index 045ab7970a..a906903d44 100644 --- a/flytekit/tools/translator.py +++ b/flytekit/tools/translator.py @@ -210,7 +210,7 @@ def get_serializable_task( ) if settings.should_fast_serialize() and isinstance(entity, PythonAutoContainerTask): entity.reset_command_fn() - return task_models.TaskSpec(template=tt) + return task_models.TaskSpec(template=tt, entity_description=entity.docs) def get_serializable_workflow( From 36a225b8d6c5593a0c9f7861f0a9b7fafa1d6ef0 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Sat, 25 Jun 2022 04:20:09 +0800 Subject: [PATCH 02/38] Doc hub Signed-off-by: Kevin Su --- flytekit/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flytekit/__init__.py b/flytekit/__init__.py index c0ea9ebe3d..a5699c1f9c 100644 --- a/flytekit/__init__.py +++ b/flytekit/__init__.py @@ -188,6 +188,7 @@ from flytekit.models.core.types import BlobType from flytekit.models.literals import Blob, BlobMetadata, Literal, Scalar from flytekit.models.types import LiteralType +from flytekit.models.documentation import Documentation, SourceCode, LongDescription from flytekit.types import directory, file, schema from flytekit.types.structured.structured_dataset import ( StructuredDataset, From 27f5e02d76f2d19852355b4bba7c5b8687212d5a Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Fri, 9 Sep 2022 19:26:55 +0800 Subject: [PATCH 03/38] Add dochub service Signed-off-by: Kevin Su --- flytekit/__init__.py | 2 +- flytekit/clients/friendly.py | 32 ++++++++++++++++ flytekit/clients/raw.py | 28 ++++++++++++++ flytekit/models/documentation.py | 63 ++++++++++++-------------------- flytekit/models/task.py | 12 +----- flytekit/remote/remote.py | 2 + flytekit/tools/translator.py | 2 +- 7 files changed, 89 insertions(+), 52 deletions(-) diff --git a/flytekit/__init__.py b/flytekit/__init__.py index a5699c1f9c..cbed4aa2db 100644 --- a/flytekit/__init__.py +++ b/flytekit/__init__.py @@ -186,9 +186,9 @@ from flytekit.models.common import Annotations, AuthRole, Labels from flytekit.models.core.execution import WorkflowExecutionPhase from flytekit.models.core.types import BlobType +from flytekit.models.documentation import Documentation, LongDescription, SourceCode from flytekit.models.literals import Blob, BlobMetadata, Literal, Scalar from flytekit.models.types import LiteralType -from flytekit.models.documentation import Documentation, SourceCode, LongDescription from flytekit.types import directory, file, schema from flytekit.types.structured.structured_dataset import ( StructuredDataset, diff --git a/flytekit/clients/friendly.py b/flytekit/clients/friendly.py index 8db7c93a98..7f4beedaf6 100644 --- a/flytekit/clients/friendly.py +++ b/flytekit/clients/friendly.py @@ -2,6 +2,7 @@ import typing from flyteidl.admin import common_pb2 as _common_pb2 +from flyteidl.admin import description_entity_pb2 as _description_entity_pb2 from flyteidl.admin import execution_pb2 as _execution_pb2 from flyteidl.admin import launch_plan_pb2 as _launch_plan_pb2 from flyteidl.admin import matchable_resource_pb2 as _matchable_resource_pb2 @@ -1004,3 +1005,34 @@ def get_upload_signed_url( expires_in=expires_in_pb, ) ) + + #################################################################################################################### + # + # Description Entity Endpoints + # + #################################################################################################################### + + def create_description_entity(self, description_entity_identifer, description_entity): + """ + This will create a task definition in the Admin database. Once successful, the task object can be + retrieved via the client or viewed via the UI or command-line interfaces. + + .. note :: + + Overwrites are not supported so any request for a given project, domain, name, and version that exists in + the database must match the existing definition exactly. Furthermore, as long as the request + remains identical, calling this method multiple times will result in success. + + :param flytekit.models.core.identifier.Identifier task_identifer: The identifier for this task. + :param flytekit.models.task.TaskSpec task_spec: This is the actual definition of the task that + should be created. + :raises flytekit.common.exceptions.user.FlyteEntityAlreadyExistsException: If an identical version of the + task is found, this exception is raised. The client might choose to ignore this exception because the + identical task is already registered. + :raises grpc.RpcError: + """ + super(SynchronousFlyteClient, self).create_description_entity( + _description_entity_pb2.DescriptionEntityCreateRequest( + id=description_entity_identifer.to_flyte_idl(), description_entity=description_entity.to_flyte_idl() + ) + ) diff --git a/flytekit/clients/raw.py b/flytekit/clients/raw.py index 9aba78888a..d07113b9f4 100644 --- a/flytekit/clients/raw.py +++ b/flytekit/clients/raw.py @@ -833,6 +833,34 @@ def create_upload_location( """ return self._dataproxy_stub.CreateUploadLocation(create_upload_location_request, metadata=self._metadata) + #################################################################################################################### + # + # Description Entity Endpoints + # + #################################################################################################################### + + @_handle_rpc_error() + @_handle_invalid_create_request + def create_description_entity(self, description_entity_create_request): + """ + This will create a task definition in the Admin database. Once successful, the task object can be + retrieved via the client or viewed via the UI or command-line interfaces. + + .. note :: + + Overwrites are not supported so any request for a given project, domain, name, and version that exists in + the database must match the existing definition exactly. This also means that as long as the request + remains identical, calling this method multiple times will result in success. + + :param: flyteidl.admin.task_pb2.TaskCreateRequest task_create_request: The request protobuf object. + :rtype: flyteidl.admin.task_pb2.TaskCreateResponse + :raises flytekit.common.exceptions.user.FlyteEntityAlreadyExistsException: If an identical version of the task + is found, this exception is raised. The client might choose to ignore this exception because the identical + task is already registered. + :raises grpc.RpcError: + """ + return self._stub.CreateDescriptionEntity(description_entity_create_request) + def get_token(token_endpoint, authorization_header, scope): """ diff --git a/flytekit/models/documentation.py b/flytekit/models/documentation.py index 0917a3c5b6..a7090248f4 100644 --- a/flytekit/models/documentation.py +++ b/flytekit/models/documentation.py @@ -2,7 +2,7 @@ from enum import Enum from typing import Dict, List, Optional -from flyteidl.admin import entity_description_pb2 as _entity_description_pb2 +from flyteidl.admin import description_entity_pb2 from flytekit.models import common as _common_models @@ -20,8 +20,13 @@ class DescriptionFormat(Enum): icon_link: Optional[str] = "" format: DescriptionFormat = DescriptionFormat.RST + def to_flyte_idl(self): + return description_entity_pb2.LongDescription( + values=self.values, uri=self.uri, long_format=self.format.value, icon_link=self.icon_link + ) + @classmethod - def from_flyte_idl(cls, pb2_object: _entity_description_pb2.LongDescription) -> "LongDescription": + def from_flyte_idl(cls, pb2_object: description_entity_pb2.LongDescription) -> "LongDescription": return cls( values=pb2_object.values, uri=pb2_object.uri, @@ -29,40 +34,20 @@ def from_flyte_idl(cls, pb2_object: _entity_description_pb2.LongDescription) -> icon_link=pb2_object.icon_link, ) - def to_flyte_idl(self): - return _entity_description_pb2.LongDescription( - values=self.values, uri=self.uri, long_format=self.format.value, icon_link=self.icon_link - ) - @dataclass class SourceCode(_common_models.FlyteIdlEntity): - file: Optional[str] = None - line_number: Optional[int] = None - repo: Optional[str] = None - branch: Optional[str] = None link: Optional[str] = None - language: Optional[str] = None + + def to_flyte_idl(self): + return description_entity_pb2.SourceCode( + link=self.link, + ) @classmethod - def from_flyte_idl(cls, pb2_object: _entity_description_pb2.SourceCode) -> "SourceCode": + def from_flyte_idl(cls, pb2_object: description_entity_pb2.SourceCode) -> "SourceCode": return cls( - file=pb2_object.file, - line_number=pb2_object.line_number, - repo=pb2_object.repo, - branch=pb2_object.branch, link=pb2_object.link, - language=pb2_object.language, - ) - - def to_flyte_idl(self): - return _entity_description_pb2.SourceCode( - file=self.file, - line_number=self.line_number, - repo=self.repo, - branch=self.branch, - link=self.link, - language=self.language, ) @@ -70,13 +55,22 @@ def to_flyte_idl(self): class Documentation(_common_models.FlyteIdlEntity): short_description: str - long_description: LongDescription + long_description: Optional[LongDescription] = None source_code: Optional[SourceCode] = None tags: Optional[List[str]] = None labels: Optional[Dict[str, str]] = None + def to_flyte_idl(self): + return description_entity_pb2.DescriptionEntity( + short_description=self.short_description, + long_description=self.long_description.to_flyte_idl() if self.long_description else None, + tags=self.tags, + labels=self.labels, + source_code=self.source_code.to_flyte_idl() if self.source_code else None, + ) + @classmethod - def from_flyte_idl(cls, pb2_object: _entity_description_pb2.EntityDescription) -> "Documentation": + def from_flyte_idl(cls, pb2_object: description_entity_pb2.DescriptionEntity) -> "Documentation": return cls( short_description=pb2_object.short_description, long_description=LongDescription.from_flyte_idl(pb2_object.long_description), @@ -84,12 +78,3 @@ def from_flyte_idl(cls, pb2_object: _entity_description_pb2.EntityDescription) - tags=pb2_object.tags, labels=pb2_object.labels, ) - - def to_flyte_idl(self): - return _entity_description_pb2.EntityDescription( - short_description=self.short_description, - long_description=self.long_description.to_flyte_idl() if self.long_description else None, - tags=self.tags, - labels=self.labels, - source=self.source_code.to_flyte_idl() if self.source_code else None, - ) diff --git a/flytekit/models/task.py b/flytekit/models/task.py index 521983cfc7..2befa73bd6 100644 --- a/flytekit/models/task.py +++ b/flytekit/models/task.py @@ -481,12 +481,11 @@ def from_flyte_idl(cls, pb2_object): class TaskSpec(_common.FlyteIdlEntity): - def __init__(self, template, entity_description): + def __init__(self, template): """ :param TaskTemplate template: """ self._template = template - self._entity_description = entity_description @property def template(self): @@ -495,20 +494,12 @@ def template(self): """ return self._template - @property - def entity_description(self): - """ - :rtype: Documentation - """ - return self._entity_description - def to_flyte_idl(self): """ :rtype: flyteidl.admin.tasks_pb2.TaskSpec """ return _admin_task.TaskSpec( template=self.template.to_flyte_idl(), - entity_description=self.entity_description.to_flyte_idl() if self._entity_description else None, ) @classmethod @@ -519,7 +510,6 @@ def from_flyte_idl(cls, pb2_object): """ return cls( TaskTemplate.from_flyte_idl(pb2_object.template), - Documentation.from_flyte_idl(pb2_object.entity_description), ) diff --git a/flytekit/remote/remote.py b/flytekit/remote/remote.py index a00ac97222..b6caaa78a3 100644 --- a/flytekit/remote/remote.py +++ b/flytekit/remote/remote.py @@ -462,6 +462,8 @@ def register_task( :return: """ ident = self._serialize_and_register(entity=entity, settings=serialization_settings, version=version) + print(entity.docs) + self.client.create_description_entity(ident, entity.docs) ft = self.fetch_task( ident.project, ident.domain, diff --git a/flytekit/tools/translator.py b/flytekit/tools/translator.py index a906903d44..045ab7970a 100644 --- a/flytekit/tools/translator.py +++ b/flytekit/tools/translator.py @@ -210,7 +210,7 @@ def get_serializable_task( ) if settings.should_fast_serialize() and isinstance(entity, PythonAutoContainerTask): entity.reset_command_fn() - return task_models.TaskSpec(template=tt, entity_description=entity.docs) + return task_models.TaskSpec(template=tt) def get_serializable_workflow( From 7694442c7d09748a919952d2436748baf71193c5 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Thu, 15 Sep 2022 00:07:08 +0800 Subject: [PATCH 04/38] update flyte remote Signed-off-by: Kevin Su --- flytekit/models/documentation.py | 6 ------ flytekit/remote/remote.py | 12 +++++++++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/flytekit/models/documentation.py b/flytekit/models/documentation.py index a7090248f4..0867e97a6b 100644 --- a/flytekit/models/documentation.py +++ b/flytekit/models/documentation.py @@ -57,15 +57,11 @@ class Documentation(_common_models.FlyteIdlEntity): short_description: str long_description: Optional[LongDescription] = None source_code: Optional[SourceCode] = None - tags: Optional[List[str]] = None - labels: Optional[Dict[str, str]] = None def to_flyte_idl(self): return description_entity_pb2.DescriptionEntity( short_description=self.short_description, long_description=self.long_description.to_flyte_idl() if self.long_description else None, - tags=self.tags, - labels=self.labels, source_code=self.source_code.to_flyte_idl() if self.source_code else None, ) @@ -75,6 +71,4 @@ def from_flyte_idl(cls, pb2_object: description_entity_pb2.DescriptionEntity) -> short_description=pb2_object.short_description, long_description=LongDescription.from_flyte_idl(pb2_object.long_description), source_code=SourceCode.from_flyte_idl(pb2_object.source_code), - tags=pb2_object.tags, - labels=pb2_object.labels, ) diff --git a/flytekit/remote/remote.py b/flytekit/remote/remote.py index b6caaa78a3..0500287b73 100644 --- a/flytekit/remote/remote.py +++ b/flytekit/remote/remote.py @@ -18,7 +18,7 @@ from flyteidl.core import literals_pb2 as literals_pb2 -from flytekit import Literal +from flytekit import Literal, Documentation from flytekit.clients.friendly import SynchronousFlyteClient from flytekit.clients.helpers import iterate_node_executions, iterate_task_executions from flytekit.configuration import Config, FastSerializationSettings, ImageConfig, SerializationSettings @@ -449,6 +449,13 @@ def _serialize_and_register( raise return ident + def register_description_entity(self, ident: Identifier, docs: Documentation): + try: + self.client.create_description_entity(ident, docs) + except FlyteEntityAlreadyExistsException: + print("test") + remote_logger.info(f"{docs} already exists") + def register_task( self, entity: PythonTask, serialization_settings: SerializationSettings, version: typing.Optional[str] = None ) -> FlyteTask: @@ -462,8 +469,7 @@ def register_task( :return: """ ident = self._serialize_and_register(entity=entity, settings=serialization_settings, version=version) - print(entity.docs) - self.client.create_description_entity(ident, entity.docs) + self.register_description_entity(ident, entity.docs) ft = self.fetch_task( ident.project, ident.domain, From 71760dd161418fe69431f3c3634849afbcaf0926 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Thu, 15 Sep 2022 13:20:13 +0800 Subject: [PATCH 05/38] Merged master Signed-off-by: Kevin Su --- flytekit/clients/friendly.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flytekit/clients/friendly.py b/flytekit/clients/friendly.py index f0bea6503a..170f4e8f58 100644 --- a/flytekit/clients/friendly.py +++ b/flytekit/clients/friendly.py @@ -1034,6 +1034,8 @@ def create_description_entity(self, description_entity_identifer, description_en super(SynchronousFlyteClient, self).create_description_entity( _description_entity_pb2.DescriptionEntityCreateRequest( id=description_entity_identifer.to_flyte_idl(), description_entity=description_entity.to_flyte_idl() + ) + ) def get_download_signed_url( self, native_url: str, expires_in: datetime.timedelta = None From 3948fb6cba21c15e3c65efbf5a28c73802382c8d Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Fri, 16 Sep 2022 16:46:49 +0800 Subject: [PATCH 06/38] update Signed-off-by: Kevin Su --- flytekit/clients/friendly.py | 42 ++++++++++++++++---------------- flytekit/clients/raw.py | 10 ++++---- flytekit/models/documentation.py | 18 ++++++++++++++ flytekit/models/task.py | 9 ++----- flytekit/remote/remote.py | 16 ++++++++---- 5 files changed, 57 insertions(+), 38 deletions(-) diff --git a/flytekit/clients/friendly.py b/flytekit/clients/friendly.py index 170f4e8f58..b5c1b466c8 100644 --- a/flytekit/clients/friendly.py +++ b/flytekit/clients/friendly.py @@ -1006,15 +1006,29 @@ def get_upload_signed_url( ) ) + def get_download_signed_url( + self, native_url: str, expires_in: datetime.timedelta = None + ) -> _data_proxy_pb2.CreateUploadLocationResponse: + expires_in_pb = None + if expires_in: + expires_in_pb = Duration() + expires_in_pb.FromTimedelta(expires_in) + return super(SynchronousFlyteClient, self).create_download_location( + _data_proxy_pb2.CreateDownloadLocationRequest( + native_url=native_url, + expires_in=expires_in_pb, + ) + ) + #################################################################################################################### # # Description Entity Endpoints # #################################################################################################################### - def create_description_entity(self, description_entity_identifer, description_entity): + def create_description_entity(self, description_entity_identifier, description_entity): """ - This will create a task definition in the Admin database. Once successful, the task object can be + This will create a description entity definition in the Admin database. Once successful, the description entity can be retrieved via the client or viewed via the UI or command-line interfaces. .. note :: @@ -1023,30 +1037,16 @@ def create_description_entity(self, description_entity_identifer, description_en the database must match the existing definition exactly. Furthermore, as long as the request remains identical, calling this method multiple times will result in success. - :param flytekit.models.core.identifier.Identifier task_identifer: The identifier for this task. - :param flytekit.models.task.TaskSpec task_spec: This is the actual definition of the task that + :param flytekit.models.core.identifier.Identifier description_entity_identifier: The identifier for this description entity. + :param flytekit.models.documentation.Documentation description_entity: This is the actual definition of the description entity that should be created. :raises flytekit.common.exceptions.user.FlyteEntityAlreadyExistsException: If an identical version of the - task is found, this exception is raised. The client might choose to ignore this exception because the - identical task is already registered. + description entity is found, this exception is raised. The client might choose to ignore this exception because the + identical description entity is already registered. :raises grpc.RpcError: """ super(SynchronousFlyteClient, self).create_description_entity( _description_entity_pb2.DescriptionEntityCreateRequest( - id=description_entity_identifer.to_flyte_idl(), description_entity=description_entity.to_flyte_idl() - ) - ) - - def get_download_signed_url( - self, native_url: str, expires_in: datetime.timedelta = None - ) -> _data_proxy_pb2.CreateUploadLocationResponse: - expires_in_pb = None - if expires_in: - expires_in_pb = Duration() - expires_in_pb.FromTimedelta(expires_in) - return super(SynchronousFlyteClient, self).create_download_location( - _data_proxy_pb2.CreateDownloadLocationRequest( - native_url=native_url, - expires_in=expires_in_pb, + id=description_entity_identifier.to_flyte_idl(), description_entity=description_entity.to_flyte_idl() ) ) diff --git a/flytekit/clients/raw.py b/flytekit/clients/raw.py index b3f5e205f1..d1de3479c1 100644 --- a/flytekit/clients/raw.py +++ b/flytekit/clients/raw.py @@ -863,7 +863,7 @@ def create_upload_location( @_handle_invalid_create_request def create_description_entity(self, description_entity_create_request): """ - This will create a task definition in the Admin database. Once successful, the task object can be + This will create a description entity definition in the Admin database. Once successful, the description entity object can be retrieved via the client or viewed via the UI or command-line interfaces. .. note :: @@ -872,11 +872,11 @@ def create_description_entity(self, description_entity_create_request): the database must match the existing definition exactly. This also means that as long as the request remains identical, calling this method multiple times will result in success. - :param: flyteidl.admin.task_pb2.TaskCreateRequest task_create_request: The request protobuf object. - :rtype: flyteidl.admin.task_pb2.TaskCreateResponse - :raises flytekit.common.exceptions.user.FlyteEntityAlreadyExistsException: If an identical version of the task + :param: flyteidl.admin.task_pb2.DescriptionEntityCreateRequest description_entity_create_request: The request protobuf object. + :rtype: flyteidl.admin.task_pb2.DescriptionEntityCreateResponse + :raises flytekit.common.exceptions.user.FlyteEntityAlreadyExistsException: If an identical version of the description entity is found, this exception is raised. The client might choose to ignore this exception because the identical - task is already registered. + description entity is already registered. :raises grpc.RpcError: """ return self._stub.CreateDescriptionEntity(description_entity_create_request) diff --git a/flytekit/models/documentation.py b/flytekit/models/documentation.py index 0867e97a6b..970870c387 100644 --- a/flytekit/models/documentation.py +++ b/flytekit/models/documentation.py @@ -9,6 +9,12 @@ @dataclass class LongDescription(_common_models.FlyteIdlEntity): + """ + Full user description with formatting preserved. This can be rendered + by clients, such as the console or command line tools with in-tact + formatting. + """ + class DescriptionFormat(Enum): UNKNOWN = 0 MARKDOWN = 1 @@ -37,6 +43,10 @@ def from_flyte_idl(cls, pb2_object: description_entity_pb2.LongDescription) -> " @dataclass class SourceCode(_common_models.FlyteIdlEntity): + """ + Link to source code used to define this task or workflow. + """ + link: Optional[str] = None def to_flyte_idl(self): @@ -53,6 +63,14 @@ def from_flyte_idl(cls, pb2_object: description_entity_pb2.SourceCode) -> "Sourc @dataclass class Documentation(_common_models.FlyteIdlEntity): + """ + DescriptionEntity contains detailed description for the task/workflow/launch plan. + Documentation could provide insight into the algorithms, business use case, etc. + Args: + short_description (str): One-liner overview of the entity. + long_description (Optional[LongDescription]): Full user description with formatting preserved. + source_code (Optional[SourceCode]): link to source code used to define this entity + """ short_description: str long_description: Optional[LongDescription] = None diff --git a/flytekit/models/task.py b/flytekit/models/task.py index 2befa73bd6..f2ff5efd89 100644 --- a/flytekit/models/task.py +++ b/flytekit/models/task.py @@ -13,7 +13,6 @@ from flytekit.models import literals as _literals from flytekit.models import security as _sec from flytekit.models.core import identifier as _identifier -from flytekit.models.documentation import Documentation class Resources(_common.FlyteIdlEntity): @@ -498,9 +497,7 @@ def to_flyte_idl(self): """ :rtype: flyteidl.admin.tasks_pb2.TaskSpec """ - return _admin_task.TaskSpec( - template=self.template.to_flyte_idl(), - ) + return _admin_task.TaskSpec(template=self.template.to_flyte_idl()) @classmethod def from_flyte_idl(cls, pb2_object): @@ -508,9 +505,7 @@ def from_flyte_idl(cls, pb2_object): :param flyteidl.admin.tasks_pb2.TaskSpec pb2_object: :rtype: TaskSpec """ - return cls( - TaskTemplate.from_flyte_idl(pb2_object.template), - ) + return cls(TaskTemplate.from_flyte_idl(pb2_object.template)) class Task(_common.FlyteIdlEntity): diff --git a/flytekit/remote/remote.py b/flytekit/remote/remote.py index 58839ce670..cd63d44dc8 100644 --- a/flytekit/remote/remote.py +++ b/flytekit/remote/remote.py @@ -19,7 +19,7 @@ from flyteidl.core import literals_pb2 as literals_pb2 -from flytekit import Literal, Documentation +from flytekit import Documentation, Literal from flytekit.clients.friendly import SynchronousFlyteClient from flytekit.clients.helpers import iterate_node_executions, iterate_task_executions from flytekit.configuration import Config, FastSerializationSettings, ImageConfig, SerializationSettings @@ -450,12 +450,18 @@ def _serialize_and_register( raise return ident - def register_description_entity(self, ident: Identifier, docs: Documentation): + def register_description_entity(self, identifier: Identifier, documentation: Documentation): + """ + Register a description entity with Remote + For any conflicting parameters method arguments are regarded as overrides + + :param Identifier identifier: Identifier of the registered entity. e.g. task/workflow/launchPlan + :param Documentation documentation: docs contains detailed description for the task/workflow/launch. + """ try: - self.client.create_description_entity(ident, docs) + self.client.create_description_entity(identifier, documentation) except FlyteEntityAlreadyExistsException: - print("test") - remote_logger.info(f"{docs} already exists") + remote_logger.info(f"{documentation} already exists") def register_task( self, entity: PythonTask, serialization_settings: SerializationSettings, version: typing.Optional[str] = None From 559754f48c054bb6c4bc17ef04646125bc8f85cc Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 27 Sep 2022 21:54:36 +0800 Subject: [PATCH 07/38] Fix tests Signed-off-by: Kevin Su --- .github/workflows/pythonbuild.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml index 2e2ef772b2..5a01cbd627 100644 --- a/.github/workflows/pythonbuild.yml +++ b/.github/workflows/pythonbuild.yml @@ -46,6 +46,7 @@ jobs: - name: Install dependencies run: | make setup${{ matrix.spark-version-suffix }} + pip install "git+https://github.com/flyteorg/flyteidl@doc-hub" pip freeze - name: Test with coverage run: | @@ -131,6 +132,7 @@ jobs: pip install -r requirements.txt if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi pip install --no-deps -U https://github.com/flyteorg/flytekit/archive/${{ github.sha }}.zip#egg=flytekit + pip install "git+https://github.com/flyteorg/flyteidl@doc-hub" pip freeze - name: Test with coverage run: | @@ -156,6 +158,7 @@ jobs: run: | python -m pip install --upgrade pip==21.2.4 pip install -r dev-requirements.txt + pip install "git+https://github.com/flyteorg/flyteidl@doc-hub" - name: Lint run: | make lint @@ -177,5 +180,6 @@ jobs: run: | python -m pip install --upgrade pip==21.2.4 setuptools wheel pip install -r doc-requirements.txt + pip install "git+https://github.com/flyteorg/flyteidl@doc-hub" - name: Build the documentation run: make -C docs html From 6499c427486c7fcb72db80dc040d5c6138f47b3b Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 27 Sep 2022 23:38:16 +0800 Subject: [PATCH 08/38] Fix tests Signed-off-by: Kevin Su --- flytekit/core/interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flytekit/core/interface.py b/flytekit/core/interface.py index bc759571c6..c721e2e160 100644 --- a/flytekit/core/interface.py +++ b/flytekit/core/interface.py @@ -316,7 +316,7 @@ def transform_function_to_interface(fn: typing.Callable, docstring: Optional[Doc def transform_variable_map( variable_map: Dict[str, type], - descriptions: Dict[str, str], + descriptions: Dict[str, str] = {}, ) -> Dict[str, _interface_models.Variable]: """ Given a map of str (names of inputs for instance) to their Python native types, return a map of the name to a From fb69f0b6797ed6fa2d94e5e54defe8c3bbaf4d05 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 27 Sep 2022 23:59:32 +0800 Subject: [PATCH 09/38] Fixed tests Signed-off-by: Kevin Su --- flytekit/models/documentation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flytekit/models/documentation.py b/flytekit/models/documentation.py index 970870c387..d62ea047d4 100644 --- a/flytekit/models/documentation.py +++ b/flytekit/models/documentation.py @@ -1,6 +1,6 @@ from dataclasses import dataclass from enum import Enum -from typing import Dict, List, Optional +from typing import Optional from flyteidl.admin import description_entity_pb2 From 1c65d59c8ce26ae61f21873ff248616eb0bd5281 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Wed, 28 Sep 2022 00:38:41 +0800 Subject: [PATCH 10/38] Use doc string description Signed-off-by: Kevin Su --- flytekit/core/base_task.py | 13 ++++++++++++- flytekit/core/interface.py | 1 - flytekit/models/documentation.py | 6 +++--- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/flytekit/core/base_task.py b/flytekit/core/base_task.py index 57568929c6..36cd2740dd 100644 --- a/flytekit/core/base_task.py +++ b/flytekit/core/base_task.py @@ -46,7 +46,7 @@ from flytekit.models import literals as _literal_models from flytekit.models import task as _task_model from flytekit.models.core import workflow as _workflow_model -from flytekit.models.documentation import Documentation +from flytekit.models.documentation import Documentation, LongDescription from flytekit.models.interface import Variable from flytekit.models.security import SecurityContext @@ -398,6 +398,17 @@ def __init__( self._environment = environment if environment else {} self._task_config = task_config self._disable_deck = disable_deck + if self._python_interface.docstring: + short_description = self._python_interface.docstring.short_description + long_description = None + if self._python_interface.docstring.long_description: + long_description = LongDescription(value=self._python_interface.docstring.long_description) + + if self.docs is None: + self._docs = Documentation(short_description=short_description, long_description=long_description) + else: + self._docs.short_description = short_description + self._docs.long_description = long_description # TODO lets call this interface and the other as flyte_interface? @property diff --git a/flytekit/core/interface.py b/flytekit/core/interface.py index c721e2e160..d422382aa0 100644 --- a/flytekit/core/interface.py +++ b/flytekit/core/interface.py @@ -207,7 +207,6 @@ def transform_interface_to_typed_interface( """ if interface is None: return None - if interface.docstring is None: input_descriptions = output_descriptions = {} else: diff --git a/flytekit/models/documentation.py b/flytekit/models/documentation.py index d62ea047d4..46106afc05 100644 --- a/flytekit/models/documentation.py +++ b/flytekit/models/documentation.py @@ -21,20 +21,20 @@ class DescriptionFormat(Enum): HTML = 2 RST = 3 - values: Optional[str] = "" + value: Optional[str] = "" uri: Optional[str] = "" icon_link: Optional[str] = "" format: DescriptionFormat = DescriptionFormat.RST def to_flyte_idl(self): return description_entity_pb2.LongDescription( - values=self.values, uri=self.uri, long_format=self.format.value, icon_link=self.icon_link + value=self.value, uri=self.uri, format=self.format.value, icon_link=self.icon_link ) @classmethod def from_flyte_idl(cls, pb2_object: description_entity_pb2.LongDescription) -> "LongDescription": return cls( - values=pb2_object.values, + value=pb2_object.value, uri=pb2_object.uri, format=LongDescription.DescriptionFormat(pb2_object.long_format), icon_link=pb2_object.icon_link, From 0f860239deae3ee4d3a7a014d6632842faea9026 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Wed, 5 Oct 2022 16:53:27 -0700 Subject: [PATCH 11/38] remove create_description_entity endpoint Signed-off-by: Kevin Su --- flytekit/clients/friendly.py | 31 ------------------------------- flytekit/clients/raw.py | 28 ---------------------------- flytekit/models/task.py | 16 +++++++++++++--- flytekit/remote/remote.py | 14 -------------- flytekit/tools/translator.py | 3 ++- 5 files changed, 15 insertions(+), 77 deletions(-) diff --git a/flytekit/clients/friendly.py b/flytekit/clients/friendly.py index b5c1b466c8..4f8072fc1c 100644 --- a/flytekit/clients/friendly.py +++ b/flytekit/clients/friendly.py @@ -1019,34 +1019,3 @@ def get_download_signed_url( expires_in=expires_in_pb, ) ) - - #################################################################################################################### - # - # Description Entity Endpoints - # - #################################################################################################################### - - def create_description_entity(self, description_entity_identifier, description_entity): - """ - This will create a description entity definition in the Admin database. Once successful, the description entity can be - retrieved via the client or viewed via the UI or command-line interfaces. - - .. note :: - - Overwrites are not supported so any request for a given project, domain, name, and version that exists in - the database must match the existing definition exactly. Furthermore, as long as the request - remains identical, calling this method multiple times will result in success. - - :param flytekit.models.core.identifier.Identifier description_entity_identifier: The identifier for this description entity. - :param flytekit.models.documentation.Documentation description_entity: This is the actual definition of the description entity that - should be created. - :raises flytekit.common.exceptions.user.FlyteEntityAlreadyExistsException: If an identical version of the - description entity is found, this exception is raised. The client might choose to ignore this exception because the - identical description entity is already registered. - :raises grpc.RpcError: - """ - super(SynchronousFlyteClient, self).create_description_entity( - _description_entity_pb2.DescriptionEntityCreateRequest( - id=description_entity_identifier.to_flyte_idl(), description_entity=description_entity.to_flyte_idl() - ) - ) diff --git a/flytekit/clients/raw.py b/flytekit/clients/raw.py index d1de3479c1..3ac0fb90c3 100644 --- a/flytekit/clients/raw.py +++ b/flytekit/clients/raw.py @@ -853,34 +853,6 @@ def create_upload_location( """ return self._dataproxy_stub.CreateUploadLocation(create_upload_location_request, metadata=self._metadata) - #################################################################################################################### - # - # Description Entity Endpoints - # - #################################################################################################################### - - @_handle_rpc_error() - @_handle_invalid_create_request - def create_description_entity(self, description_entity_create_request): - """ - This will create a description entity definition in the Admin database. Once successful, the description entity object can be - retrieved via the client or viewed via the UI or command-line interfaces. - - .. note :: - - Overwrites are not supported so any request for a given project, domain, name, and version that exists in - the database must match the existing definition exactly. This also means that as long as the request - remains identical, calling this method multiple times will result in success. - - :param: flyteidl.admin.task_pb2.DescriptionEntityCreateRequest description_entity_create_request: The request protobuf object. - :rtype: flyteidl.admin.task_pb2.DescriptionEntityCreateResponse - :raises flytekit.common.exceptions.user.FlyteEntityAlreadyExistsException: If an identical version of the description entity - is found, this exception is raised. The client might choose to ignore this exception because the identical - description entity is already registered. - :raises grpc.RpcError: - """ - return self._stub.CreateDescriptionEntity(description_entity_create_request) - @_handle_rpc_error(retry=True) def create_download_location( self, create_download_location_request: _dataproxy_pb2.CreateDownloadLocationRequest diff --git a/flytekit/models/task.py b/flytekit/models/task.py index f2ff5efd89..9eb0102b95 100644 --- a/flytekit/models/task.py +++ b/flytekit/models/task.py @@ -8,6 +8,7 @@ from google.protobuf import json_format as _json_format from google.protobuf import struct_pb2 as _struct +from flytekit.models.documentation import Documentation from flytekit.models import common as _common from flytekit.models import interface as _interface from flytekit.models import literals as _literals @@ -480,11 +481,13 @@ def from_flyte_idl(cls, pb2_object): class TaskSpec(_common.FlyteIdlEntity): - def __init__(self, template): + def __init__(self, template: TaskTemplate, docs: typing.Optional[Documentation] = None): """ :param TaskTemplate template: + :param Documentation docs: """ self._template = template + self._docs = docs @property def template(self): @@ -493,11 +496,18 @@ def template(self): """ return self._template + @property + def docs(self): + """ + :rtype: Documentation + """ + return self._docs + def to_flyte_idl(self): """ :rtype: flyteidl.admin.tasks_pb2.TaskSpec """ - return _admin_task.TaskSpec(template=self.template.to_flyte_idl()) + return _admin_task.TaskSpec(template=self.template.to_flyte_idl(), description_entity=self.docs.to_flyte_idl() if self.docs else None) @classmethod def from_flyte_idl(cls, pb2_object): @@ -505,7 +515,7 @@ def from_flyte_idl(cls, pb2_object): :param flyteidl.admin.tasks_pb2.TaskSpec pb2_object: :rtype: TaskSpec """ - return cls(TaskTemplate.from_flyte_idl(pb2_object.template)) + return cls(TaskTemplate.from_flyte_idl(pb2_object.template), Documentation.from_flyte_idl(pb2_object.description_entity) if pb2_object.description_entity else None) class Task(_common.FlyteIdlEntity): diff --git a/flytekit/remote/remote.py b/flytekit/remote/remote.py index cd63d44dc8..e60f88d9dc 100644 --- a/flytekit/remote/remote.py +++ b/flytekit/remote/remote.py @@ -450,19 +450,6 @@ def _serialize_and_register( raise return ident - def register_description_entity(self, identifier: Identifier, documentation: Documentation): - """ - Register a description entity with Remote - For any conflicting parameters method arguments are regarded as overrides - - :param Identifier identifier: Identifier of the registered entity. e.g. task/workflow/launchPlan - :param Documentation documentation: docs contains detailed description for the task/workflow/launch. - """ - try: - self.client.create_description_entity(identifier, documentation) - except FlyteEntityAlreadyExistsException: - remote_logger.info(f"{documentation} already exists") - def register_task( self, entity: PythonTask, serialization_settings: SerializationSettings, version: typing.Optional[str] = None ) -> FlyteTask: @@ -476,7 +463,6 @@ def register_task( :return: """ ident = self._serialize_and_register(entity=entity, settings=serialization_settings, version=version) - self.register_description_entity(ident, entity.docs) ft = self.fetch_task( ident.project, ident.domain, diff --git a/flytekit/tools/translator.py b/flytekit/tools/translator.py index 57f863f055..2a76af17ad 100644 --- a/flytekit/tools/translator.py +++ b/flytekit/tools/translator.py @@ -210,7 +210,8 @@ def get_serializable_task( ) if settings.should_fast_serialize() and isinstance(entity, PythonAutoContainerTask): entity.reset_command_fn() - return task_models.TaskSpec(template=tt) + + return task_models.TaskSpec(template=tt, docs=entity.docs) def get_serializable_workflow( From 58598aa25905e1096f08596809c423b9cfa5deed Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Thu, 6 Oct 2022 11:37:16 -0700 Subject: [PATCH 12/38] more tests Signed-off-by: Kevin Su --- flytekit/clients/friendly.py | 1 - flytekit/models/documentation.py | 21 +++---- flytekit/models/task.py | 11 +++- flytekit/remote/remote.py | 2 +- .../unit/models/test_documentation.py | 29 ++++++++++ tests/flytekit/unit/models/test_tasks.py | 55 +++++++++++++++++++ 6 files changed, 104 insertions(+), 15 deletions(-) create mode 100644 tests/flytekit/unit/models/test_documentation.py diff --git a/flytekit/clients/friendly.py b/flytekit/clients/friendly.py index 4f8072fc1c..d542af5f7e 100644 --- a/flytekit/clients/friendly.py +++ b/flytekit/clients/friendly.py @@ -2,7 +2,6 @@ import typing from flyteidl.admin import common_pb2 as _common_pb2 -from flyteidl.admin import description_entity_pb2 as _description_entity_pb2 from flyteidl.admin import execution_pb2 as _execution_pb2 from flyteidl.admin import launch_plan_pb2 as _launch_plan_pb2 from flyteidl.admin import matchable_resource_pb2 as _matchable_resource_pb2 diff --git a/flytekit/models/documentation.py b/flytekit/models/documentation.py index 46106afc05..00ad48443e 100644 --- a/flytekit/models/documentation.py +++ b/flytekit/models/documentation.py @@ -28,7 +28,10 @@ class DescriptionFormat(Enum): def to_flyte_idl(self): return description_entity_pb2.LongDescription( - value=self.value, uri=self.uri, format=self.format.value, icon_link=self.icon_link + value=self.value if self.value else None, + uri=self.uri if self.uri else None, + format=self.format.value, + icon_link=self.icon_link, ) @classmethod @@ -36,7 +39,7 @@ def from_flyte_idl(cls, pb2_object: description_entity_pb2.LongDescription) -> " return cls( value=pb2_object.value, uri=pb2_object.uri, - format=LongDescription.DescriptionFormat(pb2_object.long_format), + format=LongDescription.DescriptionFormat(pb2_object.format), icon_link=pb2_object.icon_link, ) @@ -50,15 +53,11 @@ class SourceCode(_common_models.FlyteIdlEntity): link: Optional[str] = None def to_flyte_idl(self): - return description_entity_pb2.SourceCode( - link=self.link, - ) + return description_entity_pb2.SourceCode(link=self.link) @classmethod def from_flyte_idl(cls, pb2_object: description_entity_pb2.SourceCode) -> "SourceCode": - return cls( - link=pb2_object.link, - ) + return cls(link=pb2_object.link) if pb2_object.link else None @dataclass @@ -87,6 +86,8 @@ def to_flyte_idl(self): def from_flyte_idl(cls, pb2_object: description_entity_pb2.DescriptionEntity) -> "Documentation": return cls( short_description=pb2_object.short_description, - long_description=LongDescription.from_flyte_idl(pb2_object.long_description), - source_code=SourceCode.from_flyte_idl(pb2_object.source_code), + long_description=LongDescription.from_flyte_idl(pb2_object.long_description) + if pb2_object.long_description + else None, + source_code=SourceCode.from_flyte_idl(pb2_object.source_code) if pb2_object.source_code else None, ) diff --git a/flytekit/models/task.py b/flytekit/models/task.py index 9eb0102b95..e8c1aba469 100644 --- a/flytekit/models/task.py +++ b/flytekit/models/task.py @@ -8,12 +8,12 @@ from google.protobuf import json_format as _json_format from google.protobuf import struct_pb2 as _struct -from flytekit.models.documentation import Documentation from flytekit.models import common as _common from flytekit.models import interface as _interface from flytekit.models import literals as _literals from flytekit.models import security as _sec from flytekit.models.core import identifier as _identifier +from flytekit.models.documentation import Documentation class Resources(_common.FlyteIdlEntity): @@ -507,7 +507,9 @@ def to_flyte_idl(self): """ :rtype: flyteidl.admin.tasks_pb2.TaskSpec """ - return _admin_task.TaskSpec(template=self.template.to_flyte_idl(), description_entity=self.docs.to_flyte_idl() if self.docs else None) + return _admin_task.TaskSpec( + template=self.template.to_flyte_idl(), description_entity=self.docs.to_flyte_idl() if self.docs else None + ) @classmethod def from_flyte_idl(cls, pb2_object): @@ -515,7 +517,10 @@ def from_flyte_idl(cls, pb2_object): :param flyteidl.admin.tasks_pb2.TaskSpec pb2_object: :rtype: TaskSpec """ - return cls(TaskTemplate.from_flyte_idl(pb2_object.template), Documentation.from_flyte_idl(pb2_object.description_entity) if pb2_object.description_entity else None) + return cls( + TaskTemplate.from_flyte_idl(pb2_object.template), + Documentation.from_flyte_idl(pb2_object.description_entity) if pb2_object.description_entity else None, + ) class Task(_common.FlyteIdlEntity): diff --git a/flytekit/remote/remote.py b/flytekit/remote/remote.py index e60f88d9dc..f02226decc 100644 --- a/flytekit/remote/remote.py +++ b/flytekit/remote/remote.py @@ -19,7 +19,7 @@ from flyteidl.core import literals_pb2 as literals_pb2 -from flytekit import Documentation, Literal +from flytekit import Literal from flytekit.clients.friendly import SynchronousFlyteClient from flytekit.clients.helpers import iterate_node_executions, iterate_task_executions from flytekit.configuration import Config, FastSerializationSettings, ImageConfig, SerializationSettings diff --git a/tests/flytekit/unit/models/test_documentation.py b/tests/flytekit/unit/models/test_documentation.py new file mode 100644 index 0000000000..73aea6aa96 --- /dev/null +++ b/tests/flytekit/unit/models/test_documentation.py @@ -0,0 +1,29 @@ +from flytekit.models.documentation import Documentation, LongDescription, SourceCode + + +def test_long_description(): + value = "long" + icon_link = "http://icon" + obj = LongDescription(value=value, icon_link=icon_link) + assert LongDescription.from_flyte_idl(obj.to_flyte_idl()) == obj + assert obj.value == value + assert obj.icon_link == icon_link + assert obj.format == LongDescription.DescriptionFormat.RST + + +def test_source_code(): + link = "https://github.com/flyteorg/flytekit" + obj = SourceCode(link=link) + assert SourceCode.from_flyte_idl(obj.to_flyte_idl()) == obj + assert obj.link == link + + +def test_documentation(): + short_description = "short" + long_description = LongDescription(value="long", icon_link="http://icon") + source_code = SourceCode(link="https://github.com/flyteorg/flytekit") + obj = Documentation(short_description=short_description, long_description=long_description, source_code=source_code) + assert Documentation.from_flyte_idl(obj.to_flyte_idl()) == obj + assert obj.short_description == short_description + assert obj.long_description == long_description + assert obj.source_code == source_code diff --git a/tests/flytekit/unit/models/test_tasks.py b/tests/flytekit/unit/models/test_tasks.py index fcebf465f9..2fdbcbf7c7 100644 --- a/tests/flytekit/unit/models/test_tasks.py +++ b/tests/flytekit/unit/models/test_tasks.py @@ -7,6 +7,7 @@ import flytekit.models.interface as interface_models import flytekit.models.literals as literal_models +from flytekit import Documentation, LongDescription, SourceCode from flytekit.models import literals, task, types from flytekit.models.core import identifier from tests.flytekit.common import parameterizers @@ -123,6 +124,60 @@ def test_task_template(in_tuple): assert obj.config == {"a": "b"} +def test_task_spec(): + task_metadata = task.TaskMetadata( + True, + task.RuntimeMetadata(task.RuntimeMetadata.RuntimeType.FLYTE_SDK, "1.0.0", "python"), + timedelta(days=1), + literals.RetryStrategy(3), + True, + "0.1.1b0", + "This is deprecated!", + True, + ) + + int_type = types.LiteralType(types.SimpleType.INTEGER) + interfaces = interface_models.TypedInterface( + {"a": interface_models.Variable(int_type, "description1")}, + { + "b": interface_models.Variable(int_type, "description2"), + "c": interface_models.Variable(int_type, "description3"), + }, + ) + + resource = [task.Resources.ResourceEntry(task.Resources.ResourceName.CPU, "1")] + resources = task.Resources(resource, resource) + + template = task.TaskTemplate( + identifier.Identifier(identifier.ResourceType.TASK, "project", "domain", "name", "version"), + "python", + task_metadata, + interfaces, + {"a": 1, "b": {"c": 2, "d": 3}}, + container=task.Container( + "my_image", + ["this", "is", "a", "cmd"], + ["this", "is", "an", "arg"], + resources, + {"a": "b"}, + {"d": "e"}, + ), + config={"a": "b"}, + ) + + short_description = "short" + long_description = LongDescription(value="long", icon_link="http://icon") + source_code = SourceCode(link="https://github.com/flyteorg/flytekit") + docs = Documentation( + short_description=short_description, long_description=long_description, source_code=source_code + ) + + obj = task.TaskSpec(template, docs) + assert task.TaskSpec.from_flyte_idl(obj.to_flyte_idl()) == obj + assert obj.docs == docs + assert obj.template == template + + def test_task_template__k8s_pod_target(): int_type = types.LiteralType(types.SimpleType.INTEGER) obj = task.TaskTemplate( From 28d66e47ba432e59ef755df5053c748593ddb3b0 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Thu, 6 Oct 2022 15:03:08 -0700 Subject: [PATCH 13/38] more tets Signed-off-by: Kevin Su --- tests/flytekit/unit/core/test_interface.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/flytekit/unit/core/test_interface.py b/tests/flytekit/unit/core/test_interface.py index 62c45c346e..f3202b21b4 100644 --- a/tests/flytekit/unit/core/test_interface.py +++ b/tests/flytekit/unit/core/test_interface.py @@ -2,6 +2,7 @@ import typing from typing import Dict, List +from flytekit import task from flytekit.core import context_manager from flytekit.core.docstring import Docstring from flytekit.core.interface import ( @@ -323,3 +324,20 @@ def z(a: Foo) -> Foo: assert params.parameters["a"].default is None assert our_interface.outputs["o0"].__origin__ == FlytePickle assert our_interface.inputs["a"].__origin__ == FlytePickle + + +def test_doc_string(): + @task + def t1(a: int) -> int: + """Set the temperature value. + + The value of the temp parameter is stored as a value in + the class variable temperature. + """ + return a + + assert t1.docs.short_description == "Set the temperature value." + assert ( + t1.docs.long_description.value + == "The value of the temp parameter is stored as a value in\nthe class variable temperature." + ) From 40ca98211866cfc462776e6281535529e5bae470 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Thu, 6 Oct 2022 17:54:45 -0700 Subject: [PATCH 14/38] update workflow spec Signed-off-by: Kevin Su --- flytekit/core/workflow.py | 26 ++++++++++++++++++- flytekit/models/admin/workflow.py | 20 +++++++++++++- flytekit/tools/translator.py | 2 +- .../unit/models/test_workflow_closure.py | 15 +++++++++++ 4 files changed, 60 insertions(+), 3 deletions(-) diff --git a/flytekit/core/workflow.py b/flytekit/core/workflow.py index 0f36f86ec3..9c362c5ab2 100644 --- a/flytekit/core/workflow.py +++ b/flytekit/core/workflow.py @@ -39,6 +39,7 @@ from flytekit.models import interface as _interface_models from flytekit.models import literals as _literal_models from flytekit.models.core import workflow as _workflow_model +from flytekit.models.documentation import Documentation, LongDescription GLOBAL_START_NODE = Node( id=_common_constants.GLOBAL_INPUT_NODE_ID, @@ -168,6 +169,7 @@ def __init__( workflow_metadata: WorkflowMetadata, workflow_metadata_defaults: WorkflowMetadataDefaults, python_interface: Interface, + docs: Documentation, **kwargs, ): self._name = name @@ -179,6 +181,20 @@ def __init__( self._unbound_inputs = set() self._nodes = [] self._output_bindings: Optional[List[_literal_models.Binding]] = [] + self._docs = docs + + if self._python_interface.docstring: + short_description = self._python_interface.docstring.short_description + long_description = None + if self._python_interface.docstring.long_description: + long_description = LongDescription(value=self._python_interface.docstring.long_description) + + if self.docs is None: + self._docs = Documentation(short_description=short_description, long_description=long_description) + else: + self._docs.short_description = short_description + self._docs.long_description = long_description + FlyteEntities.entities.append(self) super().__init__(**kwargs) @@ -186,6 +202,10 @@ def __init__( def name(self) -> str: return self._name + @property + def docs(self): + return self._docs + @property def short_name(self) -> str: return extract_obj_name(self._name) @@ -571,7 +591,8 @@ def __init__( workflow_function: Callable, metadata: Optional[WorkflowMetadata], default_metadata: Optional[WorkflowMetadataDefaults], - docstring: Docstring = None, + docstring: Optional[Docstring] = None, + docs: Optional[Documentation] = None, ): name, _, _, _ = extract_task_module(workflow_function) self._workflow_function = workflow_function @@ -586,6 +607,7 @@ def __init__( workflow_metadata=metadata, workflow_metadata_defaults=default_metadata, python_interface=native_interface, + docs=docs, ) @property @@ -690,6 +712,7 @@ def workflow( _workflow_function=None, failure_policy: Optional[WorkflowFailurePolicy] = None, interruptible: bool = False, + docs: Optional[Documentation] = None, ): """ This decorator declares a function to be a Flyte workflow. Workflows are declarative entities that construct a DAG @@ -730,6 +753,7 @@ def wrapper(fn): metadata=workflow_metadata, default_metadata=workflow_metadata_defaults, docstring=Docstring(callable_=fn), + docs=docs, ) workflow_instance.compile() update_wrapper(workflow_instance, fn) diff --git a/flytekit/models/admin/workflow.py b/flytekit/models/admin/workflow.py index f34e692123..91870b9d02 100644 --- a/flytekit/models/admin/workflow.py +++ b/flytekit/models/admin/workflow.py @@ -1,13 +1,21 @@ +import typing + from flyteidl.admin import workflow_pb2 as _admin_workflow from flytekit.models import common as _common from flytekit.models.core import compiler as _compiler_models from flytekit.models.core import identifier as _identifier from flytekit.models.core import workflow as _core_workflow +from flytekit.models.documentation import Documentation class WorkflowSpec(_common.FlyteIdlEntity): - def __init__(self, template, sub_workflows): + def __init__( + self, + template: _core_workflow.WorkflowTemplate, + sub_workflows: typing.List[_core_workflow.WorkflowTemplate], + docs: typing.Optional[Documentation] = None, + ): """ This object fully encapsulates the specification of a workflow :param flytekit.models.core.workflow.WorkflowTemplate template: @@ -15,6 +23,7 @@ def __init__(self, template, sub_workflows): """ self._template = template self._sub_workflows = sub_workflows + self._docs = docs @property def template(self): @@ -30,6 +39,13 @@ def sub_workflows(self): """ return self._sub_workflows + @property + def docs(self): + """ + :rtype: Documentation + """ + return self._docs + def to_flyte_idl(self): """ :rtype: flyteidl.admin.workflow_pb2.WorkflowSpec @@ -37,6 +53,7 @@ def to_flyte_idl(self): return _admin_workflow.WorkflowSpec( template=self._template.to_flyte_idl(), sub_workflows=[s.to_flyte_idl() for s in self._sub_workflows], + description_entity=self._docs.to_flyte_idl() if self._docs else None, ) @classmethod @@ -48,6 +65,7 @@ def from_flyte_idl(cls, pb2_object): return cls( _core_workflow.WorkflowTemplate.from_flyte_idl(pb2_object.template), [_core_workflow.WorkflowTemplate.from_flyte_idl(s) for s in pb2_object.sub_workflows], + Documentation.from_flyte_idl(pb2_object.description_entity) if pb2_object.description_entity else None, ) diff --git a/flytekit/tools/translator.py b/flytekit/tools/translator.py index 2a76af17ad..1893e52b19 100644 --- a/flytekit/tools/translator.py +++ b/flytekit/tools/translator.py @@ -291,7 +291,7 @@ def get_serializable_workflow( outputs=entity.output_bindings, ) return admin_workflow_models.WorkflowSpec( - template=wf_t, sub_workflows=sorted(set(sub_wfs), key=lambda x: x.short_string()) + template=wf_t, sub_workflows=sorted(set(sub_wfs), key=lambda x: x.short_string()), docs=entity.docs ) diff --git a/tests/flytekit/unit/models/test_workflow_closure.py b/tests/flytekit/unit/models/test_workflow_closure.py index 3a42f5af81..7346e7495c 100644 --- a/tests/flytekit/unit/models/test_workflow_closure.py +++ b/tests/flytekit/unit/models/test_workflow_closure.py @@ -5,8 +5,10 @@ from flytekit.models import task as _task from flytekit.models import types as _types from flytekit.models import workflow_closure as _workflow_closure +from flytekit.models.admin.workflow import WorkflowSpec from flytekit.models.core import identifier as _identifier from flytekit.models.core import workflow as _workflow +from flytekit.models.documentation import Documentation, LongDescription, SourceCode def test_workflow_closure(): @@ -81,3 +83,16 @@ def test_workflow_closure(): obj2 = _workflow_closure.WorkflowClosure.from_flyte_idl(obj.to_flyte_idl()) assert obj == obj2 + + short_description = "short" + long_description = LongDescription(value="long", icon_link="http://icon") + source_code = SourceCode(link="https://github.com/flyteorg/flytekit") + docs = Documentation( + short_description=short_description, long_description=long_description, source_code=source_code + ) + + workflow_spec = WorkflowSpec(template=template, sub_workflows=[], docs=docs) + assert WorkflowSpec.from_flyte_idl(workflow_spec.to_flyte_idl()) == workflow_spec + assert workflow_spec.docs.short_description == short_description + assert workflow_spec.docs.long_description == long_description + assert workflow_spec.docs.source_code == source_code From 0bb512255c01d48e5233de1e68d8e445ad91f15a Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Fri, 7 Oct 2022 09:52:19 -0700 Subject: [PATCH 15/38] fix test Signed-off-by: Kevin Su --- flytekit/core/workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flytekit/core/workflow.py b/flytekit/core/workflow.py index 9c362c5ab2..02de541152 100644 --- a/flytekit/core/workflow.py +++ b/flytekit/core/workflow.py @@ -169,7 +169,7 @@ def __init__( workflow_metadata: WorkflowMetadata, workflow_metadata_defaults: WorkflowMetadataDefaults, python_interface: Interface, - docs: Documentation, + docs: Optional[Documentation] = None, **kwargs, ): self._name = name From 15352042f6f892bb0bb74b0de0f35261868d2549 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 11 Oct 2022 13:34:05 -0700 Subject: [PATCH 16/38] Upload long description file through data proxy Signed-off-by: Kevin Su --- flytekit/core/base_task.py | 16 ++++++++-------- flytekit/core/workflow.py | 16 ++++++++-------- flytekit/remote/remote.py | 20 +++++++++++++++++--- flytekit/tools/script_mode.py | 10 ++++++++++ flytekit/tools/translator.py | 1 + 5 files changed, 44 insertions(+), 19 deletions(-) diff --git a/flytekit/core/base_task.py b/flytekit/core/base_task.py index 36cd2740dd..f6d386d91d 100644 --- a/flytekit/core/base_task.py +++ b/flytekit/core/base_task.py @@ -399,16 +399,16 @@ def __init__( self._task_config = task_config self._disable_deck = disable_deck if self._python_interface.docstring: - short_description = self._python_interface.docstring.short_description - long_description = None - if self._python_interface.docstring.long_description: - long_description = LongDescription(value=self._python_interface.docstring.long_description) - if self.docs is None: - self._docs = Documentation(short_description=short_description, long_description=long_description) + self._docs = Documentation( + short_description=self._python_interface.docstring.short_description, + long_description=LongDescription(value=self._python_interface.docstring.long_description), + ) else: - self._docs.short_description = short_description - self._docs.long_description = long_description + if self._python_interface.docstring.short_description: + self._docs.short_description = self._python_interface.docstring.short_description + if self._python_interface.docstring.long_description: + self._docs.long_description = self._python_interface.docstring.long_description # TODO lets call this interface and the other as flyte_interface? @property diff --git a/flytekit/core/workflow.py b/flytekit/core/workflow.py index 02de541152..1daa5f531f 100644 --- a/flytekit/core/workflow.py +++ b/flytekit/core/workflow.py @@ -184,16 +184,16 @@ def __init__( self._docs = docs if self._python_interface.docstring: - short_description = self._python_interface.docstring.short_description - long_description = None - if self._python_interface.docstring.long_description: - long_description = LongDescription(value=self._python_interface.docstring.long_description) - if self.docs is None: - self._docs = Documentation(short_description=short_description, long_description=long_description) + self._docs = Documentation( + short_description=self._python_interface.docstring.short_description, + long_description=LongDescription(value=self._python_interface.docstring.long_description), + ) else: - self._docs.short_description = short_description - self._docs.long_description = long_description + if self._python_interface.docstring.short_description: + self._docs.short_description = self._python_interface.docstring.short_description + if self._python_interface.docstring.long_description: + self._docs.long_description = self._python_interface.docstring.long_description FlyteEntities.entities.append(self) super().__init__(**kwargs) diff --git a/flytekit/remote/remote.py b/flytekit/remote/remote.py index f02226decc..2c29619b40 100644 --- a/flytekit/remote/remote.py +++ b/flytekit/remote/remote.py @@ -23,7 +23,7 @@ from flytekit.clients.friendly import SynchronousFlyteClient from flytekit.clients.helpers import iterate_node_executions, iterate_task_executions from flytekit.configuration import Config, FastSerializationSettings, ImageConfig, SerializationSettings -from flytekit.core import constants, utils +from flytekit.core import constants, context_manager, utils from flytekit.core.base_task import PythonTask from flytekit.core.context_manager import FlyteContext, FlyteContextManager from flytekit.core.data_persistence import FileAccessProvider @@ -60,7 +60,7 @@ from flytekit.remote.remote_callable import RemoteEntity from flytekit.remote.task import FlyteTask from flytekit.remote.workflow import FlyteWorkflow -from flytekit.tools.script_mode import fast_register_single_script, hash_file +from flytekit.tools.script_mode import fast_register_single_script, hash_file, upload_single_file from flytekit.tools.translator import FlyteLocalEntity, Options, get_serializable, get_serializable_launch_plan ExecutionDataResponse = typing.Union[WorkflowExecutionGetDataResponse, NodeExecutionGetDataResponse] @@ -607,7 +607,6 @@ def register_script( filename="scriptmode.tar.gz", ), ) - serialization_settings = SerializationSettings( project=project, domain=domain, @@ -619,6 +618,21 @@ def register_script( ), ) + # Upload long description file if it's present locally, and override the uri in the entity. + if entity.docs and entity.docs.long_description and entity.docs.long_description.uri: + ctx = context_manager.FlyteContextManager.current_context() + if not ctx.file_access.is_remote(entity.docs.long_description.uri): + upload_location, _ = upload_single_file( + entity.docs.long_description.uri, + functools.partial( + self.client.get_upload_signed_url, + project=project or self.default_project, + domain=domain or self.default_domain, + filename=os.path.basename(entity.docs.long_description.uri), + ), + ) + entity.docs.long_description.uri = upload_location.native_url + if version is None: # The md5 version that we send to S3/GCS has to match the file contents exactly, # but we don't have to use it when registering with the Flyte backend. diff --git a/flytekit/tools/script_mode.py b/flytekit/tools/script_mode.py index 29b617824c..d8f94c4c84 100644 --- a/flytekit/tools/script_mode.py +++ b/flytekit/tools/script_mode.py @@ -114,6 +114,16 @@ def fast_register_single_script( return upload_location, md5 +def upload_single_file( + file: typing.Union[str, Path], create_upload_location_fn: typing.Callable +) -> _data_proxy_pb2.CreateUploadLocationResponse: + flyte_ctx = context_manager.FlyteContextManager.current_context() + md5, _ = hash_file(file) + upload_location = create_upload_location_fn(content_md5=md5) + flyte_ctx.file_access.put_data(file, upload_location.signed_url) + return upload_location + + def hash_file(file_path: typing.Union[os.PathLike, str]) -> (bytes, str): """ Hash a file and produce a digest to be used as a version diff --git a/flytekit/tools/translator.py b/flytekit/tools/translator.py index 1893e52b19..30e36a10e4 100644 --- a/flytekit/tools/translator.py +++ b/flytekit/tools/translator.py @@ -290,6 +290,7 @@ def get_serializable_workflow( nodes=upstream_node_models, outputs=entity.output_bindings, ) + return admin_workflow_models.WorkflowSpec( template=wf_t, sub_workflows=sorted(set(sub_wfs), key=lambda x: x.short_string()), docs=entity.docs ) From 893017797d07b21dbb609fca52022c51de8d5bc1 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Thu, 13 Oct 2022 12:57:28 -0700 Subject: [PATCH 17/38] rename Signed-off-by: Kevin Su --- flytekit/__init__.py | 2 +- flytekit/core/base_task.py | 4 ++-- flytekit/core/workflow.py | 4 ++-- flytekit/models/documentation.py | 14 +++++++------- requirements.txt | 12 ++---------- tests/flytekit/unit/models/test_documentation.py | 10 +++++----- tests/flytekit/unit/models/test_tasks.py | 4 ++-- .../flytekit/unit/models/test_workflow_closure.py | 4 ++-- 8 files changed, 23 insertions(+), 31 deletions(-) diff --git a/flytekit/__init__.py b/flytekit/__init__.py index ada6f89527..4f286901cb 100644 --- a/flytekit/__init__.py +++ b/flytekit/__init__.py @@ -188,7 +188,7 @@ from flytekit.models.common import Annotations, AuthRole, Labels from flytekit.models.core.execution import WorkflowExecutionPhase from flytekit.models.core.types import BlobType -from flytekit.models.documentation import Documentation, LongDescription, SourceCode +from flytekit.models.documentation import Description, Documentation, SourceCode from flytekit.models.literals import Blob, BlobMetadata, Literal, Scalar from flytekit.models.types import LiteralType from flytekit.types import directory, file, numpy, schema diff --git a/flytekit/core/base_task.py b/flytekit/core/base_task.py index f6d386d91d..4acdb54cb1 100644 --- a/flytekit/core/base_task.py +++ b/flytekit/core/base_task.py @@ -46,7 +46,7 @@ from flytekit.models import literals as _literal_models from flytekit.models import task as _task_model from flytekit.models.core import workflow as _workflow_model -from flytekit.models.documentation import Documentation, LongDescription +from flytekit.models.documentation import Description, Documentation from flytekit.models.interface import Variable from flytekit.models.security import SecurityContext @@ -402,7 +402,7 @@ def __init__( if self.docs is None: self._docs = Documentation( short_description=self._python_interface.docstring.short_description, - long_description=LongDescription(value=self._python_interface.docstring.long_description), + long_description=Description(value=self._python_interface.docstring.long_description), ) else: if self._python_interface.docstring.short_description: diff --git a/flytekit/core/workflow.py b/flytekit/core/workflow.py index 1daa5f531f..7da361a03e 100644 --- a/flytekit/core/workflow.py +++ b/flytekit/core/workflow.py @@ -39,7 +39,7 @@ from flytekit.models import interface as _interface_models from flytekit.models import literals as _literal_models from flytekit.models.core import workflow as _workflow_model -from flytekit.models.documentation import Documentation, LongDescription +from flytekit.models.documentation import Description, Documentation GLOBAL_START_NODE = Node( id=_common_constants.GLOBAL_INPUT_NODE_ID, @@ -187,7 +187,7 @@ def __init__( if self.docs is None: self._docs = Documentation( short_description=self._python_interface.docstring.short_description, - long_description=LongDescription(value=self._python_interface.docstring.long_description), + long_description=Description(value=self._python_interface.docstring.long_description), ) else: if self._python_interface.docstring.short_description: diff --git a/flytekit/models/documentation.py b/flytekit/models/documentation.py index 00ad48443e..3d0b4df6d9 100644 --- a/flytekit/models/documentation.py +++ b/flytekit/models/documentation.py @@ -8,7 +8,7 @@ @dataclass -class LongDescription(_common_models.FlyteIdlEntity): +class Description(_common_models.FlyteIdlEntity): """ Full user description with formatting preserved. This can be rendered by clients, such as the console or command line tools with in-tact @@ -27,7 +27,7 @@ class DescriptionFormat(Enum): format: DescriptionFormat = DescriptionFormat.RST def to_flyte_idl(self): - return description_entity_pb2.LongDescription( + return description_entity_pb2.Description( value=self.value if self.value else None, uri=self.uri if self.uri else None, format=self.format.value, @@ -35,11 +35,11 @@ def to_flyte_idl(self): ) @classmethod - def from_flyte_idl(cls, pb2_object: description_entity_pb2.LongDescription) -> "LongDescription": + def from_flyte_idl(cls, pb2_object: description_entity_pb2.Description) -> "Description": return cls( value=pb2_object.value, uri=pb2_object.uri, - format=LongDescription.DescriptionFormat(pb2_object.format), + format=Description.DescriptionFormat(pb2_object.format), icon_link=pb2_object.icon_link, ) @@ -67,12 +67,12 @@ class Documentation(_common_models.FlyteIdlEntity): Documentation could provide insight into the algorithms, business use case, etc. Args: short_description (str): One-liner overview of the entity. - long_description (Optional[LongDescription]): Full user description with formatting preserved. + long_description (Optional[Description]): Full user description with formatting preserved. source_code (Optional[SourceCode]): link to source code used to define this entity """ short_description: str - long_description: Optional[LongDescription] = None + long_description: Optional[Description] = None source_code: Optional[SourceCode] = None def to_flyte_idl(self): @@ -86,7 +86,7 @@ def to_flyte_idl(self): def from_flyte_idl(cls, pb2_object: description_entity_pb2.DescriptionEntity) -> "Documentation": return cls( short_description=pb2_object.short_description, - long_description=LongDescription.from_flyte_idl(pb2_object.long_description) + long_description=Description.from_flyte_idl(pb2_object.long_description) if pb2_object.long_description else None, source_code=SourceCode.from_flyte_idl(pb2_object.source_code) if pb2_object.source_code else None, diff --git a/requirements.txt b/requirements.txt index 32b4ae49a6..33aedc5747 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with python 3.7 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # -# make requirements.txt +# pip-compile requirements.in # -e file:.#egg=flytekit # via -r requirements.in @@ -64,9 +64,7 @@ idna==3.3 # via requests importlib-metadata==4.12.0 # via - # click # flytekit - # jsonschema # keyring jinja2==3.1.2 # via @@ -98,7 +96,6 @@ natsort==8.1.0 numpy==1.21.6 # via # -r requirements.in - # flytekit # pandas # pyarrow packaging==21.3 @@ -163,8 +160,6 @@ responses==0.21.0 # via flytekit retry==0.9.2 # via flytekit -singledispatchmethod==1.0 - # via flytekit six==1.16.0 # via # grpcio @@ -179,10 +174,7 @@ text-unidecode==1.3 # via python-slugify typing-extensions==4.3.0 # via - # arrow # flytekit - # importlib-metadata - # responses # typing-inspect typing-inspect==0.8.0 # via dataclasses-json diff --git a/tests/flytekit/unit/models/test_documentation.py b/tests/flytekit/unit/models/test_documentation.py index 73aea6aa96..7702df0452 100644 --- a/tests/flytekit/unit/models/test_documentation.py +++ b/tests/flytekit/unit/models/test_documentation.py @@ -1,14 +1,14 @@ -from flytekit.models.documentation import Documentation, LongDescription, SourceCode +from flytekit.models.documentation import Description, Documentation, SourceCode def test_long_description(): value = "long" icon_link = "http://icon" - obj = LongDescription(value=value, icon_link=icon_link) - assert LongDescription.from_flyte_idl(obj.to_flyte_idl()) == obj + obj = Description(value=value, icon_link=icon_link) + assert Description.from_flyte_idl(obj.to_flyte_idl()) == obj assert obj.value == value assert obj.icon_link == icon_link - assert obj.format == LongDescription.DescriptionFormat.RST + assert obj.format == Description.DescriptionFormat.RST def test_source_code(): @@ -20,7 +20,7 @@ def test_source_code(): def test_documentation(): short_description = "short" - long_description = LongDescription(value="long", icon_link="http://icon") + long_description = Description(value="long", icon_link="http://icon") source_code = SourceCode(link="https://github.com/flyteorg/flytekit") obj = Documentation(short_description=short_description, long_description=long_description, source_code=source_code) assert Documentation.from_flyte_idl(obj.to_flyte_idl()) == obj diff --git a/tests/flytekit/unit/models/test_tasks.py b/tests/flytekit/unit/models/test_tasks.py index 2fdbcbf7c7..fed32b63aa 100644 --- a/tests/flytekit/unit/models/test_tasks.py +++ b/tests/flytekit/unit/models/test_tasks.py @@ -7,7 +7,7 @@ import flytekit.models.interface as interface_models import flytekit.models.literals as literal_models -from flytekit import Documentation, LongDescription, SourceCode +from flytekit import Description, Documentation, SourceCode from flytekit.models import literals, task, types from flytekit.models.core import identifier from tests.flytekit.common import parameterizers @@ -166,7 +166,7 @@ def test_task_spec(): ) short_description = "short" - long_description = LongDescription(value="long", icon_link="http://icon") + long_description = Description(value="long", icon_link="http://icon") source_code = SourceCode(link="https://github.com/flyteorg/flytekit") docs = Documentation( short_description=short_description, long_description=long_description, source_code=source_code diff --git a/tests/flytekit/unit/models/test_workflow_closure.py b/tests/flytekit/unit/models/test_workflow_closure.py index 7346e7495c..d229d0d5c9 100644 --- a/tests/flytekit/unit/models/test_workflow_closure.py +++ b/tests/flytekit/unit/models/test_workflow_closure.py @@ -8,7 +8,7 @@ from flytekit.models.admin.workflow import WorkflowSpec from flytekit.models.core import identifier as _identifier from flytekit.models.core import workflow as _workflow -from flytekit.models.documentation import Documentation, LongDescription, SourceCode +from flytekit.models.documentation import Description, Documentation, SourceCode def test_workflow_closure(): @@ -85,7 +85,7 @@ def test_workflow_closure(): assert obj == obj2 short_description = "short" - long_description = LongDescription(value="long", icon_link="http://icon") + long_description = Description(value="long", icon_link="http://icon") source_code = SourceCode(link="https://github.com/flyteorg/flytekit") docs = Documentation( short_description=short_description, long_description=long_description, source_code=source_code From 1ac181f7c97a74629c3b8bd6dc0f82b47a43da34 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Wed, 19 Oct 2022 15:58:32 -0700 Subject: [PATCH 18/38] update Signed-off-by: Kevin Su --- flytekit/core/base_task.py | 2 +- flytekit/core/workflow.py | 2 +- flytekit/remote/remote.py | 44 ++++++++++++++++++++++++-------------- 3 files changed, 30 insertions(+), 18 deletions(-) diff --git a/flytekit/core/base_task.py b/flytekit/core/base_task.py index 4acdb54cb1..77e5748aa0 100644 --- a/flytekit/core/base_task.py +++ b/flytekit/core/base_task.py @@ -408,7 +408,7 @@ def __init__( if self._python_interface.docstring.short_description: self._docs.short_description = self._python_interface.docstring.short_description if self._python_interface.docstring.long_description: - self._docs.long_description = self._python_interface.docstring.long_description + self._docs.long_description = Description(value=self._python_interface.docstring.long_description) # TODO lets call this interface and the other as flyte_interface? @property diff --git a/flytekit/core/workflow.py b/flytekit/core/workflow.py index 7da361a03e..2d9aa98d3f 100644 --- a/flytekit/core/workflow.py +++ b/flytekit/core/workflow.py @@ -193,7 +193,7 @@ def __init__( if self._python_interface.docstring.short_description: self._docs.short_description = self._python_interface.docstring.short_description if self._python_interface.docstring.long_description: - self._docs.long_description = self._python_interface.docstring.long_description + self._docs = Description(value=self._python_interface.docstring.long_description) FlyteEntities.entities.append(self) super().__init__(**kwargs) diff --git a/flytekit/remote/remote.py b/flytekit/remote/remote.py index 2c29619b40..becbe59cf7 100644 --- a/flytekit/remote/remote.py +++ b/flytekit/remote/remote.py @@ -10,6 +10,7 @@ import hashlib import os import pathlib +import sys import time import typing import uuid @@ -19,7 +20,7 @@ from flyteidl.core import literals_pb2 as literals_pb2 -from flytekit import Literal +from flytekit import Documentation, Literal from flytekit.clients.friendly import SynchronousFlyteClient from flytekit.clients.helpers import iterate_node_executions, iterate_task_executions from flytekit.configuration import Config, FastSerializationSettings, ImageConfig, SerializationSettings @@ -421,10 +422,12 @@ def _serialize_and_register( if isinstance(cp_entity, task_models.TaskSpec): ident = self._resolve_identifier(ResourceType.TASK, entity.name, version, settings) self.client.create_task(task_identifer=ident, task_spec=cp_entity) + self.offload_long_description(cp_entity.docs, ident.project, ident.domain) elif isinstance(cp_entity, admin_workflow_models.WorkflowSpec): ident = self._resolve_identifier(ResourceType.WORKFLOW, entity.name, version, settings) try: self.client.create_workflow(workflow_identifier=ident, workflow_spec=cp_entity) + self.offload_long_description(cp_entity.docs, ident.project, ident.domain) except FlyteEntityAlreadyExistsException: remote_logger.info(f"{entity.name} already exists") # Let us also create a default launch-plan, ideally the default launchplan should be added @@ -450,6 +453,30 @@ def _serialize_and_register( raise return ident + def offload_long_description(self, docs: Documentation, project: str, domain: str): + if docs and docs.long_description: + ctx = context_manager.FlyteContextManager.current_context() + if docs.long_description.value: + # Offload the long description if the size > 4KB + if sys.getsizeof(docs.long_description.value) > 4 * 1024 * 1024: + local_path = ctx.file_access.get_random_local_path() + with open(local_path, "w") as f: + f.write(docs.long_description.value) + docs.long_description.uri = local_path + docs.long_description.value = None + + # Upload long description file if it's present locally, and override the uri in the entity. + if docs.long_description.uri and not ctx.file_access.is_remote(docs.long_description.uri): + md5, _ = hash_file(docs.long_description.uri) + upload_location = self.client.get_upload_signed_url( + content_md5=md5, + project=project, + domain=domain, + filename=os.path.basename(docs.long_description.uri), + ) + ctx.file_access.put_data(docs.long_description.uri, upload_location.signed_url) + docs.long_description.uri = upload_location.native_url + def register_task( self, entity: PythonTask, serialization_settings: SerializationSettings, version: typing.Optional[str] = None ) -> FlyteTask: @@ -618,21 +645,6 @@ def register_script( ), ) - # Upload long description file if it's present locally, and override the uri in the entity. - if entity.docs and entity.docs.long_description and entity.docs.long_description.uri: - ctx = context_manager.FlyteContextManager.current_context() - if not ctx.file_access.is_remote(entity.docs.long_description.uri): - upload_location, _ = upload_single_file( - entity.docs.long_description.uri, - functools.partial( - self.client.get_upload_signed_url, - project=project or self.default_project, - domain=domain or self.default_domain, - filename=os.path.basename(entity.docs.long_description.uri), - ), - ) - entity.docs.long_description.uri = upload_location.native_url - if version is None: # The md5 version that we send to S3/GCS has to match the file contents exactly, # but we don't have to use it when registering with the Flyte backend. From 5625465d7c4386cb69e5afdb17498ee752377fe4 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Thu, 20 Oct 2022 14:22:58 -0700 Subject: [PATCH 19/38] update source code Signed-off-by: Kevin Su --- flytekit/configuration/__init__.py | 4 ++++ flytekit/models/documentation.py | 2 +- flytekit/remote/remote.py | 19 +++++++++++++++---- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/flytekit/configuration/__init__.py b/flytekit/configuration/__init__.py index ad8567a94d..a514d25d0b 100644 --- a/flytekit/configuration/__init__.py +++ b/flytekit/configuration/__init__.py @@ -636,6 +636,7 @@ class SerializationSettings(object): domain: typing.Optional[str] = None version: typing.Optional[str] = None env: Optional[Dict[str, str]] = None + git_repo: Optional[str] = None python_interpreter: str = DEFAULT_RUNTIME_PYTHON_INTERPRETER flytekit_virtualenv_root: Optional[str] = None fast_serialization_settings: Optional[FastSerializationSettings] = None @@ -708,6 +709,7 @@ def new_builder(self) -> Builder: version=self.version, image_config=self.image_config, env=self.env.copy() if self.env else None, + git_repo=self.git_repo, flytekit_virtualenv_root=self.flytekit_virtualenv_root, python_interpreter=self.python_interpreter, fast_serialization_settings=self.fast_serialization_settings, @@ -757,6 +759,7 @@ class Builder(object): version: str image_config: ImageConfig env: Optional[Dict[str, str]] = None + git_repo: Optional[str] = None flytekit_virtualenv_root: Optional[str] = None python_interpreter: Optional[str] = None fast_serialization_settings: Optional[FastSerializationSettings] = None @@ -772,6 +775,7 @@ def build(self) -> SerializationSettings: version=self.version, image_config=self.image_config, env=self.env, + git_repo=self.git_repo, flytekit_virtualenv_root=self.flytekit_virtualenv_root, python_interpreter=self.python_interpreter, fast_serialization_settings=self.fast_serialization_settings, diff --git a/flytekit/models/documentation.py b/flytekit/models/documentation.py index 3d0b4df6d9..32de1473af 100644 --- a/flytekit/models/documentation.py +++ b/flytekit/models/documentation.py @@ -71,7 +71,7 @@ class Documentation(_common_models.FlyteIdlEntity): source_code (Optional[SourceCode]): link to source code used to define this entity """ - short_description: str + short_description: Optional[str] = None long_description: Optional[Description] = None source_code: Optional[SourceCode] = None diff --git a/flytekit/remote/remote.py b/flytekit/remote/remote.py index becbe59cf7..793a448787 100644 --- a/flytekit/remote/remote.py +++ b/flytekit/remote/remote.py @@ -8,6 +8,7 @@ import base64 import functools import hashlib +import inspect import os import pathlib import sys @@ -19,8 +20,9 @@ from datetime import datetime, timedelta from flyteidl.core import literals_pb2 as literals_pb2 +from git import Repo -from flytekit import Documentation, Literal +from flytekit import Literal from flytekit.clients.friendly import SynchronousFlyteClient from flytekit.clients.helpers import iterate_node_executions, iterate_task_executions from flytekit.configuration import Config, FastSerializationSettings, ImageConfig, SerializationSettings @@ -47,6 +49,7 @@ from flytekit.models.core import workflow as workflow_model from flytekit.models.core.identifier import Identifier, ResourceType, WorkflowExecutionIdentifier from flytekit.models.core.workflow import NodeMetadata +from flytekit.models.documentation import Documentation, SourceCode from flytekit.models.execution import ( ExecutionMetadata, ExecutionSpec, @@ -421,13 +424,13 @@ def _serialize_and_register( try: if isinstance(cp_entity, task_models.TaskSpec): ident = self._resolve_identifier(ResourceType.TASK, entity.name, version, settings) + self.update_description_entity(cp_entity.docs, settings, ident.project, ident.domain) self.client.create_task(task_identifer=ident, task_spec=cp_entity) - self.offload_long_description(cp_entity.docs, ident.project, ident.domain) elif isinstance(cp_entity, admin_workflow_models.WorkflowSpec): ident = self._resolve_identifier(ResourceType.WORKFLOW, entity.name, version, settings) try: + self.update_description_entity(cp_entity.docs, settings, ident.project, ident.domain) self.client.create_workflow(workflow_identifier=ident, workflow_spec=cp_entity) - self.offload_long_description(cp_entity.docs, ident.project, ident.domain) except FlyteEntityAlreadyExistsException: remote_logger.info(f"{entity.name} already exists") # Let us also create a default launch-plan, ideally the default launchplan should be added @@ -453,7 +456,12 @@ def _serialize_and_register( raise return ident - def offload_long_description(self, docs: Documentation, project: str, domain: str): + def update_description_entity( + self, docs: Documentation, settings: typing.Optional[SerializationSettings], project: str, domain: str + ): + # 1. Offload the long description if the size > 4KB + # 2. Upload long description file if it's present locally, and override the uri in the entity + # 3. Extract the repo URL from the git config, and assign it to the link of the source code of the description entity if docs and docs.long_description: ctx = context_manager.FlyteContextManager.current_context() if docs.long_description.value: @@ -477,6 +485,8 @@ def offload_long_description(self, docs: Documentation, project: str, domain: st ctx.file_access.put_data(docs.long_description.uri, upload_location.signed_url) docs.long_description.uri = upload_location.native_url + docs.source_code = SourceCode(link=settings.git_repo) + def register_task( self, entity: PythonTask, serialization_settings: SerializationSettings, version: typing.Optional[str] = None ) -> FlyteTask: @@ -638,6 +648,7 @@ def register_script( project=project, domain=domain, image_config=image_config, + git_repo="github.com/" + Repo(source_path).remotes.origin.url.split(".git")[0].split(":")[-1], fast_serialization_settings=FastSerializationSettings( enabled=True, destination_dir=destination_dir, From 473f6e670cc50d8083398cd95d1027dd5eabf88f Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Fri, 21 Oct 2022 13:50:32 -0700 Subject: [PATCH 20/38] update Signed-off-by: Kevin Su --- flytekit/models/admin/workflow.py | 2 +- flytekit/models/task.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flytekit/models/admin/workflow.py b/flytekit/models/admin/workflow.py index 91870b9d02..3033060fce 100644 --- a/flytekit/models/admin/workflow.py +++ b/flytekit/models/admin/workflow.py @@ -53,7 +53,7 @@ def to_flyte_idl(self): return _admin_workflow.WorkflowSpec( template=self._template.to_flyte_idl(), sub_workflows=[s.to_flyte_idl() for s in self._sub_workflows], - description_entity=self._docs.to_flyte_idl() if self._docs else None, + description=self._docs.to_flyte_idl() if self._docs else None, ) @classmethod diff --git a/flytekit/models/task.py b/flytekit/models/task.py index e8c1aba469..6191cddb3c 100644 --- a/flytekit/models/task.py +++ b/flytekit/models/task.py @@ -508,7 +508,7 @@ def to_flyte_idl(self): :rtype: flyteidl.admin.tasks_pb2.TaskSpec """ return _admin_task.TaskSpec( - template=self.template.to_flyte_idl(), description_entity=self.docs.to_flyte_idl() if self.docs else None + template=self.template.to_flyte_idl(), description=self.docs.to_flyte_idl() if self.docs else None ) @classmethod From 3acfaf246ab85095cf9ca39296d785b21f795dc3 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Fri, 18 Nov 2022 15:30:10 -0800 Subject: [PATCH 21/38] merged master Signed-off-by: Kevin Su --- .github/workflows/pythonbuild.yml | 4 ---- dev-requirements.txt | 2 +- flytekit/core/task.py | 4 ---- requirements-spark2.txt | 2 +- requirements.txt | 2 +- setup.py | 2 +- tests/flytekit/unit/core/test_interface.py | 1 + 7 files changed, 5 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml index 045c8f34ba..fc8a554cdd 100644 --- a/.github/workflows/pythonbuild.yml +++ b/.github/workflows/pythonbuild.yml @@ -46,7 +46,6 @@ jobs: - name: Install dependencies run: | make setup${{ matrix.spark-version-suffix }} - pip install "git+https://github.com/flyteorg/flyteidl@doc-hub" pip freeze - name: Test with coverage run: | @@ -132,7 +131,6 @@ jobs: pip install -r requirements.txt if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi pip install --no-deps -U https://github.com/flyteorg/flytekit/archive/${{ github.sha }}.zip#egg=flytekit - pip install "git+https://github.com/flyteorg/flyteidl@doc-hub" pip freeze - name: Test with coverage run: | @@ -157,7 +155,6 @@ jobs: run: | python -m pip install --upgrade pip==21.2.4 pip install -r dev-requirements.txt - pip install "git+https://github.com/flyteorg/flyteidl@doc-hub" - name: Lint run: | make lint @@ -179,6 +176,5 @@ jobs: run: | python -m pip install --upgrade pip==21.2.4 setuptools wheel pip install -r doc-requirements.txt - pip install "git+https://github.com/flyteorg/flyteidl@doc-hub" - name: Build the documentation run: make -C docs html diff --git a/dev-requirements.txt b/dev-requirements.txt index 5bf005c94a..18c7d93c3f 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -122,7 +122,7 @@ exceptiongroup==1.0.1 # via pytest filelock==3.8.0 # via virtualenv -flyteidl==1.2.4 +flyteidl==1.2.5 # via # -c requirements.txt # flytekit diff --git a/flytekit/core/task.py b/flytekit/core/task.py index a272f883fb..bb24181338 100644 --- a/flytekit/core/task.py +++ b/flytekit/core/task.py @@ -90,12 +90,8 @@ def task( secret_requests: Optional[List[Secret]] = None, execution_mode: Optional[PythonFunctionTask.ExecutionBehavior] = PythonFunctionTask.ExecutionBehavior.DEFAULT, task_resolver: Optional[TaskResolverMixin] = None, -<<<<<<< HEAD - disable_deck: Optional[bool] = False, docs: Optional[Documentation] = None, -======= disable_deck: bool = True, ->>>>>>> 3acbc48b9418b45f3bd7e0c30a5a53f570647980 ) -> Union[Callable, PythonFunctionTask]: """ This is the core decorator to use for any task type in flytekit. diff --git a/requirements-spark2.txt b/requirements-spark2.txt index e69d746efc..e4dee24689 100644 --- a/requirements-spark2.txt +++ b/requirements-spark2.txt @@ -52,7 +52,7 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.15 # via flytekit -flyteidl==1.2.4 +flyteidl==1.2.5 # via flytekit googleapis-common-protos==1.56.4 # via diff --git a/requirements.txt b/requirements.txt index 3544c443e3..c5ea6d3160 100644 --- a/requirements.txt +++ b/requirements.txt @@ -50,7 +50,7 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.15 # via flytekit -flyteidl==1.2.4 +flyteidl==1.2.5 # via flytekit googleapis-common-protos==1.56.4 # via diff --git a/setup.py b/setup.py index 347d94be92..400db30546 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ ] }, install_requires=[ - "flyteidl>=1.2.0,<1.3.0", + "flyteidl>=1.2.5,<1.3.0", "wheel>=0.30.0,<1.0.0", "pandas>=1.0.0,<2.0.0", "pyarrow>=4.0.0,<7.0.0", diff --git a/tests/flytekit/unit/core/test_interface.py b/tests/flytekit/unit/core/test_interface.py index b44e6b82b1..0a23df627e 100644 --- a/tests/flytekit/unit/core/test_interface.py +++ b/tests/flytekit/unit/core/test_interface.py @@ -148,6 +148,7 @@ def q() -> os.PathLike: def test_file_types(): + def t1() -> FlyteFile[typing.TypeVar("svg")]: ... From 08eda7a997d742f98e6904dd48572e8ead9ec077 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Fri, 18 Nov 2022 15:46:08 -0800 Subject: [PATCH 22/38] nit Signed-off-by: Kevin Su --- flytekit/remote/remote.py | 42 ++++++++--------------------------- flytekit/tools/script_mode.py | 10 --------- 2 files changed, 9 insertions(+), 43 deletions(-) diff --git a/flytekit/remote/remote.py b/flytekit/remote/remote.py index 909f606a64..2f627be365 100644 --- a/flytekit/remote/remote.py +++ b/flytekit/remote/remote.py @@ -416,6 +416,7 @@ def raw_register( if isinstance(cp_entity, task_models.TaskSpec): ident = self._resolve_identifier(ResourceType.TASK, cp_entity.template.id.name, version, settings) try: + self.update_description_entity(cp_entity.docs, settings, ident.project, ident.domain) self.client.create_task(task_identifer=ident, task_spec=cp_entity) except FlyteEntityAlreadyExistsException: remote_logger.info(f" {ident} Already Exists!") @@ -424,6 +425,7 @@ def raw_register( if isinstance(cp_entity, admin_workflow_models.WorkflowSpec): ident = self._resolve_identifier(ResourceType.WORKFLOW, cp_entity.template.id.name, version, settings) try: + self.update_description_entity(cp_entity.docs, settings, ident.project, ident.domain) self.client.create_workflow(workflow_identifier=ident, workflow_spec=cp_entity) except FlyteEntityAlreadyExistsException: remote_logger.info(f" {ident} Already Exists!") @@ -498,39 +500,13 @@ def _serialize_and_register( settings, f"No serialization settings set, but workflow contains entities that need to be registered. {cp_entity.id.name}", ) - try: - if isinstance(cp_entity, task_models.TaskSpec): - ident = self._resolve_identifier(ResourceType.TASK, entity.name, version, settings) - self.update_description_entity(cp_entity.docs, settings, ident.project, ident.domain) - self.client.create_task(task_identifer=ident, task_spec=cp_entity) - elif isinstance(cp_entity, admin_workflow_models.WorkflowSpec): - ident = self._resolve_identifier(ResourceType.WORKFLOW, entity.name, version, settings) - try: - self.update_description_entity(cp_entity.docs, settings, ident.project, ident.domain) - self.client.create_workflow(workflow_identifier=ident, workflow_spec=cp_entity) - except FlyteEntityAlreadyExistsException: - remote_logger.info(f"{entity.name} already exists") - # Let us also create a default launch-plan, ideally the default launchplan should be added - # to the orderedDict, but we do not. - default_lp = LaunchPlan.get_default_launch_plan(self.context, entity) - lp_entity = get_serializable_launch_plan( - OrderedDict(), - settings or serialization_settings, - default_lp, - recurse_downstream=False, - options=options, - ) - self.client.create_launch_plan(lp_entity.id, lp_entity.spec) - elif isinstance(cp_entity, launch_plan_models.LaunchPlan): - ident = self._resolve_identifier(ResourceType.LAUNCH_PLAN, entity.name, version, settings) - self.client.create_launch_plan(launch_plan_identifer=ident, launch_plan_spec=cp_entity.spec) - else: - raise AssertionError(f"Unknown entity of type {type(cp_entity)}") - except FlyteEntityAlreadyExistsException: - remote_logger.info(f"{entity.name} already exists") - except Exception as e: - remote_logger.info(f"Failed to register entity {entity.name} with error {e}") - raise + # if isinstance(cp_entity, task_models.TaskSpec): + # ident = self._resolve_identifier(ResourceType.TASK, entity.name, version, settings) + # self.update_description_entity(cp_entity.docs, settings, ident.project, ident.domain) + # self.client.create_task(task_identifer=ident, task_spec=cp_entity) + # elif isinstance(cp_entity, admin_workflow_models.WorkflowSpec): + # ident = self._resolve_identifier(ResourceType.WORKFLOW, entity.name, version, settings) + # self.update_description_entity(cp_entity.docs, settings, ident.project, ident.domain) ident = self.raw_register( cp_entity, diff --git a/flytekit/tools/script_mode.py b/flytekit/tools/script_mode.py index d8f94c4c84..29b617824c 100644 --- a/flytekit/tools/script_mode.py +++ b/flytekit/tools/script_mode.py @@ -114,16 +114,6 @@ def fast_register_single_script( return upload_location, md5 -def upload_single_file( - file: typing.Union[str, Path], create_upload_location_fn: typing.Callable -) -> _data_proxy_pb2.CreateUploadLocationResponse: - flyte_ctx = context_manager.FlyteContextManager.current_context() - md5, _ = hash_file(file) - upload_location = create_upload_location_fn(content_md5=md5) - flyte_ctx.file_access.put_data(file, upload_location.signed_url) - return upload_location - - def hash_file(file_path: typing.Union[os.PathLike, str]) -> (bytes, str): """ Hash a file and produce a digest to be used as a version From 573696386da04b46ee6d5e5130daab8ee22836e9 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Fri, 18 Nov 2022 16:17:49 -0800 Subject: [PATCH 23/38] lint Signed-off-by: Kevin Su --- tests/flytekit/unit/core/test_interface.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/flytekit/unit/core/test_interface.py b/tests/flytekit/unit/core/test_interface.py index 0a23df627e..db05de0ddb 100644 --- a/tests/flytekit/unit/core/test_interface.py +++ b/tests/flytekit/unit/core/test_interface.py @@ -2,9 +2,9 @@ import typing from typing import Dict, List -from flytekit import task from typing_extensions import Annotated # type: ignore +from flytekit import task from flytekit.core import context_manager from flytekit.core.docstring import Docstring from flytekit.core.interface import ( @@ -148,7 +148,6 @@ def q() -> os.PathLike: def test_file_types(): - def t1() -> FlyteFile[typing.TypeVar("svg")]: ... From c17f832fba672cde833e35bd46c5b7006db3c7a4 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 19 Dec 2022 11:32:52 -0800 Subject: [PATCH 24/38] nit Signed-off-by: Kevin Su --- flytekit/remote/remote.py | 16 ++++++++-------- flytekit/tools/repo.py | 2 ++ requirements.txt | 7 +------ 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/flytekit/remote/remote.py b/flytekit/remote/remote.py index 2f627be365..39fa63226c 100644 --- a/flytekit/remote/remote.py +++ b/flytekit/remote/remote.py @@ -117,6 +117,13 @@ def _get_entity_identifier( ) +def _get_git_repo_url(source_path): + """ + Get git repo URL from remote.origin.url + """ + return "github.com/" + Repo(source_path).remotes.origin.url.split(".git")[0].split(":")[-1] + + class FlyteRemote(object): """Main entrypoint for programmatically accessing a Flyte remote backend. @@ -500,13 +507,6 @@ def _serialize_and_register( settings, f"No serialization settings set, but workflow contains entities that need to be registered. {cp_entity.id.name}", ) - # if isinstance(cp_entity, task_models.TaskSpec): - # ident = self._resolve_identifier(ResourceType.TASK, entity.name, version, settings) - # self.update_description_entity(cp_entity.docs, settings, ident.project, ident.domain) - # self.client.create_task(task_identifer=ident, task_spec=cp_entity) - # elif isinstance(cp_entity, admin_workflow_models.WorkflowSpec): - # ident = self._resolve_identifier(ResourceType.WORKFLOW, entity.name, version, settings) - # self.update_description_entity(cp_entity.docs, settings, ident.project, ident.domain) ident = self.raw_register( cp_entity, @@ -726,7 +726,7 @@ def register_script( project=project, domain=domain, image_config=image_config, - git_repo="github.com/" + Repo(source_path).remotes.origin.url.split(".git")[0].split(":")[-1], + git_repo=_get_git_repo_url(source_path), fast_serialization_settings=FastSerializationSettings( enabled=True, destination_dir=destination_dir, diff --git a/flytekit/tools/repo.py b/flytekit/tools/repo.py index 870299e5ad..363fc9ac4b 100644 --- a/flytekit/tools/repo.py +++ b/flytekit/tools/repo.py @@ -11,6 +11,7 @@ from flytekit.loggers import logger from flytekit.models import launch_plan from flytekit.remote import FlyteRemote +from flytekit.remote.remote import _get_git_repo_url from flytekit.tools import fast_registration, module_loader from flytekit.tools.script_mode import _find_project_root from flytekit.tools.serialize_helpers import get_registrable_entities, persist_registrable_entities @@ -210,6 +211,7 @@ def register( project=project, domain=domain, version=version, + git_repo=_get_git_repo_url(detected_root), image_config=image_config, fast_serialization_settings=fast_serialization_settings, ) diff --git a/requirements.txt b/requirements.txt index 16b57708f5..935120b3b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,8 @@ # -<<<<<<< HEAD # This file is autogenerated by pip-compile with python 3.9 # To update, run: -======= -# This file is autogenerated by pip-compile with Python 3.7 -# by the following command: ->>>>>>> 315956f45978ed121d04149ffe1f4d802a93c6e3 # -# pip-compile requirements.in +# make requirements.txt # -e file:.#egg=flytekit # via -r requirements.in From 645af3cdd0dd5c9bcc195b2331a57d1fa90d51da Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 19 Dec 2022 11:46:18 -0800 Subject: [PATCH 25/38] nit Signed-off-by: Kevin Su --- flytekit/core/workflow.py | 1 + flytekit/models/admin/workflow.py | 2 +- flytekit/models/task.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/flytekit/core/workflow.py b/flytekit/core/workflow.py index 0b29c9aee1..a1a1581e96 100644 --- a/flytekit/core/workflow.py +++ b/flytekit/core/workflow.py @@ -741,6 +741,7 @@ def workflow( :param _workflow_function: This argument is implicitly passed and represents the decorated function. :param failure_policy: Use the options in flytekit.WorkflowFailurePolicy :param interruptible: Whether or not tasks launched from this workflow are by default interruptible + :param docs: Description entity for the workflow """ def wrapper(fn): diff --git a/flytekit/models/admin/workflow.py b/flytekit/models/admin/workflow.py index 3033060fce..92234ece55 100644 --- a/flytekit/models/admin/workflow.py +++ b/flytekit/models/admin/workflow.py @@ -42,7 +42,7 @@ def sub_workflows(self): @property def docs(self): """ - :rtype: Documentation + :rtype: Description entity for the workflow """ return self._docs diff --git a/flytekit/models/task.py b/flytekit/models/task.py index 6191cddb3c..d537eb3e99 100644 --- a/flytekit/models/task.py +++ b/flytekit/models/task.py @@ -499,7 +499,7 @@ def template(self): @property def docs(self): """ - :rtype: Documentation + :rtype: Description entity for the task """ return self._docs From f6f082d1ea1fc382b804c7611c2eccbc550596fb Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 19 Dec 2022 13:26:43 -0800 Subject: [PATCH 26/38] Add git package Signed-off-by: Kevin Su --- flytekit/remote/remote.py | 1 - setup.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/flytekit/remote/remote.py b/flytekit/remote/remote.py index 39fa63226c..c4b692ece3 100644 --- a/flytekit/remote/remote.py +++ b/flytekit/remote/remote.py @@ -8,7 +8,6 @@ import base64 import functools import hashlib -import inspect import os import pathlib import sys diff --git a/setup.py b/setup.py index 3d9710004f..3231a95865 100644 --- a/setup.py +++ b/setup.py @@ -82,6 +82,7 @@ # TODO: We should remove mentions to the deprecated numpy # aliases. More details in https://github.com/flyteorg/flyte/issues/3166 "numpy<1.24.0", + "git", ], extras_require=extras_require, scripts=[ From a71b1be4ccdfb16e627e7f50ce5f6440bb056583 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 19 Dec 2022 14:05:00 -0800 Subject: [PATCH 27/38] Add git package Signed-off-by: Kevin Su --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3231a95865..33bf6a0fa4 100644 --- a/setup.py +++ b/setup.py @@ -82,7 +82,7 @@ # TODO: We should remove mentions to the deprecated numpy # aliases. More details in https://github.com/flyteorg/flyte/issues/3166 "numpy<1.24.0", - "git", + "gitdb", ], extras_require=extras_require, scripts=[ From d45c2d96c560d6ebfa184c1751df8ee067395afc Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 19 Dec 2022 14:31:38 -0800 Subject: [PATCH 28/38] update package Signed-off-by: Kevin Su --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 33bf6a0fa4..b42f8a8490 100644 --- a/setup.py +++ b/setup.py @@ -82,7 +82,7 @@ # TODO: We should remove mentions to the deprecated numpy # aliases. More details in https://github.com/flyteorg/flyte/issues/3166 "numpy<1.24.0", - "gitdb", + "gitpython", ], extras_require=extras_require, scripts=[ From 61820e0eddeece83149ad13c5052b62ca4878061 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Wed, 21 Dec 2022 15:23:22 -0800 Subject: [PATCH 29/38] fix tests Signed-off-by: Kevin Su --- flytekit/models/admin/workflow.py | 2 +- flytekit/models/task.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flytekit/models/admin/workflow.py b/flytekit/models/admin/workflow.py index 92234ece55..e40307b6ba 100644 --- a/flytekit/models/admin/workflow.py +++ b/flytekit/models/admin/workflow.py @@ -65,7 +65,7 @@ def from_flyte_idl(cls, pb2_object): return cls( _core_workflow.WorkflowTemplate.from_flyte_idl(pb2_object.template), [_core_workflow.WorkflowTemplate.from_flyte_idl(s) for s in pb2_object.sub_workflows], - Documentation.from_flyte_idl(pb2_object.description_entity) if pb2_object.description_entity else None, + Documentation.from_flyte_idl(pb2_object.description) if pb2_object.description else None, ) diff --git a/flytekit/models/task.py b/flytekit/models/task.py index d537eb3e99..2129cdd88f 100644 --- a/flytekit/models/task.py +++ b/flytekit/models/task.py @@ -519,7 +519,7 @@ def from_flyte_idl(cls, pb2_object): """ return cls( TaskTemplate.from_flyte_idl(pb2_object.template), - Documentation.from_flyte_idl(pb2_object.description_entity) if pb2_object.description_entity else None, + Documentation.from_flyte_idl(pb2_object.description) if pb2_object.description else None, ) From ccfa2e4b0addd15e29477a70a99649ed26cb0d0f Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Wed, 21 Dec 2022 16:27:55 -0800 Subject: [PATCH 30/38] nit Signed-off-by: Kevin Su --- flytekit/tools/repo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flytekit/tools/repo.py b/flytekit/tools/repo.py index c87e3e3980..6263d0b162 100644 --- a/flytekit/tools/repo.py +++ b/flytekit/tools/repo.py @@ -12,7 +12,7 @@ from flytekit.models import launch_plan from flytekit.models.core.identifier import Identifier from flytekit.remote import FlyteRemote -from flytekit.remote.remote import _get_git_repo_url, RegistrationSkipped +from flytekit.remote.remote import RegistrationSkipped, _get_git_repo_url from flytekit.tools import fast_registration, module_loader from flytekit.tools.script_mode import _find_project_root from flytekit.tools.serialize_helpers import get_registrable_entities, persist_registrable_entities From f73e13e6525d261b85c00c1c1b0498ae52b5aa04 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Wed, 21 Dec 2022 16:37:37 -0800 Subject: [PATCH 31/38] nit Signed-off-by: Kevin Su --- tests/flytekit/unit/core/test_context_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/flytekit/unit/core/test_context_manager.py b/tests/flytekit/unit/core/test_context_manager.py index 98af80638a..6e68c9d4be 100644 --- a/tests/flytekit/unit/core/test_context_manager.py +++ b/tests/flytekit/unit/core/test_context_manager.py @@ -207,7 +207,7 @@ def test_serialization_settings_transport(): ss = SerializationSettings.from_transport(tp) assert ss is not None assert ss == serialization_settings - assert len(tp) == 376 + assert len(tp) == 388 def test_exec_params(): From f48f83cfb7b40d3aa1060d7da76d8bd153eff844 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Wed, 21 Dec 2022 17:04:30 -0800 Subject: [PATCH 32/38] nit Signed-off-by: Kevin Su --- flytekit/remote/remote.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flytekit/remote/remote.py b/flytekit/remote/remote.py index ecc7078736..f93e0ca393 100644 --- a/flytekit/remote/remote.py +++ b/flytekit/remote/remote.py @@ -126,7 +126,11 @@ def _get_git_repo_url(source_path): """ Get git repo URL from remote.origin.url """ - return "github.com/" + Repo(source_path).remotes.origin.url.split(".git")[0].split(":")[-1] + try: + return "github.com/" + Repo(source_path).remotes.origin.url.split(".git")[0].split(":")[-1] + except Exception: + # If the file isn't in the git repo, we can't get the url from git config + return "" class FlyteRemote(object): From a3a5f7d4c93d972d31342621340ffce22a16bafd Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Wed, 28 Dec 2022 17:26:23 -0800 Subject: [PATCH 33/38] set default value to None Signed-off-by: Kevin Su --- flytekit/models/documentation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flytekit/models/documentation.py b/flytekit/models/documentation.py index 32de1473af..ba9c468e5b 100644 --- a/flytekit/models/documentation.py +++ b/flytekit/models/documentation.py @@ -21,9 +21,9 @@ class DescriptionFormat(Enum): HTML = 2 RST = 3 - value: Optional[str] = "" - uri: Optional[str] = "" - icon_link: Optional[str] = "" + value: Optional[str] = None + uri: Optional[str] = None + icon_link: Optional[str] = None format: DescriptionFormat = DescriptionFormat.RST def to_flyte_idl(self): From 44899da1732aa3b9511ebad284ee9ec7d2b533d1 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Wed, 28 Dec 2022 17:41:56 -0800 Subject: [PATCH 34/38] fix tests Signed-off-by: Kevin Su --- flytekit/models/documentation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flytekit/models/documentation.py b/flytekit/models/documentation.py index ba9c468e5b..e1bae8122e 100644 --- a/flytekit/models/documentation.py +++ b/flytekit/models/documentation.py @@ -37,10 +37,10 @@ def to_flyte_idl(self): @classmethod def from_flyte_idl(cls, pb2_object: description_entity_pb2.Description) -> "Description": return cls( - value=pb2_object.value, - uri=pb2_object.uri, + value=pb2_object.value if pb2_object.value else None, + uri=pb2_object.uri if pb2_object.uri else None, format=Description.DescriptionFormat(pb2_object.format), - icon_link=pb2_object.icon_link, + icon_link=pb2_object.icon_link if pb2_object.icon_link else None, ) From 75e5961130bfc67a06afa1233492ac8882ba17fa Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 3 Jan 2023 13:57:05 -0800 Subject: [PATCH 35/38] make get_git_repo_url public Signed-off-by: Kevin Su --- flytekit/remote/remote.py | 4 ++-- flytekit/tools/repo.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/flytekit/remote/remote.py b/flytekit/remote/remote.py index f93e0ca393..75bca632dd 100644 --- a/flytekit/remote/remote.py +++ b/flytekit/remote/remote.py @@ -122,7 +122,7 @@ def _get_entity_identifier( ) -def _get_git_repo_url(source_path): +def get_git_repo_url(source_path): """ Get git repo URL from remote.origin.url """ @@ -776,7 +776,7 @@ def register_script( project=project, domain=domain, image_config=image_config, - git_repo=_get_git_repo_url(source_path), + git_repo=get_git_repo_url(source_path), fast_serialization_settings=FastSerializationSettings( enabled=True, destination_dir=destination_dir, diff --git a/flytekit/tools/repo.py b/flytekit/tools/repo.py index 6263d0b162..82111140b0 100644 --- a/flytekit/tools/repo.py +++ b/flytekit/tools/repo.py @@ -12,7 +12,7 @@ from flytekit.models import launch_plan from flytekit.models.core.identifier import Identifier from flytekit.remote import FlyteRemote -from flytekit.remote.remote import RegistrationSkipped, _get_git_repo_url +from flytekit.remote.remote import RegistrationSkipped, get_git_repo_url from flytekit.tools import fast_registration, module_loader from flytekit.tools.script_mode import _find_project_root from flytekit.tools.serialize_helpers import get_registrable_entities, persist_registrable_entities @@ -234,7 +234,7 @@ def register( project=project, domain=domain, version=version, - git_repo=_get_git_repo_url(detected_root), + git_repo=get_git_repo_url(detected_root), image_config=image_config, fast_serialization_settings=fast_serialization_settings, ) From a791f2c4d9ec4c2d9658457a337e6d9ce5da9d48 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 3 Jan 2023 16:20:50 -0800 Subject: [PATCH 36/38] Update doc in get_serializable_flyte_workflow Signed-off-by: Kevin Su --- flytekit/remote/remote.py | 4 ++-- flytekit/tools/repo.py | 5 ++--- flytekit/tools/translator.py | 14 +++++++++++++- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/flytekit/remote/remote.py b/flytekit/remote/remote.py index 75bca632dd..f93e0ca393 100644 --- a/flytekit/remote/remote.py +++ b/flytekit/remote/remote.py @@ -122,7 +122,7 @@ def _get_entity_identifier( ) -def get_git_repo_url(source_path): +def _get_git_repo_url(source_path): """ Get git repo URL from remote.origin.url """ @@ -776,7 +776,7 @@ def register_script( project=project, domain=domain, image_config=image_config, - git_repo=get_git_repo_url(source_path), + git_repo=_get_git_repo_url(source_path), fast_serialization_settings=FastSerializationSettings( enabled=True, destination_dir=destination_dir, diff --git a/flytekit/tools/repo.py b/flytekit/tools/repo.py index 82111140b0..3c9fe64068 100644 --- a/flytekit/tools/repo.py +++ b/flytekit/tools/repo.py @@ -12,7 +12,7 @@ from flytekit.models import launch_plan from flytekit.models.core.identifier import Identifier from flytekit.remote import FlyteRemote -from flytekit.remote.remote import RegistrationSkipped, get_git_repo_url +from flytekit.remote.remote import RegistrationSkipped, _get_git_repo_url from flytekit.tools import fast_registration, module_loader from flytekit.tools.script_mode import _find_project_root from flytekit.tools.serialize_helpers import get_registrable_entities, persist_registrable_entities @@ -162,7 +162,7 @@ def load_packages_and_modules( :param options: :return: The common detected root path, the output of _find_project_root """ - + ss.git_repo = _get_git_repo_url(project_root) pkgs_and_modules = [] for pm in pkgs_or_mods: p = Path(pm).resolve() @@ -234,7 +234,6 @@ def register( project=project, domain=domain, version=version, - git_repo=get_git_repo_url(detected_root), image_config=image_config, fast_serialization_settings=fast_serialization_settings, ) diff --git a/flytekit/tools/translator.py b/flytekit/tools/translator.py index f21e08eadb..e87b0e243b 100644 --- a/flytekit/tools/translator.py +++ b/flytekit/tools/translator.py @@ -1,9 +1,10 @@ +import sys import typing from collections import OrderedDict from dataclasses import dataclass from typing import Callable, Dict, List, Optional, Tuple, Union -from flytekit import PythonFunctionTask +from flytekit import PythonFunctionTask, SourceCode from flytekit.configuration import SerializationSettings from flytekit.core import constants as _common_constants from flytekit.core.base_task import PythonTask @@ -23,6 +24,7 @@ from flytekit.models import launch_plan as _launch_plan_models from flytekit.models import security from flytekit.models.admin import workflow as admin_workflow_models +from flytekit.models.admin.workflow import WorkflowSpec from flytekit.models.core import identifier as _identifier_model from flytekit.models.core import workflow as _core_wf from flytekit.models.core import workflow as workflow_model @@ -680,6 +682,16 @@ def get_serializable( else: raise Exception(f"Non serializable type found {type(entity)} Entity {entity}") + if isinstance(entity, TaskSpec) or isinstance(entity, WorkflowSpec): + # 1. Check if the size of long description exceeds 16KB + # 2. Extract the repo URL from the git config, and assign it to the link of the source code of the description entity + if entity.docs and entity.docs.long_description: + if entity.docs.long_description.value: + if sys.getsizeof(entity.docs.long_description.value) > 16 * 1024 * 1024: + raise ValueError( + "Long Description of the flyte entity exceeds the 16KB size limit. Please specify the uri in the long description instead." + ) + entity.docs.source_code = SourceCode(link=settings.git_repo) # This needs to be at the bottom not the top - i.e. dependent tasks get added before the workflow containing it entity_mapping[entity] = cp_entity return cp_entity From a4a895dec4491dbb9abee6ca5d593f0d8a10637c Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Wed, 4 Jan 2023 13:45:32 -0800 Subject: [PATCH 37/38] remove update_description_entity Signed-off-by: Kevin Su --- flytekit/remote/remote.py | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/flytekit/remote/remote.py b/flytekit/remote/remote.py index f93e0ca393..7efe96197a 100644 --- a/flytekit/remote/remote.py +++ b/flytekit/remote/remote.py @@ -472,7 +472,6 @@ def raw_register( version = cp_entity.id.version ident = self._resolve_identifier(ResourceType.TASK, cp_entity.template.id.name, version, settings) try: - self.update_description_entity(cp_entity.docs, settings, ident.project, ident.domain) self.client.create_task(task_identifer=ident, task_spec=cp_entity) except FlyteEntityAlreadyExistsException: remote_logger.info(f" {ident} Already Exists!") @@ -483,7 +482,6 @@ def raw_register( version = cp_entity.id.version ident = self._resolve_identifier(ResourceType.WORKFLOW, cp_entity.template.id.name, version, settings) try: - self.update_description_entity(cp_entity.docs, settings, ident.project, ident.domain) self.client.create_workflow(workflow_identifier=ident, workflow_spec=cp_entity) except FlyteEntityAlreadyExistsException: remote_logger.info(f" {ident} Already Exists!") @@ -569,37 +567,6 @@ def _serialize_and_register( return ident - def update_description_entity( - self, docs: Documentation, settings: typing.Optional[SerializationSettings], project: str, domain: str - ): - # 1. Offload the long description if the size > 4KB - # 2. Upload long description file if it's present locally, and override the uri in the entity - # 3. Extract the repo URL from the git config, and assign it to the link of the source code of the description entity - if docs and docs.long_description: - ctx = context_manager.FlyteContextManager.current_context() - if docs.long_description.value: - # Offload the long description if the size > 4KB - if sys.getsizeof(docs.long_description.value) > 4 * 1024 * 1024: - local_path = ctx.file_access.get_random_local_path() - with open(local_path, "w") as f: - f.write(docs.long_description.value) - docs.long_description.uri = local_path - docs.long_description.value = None - - # Upload long description file if it's present locally, and override the uri in the entity. - if docs.long_description.uri and not ctx.file_access.is_remote(docs.long_description.uri): - md5, _ = hash_file(docs.long_description.uri) - upload_location = self.client.get_upload_signed_url( - content_md5=md5, - project=project, - domain=domain, - filename=os.path.basename(docs.long_description.uri), - ) - ctx.file_access.put_data(docs.long_description.uri, upload_location.signed_url) - docs.long_description.uri = upload_location.native_url - - docs.source_code = SourceCode(link=settings.git_repo) - def register_task( self, entity: PythonTask, serialization_settings: SerializationSettings, version: typing.Optional[str] = None ) -> FlyteTask: From 677919c3dc912310f0ec6ec29b7c099c3c4c934f Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Wed, 4 Jan 2023 14:19:09 -0800 Subject: [PATCH 38/38] lint Signed-off-by: Kevin Su --- flytekit/remote/remote.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/flytekit/remote/remote.py b/flytekit/remote/remote.py index 7efe96197a..d58ba5afba 100644 --- a/flytekit/remote/remote.py +++ b/flytekit/remote/remote.py @@ -10,7 +10,6 @@ import hashlib import os import pathlib -import sys import time import typing import uuid @@ -25,7 +24,7 @@ from flytekit.clients.friendly import SynchronousFlyteClient from flytekit.clients.helpers import iterate_node_executions, iterate_task_executions from flytekit.configuration import Config, FastSerializationSettings, ImageConfig, SerializationSettings -from flytekit.core import constants, context_manager, utils +from flytekit.core import constants, utils from flytekit.core.base_task import PythonTask from flytekit.core.context_manager import FlyteContext, FlyteContextManager from flytekit.core.data_persistence import FileAccessProvider @@ -48,7 +47,6 @@ from flytekit.models.core import workflow as workflow_model from flytekit.models.core.identifier import Identifier, ResourceType, WorkflowExecutionIdentifier from flytekit.models.core.workflow import NodeMetadata -from flytekit.models.documentation import Documentation, SourceCode from flytekit.models.execution import ( ExecutionMetadata, ExecutionSpec,