From 2544bd55e1d89fe7f5d4e9e67313f82fc14b3b2b Mon Sep 17 00:00:00 2001 From: Niels Bantilan Date: Fri, 10 Feb 2023 11:24:46 -0500 Subject: [PATCH 1/6] add reference_launch_plan to flytekit api ref Signed-off-by: Niels Bantilan --- flytekit/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flytekit/__init__.py b/flytekit/__init__.py index 1748a7a17e..67cb78c8f6 100644 --- a/flytekit/__init__.py +++ b/flytekit/__init__.py @@ -108,6 +108,7 @@ WorkflowReference reference_task reference_workflow + reference_launch_plan Core Task Types ================= From 9a87f1ad833cb2aa662ab92a6512f1b7361c0333 Mon Sep 17 00:00:00 2001 From: Niels Bantilan Date: Fri, 10 Feb 2023 12:11:46 -0500 Subject: [PATCH 2/6] import in init, add docstrings Signed-off-by: Niels Bantilan --- flytekit/__init__.py | 1 + flytekit/core/launch_plan.py | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/flytekit/__init__.py b/flytekit/__init__.py index 67cb78c8f6..c06456f9e7 100644 --- a/flytekit/__init__.py +++ b/flytekit/__init__.py @@ -183,6 +183,7 @@ from flytekit.core.resources import Resources from flytekit.core.schedule import CronSchedule, FixedRate from flytekit.core.task import Secret, reference_task, task +from flytekit.core.launch_plan import reference_launch_plan from flytekit.core.workflow import ImperativeWorkflow as Workflow from flytekit.core.workflow import WorkflowFailurePolicy, reference_workflow, workflow from flytekit.deck import Deck diff --git a/flytekit/core/launch_plan.py b/flytekit/core/launch_plan.py index 0d143e5fe8..bbbcd3e13f 100644 --- a/flytekit/core/launch_plan.py +++ b/flytekit/core/launch_plan.py @@ -455,8 +455,15 @@ def reference_launch_plan( ) -> Callable[[Callable[..., Any]], ReferenceLaunchPlan]: """ A reference launch plan is a pointer to a launch plan that already exists on your Flyte installation. This - object will not initiate a network call to Admin, which is why the user is asked to provide the expected interface. + object will not initiate a network call to Admin, which is why the user is asked to provide the expected interface + via the function definition. + If at registration time the interface provided causes an issue with compilation, an error will be returned. + + :param project: Flyte project name of the launch plan + :param domain: Flyte domain name of the launch plan + :param name: launch plan name + :param version: specific version of the launch plan to use """ def wrapper(fn) -> ReferenceLaunchPlan: From 047de20a5c373e973ffb1b76f2497ee46d5ab1a7 Mon Sep 17 00:00:00 2001 From: Niels Bantilan Date: Fri, 10 Feb 2023 16:31:58 -0500 Subject: [PATCH 3/6] add more to references Signed-off-by: Niels Bantilan --- flytekit/__init__.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/flytekit/__init__.py b/flytekit/__init__.py index c06456f9e7..bd0cb73830 100644 --- a/flytekit/__init__.py +++ b/flytekit/__init__.py @@ -154,6 +154,30 @@ Scalar LiteralType BlobType + +Task Utilities +============== + +.. autosummary:: + :nosignatures: + :template: custom.rst + :toctree: generated/ + + Deck + HashMethod + +Signaling +========= + +.. autosummary:: + :nosignatures: + :template: custom.rst + :toctree: generated/ + + approve + sleep + wait_for_input + """ import sys from typing import Generator @@ -241,7 +265,7 @@ def load_implicit_plugins(): # note the group is always ``flytekit.plugins`` setup( ... - entry_points={'flytekit.plugins’: 'fsspec=flytekitplugins.fsspec'}, + entry_points={'flytekit.plugins': 'fsspec=flytekitplugins.fsspec'}, ... ) From 3b674454b1b64f43402efcd14ef70cec42ba4a29 Mon Sep 17 00:00:00 2001 From: Niels Bantilan Date: Fri, 10 Feb 2023 16:33:20 -0500 Subject: [PATCH 4/6] fix lint Signed-off-by: Niels Bantilan --- flytekit/__init__.py | 3 +-- flytekit/core/launch_plan.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/flytekit/__init__.py b/flytekit/__init__.py index bd0cb73830..c3c2e90fdb 100644 --- a/flytekit/__init__.py +++ b/flytekit/__init__.py @@ -197,7 +197,7 @@ from flytekit.core.dynamic_workflow_task import dynamic from flytekit.core.gate import approve, sleep, wait_for_input from flytekit.core.hash import HashMethod -from flytekit.core.launch_plan import LaunchPlan +from flytekit.core.launch_plan import LaunchPlan, reference_launch_plan from flytekit.core.map_task import map_task from flytekit.core.notification import Email, PagerDuty, Slack from flytekit.core.pod_template import PodTemplate @@ -207,7 +207,6 @@ from flytekit.core.resources import Resources from flytekit.core.schedule import CronSchedule, FixedRate from flytekit.core.task import Secret, reference_task, task -from flytekit.core.launch_plan import reference_launch_plan from flytekit.core.workflow import ImperativeWorkflow as Workflow from flytekit.core.workflow import WorkflowFailurePolicy, reference_workflow, workflow from flytekit.deck import Deck diff --git a/flytekit/core/launch_plan.py b/flytekit/core/launch_plan.py index bbbcd3e13f..550dc1919e 100644 --- a/flytekit/core/launch_plan.py +++ b/flytekit/core/launch_plan.py @@ -459,7 +459,7 @@ def reference_launch_plan( via the function definition. If at registration time the interface provided causes an issue with compilation, an error will be returned. - + :param project: Flyte project name of the launch plan :param domain: Flyte domain name of the launch plan :param name: launch plan name From af25efc65a02376ee4808ab2221bf2998c31c688 Mon Sep 17 00:00:00 2001 From: Niels Bantilan Date: Fri, 10 Feb 2023 16:36:12 -0500 Subject: [PATCH 5/6] update Signed-off-by: Niels Bantilan --- flytekit/__init__.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/flytekit/__init__.py b/flytekit/__init__.py index c3c2e90fdb..0a992719d9 100644 --- a/flytekit/__init__.py +++ b/flytekit/__init__.py @@ -71,6 +71,18 @@ dynamic +Signaling +========= + +.. autosummary:: + :nosignatures: + :template: custom.rst + :toctree: generated/ + + approve + sleep + wait_for_input + Scheduling ============================ @@ -166,17 +178,17 @@ Deck HashMethod -Signaling -========= +Documentation +============= .. autosummary:: :nosignatures: :template: custom.rst :toctree: generated/ - approve - sleep - wait_for_input + Description + Documentation + SourceCode """ import sys From 1a6affd2e5a48d0df1300e1bc911107c54ace549 Mon Sep 17 00:00:00 2001 From: Niels Bantilan Date: Fri, 10 Feb 2023 17:35:10 -0500 Subject: [PATCH 6/6] fix up docstrings Signed-off-by: Niels Bantilan --- flytekit/core/gate.py | 9 ++++++--- flytekit/core/pod_template.py | 2 ++ flytekit/core/reference_entity.py | 6 ++++++ flytekit/models/common.py | 4 +++- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/flytekit/core/gate.py b/flytekit/core/gate.py index f3d90ebef8..b6cb7ca2b6 100644 --- a/flytekit/core/gate.py +++ b/flytekit/core/gate.py @@ -118,7 +118,8 @@ def local_execute(self, ctx: FlyteContext, **kwargs) -> Union[Tuple[Promise], Pr def wait_for_input(name: str, timeout: datetime.timedelta, expected_type: typing.Type): - """ + """Create a Gate object that waits for user input of the specified type. + Create a Gate object. This object will function like a task. Note that unlike a task, each time this function is called, a new Python object is created. If a workflow calls a subworkflow twice, and the subworkflow has a signal, then two Gate @@ -136,7 +137,8 @@ def wait_for_input(name: str, timeout: datetime.timedelta, expected_type: typing def sleep(duration: datetime.timedelta): - """ + """Create a sleep Gate object. + :param duration: How long to sleep for :return: """ @@ -146,7 +148,8 @@ def sleep(duration: datetime.timedelta): def approve(upstream_item: Union[Tuple[Promise], Promise, VoidPromise], name: str, timeout: datetime.timedelta): - """ + """Create a Gate object for binary approval. + Create a Gate object. This object will function like a task. Note that unlike a task, each time this function is called, a new Python object is created. If a workflow calls a subworkflow twice, and the subworkflow has a signal, then two Gate diff --git a/flytekit/core/pod_template.py b/flytekit/core/pod_template.py index be8991db45..5e9c746911 100644 --- a/flytekit/core/pod_template.py +++ b/flytekit/core/pod_template.py @@ -10,6 +10,8 @@ @dataclass class PodTemplate(object): + """Custom PodTemplate specification for a Task.""" + pod_spec: V1PodSpec = V1PodSpec(containers=[]) primary_container_name: str = PRIMARY_CONTAINER_DEFAULT_NAME labels: Optional[Dict[str, str]] = None diff --git a/flytekit/core/reference_entity.py b/flytekit/core/reference_entity.py index 77b96e6892..7247457d86 100644 --- a/flytekit/core/reference_entity.py +++ b/flytekit/core/reference_entity.py @@ -43,6 +43,8 @@ def resource_type(self) -> int: @dataclass class TaskReference(Reference): + """A reference object containing metadata that points to a remote task.""" + @property def resource_type(self) -> int: return _identifier_model.ResourceType.TASK @@ -50,6 +52,8 @@ def resource_type(self) -> int: @dataclass class LaunchPlanReference(Reference): + """A reference object containing metadata that points to a remote launch plan.""" + @property def resource_type(self) -> int: return _identifier_model.ResourceType.LAUNCH_PLAN @@ -57,6 +61,8 @@ def resource_type(self) -> int: @dataclass class WorkflowReference(Reference): + """A reference object containing metadata that points to a remote workflow.""" + @property def resource_type(self) -> int: return _identifier_model.ResourceType.WORKFLOW diff --git a/flytekit/models/common.py b/flytekit/models/common.py index 7236dd15ce..62018c1eef 100644 --- a/flytekit/models/common.py +++ b/flytekit/models/common.py @@ -414,8 +414,10 @@ def from_flyte_idl(cls, pb): class AuthRole(FlyteIdlEntity): def __init__(self, assumable_iam_role=None, kubernetes_service_account=None): - """ + """Auth configuration for IAM or K8s service account. + Either one or both of the assumable IAM role and/or the K8s service account can be set. + :param Text assumable_iam_role: IAM identity with set permissions policies. :param Text kubernetes_service_account: Provides an identity for workflow execution resources. Flyte deployment administrators are responsible for handling permissions as they