From 1417041575d054914b9b6fe303e4d9b11ead2c92 Mon Sep 17 00:00:00 2001 From: SandraGH5 <80421934+SandraGH5@users.noreply.github.com> Date: Fri, 14 May 2021 06:53:51 -0700 Subject: [PATCH] Update lp_schedules.py (#195) * Update lp_schedules.py Please recheck examples and that the flow of information makes sense. Also that the headings are denoted correctly (text size). Thanks. * Update cookbook/deployment/workflow/lp_schedules.py Co-authored-by: Samhita Alla * Update cookbook/deployment/workflow/lp_schedules.py Co-authored-by: Samhita Alla * Update cookbook/deployment/workflow/lp_schedules.py Co-authored-by: Samhita Alla * Update cookbook/deployment/workflow/lp_schedules.py Co-authored-by: Samhita Alla * Update cookbook/deployment/workflow/lp_schedules.py Co-authored-by: Samhita Alla * Update cookbook/deployment/workflow/lp_schedules.py Co-authored-by: Samhita Alla * Update cookbook/deployment/workflow/lp_schedules.py Co-authored-by: Samhita Alla * Update cookbook/deployment/workflow/lp_schedules.py Co-authored-by: Samhita Alla * Update cookbook/deployment/workflow/lp_schedules.py Co-authored-by: Samhita Alla * Update cookbook/deployment/workflow/lp_schedules.py Co-authored-by: Samhita Alla * Update cookbook/deployment/workflow/lp_schedules.py Co-authored-by: Samhita Alla * Update cookbook/deployment/workflow/lp_schedules.py Co-authored-by: Samhita Alla * Update cookbook/deployment/workflow/lp_schedules.py Co-authored-by: Samhita Alla * Update cookbook/deployment/workflow/lp_schedules.py Co-authored-by: Samhita Alla * Update cookbook/deployment/workflow/lp_schedules.py Co-authored-by: Samhita Alla * Update lp_schedules.py * Update cookbook/deployment/workflow/lp_schedules.py Co-authored-by: Samhita Alla * Update cookbook/deployment/workflow/lp_schedules.py Co-authored-by: Samhita Alla * Update lp_schedules.py * Update cookbook/deployment/workflow/lp_schedules.py Co-authored-by: Samhita Alla * Update lp_schedules.py * Update lp_schedules.py * Update lp_schedules.py * Update cookbook/deployment/workflow/lp_schedules.py Co-authored-by: Yee Hing Tong * update lp_schedules (#210) Signed-off-by: Samhita Alla Co-authored-by: Samhita Alla Co-authored-by: Yee Hing Tong --- cookbook/deployment/workflow/lp_schedules.py | 170 ++++++++++++++++--- 1 file changed, 151 insertions(+), 19 deletions(-) diff --git a/cookbook/deployment/workflow/lp_schedules.py b/cookbook/deployment/workflow/lp_schedules.py index f1d291e685..b66d64af71 100644 --- a/cookbook/deployment/workflow/lp_schedules.py +++ b/cookbook/deployment/workflow/lp_schedules.py @@ -1,15 +1,15 @@ """ - -Scheduling workflow executions with launch plans ----------------------------------------------------- +Scheduling Workflow Executions With Launch Plans +------------------------------------------------ For background on launch plans, refer to :any:`launch_plans`. -For up-to-date documentation on schedules, see the `official docs `_ +Launch plans can be set to run automatically on a schedule if the Flyte platform is properly configured. +For workflows that depend on knowing the kick-off time, Flyte also supports passing in the scheduled time (not the actual time, which may be a few seconds off) as an argument to the workflow. """ # %% -# Let's consider the following example workflow: +# Consider the following example workflow: from datetime import datetime from flytekit import task, workflow @@ -27,16 +27,19 @@ def date_formatter_wf(kickoff_time: datetime): # %% -# Cron Expression -# =============== +# The `date_formatter_wf` workflow can be scheduled using either the `CronSchedule` or the `FixedRate` object. +# +# Cron Schedules +# ############## +# # Cron expression strings use the `AWS syntax `_. # These are validated at launch plan registration time. from flytekit import CronSchedule, LaunchPlan # creates a launch plan that runs at 10am UTC every day. -cron_lp = LaunchPlan.create( - "my_cron_scheduled_lp", - date_formatter_wf, +cron_lp = LaunchPlan.get_or_create( + name="my_cron_scheduled_lp", + workflow=date_formatter_wf, schedule=CronSchedule( # Note that kickoff_time_input_arg matches the workflow input we defined above: kickoff_time cron_expression="0 10 * * ? *", @@ -45,11 +48,16 @@ def date_formatter_wf(kickoff_time: datetime): ) # %% -# Fixed Rate -# ========== -# If you prefer to use an interval rather than the cron syntax to schedule your workflows, this is currently supported -# for Flyte deployments hosted on AWS. -# To run ``date_formatter_wf`` every 10 minutes read on below: +# The ``kickoff_time_input_arg`` corresponds to the workflow input ``kickoff_time``. This means that the workflow gets triggered only after the specified kickoff time, and it thereby runs at 10 AM UTC every day. + +# %% +# Fixed Rate Intervals +# #################### +# +# If you prefer to use an interval rather than a cron scheduler to schedule your workflows, you can use the fixed-rate scheduler. +# A fixed-rate scheduler runs at the specified interval and is currently supported for Flyte deployments hosted on AWS. +# +# Here's an example: from datetime import timedelta @@ -67,9 +75,9 @@ def positive_wf(name: str): print(f"{reminder}") -fixed_rate_lp = LaunchPlan.create( - "my_fixed_rate_lp", - positive_wf, +fixed_rate_lp = LaunchPlan.get_or_create( + name="my_fixed_rate_lp", + workflow=positive_wf, # Note that the workflow above doesn't accept any kickoff time arguments. # We just omit the ``kickoff_time_input_arg`` from the FixedRate schedule invocation schedule=FixedRate(duration=timedelta(minutes=10)), @@ -77,5 +85,129 @@ def positive_wf(name: str): ) # %% +# This fixed-rate scheduler runs every ten minutes. Similar to a cron scheduler, a fixed-rate scheduler also accepts ``kickoff_time_input_arg`` (which is omitted in this example). +# +# Activating a Schedule +# ##################### +# # Once you've initialized your launch plan, don't forget to set it to active so that the schedule is run. -# TBD (katrogan) +# You can use pyflyte in container: +# +# .. code-block:: bash +# +# pyflyte lp -p {{ your project }} -d {{ your domain }} activate-all + +# %% +# (or) +# +# With flyte-cli: +# +# - View and activate launch plans: flyte-cli -i -h localhost:30081 -p flyteexamples -d development list-launch-plan-versions +# +# .. code-block:: bash +# +# flyte-cli -i -h localhost:30081 -p flyteexamples -d development list-launch-plan-versions + +# %% +# - Extract the URN returned for the launch plan you're interested in and make the call to activate it: +# +# .. code-block:: bash +# +# flyte-cli update-launch-plan -i -h localhost:30081 --state active -u {{ urn }} +# +# .. tip:: +# The equivalent command in `flytectl `__ is: +# +# .. code-block:: bash +# +# flytectl update launchplan -p flyteexamples -d development {{ name_of_lp }} --activate`` +# +# Example: +# +# .. code-block:: bash +# +# flytectl update launchplan -p flyteexamples -d development core.basic.lp.go_greet --activate + +# %% +# - Verify your active launch plans: +# +# .. code-block:: bash +# +# flyte-cli -i -h localhost:30081 -p flyteexamples -d development list-active-launch-plans +# +# .. tip:: +# The equivalent command in `flytectl `__ is: +# +# .. code-block:: bash +# +# flytectl get launchplan -p flytesnacks -d development`` + +# %% +# Platform Configuration Changes +# ############################## +# +# Scheduling features require additional infrastructure to run, so these will have to be created and configured. +# +# Setting up Scheduled Workflows +# ============================== +# To run workflow executions based on user-specified schedules, you'll need to fill out the top-level ``scheduler`` portion of the flyteadmin application configuration. +# +# In particular, you'll need to configure the two components responsible for scheduling workflows and processing schedule event triggers. +# +# .. note:: +# This functionality is currently only supported for AWS installs. +# +# Event Scheduler +# ^^^^^^^^^^^^^^^ +# +# To schedule workflow executions, you'll need to set up an `AWS SQS `_ queue. A standard-type queue should suffice. The flyteadmin event scheduler creates `AWS CloudWatch `_ event rules that invoke your SQS queue as a target. +# +# With that in mind, let's take a look at an example ``eventScheduler`` config section and dive into what each value represents: +# +# .. code-block:: bash +# +# scheduler: +# eventScheduler: +# scheme: "aws" +# region: "us-east-1" +# scheduleRole: "arn:aws:iam::{{ YOUR ACCOUNT ID }}:role/{{ ROLE }}" +# targetName: "arn:aws:sqs:us-east-1:{{ YOUR ACCOUNT ID }}:{{ YOUR QUEUE NAME }}" +# scheduleNamePrefix: "flyte" + +# %% +# * **scheme**: in this case because AWS is the only cloud back-end supported for scheduling workflows, only ``"aws"`` is a valid value. By default, the no-op scheduler is used. +# * **region**: this specifies which region initialized AWS clients should will use when creating CloudWatch rules +# * **scheduleRole** This is the IAM role ARN with permissions set to ``Allow`` +# * ``events:PutRule`` +# * ``events:PutTargets`` +# * ``events:DeleteRule`` +# * ``events:RemoveTargets`` +# * **targetName** this is the ARN for the SQS Queue you've allocated to scheduling workflows +# * **scheduleNamePrefix** this is an entirely optional prefix used when creating schedule rules. Because of AWS naming length restrictions, scheduled rules are a random hash and having a shared prefix makes these names more readable and indicates who generated the rules. +# +# Workflow Executor +# ^^^^^^^^^^^^^^^^^ +# Scheduled events which trigger need to be handled by the workflow executor, which subscribes to triggered events from the SQS queue configured above. +# +# .. NOTE:: +# +# Failure to configure a workflow executor will result in all your scheduled events piling up silently without ever kicking off workflow executions. +# +# Again, let's break down a sample config: +# +# .. code-block:: bash +# +# scheduler: +# eventScheduler: +# ... +# workflowExecutor: +# scheme: "aws" +# region: "us-east-1" +# scheduleQueueName: "{{ YOUR QUEUE NAME }}" +# accountId: "{{ YOUR ACCOUNT ID }}" + +# %% +# * **scheme**: in this case because AWS is the only cloud back-end supported for executing scheduled workflows, only ``"aws"`` is a valid value. By default, the no-op executor is used. +# * **region**: this specifies which region AWS clients should will use when creating an SQS subscriber client +# * **scheduleQueueName**: this is the name of the SQS Queue you've allocated to scheduling workflows +# * **accountId**: Your AWS `account id `_