diff --git a/cookbook/core/scheduled_workflows/README.rst b/cookbook/core/scheduled_workflows/README.rst new file mode 100644 index 000000000..77bfa3227 --- /dev/null +++ b/cookbook/core/scheduled_workflows/README.rst @@ -0,0 +1,8 @@ +.. _scheduled_workflows: + +Scheduling Workflows +-------------------- + +This module explains on how to create and launch scheduled workflows. +Flyte supports both cron based and fixed rate schedules. + diff --git a/cookbook/core/scheduled_workflows/__init__.py b/cookbook/core/scheduled_workflows/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cookbook/deployment/lp_schedules.py b/cookbook/core/scheduled_workflows/lp_schedules.py similarity index 84% rename from cookbook/deployment/lp_schedules.py rename to cookbook/core/scheduled_workflows/lp_schedules.py index b58ff03d5..a793fbc97 100644 --- a/cookbook/deployment/lp_schedules.py +++ b/cookbook/core/scheduled_workflows/lp_schedules.py @@ -1,11 +1,14 @@ """ Scheduling Workflows -------------------- - For background on launch plans, refer to :any:`launch_plans`. - -Launch plans can be set to run automatically on a schedule if the Flyte platform is properly configured. +Launch plans can be set to run automatically on a schedule using the flyte native scheduler. For workflows that depend on knowing the kick-off time, Flyte also supports passing in the scheduled time (not the actual time, which may be a few seconds off) as an argument to the workflow. + +.. note:: + + Native scheduler doesn't support `AWS syntax `_. + """ # %% @@ -32,8 +35,8 @@ def date_formatter_wf(kickoff_time: datetime): # Cron Schedules # ############## # -# Cron expression strings use the `AWS syntax `_. -# These are validated at launch plan registration time. +# Cron expression strings use the following `syntax `_. +# An incorrect cron schedule expression would lead to failure in triggering the schedule from flytekit import CronSchedule, LaunchPlan # creates a launch plan that runs at 10am UTC every day. @@ -42,7 +45,8 @@ def date_formatter_wf(kickoff_time: datetime): workflow=date_formatter_wf, schedule=CronSchedule( # Note that kickoff_time_input_arg matches the workflow input we defined above: kickoff_time - cron_expression="0 10 * * ? *", + # But in case you are using the AWS scheme of schedules and not using the native scheduler then switch over the schedule parameter with cron_expression + schedule="*/1 * * * *", # Following schedule runs every min kickoff_time_input_arg="kickoff_time", ), ) @@ -55,7 +59,7 @@ def date_formatter_wf(kickoff_time: datetime): # #################### # # If you prefer to use an interval rather than a cron scheduler to schedule your workflows, you can use the fixed-rate scheduler. -# A fixed-rate scheduler runs at the specified interval and is currently supported for Flyte deployments hosted on AWS. +# A fixed-rate scheduler runs at the specified interval. # # Here's an example: @@ -143,10 +147,10 @@ def positive_wf(name: str): # flytectl get launchplan -p flytesnacks -d development`` # %% -# Platform Configuration Changes -# ############################## +# Platform Configuration Changes For AWS Scheduler +# ################################################ # -# Scheduling features require additional infrastructure to run, so these will have to be created and configured. +# Scheduling feature can be run using the flyte native scheduler which comes with flyte but if you intend to use the AWS scheduler then it require additional infrastructure to run, so these will have to be created and configured.The following sections are only required if you use AWS scheme for the scheduler. You can even run the flyte native scheduler on AWS though # # Setting up Scheduled Workflows # ============================== @@ -207,7 +211,7 @@ def positive_wf(name: str): # accountId: "{{ YOUR ACCOUNT ID }}" # %% -# * **scheme**: in this case because AWS is the only cloud back-end supported for executing scheduled workflows, only ``"aws"`` is a valid value. By default, the no-op executor is used. +# * **scheme**: in this case because AWS is the only cloud back-end supported for executing scheduled workflows, only ``"aws"`` is a valid value. By default, the no-op executor is used and in case of sandbox we use ``"local"`` scheme which uses the flyte native scheduler. # * **region**: this specifies which region AWS clients should will use when creating an SQS subscriber client # * **scheduleQueueName**: this is the name of the SQS Queue you've allocated to scheduling workflows # * **accountId**: Your AWS `account id `_ diff --git a/cookbook/docs/conf.py b/cookbook/docs/conf.py index a115cb076..756cd38dc 100644 --- a/cookbook/docs/conf.py +++ b/cookbook/docs/conf.py @@ -56,6 +56,7 @@ class CustomSorter(FileNameSortKey): "typed_schema.py", "custom_objects.py", "enums.py", + "lp_schedules.py", # Testing "mocking.py", # Containerization @@ -74,7 +75,6 @@ class CustomSorter(FileNameSortKey): # Deployment ## Workflow "deploying_workflows.py", - "lp_schedules.py", "customizing_resources.py", "lp_notifications.py", "fast_registration.py", @@ -241,6 +241,7 @@ def __call__(self, filename): examples_dirs = [ "../core/flyte_basics", "../core/control_flow", + "../core/scheduled_workflows", "../core/type_system", "../case_studies/ml_training/pima_diabetes", "../case_studies/ml_training/house_price_prediction", @@ -270,6 +271,7 @@ def __call__(self, filename): gallery_dirs = [ "auto/core/flyte_basics", "auto/core/control_flow", + "auto/core/scheduled_workflows", "auto/core/type_system", "auto/case_studies/ml_training/pima_diabetes", "auto/case_studies/ml_training/house_price_prediction", diff --git a/cookbook/docs/index.rst b/cookbook/docs/index.rst index 181507053..33fe29c0c 100644 --- a/cookbook/docs/index.rst +++ b/cookbook/docs/index.rst @@ -50,6 +50,15 @@ Table of Contents --- + .. link-button:: auto/core/scheduled_workflows/index + :type: ref + :text: ⏱ Scheduled Workflows + :classes: btn-block stretched-link + ^^^^^^^^^^^^ + Learn about scheduled workflows. + + --- + .. link-button:: auto/testing/index :type: ref :text: ⚗️ Testing