From 66b31418d7fd9051c488d1c6a6b3ab8aee75498b Mon Sep 17 00:00:00 2001 From: Niels Bantilan Date: Tue, 12 Apr 2022 13:51:45 -0400 Subject: [PATCH] move getting started pages to "Building Larger Apps" (#713) * move getting started pages to "repo-based projects" * rename to "Building Large Apps" * clean up copy and instructions Signed-off-by: Niels Bantilan * copy updates Signed-off-by: Niels Bantilan --- cookbook/docs-requirements.in | 1 + cookbook/docs-requirements.txt | 93 ++++--- .../docs/_static/sphx_gallery_autogen.css | 4 +- cookbook/docs/conf.py | 7 + cookbook/docs/index.rst | 12 +- cookbook/larger_apps/README.rst | 15 ++ cookbook/larger_apps/__init__.py | 0 cookbook/larger_apps/larger_apps_deploy.py | 191 +++++++++++++++ cookbook/larger_apps/larger_apps_iterate.py | 227 ++++++++++++++++++ cookbook/larger_apps/larger_apps_setup.py | 88 +++++++ 10 files changed, 596 insertions(+), 42 deletions(-) create mode 100644 cookbook/larger_apps/README.rst create mode 100644 cookbook/larger_apps/__init__.py create mode 100644 cookbook/larger_apps/larger_apps_deploy.py create mode 100644 cookbook/larger_apps/larger_apps_iterate.py create mode 100644 cookbook/larger_apps/larger_apps_setup.py diff --git a/cookbook/docs-requirements.in b/cookbook/docs-requirements.in index 5871b93e48..ad5b693804 100644 --- a/cookbook/docs-requirements.in +++ b/cookbook/docs-requirements.in @@ -13,5 +13,6 @@ sphinx_fontawesome sphinx-panels sphinxcontrib-mermaid sphinxcontrib-yt +sphinx-tabs docutils astroid diff --git a/cookbook/docs-requirements.txt b/cookbook/docs-requirements.txt index 494b5ecf1e..1f14e948af 100644 --- a/cookbook/docs-requirements.txt +++ b/cookbook/docs-requirements.txt @@ -1,20 +1,20 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # -# /Library/Developer/CommandLineTools/usr/bin/make docs-requirements.txt +# /Applications/Xcode.app/Contents/Developer/usr/bin/make docs-requirements.txt # alabaster==0.7.12 # via sphinx -arrow==1.2.1 +arrow==1.2.2 # via jinja2-time -astroid==2.9.2 +astroid==2.11.2 # via # -r docs-requirements.in # sphinx-autoapi babel==2.9.1 # via sphinx -beautifulsoup4==4.10.0 +beautifulsoup4==4.11.0 # via # furo # sphinx-code-include @@ -24,11 +24,11 @@ certifi==2021.10.8 # via requests chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.10 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.1.2 # via # cookiecutter # flytekit @@ -36,11 +36,11 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.1.0 +croniter==1.3.4 # via flytekit cycler==0.11.0 # via matplotlib -dataclasses-json==0.5.6 +dataclasses-json==0.5.7 # via flytekit decorator==5.1.1 # via retry @@ -58,23 +58,28 @@ docutils==0.17.1 # sphinx # sphinx-panels # sphinx-rtd-theme -flyteidl==0.21.17 + # sphinx-tabs +flyteidl==0.24.17 # via flytekit -flytekit==0.26.0 +flytekit==0.31.0 # via -r ./common/requirements-common.in -fonttools==4.28.5 +fonttools==4.32.0 # via matplotlib furo @ git+https://github.com/flyteorg/furo@main # via -r docs-requirements.in -grpcio==1.43.0 +googleapis-common-protos==1.56.0 + # via flyteidl +grpcio==1.44.0 # via flytekit idna==3.3 # via requests imagesize==1.3.0 # via sphinx -importlib-metadata==4.10.0 - # via keyring -jinja2==3.0.3 +importlib-metadata==4.11.3 + # via + # keyring + # sphinx +jinja2==3.1.1 # via # cookiecutter # jinja2-time @@ -84,13 +89,13 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -kiwisolver==1.3.2 +kiwisolver==1.4.2 # via matplotlib lazy-object-proxy==1.7.1 # via astroid -markupsafe==2.0.1 +markupsafe==2.1.1 # via jinja2 -marshmallow==3.14.1 +marshmallow==3.15.0 # via # dataclasses-json # marshmallow-enum @@ -103,29 +108,34 @@ matplotlib==3.5.1 # via -r ./common/requirements-common.in mypy-extensions==0.4.3 # via typing-inspect -natsort==8.0.2 +natsort==8.1.0 # via flytekit -numpy==1.22.0 +numpy==1.22.3 # via # matplotlib # pandas # pyarrow packaging==21.3 # via + # marshmallow # matplotlib # sphinx -pandas==1.3.5 +pandas==1.4.2 # via flytekit -pillow==9.0.0 +pillow==9.1.0 # via # -r docs-requirements.in # matplotlib poyo==0.5.0 # via cookiecutter -protobuf==3.19.1 +protobuf==3.20.0 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 @@ -134,11 +144,12 @@ pygments==2.11.2 # via # sphinx # sphinx-prompt -pyparsing==3.0.6 + # sphinx-tabs +pyparsing==3.0.7 # via # matplotlib # packaging -python-dateutil==2.8.1 +python-dateutil==2.8.2 # via # croniter # flytekit @@ -146,18 +157,18 @@ python-dateutil==2.8.1 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit -pytz==2021.3 +pytz==2022.1 # via # babel # flytekit # pandas pyyaml==6.0 # via sphinx-autoapi -regex==2021.11.10 +regex==2022.3.15 # via docker-image-py requests==2.27.1 # via @@ -165,26 +176,24 @@ requests==2.27.1 # flytekit # responses # sphinx -responses==0.16.0 +responses==0.20.0 # via flytekit retry==0.9.2 # via flytekit six==1.16.0 # via # cookiecutter - # flytekit # grpcio # python-dateutil - # responses # sphinx-code-include # sphinxext-remoteliteralinclude snowballstemmer==2.2.0 # via sphinx sortedcontainers==2.4.0 # via flytekit -soupsieve==2.3.1 +soupsieve==2.3.2 # via beautifulsoup4 -sphinx==4.3.2 +sphinx==4.5.0 # via # -r docs-requirements.in # furo @@ -196,13 +205,14 @@ sphinx==4.3.2 # sphinx-panels # sphinx-prompt # sphinx-rtd-theme + # sphinx-tabs # sphinxcontrib-yt # sphinxext-remoteliteralinclude sphinx-autoapi==1.8.4 # via -r docs-requirements.in sphinx-code-include==1.1.1 # via -r docs-requirements.in -sphinx-copybutton==0.4.0 +sphinx-copybutton==0.5.0 # via -r docs-requirements.in sphinx-fontawesome==0.0.6 # via -r docs-requirements.in @@ -214,6 +224,8 @@ sphinx-prompt==1.5.0 # via -r docs-requirements.in sphinx-rtd-theme==1.0.0 # via -r docs-requirements.in +sphinx-tabs==3.3.1 + # via -r docs-requirements.in sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 @@ -236,15 +248,16 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # astroid + # flytekit # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -unidecode==1.3.2 +unidecode==1.3.4 # via sphinx-autoapi -urllib3==1.26.7 +urllib3==1.26.9 # via # flytekit # requests @@ -253,12 +266,12 @@ wheel==0.37.1 # via # -r ./common/requirements-common.in # flytekit -wrapt==1.13.3 +wrapt==1.14.0 # via # astroid # deprecated # flytekit -zipp==3.7.0 +zipp==3.8.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/cookbook/docs/_static/sphx_gallery_autogen.css b/cookbook/docs/_static/sphx_gallery_autogen.css index fae4eb8b4b..f41ec295fb 100644 --- a/cookbook/docs/_static/sphx_gallery_autogen.css +++ b/cookbook/docs/_static/sphx_gallery_autogen.css @@ -11,7 +11,9 @@ #sphx-glr-download-auto-remote-access-run-workflow-py, #sphx-glr-download-auto-remote-access-debugging-workflows-tasks-py, #sphx-glr-download-auto-core-extend-flyte-backend-plugins-py, -#sphx-glr-download-auto-core-extend-flyte-container-interface-py { +#sphx-glr-download-auto-core-extend-flyte-container-interface-py, +#sphx-glr-download-auto-larger-apps-larger-apps-iterate-py, +#sphx-glr-download-auto-larger-apps-larger-apps-deploy-py { height: 0px; visibility: hidden; } diff --git a/cookbook/docs/conf.py b/cookbook/docs/conf.py index 2b572538fb..d859a0e7d7 100644 --- a/cookbook/docs/conf.py +++ b/cookbook/docs/conf.py @@ -136,6 +136,10 @@ class CustomSorter(FileNameSortKey): "backend_plugins.py", # NOTE: for some reason this needs to be listed first here to show up last on the TOC "run_custom_types.py", "custom_task_plugin.py", + # Repo-based Projects + "larger_apps_setup.py", + "larger_apps_deploy.py", + "larger_apps_iterate.py", # Tutorials ## ML Training "diabetes.py", @@ -192,6 +196,7 @@ def __call__(self, filename): "sphinx_panels", "sphinxcontrib.mermaid", "sphinxcontrib.yt", + "sphinx_tabs.tabs", ] # Add any paths that contain templates here, relative to this directory. @@ -289,6 +294,7 @@ def __call__(self, filename): "../integrations/external_services/hive", "../integrations/external_services/snowflake", "../core/extend_flyte", + "../larger_apps", ] gallery_dirs = [ "auto/core/flyte_basics", @@ -325,6 +331,7 @@ def __call__(self, filename): "auto/integrations/external_services/hive", "auto/integrations/external_services/snowflake", "auto/core/extend_flyte", + "auto/larger_apps", ] # image_scrapers = ('matplotlib',) diff --git a/cookbook/docs/index.rst b/cookbook/docs/index.rst index 3c71cf8d29..f58e1dd4d7 100644 --- a/cookbook/docs/index.rst +++ b/cookbook/docs/index.rst @@ -226,6 +226,15 @@ Table of Contents ^^^^^^^^^^^^^^^ Define custom plugins that aren't currently supported in the Flyte ecosystem. + --- + + .. link-button:: auto/larger_apps/index + :type: ref + :text: ⛰ Building Large Apps + :classes: btn-block stretched-link + ^^^^^^^^^^^^^^^ + Build, deploy, and iterate on large projects by organizing your Flyte app. + .. toctree:: :maxdepth: 1 :hidden: @@ -252,7 +261,8 @@ Table of Contents Remote Access Production Config Scheduling Workflows - Extending flyte + Extending Flyte + Building Large Apps contribute .. toctree:: diff --git a/cookbook/larger_apps/README.rst b/cookbook/larger_apps/README.rst new file mode 100644 index 0000000000..339ae63b8c --- /dev/null +++ b/cookbook/larger_apps/README.rst @@ -0,0 +1,15 @@ +.. _larger_apps: + +Building Large Apps +-------------------- + +So far in the *User Guide* you've been running Flyte workflows as one-off +scripts, which is useful for quick prototyping and iteration of small ideas +on a Flyte cluster. + +However, if you need to build a larger Flyte app with sub-modules or +sub-packages to organize the logic of your tasks and workflows, you can use +`flyte python templates `__ +to quickly materialize the directory structure with boilerplate files that +we recommend for structuring an app with multiple workflows or helper +modules. diff --git a/cookbook/larger_apps/__init__.py b/cookbook/larger_apps/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cookbook/larger_apps/larger_apps_deploy.py b/cookbook/larger_apps/larger_apps_deploy.py new file mode 100644 index 0000000000..338b4fe574 --- /dev/null +++ b/cookbook/larger_apps/larger_apps_deploy.py @@ -0,0 +1,191 @@ +""" +.. _larger_apps_deploy: + +Deploy to the Cloud +-------------------------------- + +Prerequisites +^^^^^^^^^^^^^^^^ +Make sure you have `Docker `__ and Docker +Daemon is running. + +.. note:: + + Being connected to a VPN may cause problems downloading the image. + +Setup and Configuration +^^^^^^^^^^^^^^^^^^^^^^^^ + +First, install `flytectl `__. + +Then you can setup a local :ref:`Flyte Sandbox ` cluster or configure ``flytectl`` to use a +pre-provisioned remote Flyte cluster. + +.. tip:: + + Learn how to deploy to a Flyte Cluster using the :ref:`Deployment Guides `. + +.. tabs:: + .. group-tab:: Flyte Sandbox + + To start the Flyte Sandbox, run: + + .. prompt:: bash $ + + flytectl sandbox start --source . + + .. note:: + + The ``'.'`` represents the current directory, which would be ``my_flyte_project`` in this case. + + If you're having trouble with starting the sandbox cluster, refer to :ref:`troubleshoot`. + + .. group-tab:: Remote Flyte Cluster + + Setup flytectl remote cluster config + + .. prompt:: bash $ + + flytectl config init --host={FLYTEADMIN_URL} --storage + + Where ``{FLYTEADMIN_URL}`` is your custom url. + + +Build & Deploy Your Application to the Cluster +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Flyte uses Docker containers to package the workflows and tasks, and sends them to the remote Flyte cluster. Therefore, +there is a ``Dockerfile`` already included in the cloned repo. You can build the Docker container and push the built +image to a registry. + +.. tabs:: + .. group-tab:: Flyte Sandbox + + Since ``flyte-sandbox`` runs locally in a Docker container, you do not need to push the Docker image. You can + combine the build and push step by simply building the image inside the Flyte-sandbox container. This can be done + using the following command: + + .. prompt:: bash $ + + flytectl sandbox exec -- docker build . --tag "my_flyte_project:v1" + + .. tip:: + Why are we not pushing the Docker image? To understand the details, refer to :ref:`deployment-sandbox`. + + .. group-tab:: Remote Flyte Cluster + + If you are using a remote Flyte cluster, then you need to build your container and push it to a registry that + is accessible by the Flyte Kubernetes cluster. + + .. prompt:: bash $ + + docker build . --tag + docker push + + .. tip:: Recommended + + The ``flytekit-python-template`` ships with a helper script called `docker_build_and_tag.sh + `__ , which makes it + possible to build and image, tag it correctly and optionally use the git-SHA as the version. We recommend using + such a script to track versions more effectively and using a CI/CD pipeline to deploy your code. + + .. prompt:: bash $ + + ./docker_build_and_tag.sh -r -a [-v ] + +Next, package the workflow using the ``pyflyte`` cli bundled with Flytekit and upload it to the Flyte backend. Note that +the image is the same as the one built in the previous step. + +.. tabs:: + + .. group-tab:: Flyte Sandbox + + .. prompt:: bash $ + + pyflyte --pkgs flyte.workflows package --image "my_flyte_project:v1" + + .. group-tab:: Remote Flyte Cluster + + .. prompt:: bash $ + + pyflyte --pkgs flyte.workflows package --image + +Upload this package to the Flyte backend. We refer to this as ``registration``. The version here ``v1`` does not have to +match the version used in the commands above, but it is generally recommended to match the versions to make it easier to +track. + +.. note:: + + Note that we are simply using an existing project ``flytesnacks`` and an existing domain ``development`` to register + the workflows and tasks. It is possible to create your own project + and configure domains. Refer to :ref:`control-plane` to understand projects and domains. + +.. prompt:: bash $ + + flytectl register files --project flytesnacks --domain development --archive flyte-package.tgz --version v1 + +Finally, visualize the registered workflow. + +.. prompt:: bash $ + + flytectl get workflows --project flytesnacks --domain development flyte.workflows.example.my_wf --version v1 -o doturl + + +.. _getting-started-execute: + +Execute on the Flyte Cluster +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Launch and monitor from the CLI using Flytectl. +More details can be found `here `__. + +Generate an execution spec file. + +.. prompt:: bash $ + + flytectl get launchplan --project flytesnacks --domain development flyte.workflows.example.my_wf --latest --execFile exec_spec.yaml + +Create an execution using the exec spec file. + +.. prompt:: bash $ + + flytectl create execution --project flytesnacks --domain development --execFile exec_spec.yaml + +You should see an output that looks like: + +.. prompt:: text + + execution identifier project:"flytesnacks" domain:"development" name:"" + +Monitor the execution by providing the execution name from the ``create execution`` command. + +.. prompt:: bash $ + + flytectl get execution --project flytesnacks --domain development + + +**Alternatively, you can FlyteConsole to launch an execution.** + +.. tabs:: + + .. group-tab:: Flyte Sandbox + + Visit ``http://localhost:30081`` on your browser + + .. group-tab:: Remote Flyte Cluster + + Visit ``{FLYTEADMIN_URL}/console`` on your browser + +Then use the FlyteConsole to launch an execution: + +.. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flytesnacks/index/getting_started_reg.gif + :alt: A quick visual tour for launching a workflow and checking the outputs when they're done. + + +Conclusion +^^^^^^^^^^^ + +We've successfully packaged your workflow and tasks and pushed them to a Flyte cluster! 🎉 +Next, let's learn how to :ref:`iterate on and re-deploy ` our Flyte app. + +""" diff --git a/cookbook/larger_apps/larger_apps_iterate.py b/cookbook/larger_apps/larger_apps_iterate.py new file mode 100644 index 0000000000..f55d30fa52 --- /dev/null +++ b/cookbook/larger_apps/larger_apps_iterate.py @@ -0,0 +1,227 @@ +""" +.. _larger_apps_iterate: + +Iterate and Re-deploy +---------------------- + +In this guide, you'll learn how to iterate on and re-deploy your tasks and workflows. + +Modify Code and Test Locally +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Open ``example.py`` in your favorite editor. + +.. dropdown:: flyte/workflows/example.py + + .. rli:: https://raw.githubusercontent.com/flyteorg/flytekit-python-template/simplify-template/myapp/workflows/example.py + :language: python + +Add ``message: str`` as an argument to both ``my_wf`` and ``say_hello`` functions. Then update the body of +``say_hello`` to consume that argument. + +.. code-block:: python + + @task + def say_hello(message: str) -> str: + return f"hello world, {message}" + +.. code-block:: python + + @workflow + def my_wf(message: str) -> str: + res = say_hello(message=message) + return res + +Update the simple test at the bottom of the file to pass in a message, e.g. + +.. code-block:: python + + print(f"Running my_wf(message='what a nice day it is!') {my_wf(message='what a nice day it is!')}") + +When you run this file locally, it should output ``hello world, what a nice day it is!``. + +.. prompt:: bash (flyte)$ + + python flyte/workflows/example.py + + +.. dropdown:: Expected output + + .. prompt:: text + + Running my_wf(message='what a nice day it is!') hello world, what a nice day it is! + + +Quickly Re-deploy Your Application +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To re-deploy this workflow to the sandbox Flyte cluster, you can repeat the steps previously covered in +:ref:`getting-started-build-deploy`. Flyte provides a **faster** way to iterate on your workflows. Since you have not +updated any of the dependencies in your requirements file, it is possible to push just the code to Flyte backend without +re-building the entire Docker container. To do so, run the following commands. + +.. prompt:: bash (flyte)$ + + pyflyte --pkgs flyte.workflows package --image my_flyte_project:v1 --fast --force + +.. note:: + + ``--fast`` flag will take the code from your local machine and provide it for execution without having to build the container and push it. The ``--force`` flag allows overriding your previously created package. + +.. caution:: + + The ``fast`` registration method can only be used if you do not modify any requirements (that is, you re-use an existing environment). But, if you add a dependency to your requirements file or env you have to follow the :ref:`getting-started-build-deploy` method. + +The code can now be deployed using Flytectl, similar to what we've done previously. ``flytectl`` automatically understands +that the package is for fast registration. + +For this to work, a new ``storage`` block has to be added to the Flytectl configuration with appropriate permissions at +runtime. The storage block configures Flytectl to write to a specific ``S3 / GCS bucket``. + +.. tip:: + + If you're using the sandbox, this is automatically configured by Flytectl. + + The dropdown below provides more information on the required configuration depending on your cloud infrastructure. + + .. dropdown:: Flytectl configuration with ``storage`` block for Fast registration + + .. tabbed:: Local Flyte Sandbox + + Automatically configured for you by ``flytectl sandbox`` command. + + .. code-block:: yaml + + admin: + # For GRPC endpoints you might want to use dns:///flyte.myexample.com + endpoint: dns:///localhost:30081 + insecure: true + storage: + connection: + access-key: minio + auth-type: accesskey + disable-ssl: true + endpoint: http://localhost:30084 + region: my-region-here + secret-key: miniostorage + container: my-s3-bucket + type: minio + + .. tabbed:: S3 Configuration + + .. code-block:: yaml + + admin: + # For GRPC endpoints you might want to use dns:///flyte.myexample.com + endpoint: dns:/// + authType: Pkce # authType: Pkce # if using authentication or just drop this. + insecure: true # insecure: True # Set to true if the endpoint isn't accessible through TLS/SSL connection (not recommended except on local sandbox deployment) + storage: + type: stow + stow: + kind: s3 + config: + auth_type: iam + region: # Example: us-east-2 + container: # Example my-bucket. Flyte k8s cluster / service account for execution should have read access to this bucket + + + .. tabbed:: GCS Configuration + + .. code-block:: yaml + + admin: + # For GRPC endpoints you might want to use dns:///flyte.myexample.com + endpoint: dns:/// + authType: Pkce # authType: Pkce # if using authentication or just drop this. + insecure: false # insecure: True # Set to true if the endpoint isn't accessible through TLS/SSL connection (not recommended except on local sandbox deployment) + storage: + type: stow + stow: + kind: google + config: + json: "" + project_id: # replace with the GCP project ID + scopes: https://www.googleapis.com/auth/devstorage.read_write + container: # Example my-bucket. Flyte k8s cluster / service account for execution should have access to this bucket + + .. tabbed:: Others + + For other supported storage backends like Oracle, Azure, etc., refer to the configuration structure `here `__. + + +Next, you can simply register your packaged workflows with: + +.. prompt:: bash $ + + flytectl register files --project flytesnacks --domain development --archive flyte-package.tgz --version v1-fast1 + + +Execute Your Re-deployed Workflow +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Finally, you can execute the updated workflow programmatically with ``flytectl``. + +To pass arguments to the workflow, update the execution spec file that we previously generated in the +:ref:`Deploying to the Coud ` step. + +Generate an execution spec file. This will prompt you to overwrite and answer 'y' on it. + +.. prompt:: bash $ + + flytectl get launchplan --project flytesnacks --domain development flyte.workflows.example.my_wf --latest --execFile exec_spec.yaml + +Modify the execution spec file and update the input params and save the file. Notice that the version would be changed to your latest one. + +.. code-block:: yaml + + .... + inputs: + message: "what's up doc? 🐰🥕" + .... + version: v1-fast1 + +Create an execution using the exec spec file. + +.. prompt:: bash $ + + flytectl create execution --project flytesnacks --domain development --execFile exec_spec.yaml + +You should see an output that looks like: + +.. prompt:: text + + execution identifier project:"flytesnacks" domain:"development" name:"" + +Monitor the execution by providing the execution name from the ``create execution`` command. + +.. prompt:: bash $ + + flytectl get execution --project flytesnacks --domain development + +.. tip:: + + Alternatively, visit `the Flyte sandbox console `__, + click launch, and provide a ``message`` as input. + + .. TODO: update so that it reflects the new workflow with the message input https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/getting_started/getting_started_fastreg.gif + + +Recap +^^^^^^ + +In this guide, we: + +1. Setup a Flyte project with ``pyflyte init my_flyte_project`` and + ran your workflows locally. +2. Started a Flyte sandbox cluster and ran a Flyte workflow on a cluster. +3. Iterated on a Flyte workflow and updated the workflows on the cluster. + + +What's Next? +^^^^^^^^^^^^^ + +To experience the full power of Flyte on distributed compute, take a look at the +`Deployment Guides `__. + +""" diff --git a/cookbook/larger_apps/larger_apps_setup.py b/cookbook/larger_apps/larger_apps_setup.py new file mode 100644 index 0000000000..b4dd8e2f42 --- /dev/null +++ b/cookbook/larger_apps/larger_apps_setup.py @@ -0,0 +1,88 @@ +""" +.. _larger_apps_build: + +Setup a Project +---------------- + +Prerequisites +^^^^^^^^^^^^^^^^ +Make sure you have `git `__ and +`python `__ >= 3.7 installed. + +Start a new project +^^^^^^^^^^^^^^^^^^^^ + +Create a `virtual environment `__ : + +.. prompt:: bash $ + + python -m venv ~/venvs/flyte + source ~/venvs/flyte/bin/activate + +Then install `Flytekit `__ on it: + +.. prompt:: bash (flyte)$ + + pip install flytekit + +Then run ``pyflyte init ``, where ```` is the +directory that will be created containing the scaffolding for a flyte-ready +project. For this guide we're going call it ``my_flyte_project``: + +.. prompt:: bash (flyte)$ + + pyflyte init my_flyte_project + cd my_flyte_project + +The ``my_flyte_project`` directory comes with a sample workflow, which can be +found under ``flyte/workflows/example.py``. The structure below shows the most +important files and how a typical Flyte app should be laid out. + +.. dropdown:: A typical Flyte app should have these files + + .. code-block:: text + + my_flyte_project + ├── Dockerfile + ├── docker_build_and_tag.sh + ├── flyte + │ ├── __init__.py + │ └── workflows + │ ├── __init__.py + │ └── example.py + └── requirements.txt + + .. note:: + + * You can use `pip-compile` to build your requirements file. + * The Dockerfile that comes with this is not GPU ready, but is a simple Dockerfile that should work for most of your apps. + +Run the Workflow Locally +^^^^^^^^^^^^^^^^^^^^^^^^ + +The workflow can be run locally, simply by running it as a Python script: + +.. prompt:: bash (flyte)$ + + python flyte/workflows/example.py + +.. note:: + + The workflow needs to be invoked after the ``if __name__ == "__main__"`` + entrypoint at the bottom of ``flyte/workflows/example.py``. + + +Expected output: + +.. prompt:: text + + Running my_wf() hello world + + +Conclusion +^^^^^^^^^^^ + +We've successfully created a Flyte project and executed your workflows on a python runtime environment! 🎉 +Next, let's learn how to :ref:`deploy to the cloud `. + +"""