move getting started pages to "Building Larger Apps" (flyteorg#713)

* move getting started pages to "repo-based projects" * rename to "Building Large Apps" * clean up copy and instructions Signed-off-by: Niels Bantilan <[email protected]> * copy updates Signed-off-by: Niels Bantilan <[email protected]>
eapolinario · Apr 12, 2022 · 66b3141 · 66b3141
1 parent aed9309
commit 66b3141
Show file tree

Hide file tree

Showing 10 changed files with 596 additions and 42 deletions.
diff --git a/cookbook/docs-requirements.in b/cookbook/docs-requirements.in
@@ -13,5 +13,6 @@ sphinx_fontawesome
 sphinx-panels
 sphinxcontrib-mermaid
 sphinxcontrib-yt
+sphinx-tabs
 docutils
 astroid
diff --git a/cookbook/docs-requirements.txt b/cookbook/docs-requirements.txt
@@ -1,20 +1,20 @@
 #
-# This file is autogenerated by pip-compile with python 3.8
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
-#    /Library/Developer/CommandLineTools/usr/bin/make docs-requirements.txt
+#    /Applications/Xcode.app/Contents/Developer/usr/bin/make docs-requirements.txt
 #
 alabaster==0.7.12
     # via sphinx
-arrow==1.2.1
+arrow==1.2.2
     # via jinja2-time
-astroid==2.9.2
+astroid==2.11.2
     # via
     #   -r docs-requirements.in
     #   sphinx-autoapi
 babel==2.9.1
     # via sphinx
-beautifulsoup4==4.10.0
+beautifulsoup4==4.11.0
     # via
     #   furo
     #   sphinx-code-include
@@ -24,23 +24,23 @@ certifi==2021.10.8
     # via requests
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.10
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.1.2
     # via
     #   cookiecutter
     #   flytekit
 cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.1.0
+croniter==1.3.4
     # via flytekit
 cycler==0.11.0
     # via matplotlib
-dataclasses-json==0.5.6
+dataclasses-json==0.5.7
     # via flytekit
 decorator==5.1.1
     # via retry
@@ -58,23 +58,28 @@ docutils==0.17.1
     #   sphinx
     #   sphinx-panels
     #   sphinx-rtd-theme
-flyteidl==0.21.17
+    #   sphinx-tabs
+flyteidl==0.24.17
     # via flytekit
-flytekit==0.26.0
+flytekit==0.31.0
     # via -r ./common/requirements-common.in
-fonttools==4.28.5
+fonttools==4.32.0
     # via matplotlib
 furo @ git+https://github.com/flyteorg/furo@main
     # via -r docs-requirements.in
-grpcio==1.43.0
+googleapis-common-protos==1.56.0
+    # via flyteidl
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
 imagesize==1.3.0
     # via sphinx
-importlib-metadata==4.10.0
-    # via keyring
-jinja2==3.0.3
+importlib-metadata==4.11.3
+    # via
+    #   keyring
+    #   sphinx
+jinja2==3.1.1
     # via
     #   cookiecutter
     #   jinja2-time
@@ -84,13 +89,13 @@ jinja2-time==0.2.0
     # via cookiecutter
 keyring==23.5.0
     # via flytekit
-kiwisolver==1.3.2
+kiwisolver==1.4.2
     # via matplotlib
 lazy-object-proxy==1.7.1
     # via astroid
-markupsafe==2.0.1
+markupsafe==2.1.1
     # via jinja2
-marshmallow==3.14.1
+marshmallow==3.15.0
     # via
     #   dataclasses-json
     #   marshmallow-enum
@@ -103,29 +108,34 @@ matplotlib==3.5.1
     # via -r ./common/requirements-common.in
 mypy-extensions==0.4.3
     # via typing-inspect
-natsort==8.0.2
+natsort==8.1.0
     # via flytekit
-numpy==1.22.0
+numpy==1.22.3
     # via
     #   matplotlib
     #   pandas
     #   pyarrow
 packaging==21.3
     # via
+    #   marshmallow
     #   matplotlib
     #   sphinx
-pandas==1.3.5
+pandas==1.4.2
     # via flytekit
-pillow==9.0.0
+pillow==9.1.0
     # via
     #   -r docs-requirements.in
     #   matplotlib
 poyo==0.5.0
     # via cookiecutter
-protobuf==3.19.1
+protobuf==3.20.0
     # via
     #   flyteidl
     #   flytekit
+    #   googleapis-common-protos
+    #   protoc-gen-swagger
+protoc-gen-swagger==0.1.0
+    # via flyteidl
 py==1.11.0
     # via retry
 pyarrow==6.0.1
@@ -134,57 +144,56 @@ pygments==2.11.2
     # via
     #   sphinx
     #   sphinx-prompt
-pyparsing==3.0.6
+    #   sphinx-tabs
+pyparsing==3.0.7
     # via
     #   matplotlib
     #   packaging
-python-dateutil==2.8.1
+python-dateutil==2.8.2
     # via
     #   croniter
     #   flytekit
     #   matplotlib
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.1.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
-pytz==2021.3
+pytz==2022.1
     # via
     #   babel
     #   flytekit
     #   pandas
 pyyaml==6.0
     # via sphinx-autoapi
-regex==2021.11.10
+regex==2022.3.15
     # via docker-image-py
 requests==2.27.1
     # via
     #   cookiecutter
     #   flytekit
     #   responses
     #   sphinx
-responses==0.16.0
+responses==0.20.0
     # via flytekit
 retry==0.9.2
     # via flytekit
 six==1.16.0
     # via
     #   cookiecutter
-    #   flytekit
     #   grpcio
     #   python-dateutil
-    #   responses
     #   sphinx-code-include
     #   sphinxext-remoteliteralinclude
 snowballstemmer==2.2.0
     # via sphinx
 sortedcontainers==2.4.0
     # via flytekit
-soupsieve==2.3.1
+soupsieve==2.3.2
     # via beautifulsoup4
-sphinx==4.3.2
+sphinx==4.5.0
     # via
     #   -r docs-requirements.in
     #   furo
@@ -196,13 +205,14 @@ sphinx==4.3.2
     #   sphinx-panels
     #   sphinx-prompt
     #   sphinx-rtd-theme
+    #   sphinx-tabs
     #   sphinxcontrib-yt
     #   sphinxext-remoteliteralinclude
 sphinx-autoapi==1.8.4
     # via -r docs-requirements.in
 sphinx-code-include==1.1.1
     # via -r docs-requirements.in
-sphinx-copybutton==0.4.0
+sphinx-copybutton==0.5.0
     # via -r docs-requirements.in
 sphinx-fontawesome==0.0.6
     # via -r docs-requirements.in
@@ -214,6 +224,8 @@ sphinx-prompt==1.5.0
     # via -r docs-requirements.in
 sphinx-rtd-theme==1.0.0
     # via -r docs-requirements.in
+sphinx-tabs==3.3.1
+    # via -r docs-requirements.in
 sphinxcontrib-applehelp==1.0.2
     # via sphinx
 sphinxcontrib-devhelp==1.0.2
@@ -236,15 +248,16 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
     #   astroid
+    #   flytekit
     #   typing-inspect
 typing-inspect==0.7.1
     # via dataclasses-json
-unidecode==1.3.2
+unidecode==1.3.4
     # via sphinx-autoapi
-urllib3==1.26.7
+urllib3==1.26.9
     # via
     #   flytekit
     #   requests
@@ -253,12 +266,12 @@ wheel==0.37.1
     # via
     #   -r ./common/requirements-common.in
     #   flytekit
-wrapt==1.13.3
+wrapt==1.14.0
     # via
     #   astroid
     #   deprecated
     #   flytekit
-zipp==3.7.0
+zipp==3.8.0
     # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:

diff --git a/cookbook/docs/_static/sphx_gallery_autogen.css b/cookbook/docs/_static/sphx_gallery_autogen.css
@@ -11,7 +11,9 @@
 #sphx-glr-download-auto-remote-access-run-workflow-py,
 #sphx-glr-download-auto-remote-access-debugging-workflows-tasks-py,
 #sphx-glr-download-auto-core-extend-flyte-backend-plugins-py,
-#sphx-glr-download-auto-core-extend-flyte-container-interface-py {
+#sphx-glr-download-auto-core-extend-flyte-container-interface-py,
+#sphx-glr-download-auto-larger-apps-larger-apps-iterate-py,
+#sphx-glr-download-auto-larger-apps-larger-apps-deploy-py {
     height: 0px;
     visibility: hidden;
 }
diff --git a/cookbook/docs/conf.py b/cookbook/docs/conf.py
@@ -136,6 +136,10 @@ class CustomSorter(FileNameSortKey):
         "backend_plugins.py",  # NOTE: for some reason this needs to be listed first here to show up last on the TOC
         "run_custom_types.py",
         "custom_task_plugin.py",
+        # Repo-based Projects
+        "larger_apps_setup.py",
+        "larger_apps_deploy.py",
+        "larger_apps_iterate.py",
         # Tutorials
         ## ML Training
         "diabetes.py",
@@ -192,6 +196,7 @@ def __call__(self, filename):
     "sphinx_panels",
     "sphinxcontrib.mermaid",
     "sphinxcontrib.yt",
+    "sphinx_tabs.tabs",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -289,6 +294,7 @@ def __call__(self, filename):
     "../integrations/external_services/hive",
     "../integrations/external_services/snowflake",
     "../core/extend_flyte",
+    "../larger_apps",
 ]
 gallery_dirs = [
     "auto/core/flyte_basics",
@@ -325,6 +331,7 @@ def __call__(self, filename):
     "auto/integrations/external_services/hive",
     "auto/integrations/external_services/snowflake",
     "auto/core/extend_flyte",
+    "auto/larger_apps",
 ]
 
 # image_scrapers = ('matplotlib',)

diff --git a/cookbook/docs/index.rst b/cookbook/docs/index.rst
@@ -226,6 +226,15 @@ Table of Contents
    ^^^^^^^^^^^^^^^
    Define custom plugins that aren't currently supported in the Flyte ecosystem.
 
+   ---
+
+   .. link-button:: auto/larger_apps/index
+      :type: ref
+      :text: ⛰ Building Large Apps
+      :classes: btn-block stretched-link
+   ^^^^^^^^^^^^^^^
+   Build, deploy, and iterate on large projects by organizing your Flyte app.
+
 .. toctree::
    :maxdepth: 1
    :hidden:
@@ -252,7 +261,8 @@ Table of Contents
    Remote Access <auto/remote_access/index>
    Production Config <auto/deployment/index>
    Scheduling Workflows <auto/core/scheduled_workflows/index>
-   Extending flyte <auto/core/extend_flyte/index>
+   Extending Flyte <auto/core/extend_flyte/index>
+   Building Large Apps <auto/larger_apps/index>
    contribute
 
 .. toctree::

diff --git a/cookbook/larger_apps/README.rst b/cookbook/larger_apps/README.rst
@@ -0,0 +1,15 @@
+.. _larger_apps:
+
+Building Large Apps
+--------------------
+
+So far in the *User Guide* you've been running Flyte workflows as one-off
+scripts, which is useful for quick prototyping and iteration of small ideas
+on a Flyte cluster.
+
+However, if you need to build a larger Flyte app with sub-modules or
+sub-packages to organize the logic of your tasks and workflows, you can use
+`flyte python templates <https://github.com/flyteorg/flytekit-python-template>`__
+to quickly materialize the directory structure with boilerplate files that
+we recommend for structuring an app with multiple workflows or helper
+modules.
diff --git a/cookbook/larger_apps/__init__.py b/cookbook/larger_apps/__init__.py