diff --git a/dev-requirements.txt b/dev-requirements.txt
index ba18954a7b..5acc7855b8 100644
--- a/dev-requirements.txt
+++ b/dev-requirements.txt
@@ -14,6 +14,7 @@ attrs==20.3.0
# pytest
black==20.8b1
# via
+ # -c requirements.txt
# -r dev-requirements.in
# flake8-black
click==7.1.2
diff --git a/doc-requirements.in b/doc-requirements.in
index 4383aa26f1..1b104414c4 100644
--- a/doc-requirements.in
+++ b/doc-requirements.in
@@ -1,6 +1,8 @@
.[all]
-e file:.#egg=flytekit
+furo
+readthedocs-sphinx-search
sphinx
sphinx-gallery
sphinx-prompt
diff --git a/doc-requirements.txt b/doc-requirements.txt
index 963d54ce5e..84195f76de 100644
--- a/doc-requirements.txt
+++ b/doc-requirements.txt
@@ -12,7 +12,11 @@ ansiwrap==0.8.4
# via papermill
appdirs==1.4.4
# via black
-astroid==2.5.1
+appnope==0.1.2
+ # via
+ # ipykernel
+ # ipython
+astroid==2.5.2
# via sphinx-autoapi
async-generator==1.10
# via nbclient
@@ -28,15 +32,16 @@ bcrypt==3.2.0
# via paramiko
beautifulsoup4==4.9.3
# via
+ # furo
# sphinx-code-include
# sphinx-material
black==20.8b1
# via papermill
bleach==3.3.0
# via nbconvert
-boto3==1.17.39
+boto3==1.17.40
# via sagemaker-training
-botocore==1.20.39
+botocore==1.20.40
# via
# boto3
# s3transfer
@@ -61,7 +66,6 @@ cryptography==3.4.7
# via
# -r doc-requirements.in
# paramiko
- # secretstorage
css-html-js-minify==2.5.5
# via sphinx-material
dataclasses-json==0.5.2
@@ -86,6 +90,8 @@ entrypoints==0.3
# papermill
flyteidl==0.18.26
# via flytekit
+furo==2021.3.20b30
+ # via -r doc-requirements.in
gevent==21.1.2
# via sagemaker-training
greenlet==1.0.0
@@ -100,7 +106,7 @@ idna==2.10
# via requests
imagesize==1.2.0
# via sphinx
-importlib-metadata==3.7.3
+importlib-metadata==3.9.1
# via keyring
inotify_simple==1.2.1
# via sagemaker-training
@@ -110,14 +116,10 @@ ipython-genutils==0.2.0
# via
# nbformat
# traitlets
-ipython==7.21.0
+ipython==7.22.0
# via ipykernel
jedi==0.18.0
# via ipython
-jeepney==0.6.0
- # via
- # keyring
- # secretstorage
jinja2==2.11.3
# via
# nbconvert
@@ -152,7 +154,7 @@ markupsafe==1.1.1
# via jinja2
marshmallow-enum==1.5.1
# via dataclasses-json
-marshmallow==3.10.0
+marshmallow==3.11.0
# via
# dataclasses-json
# marshmallow-enum
@@ -177,7 +179,7 @@ nbformat==5.1.2
# papermill
nest-asyncio==1.5.1
# via nbclient
-numpy==1.20.1
+numpy==1.20.2
# via
# flytekit
# pandas
@@ -263,6 +265,8 @@ pyyaml==5.4.1
# sphinx-autoapi
pyzmq==22.0.3
# via jupyter-client
+readthedocs-sphinx-search==0.1.0
+ # via -r doc-requirements.in
regex==2021.3.17
# via
# black
@@ -273,7 +277,7 @@ requests==2.25.1
# papermill
# responses
# sphinx
-responses==0.13.1
+responses==0.13.2
# via flytekit
retry==0.9.2
# via flytekit
@@ -281,14 +285,12 @@ retrying==1.3.3
# via sagemaker-training
s3transfer==0.3.6
# via boto3
-sagemaker-training==3.7.3
+sagemaker-training==3.7.4
# via flytekit
scantree==0.0.1
# via dirhash
scipy==1.6.2
# via sagemaker-training
-secretstorage==3.3.1
- # via keyring
six==1.15.0
# via
# bcrypt
@@ -327,6 +329,7 @@ sphinx-prompt==1.4.0
sphinx==3.5.3
# via
# -r doc-requirements.in
+ # furo
# sphinx-autoapi
# sphinx-code-include
# sphinx-copybutton
diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css
new file mode 100644
index 0000000000..786fcd2611
--- /dev/null
+++ b/docs/source/_static/custom.css
@@ -0,0 +1,33 @@
+h1, h2, h3, h4, h5, h6 {
+ font-weight: bold;
+}
+
+.sidebar-logo {
+ max-width: 30%;
+}
+
+
+.sidebar-tree .reference.external:after {
+ content: none;
+}
+
+div.sphx-glr-download a {
+ color: #4300c9;
+ background-color: rgb(241, 241, 241);
+ background-image: none;
+ border: 1px solid rgb(202, 202, 202);
+}
+
+div.sphx-glr-download a:hover {
+ background-color: rgb(230, 230, 230);
+ box-shadow: none;
+}
+
+div.sphx-glr-thumbcontainer a.headerlink {
+ display: none;
+}
+
+div.sphx-glr-thumbcontainer:hover {
+ border-color: white;
+ box-shadow: none;
+}
diff --git a/docs/source/_templates/sidebar/brand.html b/docs/source/_templates/sidebar/brand.html
new file mode 100644
index 0000000000..a170d6c6d1
--- /dev/null
+++ b/docs/source/_templates/sidebar/brand.html
@@ -0,0 +1,18 @@
+
+ {% block brand_content %}
+ {%- if logo %}
+
+ {%- endif %}
+ {%- if theme_light_logo and theme_dark_logo %}
+
+ {%- endif %}
+ {% if not theme_sidebar_hide_name %}
+ {{ docstitle }}
+ {%- endif %}
+ {% endblock brand_content %}
+
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 98a324a6c4..15df491168 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -47,6 +47,7 @@
"sphinx.ext.graphviz",
"sphinx-prompt",
"sphinx_copybutton",
+ "sphinx_search.extension",
]
# build the templated autosummary files
@@ -87,55 +88,50 @@
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
-html_theme = "sphinx_material"
+html_theme = "furo"
+html_title = "Flyte Docs"
+
html_theme_options = {
- # Set the name of the project to appear in the navigation.
- "nav_title": "Flytekit Python Reference",
- # Set you GA account ID to enable tracking
- "google_analytics_account": "G-YQL24L5CKY",
- # Specify a base_url used to generate sitemap.xml. If not
- # specified, then no sitemap will be built.
- "base_url": "https://github.com/lyft/flytekit",
- # Set the color and the accent color
- "color_primary": "deep-purple",
- "color_accent": "blue",
- # Set the repo location to get a badge with stats
- "repo_url": "https://github.com/lyft/flyte/",
- "repo_name": "flyte",
- # Visible levels of the global TOC; -1 means unlimited
- "globaltoc_depth": 1,
- # If False, expand all TOC entries
- "globaltoc_collapse": False,
- # If True, show hidden TOC entries
- "globaltoc_includehidden": False,
- # don't include home link in breadcrumb bar, since it's included
- # in the nav_links key below.
- "master_doc": False,
- # custom nav in breadcrumb bar
- "nav_links": [
- {"href": "https://flyte.readthedocs.io/", "internal": False, "title": "Flyte"},
- {"href": "https://flytecookbook.readthedocs.io", "internal": False, "title": "Flytekit Tutorials"},
- {"href": "index", "internal": True, "title": "Flytekit Python Reference"},
- ],
+ "light_css_variables": {
+ "color-brand-primary": "#4300c9",
+ "color-brand-content": "#4300c9",
+ },
+ "dark_css_variables": {
+ "color-brand-primary": "#9D68E4",
+ "color-brand-content": "#9D68E4",
+ },
}
+templates_path = ["_templates"]
+
# The default sidebars (for documents that don't match any pattern) are
# defined by theme itself. Builtin themes are using these templates by
# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
# 'searchbox.html']``.
-html_sidebars = {"**": ["logo-text.html", "globaltoc.html", "localtoc.html", "searchbox.html"]}
+# html_sidebars = {"**": ["logo-text.html", "globaltoc.html", "localtoc.html", "searchbox.html"]}
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = []
+html_static_path = ["_static"]
# Custom sidebar templates, must be a dictionary that maps document names
# to template names.
#
html_logo = "flyte_circle_gradient_1_4x4.png"
+pygments_style = "tango"
+pygments_dark_style = "paraiso-dark"
+
+html_css_files = [
+ "custom.css",
+]
+
+html_context = {
+ "home_page": "https://docs.flyte.org",
+}
+
# -- Options for HTMLHelp output ---------------------------------------------
diff --git a/docs/source/design/authoring.rst b/docs/source/design/authoring.rst
index d0b569faf5..62a953bf30 100644
--- a/docs/source/design/authoring.rst
+++ b/docs/source/design/authoring.rst
@@ -1,7 +1,7 @@
.. _design-authoring:
############################
-Flytekit Authoring Structure
+Authoring Structure
############################
Enabling users to write tasks and workflows is the core feature of flytekit, it is why it exists. This document goes over how some of the internals work.
@@ -51,13 +51,16 @@ Workflows
=========
There is currently only one :py:class:`Workflow ` class.
-.. autoclass:: flytekit.core.workflow.Workflow
+.. autoclass:: flytekit.core.workflow.PythonFunctionWorkflow
:noindex:
Launch Plan
===========
There is also only one :py:class:`LaunchPlan ` class.
+.. autoclass:: flytekit.core.launch_plan.LaunchPlan
+ :noindex:
+
**************
Call Patterns
**************
diff --git a/docs/source/design/index.rst b/docs/source/design/index.rst
index 2b9c2005fe..24eef916a1 100644
--- a/docs/source/design/index.rst
+++ b/docs/source/design/index.rst
@@ -1,20 +1,21 @@
.. _design:
############################
-Structure of Flytekit
+Overview
############################
Flytekit is comprised of a handful of different logical components, each discusssed in greater detail in each link
-* Models - These are almost Protobuf generated files.
-* Authoring - This provides the core Flyte authoring experiences, allowing users to write tasks, workflows, and launch plans.
-* Control Plane - The code here allows users to interact with the control plane through Python objecs.
-* Execution - A small shim layer basically that handles interaction with the Flyte ecosystem at execution time.
-* CLIs and Clients - Command line tools users may find themselves interacting with and the control plane client the CLIs call.
+* :ref:`Models Files ` - These are almost Protobuf generated files.
+* :ref:`Authoring ` - This provides the core Flyte authoring experiences, allowing users to write tasks, workflows, and launch plans.
+* :ref:`Control Plane ` - The code here allows users to interact with the control plane through Python objecs.
+* :ref:`Execution ` - A small shim layer basically that handles interaction with the Flyte ecosystem at execution time.
+* :ref:`CLIs and Clients ` - Command line tools users may find themselves interacting with and the control plane client the CLIs call.
.. toctree::
:maxdepth: 1
:caption: Structure and Layout of Flytekit
+ :hidden:
models
authoring
diff --git a/docs/source/design/models.rst b/docs/source/design/models.rst
index f561c06119..63be15098f 100644
--- a/docs/source/design/models.rst
+++ b/docs/source/design/models.rst
@@ -1,7 +1,7 @@
.. _design-models:
######################
-Flytekit Model Files
+Model Files
######################
***********
diff --git a/docs/source/generator.rst b/docs/source/generator.rst
deleted file mode 100644
index eb937fdbab..0000000000
--- a/docs/source/generator.rst
+++ /dev/null
@@ -1,9 +0,0 @@
-.. currentmodule:: flytekit
-
-.. autosummary::
- :nosignatures:
- :toctree: generated/
-
- core.dynamic_workflow_task
- core.notification
- core.schedule
diff --git a/docs/source/index.rst b/docs/source/index.rst
index 0035b514db..a47b7dc653 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -9,17 +9,29 @@ Flytekit Python Reference
This section of the documentation provides more detailed descriptions of the high-level design of ``flytekit`` and an
API reference for specific usage details of python functions, classes, and decorators that you import to specify tasks,
-build workflows, extend ``flytekit``.
+build workflows, and extend ``flytekit``.
+
+.. toctree::
+ :maxdepth: 4
+ :hidden:
+
+ Getting Started
+ Tutorials
+ reference/index
+ Community
.. toctree::
:maxdepth: 1
- :caption: Design
+ :caption: Flytekit SDK
+ :hidden:
+ Flytekit Python
design/index
.. toctree::
:maxdepth: 1
:caption: APIs
+ :hidden:
flytekit
testing
@@ -28,21 +40,8 @@ build workflows, extend ``flytekit``.
.. toctree::
:maxdepth: 1
:caption: Contributing
+ :hidden:
contributing
tasks.extend
types.extend
-
-
-.. toctree::
- :maxdepth: 1
- :hidden:
-
- generator
-
-Indices and tables
-==================
-
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`
diff --git a/docs/source/reference/index.rst b/docs/source/reference/index.rst
new file mode 100644
index 0000000000..f77a447060
--- /dev/null
+++ b/docs/source/reference/index.rst
@@ -0,0 +1,24 @@
+#############
+API Reference
+#############
+
+.. toctree::
+ :maxdepth: 1
+ :caption: API Reference
+ :name: apitoc
+
+ Flytekit Python
+ Flytekit Java
+ FlyteIDL
+ Flytectl
+
+.. toctree::
+ :maxdepth: 1
+ :caption: Component Reference (Code docs)
+ :name: componentreftoc
+
+ FlytePropeller
+ FlyteAdmin
+ FlytePlugins
+ DataCatalog
+
\ No newline at end of file
diff --git a/flytekit/__init__.py b/flytekit/__init__.py
index 249f87b71f..2e84c8a6a0 100644
--- a/flytekit/__init__.py
+++ b/flytekit/__init__.py
@@ -52,12 +52,11 @@
TaskMetadata - Wrapper object that allows users to specify Task
Resources - Things like CPUs/Memory, etc.
WorkflowFailurePolicy - Customizes what happens when a workflow fails.
- dynamic
Dynamic and Nested Workflows
==============================
-Please see the :py:mod:`Dynamic ` module for more information as well.
+See the :py:mod:`Dynamic ` module for more information.
.. autosummary::
:nosignatures:
@@ -67,9 +66,9 @@
Scheduling and Notifications
============================
-:py:mod:`Notifications Module `
-:py:mod:`Schedules Module `
+See the :py:mod:`Notifications Module ` and
+:py:mod:`Schedules Module ` for more information.
.. autosummary::
:nosignatures:
@@ -118,6 +117,17 @@
Secret
SecurityContext
+Core Modules
+=============
+
+.. autosummary::
+ :nosignatures:
+ :toctree: generated/
+
+ core.dynamic_workflow_task
+ core.notification
+ core.schedule
+
"""
diff --git a/requirements.txt b/requirements.txt
index 866325c7c7..0473ae8da4 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,6 +10,10 @@ ansiwrap==0.8.4
# via papermill
appdirs==1.4.4
# via black
+appnope==0.1.2
+ # via
+ # ipykernel
+ # ipython
async-generator==1.10
# via nbclient
attrs==20.3.0
@@ -24,9 +28,9 @@ black==20.8b1
# via papermill
bleach==3.3.0
# via nbconvert
-boto3==1.17.39
+boto3==1.17.40
# via sagemaker-training
-botocore==1.20.39
+botocore==1.20.40
# via
# boto3
# s3transfer
@@ -48,9 +52,7 @@ click==7.1.2
croniter==1.0.10
# via flytekit
cryptography==3.4.7
- # via
- # paramiko
- # secretstorage
+ # via paramiko
dataclasses-json==0.5.2
# via flytekit
decorator==4.4.2
@@ -81,7 +83,7 @@ hmsclient==0.1.1
# via flytekit
idna==2.10
# via requests
-importlib-metadata==3.7.3
+importlib-metadata==3.9.1
# via keyring
inotify_simple==1.2.1
# via sagemaker-training
@@ -91,14 +93,10 @@ ipython-genutils==0.2.0
# via
# nbformat
# traitlets
-ipython==7.21.0
+ipython==7.22.0
# via ipykernel
jedi==0.18.0
# via ipython
-jeepney==0.6.0
- # via
- # keyring
- # secretstorage
jinja2==2.11.3
# via nbconvert
jmespath==0.10.0
@@ -126,7 +124,7 @@ markupsafe==1.1.1
# via jinja2
marshmallow-enum==1.5.1
# via dataclasses-json
-marshmallow==3.10.0
+marshmallow==3.11.0
# via
# dataclasses-json
# marshmallow-enum
@@ -151,7 +149,7 @@ nbformat==5.1.2
# papermill
nest-asyncio==1.5.1
# via nbclient
-numpy==1.20.1
+numpy==1.20.2
# via
# flytekit
# pandas
@@ -237,7 +235,7 @@ requests==2.25.1
# flytekit
# papermill
# responses
-responses==0.13.1
+responses==0.13.2
# via flytekit
retry==0.9.2
# via flytekit
@@ -245,14 +243,12 @@ retrying==1.3.3
# via sagemaker-training
s3transfer==0.3.6
# via boto3
-sagemaker-training==3.7.3
+sagemaker-training==3.7.4
# via flytekit
scantree==0.0.1
# via dirhash
scipy==1.6.2
# via sagemaker-training
-secretstorage==3.3.1
- # via keyring
six==1.15.0
# via
# bcrypt