From 57d1d78487a3fb0690e61a4fbd85c434bd4b51da Mon Sep 17 00:00:00 2001 From: cosmicBboy Date: Mon, 29 Mar 2021 17:29:24 -0400 Subject: [PATCH 1/7] update flytekit docs theme, fix index links Signed-off-by: cosmicBboy --- dev-requirements.txt | 1 + doc-requirements.in | 1 + doc-requirements.txt | 35 +++++++------- docs/source/_static/custom.css | 33 +++++++++++++ docs/source/_templates/sidebar/brand.html | 18 +++++++ docs/source/conf.py | 57 +++++++++++------------ docs/source/design/authoring.rst | 5 +- docs/source/flytekit.rst | 5 ++ docs/source/generator.rst | 9 ---- docs/source/index.rst | 22 ++++----- docs/source/reference/index.rst | 24 ++++++++++ requirements.txt | 32 ++++++------- 12 files changed, 152 insertions(+), 90 deletions(-) create mode 100644 docs/source/_static/custom.css create mode 100644 docs/source/_templates/sidebar/brand.html delete mode 100644 docs/source/generator.rst create mode 100644 docs/source/reference/index.rst diff --git a/dev-requirements.txt b/dev-requirements.txt index ba18954a7b..5acc7855b8 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -14,6 +14,7 @@ attrs==20.3.0 # pytest black==20.8b1 # via + # -c requirements.txt # -r dev-requirements.in # flake8-black click==7.1.2 diff --git a/doc-requirements.in b/doc-requirements.in index 4383aa26f1..8604efb264 100644 --- a/doc-requirements.in +++ b/doc-requirements.in @@ -1,6 +1,7 @@ .[all] -e file:.#egg=flytekit +furo sphinx sphinx-gallery sphinx-prompt diff --git a/doc-requirements.txt b/doc-requirements.txt index 963d54ce5e..3e6f474c73 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -12,7 +12,11 @@ ansiwrap==0.8.4 # via papermill appdirs==1.4.4 # via black -astroid==2.5.1 +appnope==0.1.2 + # via + # ipykernel + # ipython +astroid==2.5.2 # via sphinx-autoapi async-generator==1.10 # via nbclient @@ -28,15 +32,16 @@ bcrypt==3.2.0 # via paramiko beautifulsoup4==4.9.3 # via + # furo # sphinx-code-include # sphinx-material black==20.8b1 # via papermill bleach==3.3.0 # via nbconvert -boto3==1.17.39 +boto3==1.17.40 # via sagemaker-training -botocore==1.20.39 +botocore==1.20.40 # via # boto3 # s3transfer @@ -61,7 +66,6 @@ cryptography==3.4.7 # via # -r doc-requirements.in # paramiko - # secretstorage css-html-js-minify==2.5.5 # via sphinx-material dataclasses-json==0.5.2 @@ -86,6 +90,8 @@ entrypoints==0.3 # papermill flyteidl==0.18.26 # via flytekit +furo==2021.3.20b30 + # via -r doc-requirements.in gevent==21.1.2 # via sagemaker-training greenlet==1.0.0 @@ -100,24 +106,20 @@ idna==2.10 # via requests imagesize==1.2.0 # via sphinx -importlib-metadata==3.7.3 +importlib-metadata==3.9.1 # via keyring inotify_simple==1.2.1 # via sagemaker-training -ipykernel==5.5.0 +ipykernel==5.5.2 # via flytekit ipython-genutils==0.2.0 # via # nbformat # traitlets -ipython==7.21.0 +ipython==7.22.0 # via ipykernel jedi==0.18.0 # via ipython -jeepney==0.6.0 - # via - # keyring - # secretstorage jinja2==2.11.3 # via # nbconvert @@ -152,7 +154,7 @@ markupsafe==1.1.1 # via jinja2 marshmallow-enum==1.5.1 # via dataclasses-json -marshmallow==3.10.0 +marshmallow==3.11.0 # via # dataclasses-json # marshmallow-enum @@ -177,7 +179,7 @@ nbformat==5.1.2 # papermill nest-asyncio==1.5.1 # via nbclient -numpy==1.20.1 +numpy==1.20.2 # via # flytekit # pandas @@ -273,7 +275,7 @@ requests==2.25.1 # papermill # responses # sphinx -responses==0.13.1 +responses==0.13.2 # via flytekit retry==0.9.2 # via flytekit @@ -281,14 +283,12 @@ retrying==1.3.3 # via sagemaker-training s3transfer==0.3.6 # via boto3 -sagemaker-training==3.7.3 +sagemaker-training==3.7.4 # via flytekit scantree==0.0.1 # via dirhash scipy==1.6.2 # via sagemaker-training -secretstorage==3.3.1 - # via keyring six==1.15.0 # via # bcrypt @@ -327,6 +327,7 @@ sphinx-prompt==1.4.0 sphinx==3.5.3 # via # -r doc-requirements.in + # furo # sphinx-autoapi # sphinx-code-include # sphinx-copybutton diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css new file mode 100644 index 0000000000..786fcd2611 --- /dev/null +++ b/docs/source/_static/custom.css @@ -0,0 +1,33 @@ +h1, h2, h3, h4, h5, h6 { + font-weight: bold; +} + +.sidebar-logo { + max-width: 30%; +} + + +.sidebar-tree .reference.external:after { + content: none; +} + +div.sphx-glr-download a { + color: #4300c9; + background-color: rgb(241, 241, 241); + background-image: none; + border: 1px solid rgb(202, 202, 202); +} + +div.sphx-glr-download a:hover { + background-color: rgb(230, 230, 230); + box-shadow: none; +} + +div.sphx-glr-thumbcontainer a.headerlink { + display: none; +} + +div.sphx-glr-thumbcontainer:hover { + border-color: white; + box-shadow: none; +} diff --git a/docs/source/_templates/sidebar/brand.html b/docs/source/_templates/sidebar/brand.html new file mode 100644 index 0000000000..a170d6c6d1 --- /dev/null +++ b/docs/source/_templates/sidebar/brand.html @@ -0,0 +1,18 @@ + diff --git a/docs/source/conf.py b/docs/source/conf.py index 98a324a6c4..657fb7f6cd 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -87,55 +87,50 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = "sphinx_material" +html_theme = "furo" +html_title = "Flyte Docs" + html_theme_options = { - # Set the name of the project to appear in the navigation. - "nav_title": "Flytekit Python Reference", - # Set you GA account ID to enable tracking - "google_analytics_account": "G-YQL24L5CKY", - # Specify a base_url used to generate sitemap.xml. If not - # specified, then no sitemap will be built. - "base_url": "https://github.com/lyft/flytekit", - # Set the color and the accent color - "color_primary": "deep-purple", - "color_accent": "blue", - # Set the repo location to get a badge with stats - "repo_url": "https://github.com/lyft/flyte/", - "repo_name": "flyte", - # Visible levels of the global TOC; -1 means unlimited - "globaltoc_depth": 1, - # If False, expand all TOC entries - "globaltoc_collapse": False, - # If True, show hidden TOC entries - "globaltoc_includehidden": False, - # don't include home link in breadcrumb bar, since it's included - # in the nav_links key below. - "master_doc": False, - # custom nav in breadcrumb bar - "nav_links": [ - {"href": "https://flyte.readthedocs.io/", "internal": False, "title": "Flyte"}, - {"href": "https://flytecookbook.readthedocs.io", "internal": False, "title": "Flytekit Tutorials"}, - {"href": "index", "internal": True, "title": "Flytekit Python Reference"}, - ], + "light_css_variables": { + "color-brand-primary": "#4300c9", + "color-brand-content": "#4300c9", + }, + "dark_css_variables": { + "color-brand-primary": "#9D68E4", + "color-brand-content": "#9D68E4", + }, } +templates_path = ["_templates"] + # The default sidebars (for documents that don't match any pattern) are # defined by theme itself. Builtin themes are using these templates by # default: ``['localtoc.html', 'relations.html', 'sourcelink.html', # 'searchbox.html']``. -html_sidebars = {"**": ["logo-text.html", "globaltoc.html", "localtoc.html", "searchbox.html"]} +# html_sidebars = {"**": ["logo-text.html", "globaltoc.html", "localtoc.html", "searchbox.html"]} # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = [] +html_static_path = ["_static"] # Custom sidebar templates, must be a dictionary that maps document names # to template names. # html_logo = "flyte_circle_gradient_1_4x4.png" +pygments_style = "tango" +pygments_dark_style = "paraiso-dark" + +html_css_files = [ + "custom.css", +] + +html_context = { + "home_page": "https://docs.flyte.org", +} + # -- Options for HTMLHelp output --------------------------------------------- diff --git a/docs/source/design/authoring.rst b/docs/source/design/authoring.rst index d0b569faf5..f2a3ce6c41 100644 --- a/docs/source/design/authoring.rst +++ b/docs/source/design/authoring.rst @@ -51,13 +51,16 @@ Workflows ========= There is currently only one :py:class:`Workflow ` class. -.. autoclass:: flytekit.core.workflow.Workflow +.. autoclass:: flytekit.core.workflow.PythonFunctionWorkflow :noindex: Launch Plan =========== There is also only one :py:class:`LaunchPlan ` class. +.. autoclass:: flytekit.core.launch_plan.LaunchPlan + :noindex: + ************** Call Patterns ************** diff --git a/docs/source/flytekit.rst b/docs/source/flytekit.rst index b982badef6..dbf6a3a326 100644 --- a/docs/source/flytekit.rst +++ b/docs/source/flytekit.rst @@ -2,3 +2,8 @@ :no-members: :no-inherited-members: :no-special-members: + +.. automodule:: flytekit.core + :no-members: + :no-inherited-members: + :no-special-members: diff --git a/docs/source/generator.rst b/docs/source/generator.rst deleted file mode 100644 index eb937fdbab..0000000000 --- a/docs/source/generator.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. currentmodule:: flytekit - -.. autosummary:: - :nosignatures: - :toctree: generated/ - - core.dynamic_workflow_task - core.notification - core.schedule diff --git a/docs/source/index.rst b/docs/source/index.rst index 0035b514db..a42ca4e51b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -11,6 +11,14 @@ This section of the documentation provides more detailed descriptions of the hig API reference for specific usage details of python functions, classes, and decorators that you import to specify tasks, build workflows, extend ``flytekit``. +.. toctree:: + :maxdepth: 4 + :hidden: + + Getting Started + Tutorials + reference/index + .. toctree:: :maxdepth: 1 :caption: Design @@ -32,17 +40,3 @@ build workflows, extend ``flytekit``. contributing tasks.extend types.extend - - -.. toctree:: - :maxdepth: 1 - :hidden: - - generator - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/docs/source/reference/index.rst b/docs/source/reference/index.rst new file mode 100644 index 0000000000..343e53e6c7 --- /dev/null +++ b/docs/source/reference/index.rst @@ -0,0 +1,24 @@ +############# +API Reference +############# + +.. toctree:: + :maxdepth: 1 + :caption: API Reference + :name: apitoc + + Flytekit Python + Flytekit JAVA + Flyte Core Language Specification (FlyteIDL) + Flytectl + +.. toctree:: + :maxdepth: 1 + :caption: Component Reference (Code docs) + :name: componentreftoc + + FlytePropeller + FlyteAdmin + FlytePlugins + DataCatalog + \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 866325c7c7..6fa22da1ad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,6 +10,10 @@ ansiwrap==0.8.4 # via papermill appdirs==1.4.4 # via black +appnope==0.1.2 + # via + # ipykernel + # ipython async-generator==1.10 # via nbclient attrs==20.3.0 @@ -24,9 +28,9 @@ black==20.8b1 # via papermill bleach==3.3.0 # via nbconvert -boto3==1.17.39 +boto3==1.17.40 # via sagemaker-training -botocore==1.20.39 +botocore==1.20.40 # via # boto3 # s3transfer @@ -48,9 +52,7 @@ click==7.1.2 croniter==1.0.10 # via flytekit cryptography==3.4.7 - # via - # paramiko - # secretstorage + # via paramiko dataclasses-json==0.5.2 # via flytekit decorator==4.4.2 @@ -81,24 +83,20 @@ hmsclient==0.1.1 # via flytekit idna==2.10 # via requests -importlib-metadata==3.7.3 +importlib-metadata==3.9.1 # via keyring inotify_simple==1.2.1 # via sagemaker-training -ipykernel==5.5.0 +ipykernel==5.5.2 # via flytekit ipython-genutils==0.2.0 # via # nbformat # traitlets -ipython==7.21.0 +ipython==7.22.0 # via ipykernel jedi==0.18.0 # via ipython -jeepney==0.6.0 - # via - # keyring - # secretstorage jinja2==2.11.3 # via nbconvert jmespath==0.10.0 @@ -126,7 +124,7 @@ markupsafe==1.1.1 # via jinja2 marshmallow-enum==1.5.1 # via dataclasses-json -marshmallow==3.10.0 +marshmallow==3.11.0 # via # dataclasses-json # marshmallow-enum @@ -151,7 +149,7 @@ nbformat==5.1.2 # papermill nest-asyncio==1.5.1 # via nbclient -numpy==1.20.1 +numpy==1.20.2 # via # flytekit # pandas @@ -237,7 +235,7 @@ requests==2.25.1 # flytekit # papermill # responses -responses==0.13.1 +responses==0.13.2 # via flytekit retry==0.9.2 # via flytekit @@ -245,14 +243,12 @@ retrying==1.3.3 # via sagemaker-training s3transfer==0.3.6 # via boto3 -sagemaker-training==3.7.3 +sagemaker-training==3.7.4 # via flytekit scantree==0.0.1 # via dirhash scipy==1.6.2 # via sagemaker-training -secretstorage==3.3.1 - # via keyring six==1.15.0 # via # bcrypt From 984e191b08b5f6136ca545236958e40946b38e92 Mon Sep 17 00:00:00 2001 From: cosmicBboy Date: Mon, 29 Mar 2021 20:11:14 -0400 Subject: [PATCH 2/7] add readthedocs sphinx search Signed-off-by: cosmicBboy --- doc-requirements.in | 1 + doc-requirements.txt | 4 +++- docs/source/conf.py | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/doc-requirements.in b/doc-requirements.in index 8604efb264..1b104414c4 100644 --- a/doc-requirements.in +++ b/doc-requirements.in @@ -2,6 +2,7 @@ -e file:.#egg=flytekit furo +readthedocs-sphinx-search sphinx sphinx-gallery sphinx-prompt diff --git a/doc-requirements.txt b/doc-requirements.txt index 3e6f474c73..84195f76de 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -110,7 +110,7 @@ importlib-metadata==3.9.1 # via keyring inotify_simple==1.2.1 # via sagemaker-training -ipykernel==5.5.2 +ipykernel==5.5.0 # via flytekit ipython-genutils==0.2.0 # via @@ -265,6 +265,8 @@ pyyaml==5.4.1 # sphinx-autoapi pyzmq==22.0.3 # via jupyter-client +readthedocs-sphinx-search==0.1.0 + # via -r doc-requirements.in regex==2021.3.17 # via # black diff --git a/docs/source/conf.py b/docs/source/conf.py index 657fb7f6cd..15df491168 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -47,6 +47,7 @@ "sphinx.ext.graphviz", "sphinx-prompt", "sphinx_copybutton", + "sphinx_search.extension", ] # build the templated autosummary files From 121415cbf70fa9f130d03efed36db00427b57d31 Mon Sep 17 00:00:00 2001 From: cosmicBboy Date: Mon, 29 Mar 2021 20:26:01 -0400 Subject: [PATCH 3/7] update flyteidl link Signed-off-by: cosmicBboy --- docs/source/reference/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/reference/index.rst b/docs/source/reference/index.rst index 343e53e6c7..1ed8dc2cef 100644 --- a/docs/source/reference/index.rst +++ b/docs/source/reference/index.rst @@ -9,7 +9,7 @@ API Reference Flytekit Python Flytekit JAVA - Flyte Core Language Specification (FlyteIDL) + FlyteIDL Flytectl .. toctree:: From 3c60373acd44d4c0ce84f20f1af7f877428ff447 Mon Sep 17 00:00:00 2001 From: cosmicBboy Date: Mon, 29 Mar 2021 20:53:29 -0400 Subject: [PATCH 4/7] update overview and homepage Signed-off-by: cosmicBboy --- docs/source/design/authoring.rst | 2 +- docs/source/design/index.rst | 13 +++++++------ docs/source/design/models.rst | 2 +- docs/source/index.rst | 8 ++++++-- docs/source/reference/index.rst | 8 ++++---- 5 files changed, 19 insertions(+), 14 deletions(-) diff --git a/docs/source/design/authoring.rst b/docs/source/design/authoring.rst index f2a3ce6c41..62a953bf30 100644 --- a/docs/source/design/authoring.rst +++ b/docs/source/design/authoring.rst @@ -1,7 +1,7 @@ .. _design-authoring: ############################ -Flytekit Authoring Structure +Authoring Structure ############################ Enabling users to write tasks and workflows is the core feature of flytekit, it is why it exists. This document goes over how some of the internals work. diff --git a/docs/source/design/index.rst b/docs/source/design/index.rst index 2b9c2005fe..24eef916a1 100644 --- a/docs/source/design/index.rst +++ b/docs/source/design/index.rst @@ -1,20 +1,21 @@ .. _design: ############################ -Structure of Flytekit +Overview ############################ Flytekit is comprised of a handful of different logical components, each discusssed in greater detail in each link -* Models - These are almost Protobuf generated files. -* Authoring - This provides the core Flyte authoring experiences, allowing users to write tasks, workflows, and launch plans. -* Control Plane - The code here allows users to interact with the control plane through Python objecs. -* Execution - A small shim layer basically that handles interaction with the Flyte ecosystem at execution time. -* CLIs and Clients - Command line tools users may find themselves interacting with and the control plane client the CLIs call. +* :ref:`Models Files ` - These are almost Protobuf generated files. +* :ref:`Authoring ` - This provides the core Flyte authoring experiences, allowing users to write tasks, workflows, and launch plans. +* :ref:`Control Plane ` - The code here allows users to interact with the control plane through Python objecs. +* :ref:`Execution ` - A small shim layer basically that handles interaction with the Flyte ecosystem at execution time. +* :ref:`CLIs and Clients ` - Command line tools users may find themselves interacting with and the control plane client the CLIs call. .. toctree:: :maxdepth: 1 :caption: Structure and Layout of Flytekit + :hidden: models authoring diff --git a/docs/source/design/models.rst b/docs/source/design/models.rst index f561c06119..63be15098f 100644 --- a/docs/source/design/models.rst +++ b/docs/source/design/models.rst @@ -1,7 +1,7 @@ .. _design-models: ###################### -Flytekit Model Files +Model Files ###################### *********** diff --git a/docs/source/index.rst b/docs/source/index.rst index a42ca4e51b..bba226589c 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -9,7 +9,7 @@ Flytekit Python Reference This section of the documentation provides more detailed descriptions of the high-level design of ``flytekit`` and an API reference for specific usage details of python functions, classes, and decorators that you import to specify tasks, -build workflows, extend ``flytekit``. +build workflows, and extend ``flytekit``. .. toctree:: :maxdepth: 4 @@ -21,13 +21,16 @@ build workflows, extend ``flytekit``. .. toctree:: :maxdepth: 1 - :caption: Design + :caption: Flytekit SDK + :hidden: + Flytekit Python design/index .. toctree:: :maxdepth: 1 :caption: APIs + :hidden: flytekit testing @@ -36,6 +39,7 @@ build workflows, extend ``flytekit``. .. toctree:: :maxdepth: 1 :caption: Contributing + :hidden: contributing tasks.extend diff --git a/docs/source/reference/index.rst b/docs/source/reference/index.rst index 1ed8dc2cef..f77a447060 100644 --- a/docs/source/reference/index.rst +++ b/docs/source/reference/index.rst @@ -7,10 +7,10 @@ API Reference :caption: API Reference :name: apitoc - Flytekit Python - Flytekit JAVA - FlyteIDL - Flytectl + Flytekit Python + Flytekit Java + FlyteIDL + Flytectl .. toctree:: :maxdepth: 1 From 84c7e019b4805c2894ceab12b2ccd5a403375b4c Mon Sep 17 00:00:00 2001 From: cosmicBboy Date: Tue, 30 Mar 2021 10:04:13 -0400 Subject: [PATCH 5/7] address comments @wild-endeavor Signed-off-by: cosmicBboy --- docs/source/flytekit.rst | 5 ----- flytekit/__init__.py | 18 ++++++++++++++---- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/docs/source/flytekit.rst b/docs/source/flytekit.rst index dbf6a3a326..b982badef6 100644 --- a/docs/source/flytekit.rst +++ b/docs/source/flytekit.rst @@ -2,8 +2,3 @@ :no-members: :no-inherited-members: :no-special-members: - -.. automodule:: flytekit.core - :no-members: - :no-inherited-members: - :no-special-members: diff --git a/flytekit/__init__.py b/flytekit/__init__.py index 249f87b71f..2e84c8a6a0 100644 --- a/flytekit/__init__.py +++ b/flytekit/__init__.py @@ -52,12 +52,11 @@ TaskMetadata - Wrapper object that allows users to specify Task Resources - Things like CPUs/Memory, etc. WorkflowFailurePolicy - Customizes what happens when a workflow fails. - dynamic Dynamic and Nested Workflows ============================== -Please see the :py:mod:`Dynamic ` module for more information as well. +See the :py:mod:`Dynamic ` module for more information. .. autosummary:: :nosignatures: @@ -67,9 +66,9 @@ Scheduling and Notifications ============================ -:py:mod:`Notifications Module ` -:py:mod:`Schedules Module ` +See the :py:mod:`Notifications Module ` and +:py:mod:`Schedules Module ` for more information. .. autosummary:: :nosignatures: @@ -118,6 +117,17 @@ Secret SecurityContext +Core Modules +============= + +.. autosummary:: + :nosignatures: + :toctree: generated/ + + core.dynamic_workflow_task + core.notification + core.schedule + """ From bec2a8aebc42da99550a567dcf1df8efcdfba2db Mon Sep 17 00:00:00 2001 From: cosmicBboy Date: Tue, 30 Mar 2021 10:17:12 -0400 Subject: [PATCH 6/7] update ipykernel Signed-off-by: cosmicBboy --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6fa22da1ad..0473ae8da4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -87,7 +87,7 @@ importlib-metadata==3.9.1 # via keyring inotify_simple==1.2.1 # via sagemaker-training -ipykernel==5.5.2 +ipykernel==5.5.0 # via flytekit ipython-genutils==0.2.0 # via From 71a87b2447c7a95e536642cb6aa2d502a699d513 Mon Sep 17 00:00:00 2001 From: cosmicBboy Date: Tue, 30 Mar 2021 11:21:22 -0400 Subject: [PATCH 7/7] add community link Signed-off-by: cosmicBboy --- docs/source/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/index.rst b/docs/source/index.rst index bba226589c..a47b7dc653 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -18,6 +18,7 @@ build workflows, and extend ``flytekit``. Getting Started Tutorials reference/index + Community .. toctree:: :maxdepth: 1