diff --git a/dev-requirements.txt b/dev-requirements.txt index ba18954a7b..5acc7855b8 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -14,6 +14,7 @@ attrs==20.3.0 # pytest black==20.8b1 # via + # -c requirements.txt # -r dev-requirements.in # flake8-black click==7.1.2 diff --git a/doc-requirements.in b/doc-requirements.in index 4383aa26f1..1b104414c4 100644 --- a/doc-requirements.in +++ b/doc-requirements.in @@ -1,6 +1,8 @@ .[all] -e file:.#egg=flytekit +furo +readthedocs-sphinx-search sphinx sphinx-gallery sphinx-prompt diff --git a/doc-requirements.txt b/doc-requirements.txt index 963d54ce5e..84195f76de 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -12,7 +12,11 @@ ansiwrap==0.8.4 # via papermill appdirs==1.4.4 # via black -astroid==2.5.1 +appnope==0.1.2 + # via + # ipykernel + # ipython +astroid==2.5.2 # via sphinx-autoapi async-generator==1.10 # via nbclient @@ -28,15 +32,16 @@ bcrypt==3.2.0 # via paramiko beautifulsoup4==4.9.3 # via + # furo # sphinx-code-include # sphinx-material black==20.8b1 # via papermill bleach==3.3.0 # via nbconvert -boto3==1.17.39 +boto3==1.17.40 # via sagemaker-training -botocore==1.20.39 +botocore==1.20.40 # via # boto3 # s3transfer @@ -61,7 +66,6 @@ cryptography==3.4.7 # via # -r doc-requirements.in # paramiko - # secretstorage css-html-js-minify==2.5.5 # via sphinx-material dataclasses-json==0.5.2 @@ -86,6 +90,8 @@ entrypoints==0.3 # papermill flyteidl==0.18.26 # via flytekit +furo==2021.3.20b30 + # via -r doc-requirements.in gevent==21.1.2 # via sagemaker-training greenlet==1.0.0 @@ -100,7 +106,7 @@ idna==2.10 # via requests imagesize==1.2.0 # via sphinx -importlib-metadata==3.7.3 +importlib-metadata==3.9.1 # via keyring inotify_simple==1.2.1 # via sagemaker-training @@ -110,14 +116,10 @@ ipython-genutils==0.2.0 # via # nbformat # traitlets -ipython==7.21.0 +ipython==7.22.0 # via ipykernel jedi==0.18.0 # via ipython -jeepney==0.6.0 - # via - # keyring - # secretstorage jinja2==2.11.3 # via # nbconvert @@ -152,7 +154,7 @@ markupsafe==1.1.1 # via jinja2 marshmallow-enum==1.5.1 # via dataclasses-json -marshmallow==3.10.0 +marshmallow==3.11.0 # via # dataclasses-json # marshmallow-enum @@ -177,7 +179,7 @@ nbformat==5.1.2 # papermill nest-asyncio==1.5.1 # via nbclient -numpy==1.20.1 +numpy==1.20.2 # via # flytekit # pandas @@ -263,6 +265,8 @@ pyyaml==5.4.1 # sphinx-autoapi pyzmq==22.0.3 # via jupyter-client +readthedocs-sphinx-search==0.1.0 + # via -r doc-requirements.in regex==2021.3.17 # via # black @@ -273,7 +277,7 @@ requests==2.25.1 # papermill # responses # sphinx -responses==0.13.1 +responses==0.13.2 # via flytekit retry==0.9.2 # via flytekit @@ -281,14 +285,12 @@ retrying==1.3.3 # via sagemaker-training s3transfer==0.3.6 # via boto3 -sagemaker-training==3.7.3 +sagemaker-training==3.7.4 # via flytekit scantree==0.0.1 # via dirhash scipy==1.6.2 # via sagemaker-training -secretstorage==3.3.1 - # via keyring six==1.15.0 # via # bcrypt @@ -327,6 +329,7 @@ sphinx-prompt==1.4.0 sphinx==3.5.3 # via # -r doc-requirements.in + # furo # sphinx-autoapi # sphinx-code-include # sphinx-copybutton diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css new file mode 100644 index 0000000000..786fcd2611 --- /dev/null +++ b/docs/source/_static/custom.css @@ -0,0 +1,33 @@ +h1, h2, h3, h4, h5, h6 { + font-weight: bold; +} + +.sidebar-logo { + max-width: 30%; +} + + +.sidebar-tree .reference.external:after { + content: none; +} + +div.sphx-glr-download a { + color: #4300c9; + background-color: rgb(241, 241, 241); + background-image: none; + border: 1px solid rgb(202, 202, 202); +} + +div.sphx-glr-download a:hover { + background-color: rgb(230, 230, 230); + box-shadow: none; +} + +div.sphx-glr-thumbcontainer a.headerlink { + display: none; +} + +div.sphx-glr-thumbcontainer:hover { + border-color: white; + box-shadow: none; +} diff --git a/docs/source/_templates/sidebar/brand.html b/docs/source/_templates/sidebar/brand.html new file mode 100644 index 0000000000..a170d6c6d1 --- /dev/null +++ b/docs/source/_templates/sidebar/brand.html @@ -0,0 +1,18 @@ + diff --git a/docs/source/conf.py b/docs/source/conf.py index 98a324a6c4..15df491168 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -47,6 +47,7 @@ "sphinx.ext.graphviz", "sphinx-prompt", "sphinx_copybutton", + "sphinx_search.extension", ] # build the templated autosummary files @@ -87,55 +88,50 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = "sphinx_material" +html_theme = "furo" +html_title = "Flyte Docs" + html_theme_options = { - # Set the name of the project to appear in the navigation. - "nav_title": "Flytekit Python Reference", - # Set you GA account ID to enable tracking - "google_analytics_account": "G-YQL24L5CKY", - # Specify a base_url used to generate sitemap.xml. If not - # specified, then no sitemap will be built. - "base_url": "https://github.com/lyft/flytekit", - # Set the color and the accent color - "color_primary": "deep-purple", - "color_accent": "blue", - # Set the repo location to get a badge with stats - "repo_url": "https://github.com/lyft/flyte/", - "repo_name": "flyte", - # Visible levels of the global TOC; -1 means unlimited - "globaltoc_depth": 1, - # If False, expand all TOC entries - "globaltoc_collapse": False, - # If True, show hidden TOC entries - "globaltoc_includehidden": False, - # don't include home link in breadcrumb bar, since it's included - # in the nav_links key below. - "master_doc": False, - # custom nav in breadcrumb bar - "nav_links": [ - {"href": "https://flyte.readthedocs.io/", "internal": False, "title": "Flyte"}, - {"href": "https://flytecookbook.readthedocs.io", "internal": False, "title": "Flytekit Tutorials"}, - {"href": "index", "internal": True, "title": "Flytekit Python Reference"}, - ], + "light_css_variables": { + "color-brand-primary": "#4300c9", + "color-brand-content": "#4300c9", + }, + "dark_css_variables": { + "color-brand-primary": "#9D68E4", + "color-brand-content": "#9D68E4", + }, } +templates_path = ["_templates"] + # The default sidebars (for documents that don't match any pattern) are # defined by theme itself. Builtin themes are using these templates by # default: ``['localtoc.html', 'relations.html', 'sourcelink.html', # 'searchbox.html']``. -html_sidebars = {"**": ["logo-text.html", "globaltoc.html", "localtoc.html", "searchbox.html"]} +# html_sidebars = {"**": ["logo-text.html", "globaltoc.html", "localtoc.html", "searchbox.html"]} # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = [] +html_static_path = ["_static"] # Custom sidebar templates, must be a dictionary that maps document names # to template names. # html_logo = "flyte_circle_gradient_1_4x4.png" +pygments_style = "tango" +pygments_dark_style = "paraiso-dark" + +html_css_files = [ + "custom.css", +] + +html_context = { + "home_page": "https://docs.flyte.org", +} + # -- Options for HTMLHelp output --------------------------------------------- diff --git a/docs/source/design/authoring.rst b/docs/source/design/authoring.rst index d0b569faf5..62a953bf30 100644 --- a/docs/source/design/authoring.rst +++ b/docs/source/design/authoring.rst @@ -1,7 +1,7 @@ .. _design-authoring: ############################ -Flytekit Authoring Structure +Authoring Structure ############################ Enabling users to write tasks and workflows is the core feature of flytekit, it is why it exists. This document goes over how some of the internals work. @@ -51,13 +51,16 @@ Workflows ========= There is currently only one :py:class:`Workflow ` class. -.. autoclass:: flytekit.core.workflow.Workflow +.. autoclass:: flytekit.core.workflow.PythonFunctionWorkflow :noindex: Launch Plan =========== There is also only one :py:class:`LaunchPlan ` class. +.. autoclass:: flytekit.core.launch_plan.LaunchPlan + :noindex: + ************** Call Patterns ************** diff --git a/docs/source/design/index.rst b/docs/source/design/index.rst index 2b9c2005fe..24eef916a1 100644 --- a/docs/source/design/index.rst +++ b/docs/source/design/index.rst @@ -1,20 +1,21 @@ .. _design: ############################ -Structure of Flytekit +Overview ############################ Flytekit is comprised of a handful of different logical components, each discusssed in greater detail in each link -* Models - These are almost Protobuf generated files. -* Authoring - This provides the core Flyte authoring experiences, allowing users to write tasks, workflows, and launch plans. -* Control Plane - The code here allows users to interact with the control plane through Python objecs. -* Execution - A small shim layer basically that handles interaction with the Flyte ecosystem at execution time. -* CLIs and Clients - Command line tools users may find themselves interacting with and the control plane client the CLIs call. +* :ref:`Models Files ` - These are almost Protobuf generated files. +* :ref:`Authoring ` - This provides the core Flyte authoring experiences, allowing users to write tasks, workflows, and launch plans. +* :ref:`Control Plane ` - The code here allows users to interact with the control plane through Python objecs. +* :ref:`Execution ` - A small shim layer basically that handles interaction with the Flyte ecosystem at execution time. +* :ref:`CLIs and Clients ` - Command line tools users may find themselves interacting with and the control plane client the CLIs call. .. toctree:: :maxdepth: 1 :caption: Structure and Layout of Flytekit + :hidden: models authoring diff --git a/docs/source/design/models.rst b/docs/source/design/models.rst index f561c06119..63be15098f 100644 --- a/docs/source/design/models.rst +++ b/docs/source/design/models.rst @@ -1,7 +1,7 @@ .. _design-models: ###################### -Flytekit Model Files +Model Files ###################### *********** diff --git a/docs/source/generator.rst b/docs/source/generator.rst deleted file mode 100644 index eb937fdbab..0000000000 --- a/docs/source/generator.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. currentmodule:: flytekit - -.. autosummary:: - :nosignatures: - :toctree: generated/ - - core.dynamic_workflow_task - core.notification - core.schedule diff --git a/docs/source/index.rst b/docs/source/index.rst index 0035b514db..a47b7dc653 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -9,17 +9,29 @@ Flytekit Python Reference This section of the documentation provides more detailed descriptions of the high-level design of ``flytekit`` and an API reference for specific usage details of python functions, classes, and decorators that you import to specify tasks, -build workflows, extend ``flytekit``. +build workflows, and extend ``flytekit``. + +.. toctree:: + :maxdepth: 4 + :hidden: + + Getting Started + Tutorials + reference/index + Community .. toctree:: :maxdepth: 1 - :caption: Design + :caption: Flytekit SDK + :hidden: + Flytekit Python design/index .. toctree:: :maxdepth: 1 :caption: APIs + :hidden: flytekit testing @@ -28,21 +40,8 @@ build workflows, extend ``flytekit``. .. toctree:: :maxdepth: 1 :caption: Contributing + :hidden: contributing tasks.extend types.extend - - -.. toctree:: - :maxdepth: 1 - :hidden: - - generator - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/docs/source/reference/index.rst b/docs/source/reference/index.rst new file mode 100644 index 0000000000..f77a447060 --- /dev/null +++ b/docs/source/reference/index.rst @@ -0,0 +1,24 @@ +############# +API Reference +############# + +.. toctree:: + :maxdepth: 1 + :caption: API Reference + :name: apitoc + + Flytekit Python + Flytekit Java + FlyteIDL + Flytectl + +.. toctree:: + :maxdepth: 1 + :caption: Component Reference (Code docs) + :name: componentreftoc + + FlytePropeller + FlyteAdmin + FlytePlugins + DataCatalog + \ No newline at end of file diff --git a/flytekit/__init__.py b/flytekit/__init__.py index 249f87b71f..2e84c8a6a0 100644 --- a/flytekit/__init__.py +++ b/flytekit/__init__.py @@ -52,12 +52,11 @@ TaskMetadata - Wrapper object that allows users to specify Task Resources - Things like CPUs/Memory, etc. WorkflowFailurePolicy - Customizes what happens when a workflow fails. - dynamic Dynamic and Nested Workflows ============================== -Please see the :py:mod:`Dynamic ` module for more information as well. +See the :py:mod:`Dynamic ` module for more information. .. autosummary:: :nosignatures: @@ -67,9 +66,9 @@ Scheduling and Notifications ============================ -:py:mod:`Notifications Module ` -:py:mod:`Schedules Module ` +See the :py:mod:`Notifications Module ` and +:py:mod:`Schedules Module ` for more information. .. autosummary:: :nosignatures: @@ -118,6 +117,17 @@ Secret SecurityContext +Core Modules +============= + +.. autosummary:: + :nosignatures: + :toctree: generated/ + + core.dynamic_workflow_task + core.notification + core.schedule + """ diff --git a/requirements.txt b/requirements.txt index 866325c7c7..0473ae8da4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,6 +10,10 @@ ansiwrap==0.8.4 # via papermill appdirs==1.4.4 # via black +appnope==0.1.2 + # via + # ipykernel + # ipython async-generator==1.10 # via nbclient attrs==20.3.0 @@ -24,9 +28,9 @@ black==20.8b1 # via papermill bleach==3.3.0 # via nbconvert -boto3==1.17.39 +boto3==1.17.40 # via sagemaker-training -botocore==1.20.39 +botocore==1.20.40 # via # boto3 # s3transfer @@ -48,9 +52,7 @@ click==7.1.2 croniter==1.0.10 # via flytekit cryptography==3.4.7 - # via - # paramiko - # secretstorage + # via paramiko dataclasses-json==0.5.2 # via flytekit decorator==4.4.2 @@ -81,7 +83,7 @@ hmsclient==0.1.1 # via flytekit idna==2.10 # via requests -importlib-metadata==3.7.3 +importlib-metadata==3.9.1 # via keyring inotify_simple==1.2.1 # via sagemaker-training @@ -91,14 +93,10 @@ ipython-genutils==0.2.0 # via # nbformat # traitlets -ipython==7.21.0 +ipython==7.22.0 # via ipykernel jedi==0.18.0 # via ipython -jeepney==0.6.0 - # via - # keyring - # secretstorage jinja2==2.11.3 # via nbconvert jmespath==0.10.0 @@ -126,7 +124,7 @@ markupsafe==1.1.1 # via jinja2 marshmallow-enum==1.5.1 # via dataclasses-json -marshmallow==3.10.0 +marshmallow==3.11.0 # via # dataclasses-json # marshmallow-enum @@ -151,7 +149,7 @@ nbformat==5.1.2 # papermill nest-asyncio==1.5.1 # via nbclient -numpy==1.20.1 +numpy==1.20.2 # via # flytekit # pandas @@ -237,7 +235,7 @@ requests==2.25.1 # flytekit # papermill # responses -responses==0.13.1 +responses==0.13.2 # via flytekit retry==0.9.2 # via flytekit @@ -245,14 +243,12 @@ retrying==1.3.3 # via sagemaker-training s3transfer==0.3.6 # via boto3 -sagemaker-training==3.7.3 +sagemaker-training==3.7.4 # via flytekit scantree==0.0.1 # via dirhash scipy==1.6.2 # via sagemaker-training -secretstorage==3.3.1 - # via keyring six==1.15.0 # via # bcrypt