diff --git a/flyteidl/.gitignore b/flyteidl/.gitignore index dceb6a24723..dde9a93f69e 100644 --- a/flyteidl/.gitignore +++ b/flyteidl/.gitignore @@ -10,4 +10,4 @@ dist gen/pb_python/flyteidl.egg-info/ .virtualgo -/build/ +docs/build/ diff --git a/flyteidl/.readthedocs.yml b/flyteidl/.readthedocs.yml new file mode 100644 index 00000000000..7433053ad71 --- /dev/null +++ b/flyteidl/.readthedocs.yml @@ -0,0 +1,16 @@ +# .readthedocs.yml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: conf.py + +# Optionally set the version of Python and requirements required to build your docs +python: + version: 3.8 + install: + - requirements: doc-requirements.txt diff --git a/flyteidl/Makefile b/flyteidl/Makefile index 36a6edfe386..f9cfdc06a3c 100644 --- a/flyteidl/Makefile +++ b/flyteidl/Makefile @@ -2,6 +2,10 @@ export REPOSITORY=flyteidl include boilerplate/lyft/golang_test_targets/Makefile +define PIP_COMPILE +pip-compile $(1) --upgrade --verbose +endef + .PHONY: update_boilerplate update_boilerplate: @boilerplate/update.sh @@ -28,3 +32,11 @@ test_unit: .PHONY: build_python build_python: @python setup.py sdist + +.PHONY: install-piptools +install-piptools: + pip install -U pip-tools + +.PHONY: doc-requirements.txt +doc-requirements.txt: doc-requirements.in install-piptools + $(call PIP_COMPILE,doc-requirements.in) diff --git a/flyteidl/README.rst b/flyteidl/README.rst index 030c5ecc943..13ec8834370 100644 --- a/flyteidl/README.rst +++ b/flyteidl/README.rst @@ -1,22 +1,33 @@ ================ Flyte IDL ================ -This repository contains all of Flyte's IDLs. +This is one of the core repositories of Flyte and contains the Specification of +the Flyte Lanugage. The Specification is maintained using Googles fantastic +Protcol buffers library. The code and docs are auto-generated. -The contents of this Readme will be deprecated shortly. Please refer to the Flyte `contributor's guide `__ in the future. + - [flyte.org](https://flyte.org) + - [Flyte Docs](http://flyte.readthedocs.org/) + - [FlyteIDL Docs](http://flyteidl.readthedocs.org) -Generating Protobufs -> Code -############################# +Generate Code from protobuf +---------------------------- +#. Make sure docker is installed locally. +#. Once installed run, ``make generate`` to generate all the code and mock + client for FlyteAdmin Service -Mac OS -******* + .. prompt:: bash -1. Install Docker for Mac (https://www.docker.com/docker-mac) -2. Start 'docker' and sign in. You should see a whale icon in your toolbar. -3. cd to the root of your flyteidl repository. -4. Run ``./generate_protos.sh`` to generate just the protobuf files. Side note: running ``make generate`` will generate protos along with some other things like mock classes. + make generate -Admin Client Generation -************************* +#. You might have to run, ``make download_tooling`` to install generator + dependencies -Please see the Flyte Tools documentation on the `generator `__ for more information. + .. prompt:: bash + + make download_tooling + +#. To add new dependencies for a doc, modify ``doc_requirements.in`` and then + + .. prompt:: bash + + make doc-requirements.txt diff --git a/flyteidl/conf.py b/flyteidl/conf.py new file mode 100644 index 00000000000..b30cf12eb31 --- /dev/null +++ b/flyteidl/conf.py @@ -0,0 +1,194 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/stable/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import re +import sys + +sys.path.insert(0, os.path.abspath("gen/pb-protodoc/")) + + +# -- Project information ----------------------------------------------------- + +project = "Flyte Language Specification" +copyright = "2021, Flyte" +author = "Flyte" + +# The full version, including alpha/beta/rc tags +release = re.sub('^v', '', os.popen('git describe').read().strip()) +version = release + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.autosectionlabel", + "sphinx.ext.napoleon", + "sphinx.ext.todo", + "sphinx.ext.viewcode", + "sphinx.ext.doctest", + "sphinx.ext.intersphinx", + "sphinx.ext.coverage", + "sphinx-prompt", + "sphinx_copybutton", +] + +# build the templated autosummary files +autosummary_generate = True + +# autosectionlabel throws warnings if section names are duplicated. +# The following tells autosectionlabel to not throw a warning for +# duplicated section names that are in different documents. +autosectionlabel_prefix_document = True + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = ".rst" + +# The master toctree document. +master_doc = "index" + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path . +exclude_patterns = [] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "sphinx_material" +html_logo = "docs/images/flyte_circle_gradient_1_4x4.png" +html_theme_options = { + # Set the name of the project to appear in the navigation. + "nav_title": "Flyte", + # Set you GA account ID to enable tracking + "google_analytics_account": "G-YQL24L5CKY", + # Specify a base_url used to generate sitemap.xml. If not + # specified, then no sitemap will be built. + "base_url": "https://github.com/lyft/flyteidl", + # Set the color and the accent color + "color_primary": "deep-purple", + "color_accent": "blue", + # Set the repo location to get a badge with stats + "repo_url": "https://github.com/flyteorg/flyte/", + "repo_name": "flyte", + # Visible levels of the global TOC; -1 means unlimited + "globaltoc_depth": 1, + # If False, expand all TOC entries + "globaltoc_collapse": False, + # If True, show hidden TOC entries + "globaltoc_includehidden": False, +} + +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +html_sidebars = {"**": ["logo-text.html", "globaltoc.html", "localtoc.html", "searchbox.html"]} + + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = [] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = "flyteidldoc" + + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, "flyteidl.tex", "flyteidl Documentation", "Flyte", "manual"), +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [(master_doc, "flyteidl", "flyteidl Documentation", [author], 1)] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ( + master_doc, + "flyteidl", + "flyteidl Documentation", + author, + "flyteidl", + "Python SDK for Flyte (https://flyte.org).", + "Miscellaneous", + ), +] + + +# -- Extension configuration ------------------------------------------------- +# intersphinx configuration +intersphinx_mapping = { + "python": ("https://docs.python.org/{.major}".format(sys.version_info), None), +} diff --git a/flyteidl/developing.rst b/flyteidl/developing.rst new file mode 100644 index 00000000000..3f9b6d466d2 --- /dev/null +++ b/flyteidl/developing.rst @@ -0,0 +1,26 @@ +How to contribute to and develop Flyte IDL +=========================================== + +Generate Code from protobuf +---------------------------- +#. Make sure docker is installed locally. +#. Once installed run, ``make generate`` to generate all the code and mock + client for FlyteAdmin Service + + .. prompt:: bash + + make generate + +#. You might have to run, ``make download_tooling`` to install generator + dependencies + + .. prompt:: bash + + make download_tooling + +#. To add new dependencies for a doc, modify ``doc_requirements.in`` and then + + .. prompt:: bash + + make doc-requirements.txt + diff --git a/flyteidl/doc-requirements.in b/flyteidl/doc-requirements.in new file mode 100644 index 00000000000..cdb6725a1b7 --- /dev/null +++ b/flyteidl/doc-requirements.in @@ -0,0 +1,5 @@ +sphinx +sphinx-prompt +sphinx-material +sphinx-code-include +sphinx-copybutton diff --git a/flyteidl/doc-requirements.txt b/flyteidl/doc-requirements.txt new file mode 100644 index 00000000000..d5a288dbd0e --- /dev/null +++ b/flyteidl/doc-requirements.txt @@ -0,0 +1,88 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile doc-requirements.in +# +alabaster==0.7.12 + # via sphinx +babel==2.9.0 + # via sphinx +beautifulsoup4==4.9.3 + # via + # sphinx-code-include + # sphinx-material +certifi==2020.12.5 + # via requests +chardet==4.0.0 + # via requests +css-html-js-minify==2.5.5 + # via sphinx-material +docutils==0.16 + # via sphinx +idna==2.10 + # via requests +imagesize==1.2.0 + # via sphinx +jinja2==2.11.3 + # via sphinx +lxml==4.6.2 + # via sphinx-material +markupsafe==1.1.1 + # via jinja2 +packaging==20.9 + # via sphinx +pygments==2.7.4 + # via + # sphinx + # sphinx-prompt +pyparsing==2.4.7 + # via packaging +python-slugify[unidecode]==4.0.1 + # via sphinx-material +pytz==2021.1 + # via babel +requests==2.25.1 + # via sphinx +six==1.15.0 + # via sphinx-code-include +snowballstemmer==2.1.0 + # via sphinx +soupsieve==2.1 + # via beautifulsoup4 +sphinx-code-include==1.1.1 + # via -r doc-requirements.in +sphinx-copybutton==0.3.1 + # via -r doc-requirements.in +sphinx-material==0.0.32 + # via -r doc-requirements.in +sphinx-prompt==1.3.0 + # via -r doc-requirements.in +sphinx==3.4.3 + # via + # -r doc-requirements.in + # sphinx-code-include + # sphinx-copybutton + # sphinx-material + # sphinx-prompt +sphinxcontrib-applehelp==1.0.2 + # via sphinx +sphinxcontrib-devhelp==1.0.2 + # via sphinx +sphinxcontrib-htmlhelp==1.0.3 + # via sphinx +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==1.0.3 + # via sphinx +sphinxcontrib-serializinghtml==1.1.4 + # via sphinx +text-unidecode==1.3 + # via python-slugify +unidecode==1.2.0 + # via python-slugify +urllib3==1.26.3 + # via requests + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/flyteidl/docs/Makefile b/flyteidl/docs/Makefile new file mode 100644 index 00000000000..e31a5fb0384 --- /dev/null +++ b/flyteidl/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = flyteidl +SOURCEDIR = ../ +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/flyteidl/docs/images/flyte_circle_gradient_1_4x4.png b/flyteidl/docs/images/flyte_circle_gradient_1_4x4.png new file mode 100644 index 00000000000..49cdbbbc341 Binary files /dev/null and b/flyteidl/docs/images/flyte_circle_gradient_1_4x4.png differ diff --git a/flyteidl/docs/make.bat b/flyteidl/docs/make.bat new file mode 100644 index 00000000000..47d656bb748 --- /dev/null +++ b/flyteidl/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build +set SPHINXPROJ=simpleble + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/flyteidl/index.rst b/flyteidl/index.rst new file mode 100644 index 00000000000..584ebaa5255 --- /dev/null +++ b/flyteidl/index.rst @@ -0,0 +1,33 @@ +.. flyteidl documentation master file, created by + +Welcome to ``FlyteIDL - Flytes Languages Specification`` documentation! +========================================================================= +FlyteIDL contains the core specification of Flyte, using `Googles Protocol Buffers `_. The Specification contains + +#. The core specification for Flyte Workflows, tasks and the type system +#. The specification for FlyteAdmin's `gRPC `_ and ``REST`` endpoints +#. Some of the core plugin API's like - Spark, Sagemaker, etc + +This specification is used to generate client stubs for `FlyteKit `_, `FlyteKit Java `_, `Flytectl `_ and the `FlyteAdmin Service `_. + +.. toctree:: + :maxdepth: 1 + :caption: Flytekit Learn by Example + + Flyte Documentation + Flytekit Learn by Example + +.. toctree:: + :maxdepth: 2 + :caption: FlyteIDL API reference & Development + + gen/pb-protodoc/flyteidl/index + developing + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search`