Docs process updated (#274)

.github/workflows/tests.yml

@@ -23,3 +23,11 @@ jobs:
         go-version: ${{ matrix.go-version }}
     - name: Run end-to-end tests
       run: make end2end
+  docs:
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 1
+    steps:
+    - uses: actions/checkout@v1
+    - name: Build docs
+      run: make generate-docs

Makefile

@@ -17,16 +17,16 @@ end2end_execute:
 	@end2end/execute.sh
 
 # Use this target to build the rsts directory only. In order to build the entire flyte docs, use update_ref_docs && all_docs
-.PHONY: docs
-docs:
+.PHONY: generate-local-docs
+generate-local-docs:
 	@docker run -t -v `pwd`:/base lyft/docbuilder:v2.2.0 sphinx-build -E -b html /base/rsts/. /base/_build
 
 # Builds the entire doc tree. Assumes update_ref_docs has run and that all externals rsts are in _rsts/ dir
-.PHONY: all_docs
-all_docs:
-	@./script/generate_docs.sh
+.PHONY: generate-docs
+generate-docs: generate-dependent-repo-docs
+	@FLYTEKIT_VERSION=0.6.2 ./script/generate_docs.sh
 
 # updates referenced docs from other repositories (e.g. flyteidl, flytekit)
-.PHONY: update_ref_docs
-update_ref_docs:
-	@./script/update_ref_docs.sh
+.PHONY: generate-dependent-repo-docs
+generate-dependent-repo-docs:
+	@FLYTEKIT_VERSION=0.6.2 FLYTEIDL_VERSION=0.17.27 ./script/update_ref_docs.sh

docs_infra/in_container_html_generation.sh

@@ -13,6 +13,7 @@ set -x
 
 # We're generating documentation for flytekit, which references flyteidl.  autodoc complains if the python libraries
 # referenced are not installed, so let's install them
-pip install -U flytekit[all]
+echo "Installing flytekit==${FLYTEKIT_VERSION}"
+pip install -U flytekit[all]==${FLYTEKIT_VERSION}
 
 sphinx-build -Q -b html -c /rsts /rsts /docs/

docs_infra/in_container_rst_generation.sh

@@ -10,7 +10,7 @@ set -x
 
 # We're generating documentation for flytesdk, which references flyteidl.  autodoc complains if the python libraries
 # referenced are not installed, so let's install them
-pip install -U flytekit[all]
+pip install -U flytekit[all]==${FLYTEKIT_VERSION}
 
 # Generate the RST files for flytekit
 sphinx-apidoc --force --tocfile index --ext-autodoc --output-dir /repos/flytekit/docs /repos/flytekit/flytekit

rsts/contributor/docs/index.rst

@@ -12,23 +12,32 @@ Documentation that the steps below will compile come from:
 * RST files generated from the Flyte IDL repository
 * RST files generated from the Flytekit Python SDK
 
-Building
-**********
+Building all documentation including dependent repos
+*****************************************************
 
 In order to create this set of documentation run::
 
-    $ make update_ref_docs && make all_docs
+    $ make generate-docs
 
 What happens is:
-* ``./generate_docs.sh`` runs.  All this does is create a temp directory and clone the two aforementioned repos.
-* The Sphinx builder container will run with files from all three repos (the two cloned one and this one) mounted.
-
+  * ``./generate_docs.sh`` runs.  All this does is create a temp directory and clone the two aforementioned repos.
+  * The Sphinx builder container will run with files from all three repos (the two cloned one and this one) mounted.
   * It will generate RST files for Flytekit from the Python source code
   * Copy RST files from all three repos into a common location
   * Build HTML files into the ``docs/`` folder
 
 Please then commit the newly generated files before merging your PR.  In the future we will invest in CI to help with this.
 
+Building a local copy of documentation for RST modifications only
+******************************************************************
+This can be used if one wants to quickly verify documentation updates in the rst files housed in the flyte repo. These are the main docs that flyte publishes.
+Building all documentation can be a slow process. To speed up the iteration one can use the make target::
+  
+  $ make generate-local-docs
+
+This needs a local virtual environment with sphinx-build installed.
+
+
 Notes
 *******
 We aggregate the doc sources in a single ``index.rst`` file so that ``flytekit`` and the Flyte IDL HTML pages to be together in the same index/table of contents.

rsts/index.rst

@@ -19,4 +19,9 @@ Welcome to the documentation hub for Flyte.
    :maxdepth: 1
    :name: sourcecodedocstoc
 
-   Flyte Language Specification <https://github.com/lyft/flyteidl>
+   flyteidl/index
+   Flytekit <flytekit/index>
+   FlytePropeller <https://pkg.go.dev/mod/github.com/lyft/flytepropeller>
+   FlyteAdmin <https://pkg.go.dev/mod/github.com/lyft/flyteadmin>
+   FlytePlugins <https://pkg.go.dev/mod/github.com/lyft/flyteplugins>
+   DataCatalog <https://pkg.go.dev/mod/github.com/lyft/datacatalog>

script/generate_docs.sh

@@ -18,11 +18,8 @@ RSTS_DIR=`mktemp -d "${BASEDIR}/rsts_tmp/XXXXXXXXX"`
 cp -R rsts/* ${RSTS_DIR}
 cp -R _rsts/* ${RSTS_DIR}
 
-# The toctree in this index file requires that the idl/sdk rsts are in the same folder
-cp docs_infra/index.rst ${RSTS_DIR}
-
 # Generate documentation by running script inside the generation container
-docker run -t -v ${BASEDIR}:/base -v ${BASEDIR}/docs:/docs -v ${RSTS_DIR}:/rsts lyft/docbuilder:v2.2.0 /base/docs_infra/in_container_html_generation.sh
+docker run --rm -t -e FLYTEKIT_VERSION=${FLYTEKIT_VERSION} -v ${BASEDIR}:/base -v ${BASEDIR}/docs:/docs -v ${RSTS_DIR}:/rsts lyft/docbuilder:v2.2.0 /base/docs_infra/in_container_html_generation.sh
 
 # Cleanup
 rm -rf ${RSTS_DIR} || true

script/update_ref_docs.sh

@@ -16,17 +16,14 @@ BASEDIR="${DIR}/.."
 mkdir ${BASEDIR}/_repos || true
 REPOS_DIR=`mktemp -d "${BASEDIR}/_repos/XXXXXXXXX"`
 
-function clone_repos()
-{
-    git clone https://github.com/lyft/flytekit.git ${REPOS_DIR}/flytekit
-    git clone https://github.com/lyft/flyteidl.git ${REPOS_DIR}/flyteidl
-}
-
 # Clone all repos
-$(clone_repos)
+echo "Cloning Flyteidl"
+git clone https://github.com/lyft/flyteidl.git --single-branch --branch v${FLYTEIDL_VERSION} ${REPOS_DIR}/flyteidl
+echo "Cloning Flytekit"
+git clone https://github.com/lyft/flytekit.git --single-branch --branch v${FLYTEKIT_VERSION} ${REPOS_DIR}/flytekit
 
 # Generate documentation by running script inside the generation container
-docker run -t -v ${BASEDIR}:/base -v ${REPOS_DIR}:/repos -v ${BASEDIR}/_rsts:/_rsts lyft/docbuilder:v2.2.0 /base/docs_infra/in_container_rst_generation.sh
+docker run --rm -t -e FLYTEKIT_VERSION=${FLYTEKIT_VERSION} -v ${BASEDIR}:/base -v ${REPOS_DIR}:/repos -v ${BASEDIR}/_rsts:/_rsts lyft/docbuilder:v2.2.0 /base/docs_infra/in_container_rst_generation.sh
 
 # Cleanup
 rm -rf ${REPOS_DIR}/* || true