From 8306eca0cc2afa4686dcd7c281595bacd574328e Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Wed, 8 Apr 2020 18:18:51 -0700 Subject: [PATCH 1/6] Work in progress --- Makefile | 12 ++++++------ script/dependent_repos.txt | 3 +++ script/update_ref_docs.sh | 6 ++++-- 3 files changed, 13 insertions(+), 8 deletions(-) create mode 100644 script/dependent_repos.txt diff --git a/Makefile b/Makefile index 2d58e0506b..7b3a87a614 100644 --- a/Makefile +++ b/Makefile @@ -17,16 +17,16 @@ end2end_execute: @end2end/execute.sh # Use this target to build the rsts directory only. In order to build the entire flyte docs, use update_ref_docs && all_docs -.PHONY: docs -docs: +.PHONY: generate-local-docs +generate-local-docs: @docker run -t -v `pwd`:/base lyft/docbuilder:v2.2.0 sphinx-build -E -b html /base/rsts/. /base/_build # Builds the entire doc tree. Assumes update_ref_docs has run and that all externals rsts are in _rsts/ dir -.PHONY: all_docs -all_docs: +.PHONY: generate-dependent-repo-docs, generate-docs +generate-docs: @./script/generate_docs.sh # updates referenced docs from other repositories (e.g. flyteidl, flytekit) -.PHONY: update_ref_docs -update_ref_docs: +.PHONY: generate-dependent-repo-docs +generate-dependent-repo-docs: @./script/update_ref_docs.sh diff --git a/script/dependent_repos.txt b/script/dependent_repos.txt new file mode 100644 index 0000000000..39b474a58c --- /dev/null +++ b/script/dependent_repos.txt @@ -0,0 +1,3 @@ +github.com/lyft/flytekit +github.com/lyft/flyteidl + diff --git a/script/update_ref_docs.sh b/script/update_ref_docs.sh index 40f56b3405..8c69f4c520 100755 --- a/script/update_ref_docs.sh +++ b/script/update_ref_docs.sh @@ -18,8 +18,10 @@ REPOS_DIR=`mktemp -d "${BASEDIR}/_repos/XXXXXXXXX"` function clone_repos() { - git clone https://github.com/lyft/flytekit.git ${REPOS_DIR}/flytekit - git clone https://github.com/lyft/flyteidl.git ${REPOS_DIR}/flyteidl + REPOS=`cat ${DIR}/dependent_repos.txt` + for REPO in ${REPOS}; do + git clone https://${REPO}.git ${REPOS_DIR}/flytekit + done } # Clone all repos From d7bdf6ca08a0ffaa1dbfe17ec3f89ec16d29dedb Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Tue, 14 Apr 2020 22:22:08 -0700 Subject: [PATCH 2/6] Documentation process updated --- Makefile | 8 ++++---- docs_infra/in_container_html_generation.sh | 3 ++- docs_infra/in_container_rst_generation.sh | 2 +- rsts/contributor/docs/index.rst | 21 +++++++++++++++------ rsts/index.rst | 4 ++++ script/dependent_repos.txt | 3 --- script/generate_docs.sh | 4 ++-- script/update_ref_docs.sh | 15 +++++---------- 8 files changed, 33 insertions(+), 27 deletions(-) delete mode 100644 script/dependent_repos.txt diff --git a/Makefile b/Makefile index 7b3a87a614..19ab05f437 100644 --- a/Makefile +++ b/Makefile @@ -22,11 +22,11 @@ generate-local-docs: @docker run -t -v `pwd`:/base lyft/docbuilder:v2.2.0 sphinx-build -E -b html /base/rsts/. /base/_build # Builds the entire doc tree. Assumes update_ref_docs has run and that all externals rsts are in _rsts/ dir -.PHONY: generate-dependent-repo-docs, generate-docs -generate-docs: - @./script/generate_docs.sh +.PHONY: generate-docs +generate-docs: generate-dependent-repo-docs + @FLYTEKIT_VERSION=0.6.2 ./script/generate_docs.sh # updates referenced docs from other repositories (e.g. flyteidl, flytekit) .PHONY: generate-dependent-repo-docs generate-dependent-repo-docs: - @./script/update_ref_docs.sh + @FLYTEKIT_VERSION=0.6.2 FLYTEIDL_VERSION=0.17.27 ./script/update_ref_docs.sh diff --git a/docs_infra/in_container_html_generation.sh b/docs_infra/in_container_html_generation.sh index 2743645b0f..54cfd7529a 100755 --- a/docs_infra/in_container_html_generation.sh +++ b/docs_infra/in_container_html_generation.sh @@ -13,6 +13,7 @@ set -x # We're generating documentation for flytekit, which references flyteidl. autodoc complains if the python libraries # referenced are not installed, so let's install them -pip install -U flytekit[all] +echo "Installing flytekit==${FLYTEKIT_VERSION}" +pip install -U flytekit[all]==${FLYTEKIT_VERSION} sphinx-build -Q -b html -c /rsts /rsts /docs/ diff --git a/docs_infra/in_container_rst_generation.sh b/docs_infra/in_container_rst_generation.sh index 54f8ed3cf5..20151a92c0 100755 --- a/docs_infra/in_container_rst_generation.sh +++ b/docs_infra/in_container_rst_generation.sh @@ -10,7 +10,7 @@ set -x # We're generating documentation for flytesdk, which references flyteidl. autodoc complains if the python libraries # referenced are not installed, so let's install them -pip install -U flytekit[all] +pip install -U flytekit[all]==${FLYTEKIT_VERSION} # Generate the RST files for flytekit sphinx-apidoc --force --tocfile index --ext-autodoc --output-dir /repos/flytekit/docs /repos/flytekit/flytekit diff --git a/rsts/contributor/docs/index.rst b/rsts/contributor/docs/index.rst index 0c8a0513cc..e0fec6d755 100644 --- a/rsts/contributor/docs/index.rst +++ b/rsts/contributor/docs/index.rst @@ -12,23 +12,32 @@ Documentation that the steps below will compile come from: * RST files generated from the Flyte IDL repository * RST files generated from the Flytekit Python SDK -Building -********** +Building all documentation including dependent repos +***************************************************** In order to create this set of documentation run:: - $ make update_ref_docs && make all_docs + $ make generate-docs What happens is: -* ``./generate_docs.sh`` runs. All this does is create a temp directory and clone the two aforementioned repos. -* The Sphinx builder container will run with files from all three repos (the two cloned one and this one) mounted. - + * ``./generate_docs.sh`` runs. All this does is create a temp directory and clone the two aforementioned repos. + * The Sphinx builder container will run with files from all three repos (the two cloned one and this one) mounted. * It will generate RST files for Flytekit from the Python source code * Copy RST files from all three repos into a common location * Build HTML files into the ``docs/`` folder Please then commit the newly generated files before merging your PR. In the future we will invest in CI to help with this. +Building a local copy of documentation for RST modifications only +****************************************************************** +This can be used if one wants to quickly verify documentation updates in the rst files housed in the flyte repo. These are the main docs that flyte publishes. +Building all documentation can be a slow process. To speed up the iteration one can use the make target:: + + $ make generate-local-docs + +This needs a local virtual environment with sphinx-build installed. + + Notes ******* We aggregate the doc sources in a single ``index.rst`` file so that ``flytekit`` and the Flyte IDL HTML pages to be together in the same index/table of contents. diff --git a/rsts/index.rst b/rsts/index.rst index 868405835c..a147eff19d 100644 --- a/rsts/index.rst +++ b/rsts/index.rst @@ -20,3 +20,7 @@ Welcome to the documentation hub for Flyte. :name: sourcecodedocstoc Flyte Language Specification + FlytePropeller + FlyteAdmin + FlytePlugins + DataCatalog diff --git a/script/dependent_repos.txt b/script/dependent_repos.txt deleted file mode 100644 index 39b474a58c..0000000000 --- a/script/dependent_repos.txt +++ /dev/null @@ -1,3 +0,0 @@ -github.com/lyft/flytekit -github.com/lyft/flyteidl - diff --git a/script/generate_docs.sh b/script/generate_docs.sh index 591f6b1d0c..8a36b9dd3f 100755 --- a/script/generate_docs.sh +++ b/script/generate_docs.sh @@ -16,13 +16,13 @@ RSTS_DIR=`mktemp -d "${BASEDIR}/rsts_tmp/XXXXXXXXX"` # Copy all rst files to the same place cp -R rsts/* ${RSTS_DIR} -cp -R _rsts/* ${RSTS_DIR} +#cp -R _rsts/* ${RSTS_DIR} # The toctree in this index file requires that the idl/sdk rsts are in the same folder cp docs_infra/index.rst ${RSTS_DIR} # Generate documentation by running script inside the generation container -docker run -t -v ${BASEDIR}:/base -v ${BASEDIR}/docs:/docs -v ${RSTS_DIR}:/rsts lyft/docbuilder:v2.2.0 /base/docs_infra/in_container_html_generation.sh +docker run --rm -t -e FLYTEKIT_VERSION=${FLYTEKIT_VERSION} -v ${BASEDIR}:/base -v ${BASEDIR}/docs:/docs -v ${RSTS_DIR}:/rsts lyft/docbuilder:v2.2.0 /base/docs_infra/in_container_html_generation.sh # Cleanup rm -rf ${RSTS_DIR} || true diff --git a/script/update_ref_docs.sh b/script/update_ref_docs.sh index 8c69f4c520..171f7ec401 100755 --- a/script/update_ref_docs.sh +++ b/script/update_ref_docs.sh @@ -16,19 +16,14 @@ BASEDIR="${DIR}/.." mkdir ${BASEDIR}/_repos || true REPOS_DIR=`mktemp -d "${BASEDIR}/_repos/XXXXXXXXX"` -function clone_repos() -{ - REPOS=`cat ${DIR}/dependent_repos.txt` - for REPO in ${REPOS}; do - git clone https://${REPO}.git ${REPOS_DIR}/flytekit - done -} - # Clone all repos -$(clone_repos) +echo "Cloning Flyteidl" +git clone https://github.com/lyft/flyteidl.git --single-branch --branch v${FLYTEIDL_VERSION} ${REPOS_DIR}/flyteidl +echo "Cloning Flytekit" +git clone https://github.com/lyft/flytekit.git --single-branch --branch v${FLYTEKIT_VERSION} ${REPOS_DIR}/flytekit # Generate documentation by running script inside the generation container -docker run -t -v ${BASEDIR}:/base -v ${REPOS_DIR}:/repos -v ${BASEDIR}/_rsts:/_rsts lyft/docbuilder:v2.2.0 /base/docs_infra/in_container_rst_generation.sh +docker run --rm -t -e FLYTEKIT_VERSION=${FLYTEKIT_VERSION} -v ${BASEDIR}:/base -v ${REPOS_DIR}:/repos -v ${BASEDIR}/_rsts:/_rsts lyft/docbuilder:v2.2.0 /base/docs_infra/in_container_rst_generation.sh # Cleanup rm -rf ${REPOS_DIR}/* || true From ebb9935bae1ecc3339609daf791533f5467d61d8 Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Wed, 15 Apr 2020 22:54:12 -0700 Subject: [PATCH 3/6] uncomment _rsts --- script/generate_docs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/generate_docs.sh b/script/generate_docs.sh index 8a36b9dd3f..fbd2e27fe8 100755 --- a/script/generate_docs.sh +++ b/script/generate_docs.sh @@ -16,7 +16,7 @@ RSTS_DIR=`mktemp -d "${BASEDIR}/rsts_tmp/XXXXXXXXX"` # Copy all rst files to the same place cp -R rsts/* ${RSTS_DIR} -#cp -R _rsts/* ${RSTS_DIR} +cp -R _rsts/* ${RSTS_DIR} # The toctree in this index file requires that the idl/sdk rsts are in the same folder cp docs_infra/index.rst ${RSTS_DIR} From 29145efc357848bab855f99e3033d9e1af78b221 Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Wed, 15 Apr 2020 23:13:52 -0700 Subject: [PATCH 4/6] Fixed doc generation --- docs_infra/index.rst | 22 ---------------------- rsts/index.rst | 3 ++- script/generate_docs.sh | 3 --- 3 files changed, 2 insertions(+), 26 deletions(-) delete mode 100644 docs_infra/index.rst diff --git a/docs_infra/index.rst b/docs_infra/index.rst deleted file mode 100644 index 1c5a807052..0000000000 --- a/docs_infra/index.rst +++ /dev/null @@ -1,22 +0,0 @@ -Flyte -===== - -About Flyte - -.. toctree:: - :maxdepth: 2 - :caption: Flyte Documentation - :name: mastertoc - - introduction/index - user/index - administrator/index - contributor/index - -.. toctree:: - :caption: Generated Documentation from Source - :maxdepth: 1 - :name: sourcecodedocstoc - - flyteidl/index - Flytekit diff --git a/rsts/index.rst b/rsts/index.rst index a147eff19d..98d45413aa 100644 --- a/rsts/index.rst +++ b/rsts/index.rst @@ -19,7 +19,8 @@ Welcome to the documentation hub for Flyte. :maxdepth: 1 :name: sourcecodedocstoc - Flyte Language Specification + flyteidl/index + Flytekit FlytePropeller FlyteAdmin FlytePlugins diff --git a/script/generate_docs.sh b/script/generate_docs.sh index fbd2e27fe8..7585282709 100755 --- a/script/generate_docs.sh +++ b/script/generate_docs.sh @@ -18,9 +18,6 @@ RSTS_DIR=`mktemp -d "${BASEDIR}/rsts_tmp/XXXXXXXXX"` cp -R rsts/* ${RSTS_DIR} cp -R _rsts/* ${RSTS_DIR} -# The toctree in this index file requires that the idl/sdk rsts are in the same folder -cp docs_infra/index.rst ${RSTS_DIR} - # Generate documentation by running script inside the generation container docker run --rm -t -e FLYTEKIT_VERSION=${FLYTEKIT_VERSION} -v ${BASEDIR}:/base -v ${BASEDIR}/docs:/docs -v ${RSTS_DIR}:/rsts lyft/docbuilder:v2.2.0 /base/docs_infra/in_container_html_generation.sh From 89f53fdd8cd3fef9dcc58ca091dae55df7aaa524 Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Wed, 15 Apr 2020 23:24:44 -0700 Subject: [PATCH 5/6] github workflow docs --- .github/workflows/docs.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .github/workflows/docs.yml diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000000..48d11e0698 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,16 @@ +name: docs +on: + push: + branches: + - master + pull_request: + branches: + - master +jobs: + runs-on: ubuntu-latest + strategy: + max-parallel: 4 + steps: + - uses: actions/checkout@v1 + - name: Build docs + run: make generate-docs From b4c5000311e040d644bfa908da9b08ce58c632e1 Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Wed, 15 Apr 2020 23:32:16 -0700 Subject: [PATCH 6/6] added new job --- .github/workflows/docs.yml | 16 ---------------- .github/workflows/tests.yml | 8 ++++++++ 2 files changed, 8 insertions(+), 16 deletions(-) delete mode 100644 .github/workflows/docs.yml diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index 48d11e0698..0000000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: docs -on: - push: - branches: - - master - pull_request: - branches: - - master -jobs: - runs-on: ubuntu-latest - strategy: - max-parallel: 4 - steps: - - uses: actions/checkout@v1 - - name: Build docs - run: make generate-docs diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f7be948dc4..51d6ec4322 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -23,3 +23,11 @@ jobs: go-version: ${{ matrix.go-version }} - name: Run end-to-end tests run: make end2end + docs: + runs-on: ubuntu-latest + strategy: + max-parallel: 1 + steps: + - uses: actions/checkout@v1 + - name: Build docs + run: make generate-docs