From ff3411f3c9ea5f59809efedb5cdf0ce92a4ac66d Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Fri, 19 Jul 2019 14:54:26 -0700 Subject: [PATCH 1/7] Add docs job to publish to googleapis.dev --- .kokoro/docs/common.cfg | 48 +++++++++++++++++++++++++++++++++++++++++ .kokoro/docs/ndb.cfg | 1 + .kokoro/publish-docs.sh | 42 ++++++++++++++++++++++++++++++++++++ .repo-metadata.json | 12 +++++++++++ noxfile.py | 28 +++++++++++++++++------- 5 files changed, 123 insertions(+), 8 deletions(-) create mode 100644 .kokoro/docs/common.cfg create mode 100644 .kokoro/docs/ndb.cfg create mode 100644 .kokoro/publish-docs.sh create mode 100644 .repo-metadata.json diff --git a/.kokoro/docs/common.cfg b/.kokoro/docs/common.cfg new file mode 100644 index 00000000..25540253 --- /dev/null +++ b/.kokoro/docs/common.cfg @@ -0,0 +1,48 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-ndb/.kokoro/trampoline.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-multi" +} +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/google-cloud-python/.kokoro/publish-docs.sh" +} + +env_vars: { + key: "STAGING_BUCKET" + value: "docs-staging" +} + +# Fetch the token needed for reporting release status to GitHub +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "yoshi-automation-github-key" + } + } +} + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "docuploader_service_account" + } + } +} \ No newline at end of file diff --git a/.kokoro/docs/ndb.cfg b/.kokoro/docs/ndb.cfg new file mode 100644 index 00000000..8f43917d --- /dev/null +++ b/.kokoro/docs/ndb.cfg @@ -0,0 +1 @@ +# Format: //devtools/kokoro/config/proto/build.proto \ No newline at end of file diff --git a/.kokoro/publish-docs.sh b/.kokoro/publish-docs.sh new file mode 100644 index 00000000..8d23eddd --- /dev/null +++ b/.kokoro/publish-docs.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +set -eo pipefail + +# Disable buffering, so that the logs stream through. +export PYTHONUNBUFFERED=1 + +cd github/python-ndb + +# Remove old nox +python3.6 -m pip uninstall --yes --quiet nox-automation + +# Install nox +python3.6 -m pip install --upgrade --quiet nox +python3.6 -m nox --version + +# build docs +nox -s docs + +python3 -m pip install gcp-docuploader + +# install a json parser +sudo apt-get update +sudo apt-get -y install software-properties-common +sudo add-apt-repository universe +sudo apt-get update +sudo apt-get -y install jq + +# create metadata +python3 -m docuploader create-metadata \ + --name=$(jq --raw-output '.name // empty' .repo-metadata.json) \ + --version=$(python3 setup.py --version) \ + --language=$(jq --raw-output '.language // empty' .repo-metadata.json) \ + --distribution-name=$(python3 setup.py --name) \ + --product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \ + --github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \ + --issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json) + +cat docs.metadata + +# upload docs +python3 -m docuploader upload docs/_build/html --metadata-file docs.metadata --staging-bucket docs-staging \ No newline at end of file diff --git a/.repo-metadata.json b/.repo-metadata.json new file mode 100644 index 00000000..124a9316 --- /dev/null +++ b/.repo-metadata.json @@ -0,0 +1,12 @@ +{ + "name": "python-ndb", + "name_pretty": "NDB Client Library for Google Cloud Datastore", + "product_documentation": "https://cloud.google.com/datastore", + "client_documentation": "https://googleapis.dev/python/python-ndb/latest", + "issue_tracker": "https://github.com/googleapis/python-ndb/issues", + "release_level": "alpha", + "language": "python", + "repo": "googleapis/python-ndb", + "distribution_name": "google-cloud-ndb", + "api_id": "datastore.googleapis.com" +} \ No newline at end of file diff --git a/noxfile.py b/noxfile.py index 34bdaca6..80a74205 100644 --- a/noxfile.py +++ b/noxfile.py @@ -18,6 +18,7 @@ """ import os +import shutil import nox @@ -105,15 +106,26 @@ def blacken(session): run_black(session) -@nox.session(py=DEFAULT_INTERPRETER) +@nox.session(python=DEFAULT_INTERPRETER) def docs(session): - # Install all dependencies. - session.install("Sphinx") - session.install("sphinxcontrib.spelling") - session.install(".") - # Building the docs. - run_args = ["bash", "test_utils/test_utils/scripts/update_docs.sh"] - session.run(*run_args) + """Build the docs for this library.""" + + session.install("-e", ".") + session.install("sphinx", "alabaster", "recommonmark", "sphinxcontrib.spelling") + + shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) + session.run( + "sphinx-build", + "-W", # warnings as errors + "-T", # show full traceback on exception + "-N", # no colors + "-b", + "html", + "-d", + os.path.join("docs", "_build", "doctrees", ""), + os.path.join("docs", ""), + os.path.join("docs", "_build", "html", ""), + ) @nox.session(py=DEFAULT_INTERPRETER) From 0819a51f3a283deb200df95d823a370ce612eab3 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Fri, 19 Jul 2019 14:56:13 -0700 Subject: [PATCH 2/7] Update repo metadata. --- .repo-metadata.json | 2 -- env/include/python3.6m | 1 + env/pip-selfcheck.json | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) create mode 120000 env/include/python3.6m create mode 100644 env/pip-selfcheck.json diff --git a/.repo-metadata.json b/.repo-metadata.json index 124a9316..f924251c 100644 --- a/.repo-metadata.json +++ b/.repo-metadata.json @@ -1,12 +1,10 @@ { "name": "python-ndb", "name_pretty": "NDB Client Library for Google Cloud Datastore", - "product_documentation": "https://cloud.google.com/datastore", "client_documentation": "https://googleapis.dev/python/python-ndb/latest", "issue_tracker": "https://github.com/googleapis/python-ndb/issues", "release_level": "alpha", "language": "python", "repo": "googleapis/python-ndb", "distribution_name": "google-cloud-ndb", - "api_id": "datastore.googleapis.com" } \ No newline at end of file diff --git a/env/include/python3.6m b/env/include/python3.6m new file mode 120000 index 00000000..ba3a1bff --- /dev/null +++ b/env/include/python3.6m @@ -0,0 +1 @@ +/usr/include/python3.6m \ No newline at end of file diff --git a/env/pip-selfcheck.json b/env/pip-selfcheck.json new file mode 100644 index 00000000..cebf51f0 --- /dev/null +++ b/env/pip-selfcheck.json @@ -0,0 +1 @@ +{"last_check":"2019-04-29T20:18:38Z","pypi_version":"19.1"} \ No newline at end of file From 97cdeceffee221b1706c4f5f2a97c29a914fe591 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Fri, 19 Jul 2019 14:57:01 -0700 Subject: [PATCH 3/7] Undo change to noxfile. --- noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 80a74205..bac91f97 100644 --- a/noxfile.py +++ b/noxfile.py @@ -106,7 +106,7 @@ def blacken(session): run_black(session) -@nox.session(python=DEFAULT_INTERPRETER) +@nox.session(py=DEFAULT_INTERPRETER) def docs(session): """Build the docs for this library.""" From 1ff7b240bda11423c9cde5591933ee1e527af840 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Fri, 19 Jul 2019 14:58:01 -0700 Subject: [PATCH 4/7] Remove env. --- env/include/python3.6m | 1 - env/pip-selfcheck.json | 1 - 2 files changed, 2 deletions(-) delete mode 120000 env/include/python3.6m delete mode 100644 env/pip-selfcheck.json diff --git a/env/include/python3.6m b/env/include/python3.6m deleted file mode 120000 index ba3a1bff..00000000 --- a/env/include/python3.6m +++ /dev/null @@ -1 +0,0 @@ -/usr/include/python3.6m \ No newline at end of file diff --git a/env/pip-selfcheck.json b/env/pip-selfcheck.json deleted file mode 100644 index cebf51f0..00000000 --- a/env/pip-selfcheck.json +++ /dev/null @@ -1 +0,0 @@ -{"last_check":"2019-04-29T20:18:38Z","pypi_version":"19.1"} \ No newline at end of file From 4092f64c2a95a4da705d825006fdc467581eb496 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Fri, 19 Jul 2019 15:00:04 -0700 Subject: [PATCH 5/7] Remove update docs script. --- test_utils/test_utils/scripts/update_docs.sh | 104 ------------------- 1 file changed, 104 deletions(-) delete mode 100755 test_utils/test_utils/scripts/update_docs.sh diff --git a/test_utils/test_utils/scripts/update_docs.sh b/test_utils/test_utils/scripts/update_docs.sh deleted file mode 100755 index 33718836..00000000 --- a/test_utils/test_utils/scripts/update_docs.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/bin/bash - -# Copyright 2016 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -ev - -GH_OWNER='GoogleAPIs' -GH_PROJECT_NAME='python-ndb' - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -# Function to build the docs. -function build_docs { - rm -rf docs/_build/ - # -W -> warnings as errors - # -T -> show full traceback on exception - # -N -> no color - sphinx-build \ - -W -T -N \ - -b html \ - -d docs/_build/doctrees \ - docs/ \ - docs/_build/html/ - return $? -} - -# Function to check spelling. -function check_spelling { - sphinx-build \ - -W -N \ - -b spelling \ - -d docs/_build/doctrees \ - docs/ \ - docs/_build/html/ - return $? -} - -# Only update docs if we are on CircleCI. -if [[ "${CIRCLE_BRANCH}" == "master" ]] && [[ -z "${CIRCLE_PR_NUMBER}" ]]; then - echo "Building new docs on a merged commit." -elif [[ "$1" == "kokoro" ]]; then - echo "Building and publishing docs on Kokoro." -elif [[ -n "${CIRCLE_TAG}" ]]; then - echo "Building new docs on a tag (but will not deploy)." - build_docs - exit $? -else - echo "Not on master nor a release tag." - echo "Building new docs for testing purposes, but not deploying." - build_docs - check_spelling - exit $? -fi - -# Adding GitHub pages branch. `git submodule add` checks it -# out at HEAD. -GH_PAGES_DIR='ghpages' -git submodule add -q -b gh-pages \ - "git@github.com:${GH_OWNER}/${GH_PROJECT_NAME}" ${GH_PAGES_DIR} - -# Determine if we are building a new tag or are building docs -# for master. Then build new docs in docs/_build from master. -if [[ -n "${CIRCLE_TAG}" ]]; then - # Sphinx will use the package version by default. - build_docs -else - SPHINX_RELEASE=$(git log -1 --pretty=%h) build_docs -fi - -# Update gh-pages with the created docs. -cd ${GH_PAGES_DIR} -git rm -fr latest/ -cp -R ../docs/_build/html/ latest/ - -# Update the files push to gh-pages. -git add . -git status - -# If there are no changes, just exit cleanly. -if [[ -z "$(git status --porcelain)" ]]; then - echo "Nothing to commit. Exiting without pushing changes." - exit -fi - -# Commit to gh-pages branch to apply changes. -git config --global user.email "dpebot@google.com" -git config --global user.name "dpebot" -git commit -m "Update docs after merge to master." - -# NOTE: This may fail if two docs updates (on merges to master) -# happen in close proximity. -git push -q origin HEAD:gh-pages From 05cc7ec517ef1409d39aa2f288eab762b8159639 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Mon, 22 Jul 2019 10:18:17 -0700 Subject: [PATCH 6/7] Blacken. --- noxfile.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/noxfile.py b/noxfile.py index bac91f97..599850b2 100644 --- a/noxfile.py +++ b/noxfile.py @@ -18,7 +18,7 @@ """ import os -import shutil +import shutil import nox @@ -179,6 +179,4 @@ def system(session): if system_test_exists: session.run("py.test", "--quiet", system_test_path, *session.posargs) if system_test_folder_exists: - session.run( - "py.test", "--quiet", system_test_folder_path, *session.posargs - ) + session.run("py.test", "--quiet", system_test_folder_path, *session.posargs) From 1a0f44cd79c1fc151468122d77acc306bf43fa8a Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Mon, 22 Jul 2019 10:37:49 -0700 Subject: [PATCH 7/7] Blacken again. --- noxfile.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/noxfile.py b/noxfile.py index 599850b2..586f5907 100644 --- a/noxfile.py +++ b/noxfile.py @@ -111,7 +111,9 @@ def docs(session): """Build the docs for this library.""" session.install("-e", ".") - session.install("sphinx", "alabaster", "recommonmark", "sphinxcontrib.spelling") + session.install( + "sphinx", "alabaster", "recommonmark", "sphinxcontrib.spelling" + ) shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) session.run( @@ -179,4 +181,6 @@ def system(session): if system_test_exists: session.run("py.test", "--quiet", system_test_path, *session.posargs) if system_test_folder_exists: - session.run("py.test", "--quiet", system_test_folder_path, *session.posargs) + session.run( + "py.test", "--quiet", system_test_folder_path, *session.posargs + )