chore(java): use separate autosynth job for README (#767)

Split java README generation into a separate, per-repo continuous job running from Kokoro.

We now generate a new job that runs on commits to the primary branch that strictly manages the README.md. This should prevent us from seeing the contextless autosynth PRs which are caused by non-proto/template upstream changes.

autosynth/synth.py

@@ -374,11 +374,11 @@ def synthesize_loop(
     youngest = len(toolbox.versions) - 1
     has_changes = toolbox.synthesize_version_in_new_branch(synthesizer, youngest)
     if not has_changes:
-        if not toolbox.metadata_contains_generated_files(
-            toolbox.branch
-        ) and toolbox.metadata_contains_generated_files(
-            toolbox.sub_branch(youngest)
-        ) and not change_pusher.check_if_pr_already_exists(toolbox.branch):
+        if (
+            not toolbox.metadata_contains_generated_files(toolbox.branch)
+            and toolbox.metadata_contains_generated_files(toolbox.sub_branch(youngest))
+            and not change_pusher.check_if_pr_already_exists(toolbox.branch)
+        ):
             # Special case: the repo owner turned on obsolete file tracking.
             # Generate a one-time PR containing only metadata changes.
             executor.check_call(["git", "checkout", toolbox.branch])
@@ -553,7 +553,16 @@ def _inner_main(temp_dir: str) -> int:
     parser.add_argument(
         "--repository", default=os.environ.get("REPOSITORY"), required=True
     )
-    parser.add_argument("--synth-path", default=os.environ.get("SYNTH_PATH"))
+    parser.add_argument(
+        "--synth-path",
+        default=os.environ.get("SYNTH_PATH"),
+        help="If specified, changes the directory from which synthtool is invoked.",
+    )
+    parser.add_argument(
+        "--synth-file-name",
+        default=os.environ.get("SYNTH_FILE_NAME"),
+        help="If specified, override the synth file name and may be a path to a file. Defaults to 'synth.py'.",
+    )
     parser.add_argument("--metadata-path", default=os.environ.get("METADATA_PATH"))
     parser.add_argument("--base-log-dir", default="")
     parser.add_argument(
@@ -579,6 +588,7 @@ def _inner_main(temp_dir: str) -> int:
         f"the API or client library generator."
     )
     change_pusher: AbstractChangePusher = ChangePusher(args.repository, gh, branch)
+    synth_file_name = args.synth_file_name or "synth.py"
 
     # capture logs for later
     # The logs directory path will be rendered in Sponge and Fusion as the test name,
@@ -629,6 +639,7 @@ def _inner_main(temp_dir: str) -> int:
                 metadata_path,
                 args.extra_args,
                 deprecated_execution=args.deprecated_execution,
+                synth_py_path=synth_file_name,
             ).synthesize(synth_log_path / "sponge_log.log")
 
             if not has_changes():
@@ -655,7 +666,10 @@ def _inner_main(temp_dir: str) -> int:
 
             # Prepare to call synthesize loop.
             synthesizer = Synthesizer(
-                metadata_path, args.extra_args, args.deprecated_execution, "synth.py",
+                metadata_path,
+                args.extra_args,
+                deprecated_execution=args.deprecated_execution,
+                synth_py_path=synth_file_name,
             )
             x = SynthesizeLoopToolbox(
                 source_versions,

synthtool/gcp/common.py

@@ -296,7 +296,9 @@ def ruby_library(self, **kwargs) -> Path:
         return self._generic_library("ruby_library", **kwargs)
 
     def render(self, template_name: str, **kwargs) -> Path:
-        return self._templates.render(template_name, **kwargs)
+        template = self._templates.render(template_name, **kwargs)
+        _tracked_paths.add(template)
+        return template
 
     def _load_generic_metadata(self, metadata: Dict):
         """

synthtool/gcp/templates/java_library/.github/readme/synth.py

@@ -0,0 +1,19 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This script is used to synthesize generated the README for this library."""
+
+from synthtool.languages import java
+
+java.custom_templates(["java_library/README.md"])

synthtool/gcp/templates/java_library/.kokoro/continuous/readme.cfg

@@ -0,0 +1,55 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Format: //devtools/kokoro/config/proto/build.proto
+
+env_vars: {
+  key: "TRAMPOLINE_IMAGE"
+  value: "gcr.io/cloud-devrel-kokoro-resources/python-multi"
+}
+
+env_vars: {
+  key: "TRAMPOLINE_BUILD_FILE"
+  value: "github/{{ metadata['repo']['repo_short'] }}/.kokoro/readme.sh"
+}
+
+# Build logs will be here
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+    regex: "**/*sponge_log.log"
+  }
+}
+
+# The github token is stored here.
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 73713
+      keyname: "yoshi-automation-github-key"
+      # TODO(theacodes): remove this after secrets have globally propagated
+      backend_type: FASTCONFIGPUSH
+    }
+  }
+}
+
+# Common env vars for all repositories and builds.
+env_vars: {
+    key: "GITHUB_USER"
+    value: "yoshi-automation"
+}
+env_vars: {
+    key: "GITHUB_EMAIL"
+    value: "[email protected]"
+}

synthtool/gcp/templates/java_library/.kokoro/readme.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eo pipefail
+
+cd ${KOKORO_ARTIFACTS_DIR}/github/{{ metadata['repo']['repo_short'] }}
+
+# Disable buffering, so that the logs stream through.
+export PYTHONUNBUFFERED=1
+
+# Kokoro exposes this as a file, but the scripts expect just a plain variable.
+export GITHUB_TOKEN=$(cat ${KOKORO_KEYSTORE_DIR}/73713_yoshi-automation-github-key)
+
+# Setup git credentials
+echo "https://${GITHUB_TOKEN}:@github.com" >> ~/.git-credentials
+git config --global credential.helper 'store --file ~/.git-credentials'
+
+python3.6 -m pip install git+https://github.com/googleapis/synthtool.git#egg=gcp-synthtool
+python3.6 -m autosynth.synth \
+    --repository={{metadata['repo']['repo']}} \
+    --synth-file-name=.github/readme/synth.py \
+    --metadata-path=.github/readme/synth.metadata \
+    --pr-title="chore: regenerate README" \
+    --branch-suffix="readme"

synthtool/languages/java.py

@@ -323,19 +323,7 @@ def _merge_common_templates(
     return source_text
 
 
-def common_templates(
-    excludes: List[str] = [], template_path: Optional[Path] = None, **kwargs
-) -> None:
-    """Generate common templates for a Java Library
-
-    Fetches information about the repository from the .repo-metadata.json file,
-    information about the latest artifact versions and copies the files into
-    their expected location.
-
-    Args:
-        excludes (List[str], optional): List of template paths to ignore
-        **kwargs: Additional options for CommonTemplates.java_library()
-    """
+def _common_template_metadata() -> Dict[str, Any]:
     metadata = {}  # type: Dict[str, Any]
     repo_metadata = common._load_repo_metadata()
     if repo_metadata:
@@ -359,6 +347,44 @@ def common_templates(
     else:
         metadata["min_java_version"] = DEFAULT_MIN_SUPPORTED_JAVA_VERSION
 
-    kwargs["metadata"] = metadata
+    return metadata
+
+
+def common_templates(
+    excludes: List[str] = [], template_path: Optional[Path] = None, **kwargs
+) -> None:
+    """Generate common templates for a Java Library
+
+    Fetches information about the repository from the .repo-metadata.json file,
+    information about the latest artifact versions and copies the files into
+    their expected location.
+
+    Args:
+        excludes (List[str], optional): List of template paths to ignore
+        **kwargs: Additional options for CommonTemplates.java_library()
+    """
+    kwargs["metadata"] = _common_template_metadata()
     templates = gcp.CommonTemplates(template_path=template_path).java_library(**kwargs)
+
+    # README.md is now synthesized separately. This prevents synthtool from deleting the
+    # README as it's no longer generated here.
+    excludes.append("README.md")
+
     s.copy([templates], excludes=excludes, merge=_merge_common_templates)
+
+
+def custom_templates(files: List[str], **kwargs) -> None:
+    """Generate custom template files
+
+    Fetches information about the repository from the .repo-metadata.json file,
+    information about the latest artifact versions and copies the files into
+    their expected location.
+
+    Args:
+        files (List[str], optional): List of template paths to include
+        **kwargs: Additional options for CommonTemplates.render()
+    """
+    kwargs["metadata"] = _common_template_metadata()
+    for file in files:
+        template = gcp.CommonTemplates().render(file, **kwargs)
+        s.copy([template])

tests/test_language_java.py

@@ -100,7 +100,7 @@ def assert_valid_yaml(file):
         try:
             # generate the common templates
             java.common_templates(template_path=TEMPLATES_PATH)
-            assert os.path.isfile("README.md")
+            assert os.path.isfile("renovate.json")
 
             # lint xml, yaml files
             # use os.walk because glob ignores hidden directories