diff --git a/autosynth/synth.py b/autosynth/synth.py index 3ef551858..40fe71171 100644 --- a/autosynth/synth.py +++ b/autosynth/synth.py @@ -374,11 +374,11 @@ def synthesize_loop( youngest = len(toolbox.versions) - 1 has_changes = toolbox.synthesize_version_in_new_branch(synthesizer, youngest) if not has_changes: - if not toolbox.metadata_contains_generated_files( - toolbox.branch - ) and toolbox.metadata_contains_generated_files( - toolbox.sub_branch(youngest) - ) and not change_pusher.check_if_pr_already_exists(toolbox.branch): + if ( + not toolbox.metadata_contains_generated_files(toolbox.branch) + and toolbox.metadata_contains_generated_files(toolbox.sub_branch(youngest)) + and not change_pusher.check_if_pr_already_exists(toolbox.branch) + ): # Special case: the repo owner turned on obsolete file tracking. # Generate a one-time PR containing only metadata changes. executor.check_call(["git", "checkout", toolbox.branch]) @@ -553,7 +553,16 @@ def _inner_main(temp_dir: str) -> int: parser.add_argument( "--repository", default=os.environ.get("REPOSITORY"), required=True ) - parser.add_argument("--synth-path", default=os.environ.get("SYNTH_PATH")) + parser.add_argument( + "--synth-path", + default=os.environ.get("SYNTH_PATH"), + help="If specified, changes the directory from which synthtool is invoked.", + ) + parser.add_argument( + "--synth-file-name", + default=os.environ.get("SYNTH_FILE_NAME"), + help="If specified, override the synth file name and may be a path to a file. Defaults to 'synth.py'.", + ) parser.add_argument("--metadata-path", default=os.environ.get("METADATA_PATH")) parser.add_argument("--base-log-dir", default="") parser.add_argument( @@ -579,6 +588,7 @@ def _inner_main(temp_dir: str) -> int: f"the API or client library generator." ) change_pusher: AbstractChangePusher = ChangePusher(args.repository, gh, branch) + synth_file_name = args.synth_file_name or "synth.py" # capture logs for later # The logs directory path will be rendered in Sponge and Fusion as the test name, @@ -629,6 +639,7 @@ def _inner_main(temp_dir: str) -> int: metadata_path, args.extra_args, deprecated_execution=args.deprecated_execution, + synth_py_path=synth_file_name, ).synthesize(synth_log_path / "sponge_log.log") if not has_changes(): @@ -655,7 +666,10 @@ def _inner_main(temp_dir: str) -> int: # Prepare to call synthesize loop. synthesizer = Synthesizer( - metadata_path, args.extra_args, args.deprecated_execution, "synth.py", + metadata_path, + args.extra_args, + deprecated_execution=args.deprecated_execution, + synth_py_path=synth_file_name, ) x = SynthesizeLoopToolbox( source_versions, diff --git a/synthtool/gcp/common.py b/synthtool/gcp/common.py index 0aba4c3ad..529660a35 100644 --- a/synthtool/gcp/common.py +++ b/synthtool/gcp/common.py @@ -296,7 +296,9 @@ def ruby_library(self, **kwargs) -> Path: return self._generic_library("ruby_library", **kwargs) def render(self, template_name: str, **kwargs) -> Path: - return self._templates.render(template_name, **kwargs) + template = self._templates.render(template_name, **kwargs) + _tracked_paths.add(template) + return template def _load_generic_metadata(self, metadata: Dict): """ diff --git a/synthtool/gcp/templates/java_library/.github/readme/synth.py b/synthtool/gcp/templates/java_library/.github/readme/synth.py new file mode 100644 index 000000000..7b48cc28d --- /dev/null +++ b/synthtool/gcp/templates/java_library/.github/readme/synth.py @@ -0,0 +1,19 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This script is used to synthesize generated the README for this library.""" + +from synthtool.languages import java + +java.custom_templates(["java_library/README.md"]) diff --git a/synthtool/gcp/templates/java_library/.kokoro/continuous/readme.cfg b/synthtool/gcp/templates/java_library/.kokoro/continuous/readme.cfg new file mode 100644 index 000000000..1046a25dd --- /dev/null +++ b/synthtool/gcp/templates/java_library/.kokoro/continuous/readme.cfg @@ -0,0 +1,55 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-multi" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/{{ metadata['repo']['repo_short'] }}/.kokoro/readme.sh" +} + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.log" + } +} + +# The github token is stored here. +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "yoshi-automation-github-key" + # TODO(theacodes): remove this after secrets have globally propagated + backend_type: FASTCONFIGPUSH + } + } +} + +# Common env vars for all repositories and builds. +env_vars: { + key: "GITHUB_USER" + value: "yoshi-automation" +} +env_vars: { + key: "GITHUB_EMAIL" + value: "yoshi-automation@google.com" +} diff --git a/synthtool/gcp/templates/java_library/.kokoro/readme.sh b/synthtool/gcp/templates/java_library/.kokoro/readme.sh new file mode 100644 index 000000000..295391969 --- /dev/null +++ b/synthtool/gcp/templates/java_library/.kokoro/readme.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +cd ${KOKORO_ARTIFACTS_DIR}/github/{{ metadata['repo']['repo_short'] }} + +# Disable buffering, so that the logs stream through. +export PYTHONUNBUFFERED=1 + +# Kokoro exposes this as a file, but the scripts expect just a plain variable. +export GITHUB_TOKEN=$(cat ${KOKORO_KEYSTORE_DIR}/73713_yoshi-automation-github-key) + +# Setup git credentials +echo "https://${GITHUB_TOKEN}:@github.com" >> ~/.git-credentials +git config --global credential.helper 'store --file ~/.git-credentials' + +python3.6 -m pip install git+https://github.com/googleapis/synthtool.git#egg=gcp-synthtool +python3.6 -m autosynth.synth \ + --repository={{metadata['repo']['repo']}} \ + --synth-file-name=.github/readme/synth.py \ + --metadata-path=.github/readme/synth.metadata \ + --pr-title="chore: regenerate README" \ + --branch-suffix="readme" \ No newline at end of file diff --git a/synthtool/languages/java.py b/synthtool/languages/java.py index 670b102cb..53ae81224 100644 --- a/synthtool/languages/java.py +++ b/synthtool/languages/java.py @@ -323,19 +323,7 @@ def _merge_common_templates( return source_text -def common_templates( - excludes: List[str] = [], template_path: Optional[Path] = None, **kwargs -) -> None: - """Generate common templates for a Java Library - - Fetches information about the repository from the .repo-metadata.json file, - information about the latest artifact versions and copies the files into - their expected location. - - Args: - excludes (List[str], optional): List of template paths to ignore - **kwargs: Additional options for CommonTemplates.java_library() - """ +def _common_template_metadata() -> Dict[str, Any]: metadata = {} # type: Dict[str, Any] repo_metadata = common._load_repo_metadata() if repo_metadata: @@ -359,6 +347,44 @@ def common_templates( else: metadata["min_java_version"] = DEFAULT_MIN_SUPPORTED_JAVA_VERSION - kwargs["metadata"] = metadata + return metadata + + +def common_templates( + excludes: List[str] = [], template_path: Optional[Path] = None, **kwargs +) -> None: + """Generate common templates for a Java Library + + Fetches information about the repository from the .repo-metadata.json file, + information about the latest artifact versions and copies the files into + their expected location. + + Args: + excludes (List[str], optional): List of template paths to ignore + **kwargs: Additional options for CommonTemplates.java_library() + """ + kwargs["metadata"] = _common_template_metadata() templates = gcp.CommonTemplates(template_path=template_path).java_library(**kwargs) + + # README.md is now synthesized separately. This prevents synthtool from deleting the + # README as it's no longer generated here. + excludes.append("README.md") + s.copy([templates], excludes=excludes, merge=_merge_common_templates) + + +def custom_templates(files: List[str], **kwargs) -> None: + """Generate custom template files + + Fetches information about the repository from the .repo-metadata.json file, + information about the latest artifact versions and copies the files into + their expected location. + + Args: + files (List[str], optional): List of template paths to include + **kwargs: Additional options for CommonTemplates.render() + """ + kwargs["metadata"] = _common_template_metadata() + for file in files: + template = gcp.CommonTemplates().render(file, **kwargs) + s.copy([template]) diff --git a/tests/test_language_java.py b/tests/test_language_java.py index 6a2acb17a..7d97c58fd 100644 --- a/tests/test_language_java.py +++ b/tests/test_language_java.py @@ -100,7 +100,7 @@ def assert_valid_yaml(file): try: # generate the common templates java.common_templates(template_path=TEMPLATES_PATH) - assert os.path.isfile("README.md") + assert os.path.isfile("renovate.json") # lint xml, yaml files # use os.walk because glob ignores hidden directories