diff --git a/docker/owlbot/java/Dockerfile b/docker/owlbot/java/Dockerfile new file mode 100644 index 000000000..b917ba29f --- /dev/null +++ b/docker/owlbot/java/Dockerfile @@ -0,0 +1,46 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# build from the root of this repo: +FROM gcr.io/cloud-devrel-public-resources/java8 + +ARG JAVA_FORMAT_VERSION=1.7 + +RUN apt-get install -y --no-install-recommends jq + +COPY docker/owlbot/java/bin /owlbot/bin +COPY docker/owlbot/java/src /owlbot/src +COPY docker/owlbot/java/templates /owlbot/templates +RUN cd /owlbot/src && \ + python3 -m pip install -r requirements.txt + +ADD https://repo1.maven.org/maven2/com/google/googlejavaformat/google-java-format/${JAVA_FORMAT_VERSION}/google-java-format-${JAVA_FORMAT_VERSION}-all-deps.jar /owlbot/google-java-format.jar + +###################### Install synthtool's requirements. +COPY . /synthtool/ + +WORKDIR /synthtool +RUN python3 -m pip install -e . + +# Allow non-root users to run python +RUN chmod +rx /root/ /root/.pyenv && chmod +r /owlbot/google-java-format.jar + +# Tell synthtool to pull templates from this docker image instead of from +# the live repo. +ENV SYNTHTOOL_TEMPLATES="/synthtool/synthtool/gcp/templates" \ + PYTHON_PATH="/owlbot/src" + +WORKDIR /workspace + +CMD [ "/owlbot/bin/entrypoint.sh" ] diff --git a/docker/owlbot/java/README.md b/docker/owlbot/java/README.md new file mode 100644 index 000000000..84432676b --- /dev/null +++ b/docker/owlbot/java/README.md @@ -0,0 +1,26 @@ +# Java Post-Processing Docker Image + +Docker image used for bootstrapping/post-processing. Running this on +should: + +1. Generate common templates +2. Write any missing `pom.xml` files or update with new detected modules +3. Restore or create `clirr-ignored-differences.xml` files after a new release +4. Restore license header years on generated files. +5. Run our standard `google-java-format` plugin. + +## Usage + +### Running locally + +```bash +docker run --rm -v $(pwd):/workspace --user "$(id -u):$(id -g)" gcr.io/repo-automation-bots/owlbot-java +``` + +### Building the image + +This image is built via Cloud Build. From the root of this repository, run: + +```bash +gcloud builds submit --config=docker/owlbot/java/cloudbuild.yaml +``` diff --git a/docker/owlbot/java/bin/entrypoint.sh b/docker/owlbot/java/bin/entrypoint.sh new file mode 100755 index 000000000..414adbec4 --- /dev/null +++ b/docker/owlbot/java/bin/entrypoint.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# templates +echo "Generating templates..." +/owlbot/bin/write_templates.sh +echo "...done" + +# write or restore pom.xml files +echo "Generating missing pom.xml..." +/owlbot/bin/write_missing_pom_files.sh +echo "...done" + +# write or restore clirr-ignored-differences.xml +echo "Generating clirr-ignored-differences.xml..." +/owlbot/bin/write_clirr_ignore.sh +echo "...done" + +# TODO: re-enable this once we resolve thrashing +# restore license headers years +# echo "Restoring copyright years..." +# /owlbot/bin/restore_license_headers.sh +# echo "...done" + +# ensure formatting on all .java files in the repository +echo "Reformatting source..." +/owlbot/bin/format_source.sh +echo "...done" diff --git a/docker/owlbot/java/bin/format_source.sh b/docker/owlbot/java/bin/format_source.sh new file mode 100755 index 000000000..20d08b772 --- /dev/null +++ b/docker/owlbot/java/bin/format_source.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# Find all the java files relative to the current directory and format them +# using google-java-format +find . -name '*.java' | xargs java -jar /owlbot/google-java-format.jar --replace diff --git a/docker/owlbot/java/bin/restore_license_headers.sh b/docker/owlbot/java/bin/restore_license_headers.sh new file mode 100755 index 000000000..eb3165468 --- /dev/null +++ b/docker/owlbot/java/bin/restore_license_headers.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# list the modified files in the current commit +last_commit_files=$(git diff-tree --no-commit-id -r $(git rev-parse HEAD) --name-only --diff-filter=M) + +# list the modified, uncommited files +current_modified_files=$(git diff --name-only HEAD) + +# join and deduplicate the list +all_files=$(echo ${last_commit_files} ${current_modified_files} | sort -u) + +for file in ${all_files} +do + # look for the Copyright YYYY line within the first 10 lines + old_copyright=$(git show HEAD~1:${file} | head -n 10 | egrep -o -e "Copyright ([[:digit:]]{4})" || echo "") + new_copyright=$(cat ${file} | head -n 10 | egrep -o -e "Copyright ([[:digit:]]{4})" || echo "") + # if the header year changed in the last diff, then restore the previous year + if [ ! -z "${old_copyright}" ] && [ ! -z "${new_copyright}" ] && [ "${old_copyright}" != "${new_copyright}" ] + then + echo "Restoring copyright in ${file} to '${old_copyright}'" + # replace the first instance of the old copyright header with the new + sed -i "s/${new_copyright}/${old_copyright}/1" ${file} + fi +done diff --git a/docker/owlbot/java/bin/write_clirr_ignore.sh b/docker/owlbot/java/bin/write_clirr_ignore.sh new file mode 100755 index 000000000..5c97a557f --- /dev/null +++ b/docker/owlbot/java/bin/write_clirr_ignore.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +templates_dir=$(realpath $(dirname "${BASH_SOURCE[0]}")/../templates/clirr) +is_release=$((git log -1 --pretty=%B | grep -e "chore.*release.*-SNAPSHOT") || echo "") + +# on a snapshot bump, clear all clirr-ignore-differences files +if [ ! -z "${is_release}" ] +then + find . -name 'clirr-ignored-differences.xml' | xargs rm +fi + +# restore default clirr-ignored-differences.xml for protos if the file does not exist +for dir in `ls -d proto-google-*` +do + if [ ! -f "${dir}/clirr-ignored-differences.xml" ] + then + tmp_dir=$(mktemp -d -t ci-XXXXXXXXXX) + pushd ${dir} + pushd src/main/java + find * -name *OrBuilder.java | xargs dirname | sort -u | jq -Rns ' (inputs | rtrimstr("\n") | split("\n") ) as $data | {proto_paths: $data}' > ${tmp_dir}/paths.json + popd + python3 /owlbot/src/gen-template.py --data=${tmp_dir}/paths.json --folder=${templates_dir} + popd + fi +done diff --git a/docker/owlbot/java/bin/write_missing_pom_files.sh b/docker/owlbot/java/bin/write_missing_pom_files.sh new file mode 100755 index 000000000..a3831ac12 --- /dev/null +++ b/docker/owlbot/java/bin/write_missing_pom_files.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +python3 /owlbot/src/fix-poms.py diff --git a/docker/owlbot/java/bin/write_templates.sh b/docker/owlbot/java/bin/write_templates.sh new file mode 100755 index 000000000..84e7e8dc4 --- /dev/null +++ b/docker/owlbot/java/bin/write_templates.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# create initial .gitignore if it does not yet exist +if [ ! -f ".gitignore" ] +then + cp /owlbot/templates/gitignore ./.gitignore +fi + +if [ -f "synth.py" ] +then + python3 /owlbot/src/convert-synthtool-templates.py --synth-file=synth.py +fi diff --git a/docker/owlbot/java/cloudbuild.yaml b/docker/owlbot/java/cloudbuild.yaml new file mode 100644 index 000000000..76e647eb0 --- /dev/null +++ b/docker/owlbot/java/cloudbuild.yaml @@ -0,0 +1,26 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +steps: + - name: 'gcr.io/cloud-builders/docker' + args: [ 'build', + '-t', 'gcr.io/$PROJECT_ID/owlbot-java:$SHORT_SHA', + '-t', 'gcr.io/$PROJECT_ID/owlbot-java:latest', + '-f', 'docker/owlbot/java/Dockerfile', '.' ] + - name: gcr.io/gcp-runtimes/container-structure-test + args: + ["test", "--image", "gcr.io/$PROJECT_ID/owlbot-java:$SHORT_SHA", "--config", "docker/owlbot/java/container_test.yaml"] + +images: + - gcr.io/$PROJECT_ID/owlbot-java:$SHORT_SHA + - gcr.io/$PROJECT_ID/owlbot-java:latest diff --git a/docker/owlbot/java/cloudbuild_test.yaml b/docker/owlbot/java/cloudbuild_test.yaml new file mode 100644 index 000000000..a701c2eab --- /dev/null +++ b/docker/owlbot/java/cloudbuild_test.yaml @@ -0,0 +1,22 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +steps: + - name: 'gcr.io/cloud-builders/docker' + args: [ 'build', + '-t', 'gcr.io/$PROJECT_ID/owlbot-java:$SHORT_SHA', + '-t', 'gcr.io/$PROJECT_ID/owlbot-java:latest', + '-f', 'docker/owlbot/java/Dockerfile', '.' ] + - name: gcr.io/gcp-runtimes/container-structure-test + args: + ["test", "--image", "gcr.io/$PROJECT_ID/owlbot-java:$SHORT_SHA", "--config", "docker/owlbot/java/container_test.yaml"] diff --git a/docker/owlbot/java/container_test.yaml b/docker/owlbot/java/container_test.yaml new file mode 100644 index 000000000..006acb24a --- /dev/null +++ b/docker/owlbot/java/container_test.yaml @@ -0,0 +1,26 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +schemaVersion: 1.0.0 +commandTests: +- name: "version" + command: ["java", "-version"] + # java -version outputs to stderr... + expectedError: ["(java|openjdk) version \"1.8.*\""] +- name: "formatter" + command: ["java", "-jar", "/owlbot/google-java-format.jar", "--version"] + expectedError: ["google-java-format: Version 1.7"] +- name: "python" + command: ["python", "--version"] + expectedOutput: ["Python 3.6.1"] diff --git a/docker/owlbot/java/src/convert-synthtool-templates.py b/docker/owlbot/java/src/convert-synthtool-templates.py new file mode 100644 index 000000000..55013dbb4 --- /dev/null +++ b/docker/owlbot/java/src/convert-synthtool-templates.py @@ -0,0 +1,47 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ast +import click +from synthtool.languages import java + + +@click.command() +@click.option( + "--synth-file", help="Path to synth.py file", default="synth.py", +) +def main(synth_file: str): + excludes = [] + should_include_templates = False + with open(synth_file, "r") as fp: + tree = ast.parse(fp.read()) + + # look for a call to java.common_templates() and extract the list of excludes + for node in ast.walk(tree): + if isinstance(node, ast.Call): + if ( + node.func.value.id == "java" + and node.func.attr == "common_templates" + ): + should_include_templates = True + for keyword in node.keywords: + if keyword.arg == "excludes": + excludes = [element.s for element in keyword.value.elts] + + if should_include_templates: + java.common_templates(excludes=excludes) + + +if __name__ == "__main__": + main() diff --git a/docker/owlbot/java/src/fix-poms.py b/docker/owlbot/java/src/fix-poms.py new file mode 100644 index 000000000..d68169a4c --- /dev/null +++ b/docker/owlbot/java/src/fix-poms.py @@ -0,0 +1,400 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import inspect +import itertools +import json +from lxml import etree +import os +from typing import List, Mapping +from poms import module, templates + + +def load_versions(filename: str, default_group_id: str) -> Mapping[str, module.Module]: + if not os.path.isfile(filename): + return {} + modules = {} + with open(filename, "r") as fp: + for line in fp: + line = line.strip() + if line.startswith("#"): + continue + + parts = line.split(":") + if len(parts) == 3: + artifact_id = parts[0] + group_id = ( + default_group_id + if artifact_id.startswith("google-") + else "com.google.api.grpc" + ) + modules[artifact_id] = module.Module( + group_id=group_id, + artifact_id=artifact_id, + release_version=parts[1], + version=parts[2], + ) + + return modules + + +def _find_dependency_index(dependencies, group_id, artifact_id) -> int: + try: + return next( + i + for i, x in enumerate(dependencies.getchildren()) + if _dependency_matches(x, group_id, artifact_id) + ) + except StopIteration: + return -1 + + +def _dependency_matches(node, group_id, artifact_id) -> bool: + artifact_node = node.find("{http://maven.apache.org/POM/4.0.0}artifactId") + group_node = node.find("{http://maven.apache.org/POM/4.0.0}groupId") + + if artifact_node is None or group_node is None: + return False + + return artifact_node.text.startswith(artifact_id) and group_node.text.startswith( + group_id + ) + + +def update_cloud_pom( + filename: str, proto_modules: List[module.Module], grpc_modules: List[module.Module] +): + tree = etree.parse(filename) + root = tree.getroot() + dependencies = root.find("{http://maven.apache.org/POM/4.0.0}dependencies") + + existing_dependencies = [ + m.find("{http://maven.apache.org/POM/4.0.0}artifactId").text + for m in dependencies + if m.find("{http://maven.apache.org/POM/4.0.0}artifactId") is not None + ] + + try: + grpc_index = _find_dependency_index( + dependencies, "com.google.api.grpc", "grpc-" + ) + except StopIteration: + grpc_index = _find_dependency_index(dependencies, "junit", "junit") + # insert grpc dependencies after junit + for m in grpc_modules: + if m.artifact_id not in existing_dependencies: + print(f"adding new test dependency {m.artifact_id}") + new_dependency = etree.Element( + "{http://maven.apache.org/POM/4.0.0}dependency" + ) + new_dependency.tail = "\n " + new_dependency.text = "\n " + new_group = etree.Element("{http://maven.apache.org/POM/4.0.0}groupId") + new_group.text = m.group_id + new_group.tail = "\n " + new_artifact = etree.Element( + "{http://maven.apache.org/POM/4.0.0}artifactId" + ) + new_artifact.text = m.artifact_id + new_artifact.tail = "\n " + new_scope = etree.Element("{http://maven.apache.org/POM/4.0.0}scope") + new_scope.text = "test" + new_scope.tail = "\n " + new_dependency.append(new_group) + new_dependency.append(new_artifact) + new_dependency.append(new_scope) + dependencies.insert(grpc_index + 1, new_dependency) + + try: + proto_index = _find_dependency_index( + dependencies, "com.google.api.grpc", "proto-" + ) + except StopIteration: + print("after protobuf") + proto_index = _find_dependency_index( + dependencies, "com.google.protobuf", "protobuf-java" + ) + # insert proto dependencies after protobuf-java + for m in proto_modules: + if m.artifact_id not in existing_dependencies: + print(f"adding new dependency {m.artifact_id}") + new_dependency = etree.Element( + "{http://maven.apache.org/POM/4.0.0}dependency" + ) + new_dependency.tail = "\n " + new_dependency.text = "\n " + new_group = etree.Element("{http://maven.apache.org/POM/4.0.0}groupId") + new_group.text = m.group_id + new_group.tail = "\n " + new_artifact = etree.Element( + "{http://maven.apache.org/POM/4.0.0}artifactId" + ) + new_artifact.text = m.artifact_id + new_artifact.tail = "\n " + new_dependency.append(new_group) + new_dependency.append(new_artifact) + dependencies.insert(proto_index + 1, new_dependency) + + tree.write(filename, pretty_print=True, xml_declaration=True, encoding="utf-8") + + +def update_parent_pom(filename: str, modules: List[module.Module]): + tree = etree.parse(filename) + root = tree.getroot() + + # BEGIN: update modules + existing = root.find("{http://maven.apache.org/POM/4.0.0}modules") + + module_names = [m.artifact_id for m in modules] + extra_modules = [ + m.text for i, m in enumerate(existing) if m.text not in module_names + ] + + modules_to_write = module_names + extra_modules + num_modules = len(modules_to_write) + + existing.clear() + existing.text = "\n " + for index, m in enumerate(modules_to_write): + new_module = etree.Element("{http://maven.apache.org/POM/4.0.0}module") + new_module.text = m + if index == num_modules - 1: + new_module.tail = "\n " + else: + new_module.tail = "\n " + existing.append(new_module) + + existing.tail = "\n\n " + # END: update modules + + # BEGIN: update versions in dependencyManagement + dependencies = root.find( + "{http://maven.apache.org/POM/4.0.0}dependencyManagement" + ).find("{http://maven.apache.org/POM/4.0.0}dependencies") + + existing_dependencies = [ + m.find("{http://maven.apache.org/POM/4.0.0}artifactId").text + for m in dependencies + if m.find("{http://maven.apache.org/POM/4.0.0}artifactId") is not None + ] + insert_index = 1 + + num_modules = len(modules) + + for index, m in enumerate(modules): + if m.artifact_id in existing_dependencies: + continue + + new_dependency = etree.Element("{http://maven.apache.org/POM/4.0.0}dependency") + new_dependency.tail = "\n " + new_dependency.text = "\n " + new_group = etree.Element("{http://maven.apache.org/POM/4.0.0}groupId") + new_group.text = m.group_id + new_group.tail = "\n " + new_artifact = etree.Element("{http://maven.apache.org/POM/4.0.0}artifactId") + new_artifact.text = m.artifact_id + new_artifact.tail = "\n " + new_version = etree.Element("{http://maven.apache.org/POM/4.0.0}version") + new_version.text = m.version + comment = etree.Comment(" {x-version-update:" + m.artifact_id + ":current} ") + comment.tail = "\n " + new_dependency.append(new_group) + new_dependency.append(new_artifact) + new_dependency.append(new_version) + new_dependency.append(comment) + new_dependency.tail = "\n " + dependencies.insert(1, new_dependency) + + # END: update versions in dependencyManagement + + tree.write(filename, pretty_print=True, xml_declaration=True, encoding="utf-8") + + +def update_bom_pom(filename: str, modules: List[module.Module]): + tree = etree.parse(filename) + root = tree.getroot() + existing = root.find( + "{http://maven.apache.org/POM/4.0.0}dependencyManagement" + ).find("{http://maven.apache.org/POM/4.0.0}dependencies") + + num_modules = len(modules) + + existing.clear() + existing.text = "\n " + for index, m in enumerate(modules): + new_dependency = etree.Element("{http://maven.apache.org/POM/4.0.0}dependency") + new_dependency.tail = "\n " + new_dependency.text = "\n " + new_group = etree.Element("{http://maven.apache.org/POM/4.0.0}groupId") + new_group.text = m.group_id + new_group.tail = "\n " + new_artifact = etree.Element("{http://maven.apache.org/POM/4.0.0}artifactId") + new_artifact.text = m.artifact_id + new_artifact.tail = "\n " + new_version = etree.Element("{http://maven.apache.org/POM/4.0.0}version") + new_version.text = m.version + comment = etree.Comment(" {x-version-update:" + m.artifact_id + ":current} ") + comment.tail = "\n " + new_dependency.append(new_group) + new_dependency.append(new_artifact) + new_dependency.append(new_version) + new_dependency.append(comment) + + if index == num_modules - 1: + new_dependency.tail = "\n " + else: + new_dependency.tail = "\n " + existing.append(new_dependency) + + existing.tail = "\n " + + tree.write(filename, pretty_print=True, xml_declaration=True, encoding="utf-8") + + +def main(): + with open(".repo-metadata.json", "r") as fp: + repo_metadata = json.load(fp) + + group_id, artifact_id = repo_metadata["distribution_name"].split(":") + name = repo_metadata["name_pretty"] + + existing_modules = load_versions("versions.txt", group_id) + + if artifact_id not in existing_modules: + existing_modules[artifact_id] = module.Module( + group_id=group_id, + artifact_id=artifact_id, + version="0.0.1-SNAPSHOT", + release_version="0.0.0", + ) + main_module = existing_modules[artifact_id] + + parent_artifact_id = f"{artifact_id}-parent" + if parent_artifact_id not in existing_modules: + existing_modules[parent_artifact_id] = module.Module( + group_id=group_id, + artifact_id=parent_artifact_id, + version="0.0.1-SNAPSHOT", + release_version="0.0.0", + ) + parent_module = existing_modules[parent_artifact_id] + + for path in glob.glob("proto-google-*"): + if not path in existing_modules: + existing_modules[path] = module.Module( + group_id="com.google.api.grpc", + artifact_id=path, + version=main_module.version, + release_version=main_module.release_version, + ) + + if not os.path.isfile(f"{path}/pom.xml"): + print(f"creating missing proto pom: {path}") + templates.render( + template_name="proto_pom.xml.j2", + output_name=f"{path}/pom.xml", + module=existing_modules[path], + parent_module=parent_module, + main_module=main_module, + ) + + for path in glob.glob("grpc-google-*"): + if not path in existing_modules: + existing_modules[path] = module.Module( + group_id="com.google.api.grpc", + artifact_id=path, + version=main_module.version, + release_version=main_module.release_version, + ) + + if not os.path.isfile(f"{path}/pom.xml"): + proto_artifact_id = path.replace("grpc-", "proto-") + print(f"creating missing grpc pom: {path}") + templates.render( + template_name="grpc_pom.xml.j2", + output_name=f"{path}/pom.xml", + module=existing_modules[path], + parent_module=parent_module, + main_module=main_module, + proto_module=existing_modules[proto_artifact_id], + ) + proto_modules = [ + module + for module in existing_modules.values() + if module.artifact_id.startswith("proto-") + ] + grpc_modules = [ + module + for module in existing_modules.values() + if module.artifact_id.startswith("grpc-") + ] + modules = [main_module] + grpc_modules + proto_modules + + if os.path.isfile(f"{artifact_id}/pom.xml"): + print("updating modules in cloud pom.xml") + update_cloud_pom(f"{artifact_id}/pom.xml", proto_modules, grpc_modules) + else: + print("creating missing cloud pom.xml") + templates.render( + template_name="cloud_pom.xml.j2", + output_name=f"{artifact_id}/pom.xml", + module=main_module, + parent_module=parent_module, + repo=repo_metadata["repo"], + name=name, + description=repo_metadata["api_description"], + proto_modules=proto_modules, + grpc_modules=grpc_modules, + ) + + if os.path.isfile(f"{artifact_id}-bom/pom.xml"): + print("updating modules in bom pom.xml") + update_bom_pom(f"{artifact_id}-bom/pom.xml", modules) + else: + print("creating missing bom pom.xml") + templates.render( + template_name="bom_pom.xml.j2", + output_name=f"{artifact_id}-bom/pom.xml", + repo=repo_metadata["repo"], + name=name, + modules=modules, + main_module=main_module, + ) + + if os.path.isfile("pom.xml"): + print("updating modules in parent pom.xml") + update_parent_pom("pom.xml", modules) + else: + print("creating missing parent pom.xml") + templates.render( + template_name="parent_pom.xml.j2", + output_name="./pom.xml", + repo=repo_metadata["repo"], + modules=modules, + main_module=main_module, + name=name, + ) + + if os.path.isfile("versions.txt"): + print("updating modules in versions.txt") + else: + print("creating missing versions.txt") + templates.render( + template_name="versions.txt.j2", output_name="./versions.txt", modules=modules, + ) + + +if __name__ == "__main__": + main() diff --git a/docker/owlbot/java/src/gen-template.py b/docker/owlbot/java/src/gen-template.py new file mode 100644 index 000000000..95334b963 --- /dev/null +++ b/docker/owlbot/java/src/gen-template.py @@ -0,0 +1,81 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import json +from typing import List +import os +from pathlib import Path + +import click +import jinja2 + + +@click.command() +@click.option( + "--folder", help="Path to folder of templates", +) +@click.option("--file", help="Path to template file") +@click.option( + "--data", + help="Path to JSON file with template values", + multiple=True, + required=True, +) +@click.option( + "--output", help="Path to output", default=".", +) +def main(folder: str, file: str, data: List[str], output: str): + """Generate templates""" + variables = {} + for data_file in data: + with open(data_file, "r") as fp: + variables = {**variables, **json.load(fp)} + + if folder is not None: + location = Path(folder) + filenames = glob.glob(f"{folder}/**/*.j2", recursive=True) + elif file is not None: + location = Path(file).parent + filenames = [f"{file}.j2"] + else: + raise Exception("Need to specify either folder or file") + + output_path = Path(output) + + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(str(location)), + autoescape=False, + keep_trailing_newline=True, + ) + + for filename in filenames: + template_name = Path(filename).relative_to(location) + template = env.get_template(str(template_name)) + output = template.stream(**variables) + + destination = output_path / os.path.splitext(template_name)[0] + destination.parent.mkdir(parents=True, exist_ok=True) + + with destination.open("w") as fp: + output.dump(fp) + + # Copy file mode over + source_path = Path(template.filename) + mode = source_path.stat().st_mode + destination.chmod(mode) + + +if __name__ == "__main__": + main() diff --git a/docker/owlbot/java/src/poms/module.py b/docker/owlbot/java/src/poms/module.py new file mode 100644 index 000000000..c8cc15984 --- /dev/null +++ b/docker/owlbot/java/src/poms/module.py @@ -0,0 +1,50 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import attr +from lxml import etree +import os +from typing import List, Optional + + +@attr.s(auto_attribs=True) +class Module: + group_id: str + artifact_id: str + version: str + release_version: Optional[str] + + +def read_module(pom: str) -> Module: + tree = etree.parse(pom) + artifact_id = tree.find("{http://maven.apache.org/POM/4.0.0}artifactId").text + version = tree.find("{http://maven.apache.org/POM/4.0.0}version").text + group_id = ( + "com.google.cloud" + if artifact_id.startswith("google-cloud") + else "com.google.api.grpc" + ) + return Module(group_id=group_id, artifact_id=artifact_id, version=version,) + + +def read_modules(service: str) -> List[Module]: + thedir = f"workspace/java-{service}/" + modules = [] + for name in os.listdir(thedir): + dir = os.path.join(thedir, name) + pom = os.path.join(dir, "pom.xml") + if os.path.exists(pom): + modules.append(read_module(pom)) + + return modules diff --git a/docker/owlbot/java/src/poms/templates.py b/docker/owlbot/java/src/poms/templates.py new file mode 100644 index 000000000..287c40938 --- /dev/null +++ b/docker/owlbot/java/src/poms/templates.py @@ -0,0 +1,36 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from jinja2 import Environment, FileSystemLoader +import os +import pathlib +from typing import List + +root_directory = pathlib.Path( + os.path.realpath(os.path.dirname(os.path.realpath(__file__))) +).parent.parent +print(root_directory) +jinja_env = Environment( + loader=FileSystemLoader(str(root_directory / "templates" / "poms")), + keep_trailing_newline=True, +) + + +def render(template_name: str, output_name: str, **kwargs): + template = jinja_env.get_template(template_name) + t = template.stream(kwargs) + directory = os.path.dirname(output_name) + if not os.path.isdir(directory): + os.makedirs(directory) + t.dump(str(output_name)) diff --git a/docker/owlbot/java/src/requirements.txt b/docker/owlbot/java/src/requirements.txt new file mode 100644 index 000000000..8f7634365 --- /dev/null +++ b/docker/owlbot/java/src/requirements.txt @@ -0,0 +1,5 @@ +attrs +click==7.0 +jinja2 +lxml==4.4.1 +typing==3.7.4.1 diff --git a/docker/owlbot/java/templates/clirr/clirr-ignored-differences.xml.j2 b/docker/owlbot/java/templates/clirr/clirr-ignored-differences.xml.j2 new file mode 100644 index 000000000..652898170 --- /dev/null +++ b/docker/owlbot/java/templates/clirr/clirr-ignored-differences.xml.j2 @@ -0,0 +1,19 @@ + + + +{% for proto_path in proto_paths %} + 7012 + {{proto_path}}/*OrBuilder + * get*(*) + + + 7012 + {{proto_path}}/*OrBuilder + boolean contains*(*) + + + 7012 + {{proto_path}}/*OrBuilder + boolean has*(*) + {% endfor %} + diff --git a/docker/owlbot/java/templates/gitignore b/docker/owlbot/java/templates/gitignore new file mode 100644 index 000000000..069d08fc7 --- /dev/null +++ b/docker/owlbot/java/templates/gitignore @@ -0,0 +1,17 @@ +# Maven +target/ + +# Eclipse +.classpath +.project +.settings + +# Intellij +*.iml +.idea/ + +# python utilities +*.pyc +__pycache__ + +.flattened-pom.xml diff --git a/docker/owlbot/java/templates/poms/bom_pom.xml.j2 b/docker/owlbot/java/templates/poms/bom_pom.xml.j2 new file mode 100644 index 000000000..9b7da4388 --- /dev/null +++ b/docker/owlbot/java/templates/poms/bom_pom.xml.j2 @@ -0,0 +1,75 @@ + + + 4.0.0 + {{main_module.group_id}} + {{main_module.artifact_id}}-bom + {{main_module.version}} + pom + + com.google.cloud + google-cloud-shared-config + 0.11.0 + + + Google {{name}} BOM + https://github.com/{{repo}} + + BOM for {{name}} + + + + Google LLC + + + + + chingor13 + Jeff Ching + chingor@google.com + Google LLC + + Developer + + + + + + scm:git:https://github.com/{{repo}}.git + scm:git:git@github.com:{{repo}}.git + https://github.com/{{repo}} + + + + true + + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + {% for module in modules %} + + {{module.group_id}} + {{module.artifact_id}} + {{module.version}} + {% endfor %} + + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + true + + + + + diff --git a/docker/owlbot/java/templates/poms/cloud_pom.xml.j2 b/docker/owlbot/java/templates/poms/cloud_pom.xml.j2 new file mode 100644 index 000000000..286c139ae --- /dev/null +++ b/docker/owlbot/java/templates/poms/cloud_pom.xml.j2 @@ -0,0 +1,111 @@ + + + 4.0.0 + {{module.group_id}} + {{module.artifact_id}} + {{module.version}} + jar + Google {{name}} + https://github.com/{{repo}} + {{name}} {{description}} + + {{parent_module.group_id}} + {{parent_module.artifact_id}} + {{parent_module.version}} + + + {{module.artifact_id}} + + + + io.grpc + grpc-api + + + io.grpc + grpc-stub + + + io.grpc + grpc-protobuf + + + com.google.api + api-common + + + com.google.protobuf + protobuf-java + + + com.google.api.grpc + proto-google-common-protos + +{% for module in proto_modules %} + + com.google.api.grpc + {{module.artifact_id}} + {% endfor %} + + com.google.guava + guava + + + com.google.api + gax + + + com.google.api + gax-grpc + + + org.threeten + threetenbp + + + + + junit + junit + test + 4.13.2 + +{% for module in grpc_modules %} + + {{module.group_id}} + {{module.artifact_id}} + test + {% endfor %} + + + com.google.api + gax-grpc + testlib + test + + + + + + java9 + + [9,) + + + + javax.annotation + javax.annotation-api + + + + + + + + + org.codehaus.mojo + flatten-maven-plugin + + + + \ No newline at end of file diff --git a/docker/owlbot/java/templates/poms/grpc_pom.xml.j2 b/docker/owlbot/java/templates/poms/grpc_pom.xml.j2 new file mode 100644 index 000000000..1b2b1c16f --- /dev/null +++ b/docker/owlbot/java/templates/poms/grpc_pom.xml.j2 @@ -0,0 +1,69 @@ + + 4.0.0 + {{module.group_id}} + {{module.artifact_id}} + {{module.version}} + {{module.artifact_id}} + GRPC library for {{main_module.artifact_id}} + + {{parent_module.group_id}} + {{parent_module.artifact_id}} + {{parent_module.version}} + + + + io.grpc + grpc-api + + + io.grpc + grpc-stub + + + io.grpc + grpc-protobuf + + + com.google.protobuf + protobuf-java + + + com.google.api.grpc + proto-google-common-protos + + + {{proto_module.group_id}} + {{proto_module.artifact_id}} + + + com.google.guava + guava + + + + + + java9 + + [9,) + + + + javax.annotation + javax.annotation-api + + + + + + + + + org.codehaus.mojo + flatten-maven-plugin + + + + \ No newline at end of file diff --git a/docker/owlbot/java/templates/poms/parent_pom.xml.j2 b/docker/owlbot/java/templates/poms/parent_pom.xml.j2 new file mode 100644 index 000000000..2c7e60756 --- /dev/null +++ b/docker/owlbot/java/templates/poms/parent_pom.xml.j2 @@ -0,0 +1,167 @@ + + + 4.0.0 + {{main_module.group_id}} + {{main_module.artifact_id}}-parent + pom + {{main_module.version}} + Google {{name}} Parent + https://github.com/{{repo}} + + Java idiomatic client for Google Cloud Platform services. + + + + com.google.cloud + google-cloud-shared-config + 0.11.0 + + + + + chingor + Jeff Ching + chingor@google.com + Google + + Developer + + + + + Google LLC + + + scm:git:git@github.com:{{repo}}.git + scm:git:git@github.com:{{repo}}.git + https://github.com/{{repo}} + HEAD + + + https://github.com/{{repo}}/issues + GitHub Issues + + + + Apache-2.0 + https://www.apache.org/licenses/LICENSE-2.0.txt + + + + + UTF-8 + UTF-8 + github + {{main_module.artifact_id}}-parent + + + + +{% for module in modules %} + {{module.group_id}} + {{module.artifact_id}} + {{module.version}} + +{% endfor %} + + com.google.cloud + google-cloud-shared-dependencies + 0.20.1 + pom + import + + + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + org.objenesis:objenesis + javax.annotation:javax.annotation-api + + + + + + + + +{% for module in modules %} {{module.artifact_id}} +{% endfor %} {{main_module.artifact_id}}-bom + + + + + + org.apache.maven.plugins + maven-project-info-reports-plugin + 3.1.1 + + + + index + dependency-info + team + ci-management + issue-management + licenses + scm + dependency-management + distribution-management + summary + modules + + + + + true + ${site.installationModule} + jar + + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.2.0 + + + html + + aggregate + javadoc + + + + + none + protected + true + ${project.build.directory}/javadoc + + + Test helpers packages + com.google.cloud.testing + + + SPI packages + com.google.cloud.spi* + + + + + https://grpc.io/grpc-java/javadoc/ + https://developers.google.com/protocol-buffers/docs/reference/java/ + https://googleapis.dev/java/google-auth-library/latest/ + https://googleapis.dev/java/gax/latest/ + https://googleapis.github.io/api-common-java/${google.api-common.version}/apidocs/ + + + + + + \ No newline at end of file diff --git a/docker/owlbot/java/templates/poms/proto_pom.xml.j2 b/docker/owlbot/java/templates/poms/proto_pom.xml.j2 new file mode 100644 index 000000000..9c383533c --- /dev/null +++ b/docker/owlbot/java/templates/poms/proto_pom.xml.j2 @@ -0,0 +1,46 @@ + + 4.0.0 + {{module.group_id}} + {{module.artifact_id}} + {{module.version}} + {{module.artifact_id}} + Proto library for {{main_module.artifact_id}} + + {{parent_module.group_id}} + {{parent_module.artifact_id}} + {{parent_module.version}} + + + + com.google.protobuf + protobuf-java + + + com.google.api.grpc + proto-google-common-protos + + + com.google.api.grpc + proto-google-iam-v1 + + + com.google.api + api-common + + + com.google.guava + guava + + + + + + + org.codehaus.mojo + flatten-maven-plugin + + + + diff --git a/docker/owlbot/java/templates/poms/versions.txt.j2 b/docker/owlbot/java/templates/poms/versions.txt.j2 new file mode 100644 index 000000000..2ebaf85d3 --- /dev/null +++ b/docker/owlbot/java/templates/poms/versions.txt.j2 @@ -0,0 +1,4 @@ +# Format: +# module:released-version:current-version +{% for module in modules %} +{{module.artifact_id}}:{% if module.release_version %}{{module.release_version}}{% else %}{{module.version}}{% endif %}:{{module.version}}{% endfor %}