feat: add Java owl-bot post-processing image (#991)

docker/owlbot/java/Dockerfile

@@ -0,0 +1,46 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# build from the root of this repo:
+FROM gcr.io/cloud-devrel-public-resources/java8
+
+ARG JAVA_FORMAT_VERSION=1.7
+
+RUN apt-get install -y --no-install-recommends jq
+
+COPY docker/owlbot/java/bin /owlbot/bin
+COPY docker/owlbot/java/src /owlbot/src
+COPY docker/owlbot/java/templates /owlbot/templates
+RUN cd /owlbot/src && \
+    python3 -m pip install -r requirements.txt
+
+ADD https://repo1.maven.org/maven2/com/google/googlejavaformat/google-java-format/${JAVA_FORMAT_VERSION}/google-java-format-${JAVA_FORMAT_VERSION}-all-deps.jar /owlbot/google-java-format.jar
+
+###################### Install synthtool's requirements.
+COPY . /synthtool/
+
+WORKDIR /synthtool
+RUN python3 -m pip install -e .
+
+# Allow non-root users to run python
+RUN chmod +rx /root/ /root/.pyenv && chmod +r /owlbot/google-java-format.jar
+
+# Tell synthtool to pull templates from this docker image instead of from
+# the live repo.
+ENV SYNTHTOOL_TEMPLATES="/synthtool/synthtool/gcp/templates" \
+    PYTHON_PATH="/owlbot/src"
+
+WORKDIR /workspace
+
+CMD [ "/owlbot/bin/entrypoint.sh" ]

docker/owlbot/java/README.md

@@ -0,0 +1,26 @@
+# Java Post-Processing Docker Image
+
+Docker image used for bootstrapping/post-processing. Running this on
+should:
+
+1. Generate common templates
+2. Write any missing `pom.xml` files or update with new detected modules
+3. Restore or create `clirr-ignored-differences.xml` files after a new release
+4. Restore license header years on generated files.
+5. Run our standard `google-java-format` plugin.
+
+## Usage
+
+### Running locally
+
+```bash
+docker run --rm -v $(pwd):/workspace --user "$(id -u):$(id -g)" gcr.io/repo-automation-bots/owlbot-java
+```
+
+### Building the image
+
+This image is built via Cloud Build. From the root of this repository, run:
+
+```bash
+gcloud builds submit --config=docker/owlbot/java/cloudbuild.yaml
+```

docker/owlbot/java/bin/entrypoint.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+# templates
+echo "Generating templates..."
+/owlbot/bin/write_templates.sh
+echo "...done"
+
+# write or restore pom.xml files
+echo "Generating missing pom.xml..."
+/owlbot/bin/write_missing_pom_files.sh
+echo "...done"
+
+# write or restore clirr-ignored-differences.xml
+echo "Generating clirr-ignored-differences.xml..."
+/owlbot/bin/write_clirr_ignore.sh
+echo "...done"
+
+# TODO: re-enable this once we resolve thrashing
+# restore license headers years
+# echo "Restoring copyright years..."
+# /owlbot/bin/restore_license_headers.sh
+# echo "...done"
+
+# ensure formatting on all .java files in the repository
+echo "Reformatting source..."
+/owlbot/bin/format_source.sh
+echo "...done"

docker/owlbot/java/bin/format_source.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+# Find all the java files relative to the current directory and format them
+# using google-java-format
+find . -name '*.java' | xargs java -jar /owlbot/google-java-format.jar --replace

docker/owlbot/java/bin/restore_license_headers.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+# list the modified files in the current commit
+last_commit_files=$(git diff-tree --no-commit-id -r $(git rev-parse HEAD) --name-only --diff-filter=M)
+
+# list the modified, uncommited files
+current_modified_files=$(git diff --name-only HEAD)
+
+# join and deduplicate the list
+all_files=$(echo ${last_commit_files} ${current_modified_files} | sort -u)
+
+for file in ${all_files}
+do
+  # look for the Copyright YYYY line within the first 10 lines
+  old_copyright=$(git show HEAD~1:${file} | head -n 10 | egrep -o -e "Copyright ([[:digit:]]{4})" || echo "")
+  new_copyright=$(cat ${file} | head -n 10 | egrep -o -e "Copyright ([[:digit:]]{4})" || echo "")
+  # if the header year changed in the last diff, then restore the previous year
+  if [ ! -z "${old_copyright}" ] && [ ! -z "${new_copyright}" ] && [ "${old_copyright}" != "${new_copyright}" ]
+  then
+    echo "Restoring copyright in ${file} to '${old_copyright}'"
+    # replace the first instance of the old copyright header with the new
+    sed -i "s/${new_copyright}/${old_copyright}/1" ${file}
+  fi
+done

docker/owlbot/java/bin/write_clirr_ignore.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+templates_dir=$(realpath $(dirname "${BASH_SOURCE[0]}")/../templates/clirr)
+is_release=$((git log -1 --pretty=%B | grep -e "chore.*release.*-SNAPSHOT") || echo "")
+
+# on a snapshot bump, clear all clirr-ignore-differences files
+if [ ! -z "${is_release}" ]
+then
+  find . -name 'clirr-ignored-differences.xml' | xargs rm
+fi
+
+# restore default clirr-ignored-differences.xml for protos if the file does not exist
+for dir in `ls -d proto-google-*`
+do
+  if [ ! -f "${dir}/clirr-ignored-differences.xml" ]
+  then
+    tmp_dir=$(mktemp -d -t ci-XXXXXXXXXX)
+    pushd ${dir}
+    pushd src/main/java
+    find * -name *OrBuilder.java | xargs dirname | sort -u | jq -Rns ' (inputs | rtrimstr("\n") | split("\n") ) as $data | {proto_paths: $data}' > ${tmp_dir}/paths.json
+    popd
+    python3 /owlbot/src/gen-template.py --data=${tmp_dir}/paths.json --folder=${templates_dir}
+    popd
+  fi
+done

docker/owlbot/java/bin/write_missing_pom_files.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+python3 /owlbot/src/fix-poms.py

docker/owlbot/java/bin/write_templates.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+# create initial .gitignore if it does not yet exist
+if [ ! -f ".gitignore" ]
+then
+  cp /owlbot/templates/gitignore ./.gitignore
+fi
+
+if [ -f "synth.py" ]
+then
+  python3 /owlbot/src/convert-synthtool-templates.py --synth-file=synth.py
+fi

docker/owlbot/java/cloudbuild.yaml

@@ -0,0 +1,26 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+steps:
+  - name: 'gcr.io/cloud-builders/docker'
+    args: [ 'build',
+      '-t', 'gcr.io/$PROJECT_ID/owlbot-java:$SHORT_SHA',
+      '-t', 'gcr.io/$PROJECT_ID/owlbot-java:latest',
+      '-f', 'docker/owlbot/java/Dockerfile', '.' ]
+  - name: gcr.io/gcp-runtimes/container-structure-test
+    args:
+      ["test", "--image", "gcr.io/$PROJECT_ID/owlbot-java:$SHORT_SHA", "--config", "docker/owlbot/java/container_test.yaml"]
+
+images:
+  - gcr.io/$PROJECT_ID/owlbot-java:$SHORT_SHA
+  - gcr.io/$PROJECT_ID/owlbot-java:latest

docker/owlbot/java/cloudbuild_test.yaml

@@ -0,0 +1,22 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+steps:
+  - name: 'gcr.io/cloud-builders/docker'
+    args: [ 'build',
+      '-t', 'gcr.io/$PROJECT_ID/owlbot-java:$SHORT_SHA',
+      '-t', 'gcr.io/$PROJECT_ID/owlbot-java:latest',
+      '-f', 'docker/owlbot/java/Dockerfile', '.' ]
+  - name: gcr.io/gcp-runtimes/container-structure-test
+    args:
+      ["test", "--image", "gcr.io/$PROJECT_ID/owlbot-java:$SHORT_SHA", "--config", "docker/owlbot/java/container_test.yaml"]

docker/owlbot/java/container_test.yaml

@@ -0,0 +1,26 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+schemaVersion: 1.0.0
+commandTests:
+- name: "version"
+  command: ["java", "-version"]
+  # java -version outputs to stderr...
+  expectedError: ["(java|openjdk) version \"1.8.*\""]
+- name: "formatter"
+  command: ["java", "-jar", "/owlbot/google-java-format.jar", "--version"]
+  expectedError: ["google-java-format: Version 1.7"]
+- name: "python"
+  command: ["python", "--version"]
+  expectedOutput: ["Python 3.6.1"]

docker/owlbot/java/src/convert-synthtool-templates.py

@@ -0,0 +1,47 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import ast
+import click
+from synthtool.languages import java
+
+
+@click.command()
+@click.option(
+    "--synth-file", help="Path to synth.py file", default="synth.py",
+)
+def main(synth_file: str):
+    excludes = []
+    should_include_templates = False
+    with open(synth_file, "r") as fp:
+        tree = ast.parse(fp.read())
+
+        # look for a call to java.common_templates() and extract the list of excludes
+        for node in ast.walk(tree):
+            if isinstance(node, ast.Call):
+                if (
+                    node.func.value.id == "java"
+                    and node.func.attr == "common_templates"
+                ):
+                    should_include_templates = True
+                    for keyword in node.keywords:
+                        if keyword.arg == "excludes":
+                            excludes = [element.s for element in keyword.value.elts]
+
+    if should_include_templates:
+        java.common_templates(excludes=excludes)
+
+
+if __name__ == "__main__":
+    main()