-
Notifications
You must be signed in to change notification settings - Fork 55
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: move Java Owlbot into this repository for postprocessing (#2282)
* use local synthtool and owlbot * remove unused files * remove more unused files * remove cache files in owlbot * use java 11 for it * remove kokoro files * use glob in owlbot entrypoint * remove unused files * do not do post-process IT on mac * concise entrypoint logic * cleanup i * cleanup ii * cleanup iii * cleanup iv * remove templates * remove protos folder * remove synthtool * connect image to owlbot entrypoint * simplify synthtool docker run command * install synthtool locally * install synthtool only once * use virtualenvs to run python scripts * install pyenv in action * remove jar from history * download google-java-format * fix pyenv init * attempt to fix pyenv installation in gh action * fix manual pyenv installation * install pyenv in profile * install pyenv in bashrc as well * use bash shell explicitly in gh action * install pyenv in same step as IT * do not restart shell * set pyenv path manually * install pyenv in its own step * propagate environment variables to other steps * fix global env var setup * remove wrong env settings * explain usage of pyenv in README * simplify pyenv setup * add comment to owlbot entrypoint * rename destination_path to preprocessed_libraries_path * infer scripts_root in postprocess_library.sh * use temporary folder for preprocess step * use owlbot files from workspace * get rid of output_folder argument * use common temp dir to clone synthtool into * lock synthtool to a specific commitish * fix file transfer * fix owl-bot-staging unpacking * remove unnecessary workspace variable * rename workspace to postprocessing_target * remove owlbot sha logic * remove repository_root variable * cleanup * correct pyenv comment * clean temp sources folder on each run * safety checks for get_proto_path_from_preprocessed_sources * fix integration test * disable compute and asset/v1p2beta1 temporarily they have changes in googleapis that have not been reflected yet in google-cloud-java * fix unit tests * correct comment * do not install docker for macos * fix owlbot files check * fix license headers * remove unnecessary owlbot_sha * add explanation on why are there no macos + postprocess ITs * use `fmt:format` instead of google formatter * clean templates * remove more unnecessary elements * add README entry explaining owlbot maintenance * remove unnecessary java format version
- Loading branch information
1 parent
c9cf66b
commit f8969d2
Showing
30 changed files
with
1,828 additions
and
203 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -12,7 +12,7 @@ jobs: | |
| integration_tests: | ||
| strategy: | ||
| matrix: | ||
| java: [ 8 ] | ||
| java: [ 11 ] | ||
| os: [ ubuntu-22.04, macos-12 ] | ||
| post_processing: [ 'true', 'false' ] | ||
| runs-on: ${{ matrix.os }} | ||
|
|
@@ -26,8 +26,23 @@ jobs: | |
| - uses: actions/setup-python@v4 | ||
| with: | ||
| python-version: '3.11' | ||
| - name: install pyenv | ||
| shell: bash | ||
| run: | | ||
| set -ex | ||
| curl https://pyenv.run | bash | ||
| # setup environment | ||
| export PYENV_ROOT="$HOME/.pyenv" | ||
| export PATH="$PYENV_ROOT/bin:$PATH" | ||
| echo "PYENV_ROOT=${PYENV_ROOT}" >> $GITHUB_ENV | ||
| echo "PATH=${PATH}" >> $GITHUB_ENV | ||
| # init pyenv | ||
| eval "$(pyenv init --path)" | ||
| eval "$(pyenv init -)" | ||
| set +ex | ||
| - name: install docker (ubuntu) | ||
| if: matrix.os == 'ubuntu-22.04' | ||
| shell: bash | ||
| run: | | ||
| set -x | ||
| # install docker | ||
|
|
@@ -36,17 +51,11 @@ jobs: | |
| # launch docker | ||
| sudo systemctl start docker | ||
| - name: install docker (macos) | ||
| if: matrix.os == 'macos-12' | ||
| run: | | ||
| brew update --preinstall | ||
| brew install docker docker-compose qemu | ||
| brew upgrade qemu | ||
| colima start | ||
| docker run --user $(id -u):$(id -g) --rm hello-world | ||
| - name: Run integration tests | ||
| # we don't run ITs with postprocessing on macos because one of its dependencies "synthtool" is designed to run on linux only | ||
| if: matrix.os == 'ubuntu-22.04' || matrix.post_processing == 'false' | ||
| shell: bash | ||
| run: | | ||
| set -x | ||
| git config --global user.email "[email protected]" | ||
| git config --global user.name "Github Workflow" | ||
| library_generation/test/generate_library_integration_test.sh \ | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| 3.11.2 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| 59fe44fde9866a26e7ee4e4450fd79f67f8cf599 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,113 @@ | ||
| #!/bin/bash | ||
| # Copyright 2023 Google LLC | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # https://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| # This is the entrypoint script for java owlbot. This is not intended to be | ||
| # called directly but rather be called from postproces_library.sh | ||
| # For reference, the positional arguments are | ||
| # 1: scripts_root: location of postprocess_library.sh | ||
| # 2: versions_file: points to a versions.txt containing versions to be applied | ||
| # both to README and pom.xml files | ||
|
|
||
| # The scripts assumes the CWD is the folder where postprocessing is going to be | ||
| # applied | ||
|
|
||
| set -ex | ||
| scripts_root=$1 | ||
| versions_file=$2 | ||
|
|
||
| # Runs template and etc in current working directory | ||
| function processModule() { | ||
| # templates as well as retrieving files from owl-bot-staging | ||
| echo "Generating templates and retrieving files from owl-bot-staging directory..." | ||
| if [ -f "owlbot.py" ] | ||
| then | ||
| # defaults to run owlbot.py | ||
| python3 owlbot.py | ||
| fi | ||
| echo "...done" | ||
|
|
||
| # write or restore pom.xml files | ||
| echo "Generating missing pom.xml..." | ||
| python3 "${scripts_root}/owlbot/src/fix-poms.py" "${versions_file}" "true" | ||
| echo "...done" | ||
|
|
||
| # write or restore clirr-ignored-differences.xml | ||
| echo "Generating clirr-ignored-differences.xml..." | ||
| ${scripts_root}/owlbot/bin/write_clirr_ignore.sh "${scripts_root}" | ||
| echo "...done" | ||
|
|
||
| # fix license headers | ||
| echo "Fixing missing license headers..." | ||
| python3 "${scripts_root}/owlbot/src/fix-license-headers.py" | ||
| echo "...done" | ||
|
|
||
| # TODO: re-enable this once we resolve thrashing | ||
| # restore license headers years | ||
| # echo "Restoring copyright years..." | ||
| # /owlbot/bin/restore_license_headers.sh | ||
| # echo "...done" | ||
|
|
||
| # ensure formatting on all .java files in the repository | ||
| echo "Reformatting source..." | ||
| mvn fmt:format | ||
| echo "...done" | ||
| } | ||
|
|
||
| if [ "$(ls */.OwlBot.yaml|wc -l)" -gt 1 ];then | ||
| # Monorepo (googleapis/google-cloud-java) has multiple OwlBot.yaml config | ||
| # files in the modules. | ||
| echo "Processing monorepo" | ||
| if [ -d owl-bot-staging ]; then | ||
| # The content of owl-bot-staging is controlled by Owlbot.yaml files in | ||
| # each module in the monorepo | ||
| echo "Extracting contents from owl-bot-staging" | ||
| for module in owl-bot-staging/* ; do | ||
| if [ ! -d "$module" ]; then | ||
| continue | ||
| fi | ||
| # This relocation allows us continue to use owlbot.py without modification | ||
| # after monorepo migration. | ||
| mv "owl-bot-staging/$module" "$module/owl-bot-staging" | ||
| pushd "$module" | ||
| processModule | ||
| popd | ||
| done | ||
| rm -r owl-bot-staging | ||
| else | ||
| echo "In monorepo but no owl-bot-staging." \ | ||
| "Formatting changes in the last commit" | ||
| # Find the files that were touched by the last commit. | ||
| last_commit=$(git log -1 --format=%H) | ||
| # [A]dded, [C]reated, [M]odified, and [R]enamed | ||
| changed_files=$(git show --name-only --no-renames --diff-filter=ACMR \ | ||
| "${last_commit}") | ||
| changed_modules=$(echo "$changed_files" |grep -E '.java$' |cut -d '/' -f 1 \ | ||
| |sort -u) | ||
| for module in ${changed_modules}; do | ||
| if [ ! -f "$module/.OwlBot.yaml" ]; then | ||
| # Changes irrelevant to Owlbot-generated module (such as .github) do not | ||
| # need formatting | ||
| continue | ||
| fi | ||
| pushd "$module" | ||
| processModule | ||
| popd | ||
| done | ||
| fi | ||
| else | ||
| # Split repository | ||
| echo "Processing a split repo" | ||
| processModule | ||
| fi |
Oops, something went wrong.