From 91fa7bee10c3cb68ed38585790ac9e100e27eb65 Mon Sep 17 00:00:00 2001
From: Joe Wang <106995533+JoeWang1127@users.noreply.github.com>
Date: Fri, 19 Apr 2024 20:36:43 -0400
Subject: [PATCH] chore: update hermetic library generation workflow (#10693)

* chore: update generation workflow

* update workflow

* update new client creation workflow

* add base branch

* use shell script

* generalize repo name

* add script to update googleapis commit

* update comments

* update workflow file

* update image tag

* try to fix workflow error

* change comment

* update comment

* refactor according to code review

* restore change

* update branch
---
 .../generate-from-configuration.yaml          | 113 ------------------
 .../generate_new_client_hermetic_build.yaml   |  48 +-------
 .github/workflows/generated_files_sync.yaml   |   4 +-
 .../hermetic_library_generation.yaml          |  43 ++++---
 .../workflows/update_googleapis_commit.yaml   |  41 +++++++
 generation/hermetic_library_generation.sh     | 111 +++++++++++++++++
 generation/update_googleapis_commit.sh        |  88 ++++++++++++++
 7 files changed, 273 insertions(+), 175 deletions(-)
 delete mode 100644 .github/workflows/generate-from-configuration.yaml
 create mode 100644 .github/workflows/update_googleapis_commit.yaml
 create mode 100755 generation/hermetic_library_generation.sh
 create mode 100644 generation/update_googleapis_commit.sh

diff --git a/.github/workflows/generate-from-configuration.yaml b/.github/workflows/generate-from-configuration.yaml
deleted file mode 100644
index c1f84ea2e78b..000000000000
--- a/.github/workflows/generate-from-configuration.yaml
+++ /dev/null
@@ -1,113 +0,0 @@
-name: Generate GAPIC libraries from configuration
-on:
-  schedule:
-  - cron: '0 2 * * *' # nightly at 2 am UTC
-  workflow_dispatch:
-
-
-jobs:
-  generate-from-configuration:
-    runs-on: ubuntu-22.04
-    env:
-      # the branch into which pull request is created.
-      base_branch: main
-      # the branch with which the pull request is associated.
-      branch_name: generate-libraries-main
-      library_generation_image_tag: latest
-      repo_volumes: "-v repo-google-cloud-java:/workspace/google-cloud-java"
-    steps:
-    - uses: actions/checkout@v4
-    - name: get baseline commit
-      id: get-baseline
-      shell: bash
-      run: |
-        echo "baseline_commit=$(grep googleapis_commitish generation_config.yaml | cut -d " " -f2 | xargs)" >> "$GITHUB_ENV"
-    - name: setup branch for pull request
-      id: setup-branch
-      shell: bash
-      run: |
-        [ -z "`git config user.email`" ] && git config --global user.email "cloud-java-bot@google.com"
-        [ -z "`git config user.name`" ] && git config --global user.name "cloud-java-bot"
-        # try to find a open pull request associated with the branch
-        pr_num=$(gh pr list -s open -H "${branch_name}" -q . --json number | jq ".[] | .number")
-        # create a branch if there's no open pull request associated with the
-        # branch; otherwise checkout the pull request.
-        if [ -z "${pr_num}" ]; then
-          git checkout -b "${branch_name}"
-        else
-          gh pr checkout "${pr_num}"
-        fi
-        echo "pr_num=${pr_num}" >> "$GITHUB_ENV"
-      env:
-        GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }}
-    - name: setup docker environment
-      shell: bash
-      run: |
-        # we create a volume pointing to `pwd` (google-cloud-java) that will
-        # be referenced by the container and its children
-        if [[ $(docker volume inspect repo-google-cloud-java) != '[]' ]]; then
-          docker volume rm repo-google-cloud-java
-        fi
-        docker volume create --name "repo-google-cloud-java" --opt "type=none" --opt "device=$(pwd)" --opt "o=bind"
-    - name: update googleapis commit to latest
-      id: update-commit
-      shell: bash
-      run: |
-        mkdir tmp-googleapis
-        # use partial clone because only commit history is needed.
-        git clone --filter=blob:none https://github.com/googleapis/googleapis.git tmp-googleapis
-        pushd tmp-googleapis
-        git pull
-        latest_commit=$(git rev-parse HEAD)
-        popd
-        rm -rf tmp-googleapis
-        sed -i -e "s/^googleapis_commitish.*$/googleapis_commitish: ${latest_commit}/" generation_config.yaml
-    - name: generate from configuration
-      shell: bash
-      run: |
-        docker run --rm \
-          ${repo_volumes} \
-          -v /tmp:/tmp \
-          -v /var/run/docker.sock:/var/run/docker.sock \
-          -e "RUNNING_IN_DOCKER=true" \
-          -e "REPO_BINDING_VOLUMES=${repo_volumes}" \
-          gcr.io/cloud-devrel-public-resources/java-library-generation:"${library_generation_image_tag}" \
-          python /src/generate_repo.py generate --generation-config-yaml=/workspace/google-cloud-java/generation_config.yaml --repository-path=/workspace/google-cloud-java
-    - name: generate pull request description
-      id: generate-description
-      shell: bash
-      run: |
-        # the pr description (body) will be available in pr-description.txt after
-        # running this command.
-        set -x
-        docker run --rm \
-          ${repo_volumes} \
-          -v /tmp:/tmp \
-          -v /var/run/docker.sock:/var/run/docker.sock \
-          -e "RUNNING_IN_DOCKER=true" \
-          -e "REPO_BINDING_VOLUMES=${repo_volumes}" \
-          gcr.io/cloud-devrel-public-resources/java-library-generation:"${library_generation_image_tag}" \
-          python /src/generate_pr_description.py generate --generation-config-yaml=/workspace/google-cloud-java/generation_config.yaml --baseline-commit="${baseline_commit}"
-      env:
-        baseline_commit: ${{ env.baseline_commit }}
-    - name: create or update the pull request
-      shell: bash
-      run: |
-        title="chore: generate libraries at $(date)"
-        git add java-* pom.xml gapic-libraries-bom/pom.xml versions.txt generation_config.yaml
-        # use --allow-empty because (rarely) there's no change.
-        git commit --allow-empty -m "${title}"
-        if [ -z "${pr_num}" ]; then
-          git remote add monorepo https://cloud-java-bot:${GH_TOKEN}@github.com/${{ github.repository }}.git
-          git fetch -q --unshallow monorepo
-          git push -f monorepo "${branch_name}"
-          set -x
-          gh pr create --base "${base_branch}" --title "${title}" --head "${branch_name}" --body "$(cat pr_description.txt)"
-        else
-          git push
-          gh pr edit "${pr_num}" --title "${title}"
-          gh pr edit "${pr_num}" --body "$(cat pr_description.txt)"
-        fi
-      env:
-        GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }}
-        pr_num: ${{ env.pr_num }}
diff --git a/.github/workflows/generate_new_client_hermetic_build.yaml b/.github/workflows/generate_new_client_hermetic_build.yaml
index 493fb7b93964..f66f597890df 100644
--- a/.github/workflows/generate_new_client_hermetic_build.yaml
+++ b/.github/workflows/generate_new_client_hermetic_build.yaml
@@ -1,7 +1,7 @@
 name: Generate new GAPIC client library (Hermetic Build)
 on:
   workflow_dispatch:
-      # some inputs are ommited due to limit of 10 input arguments
+      # some inputs are omitted due to limit of 10 input arguments
       inputs:
         api_shortname:
           required: true
@@ -53,7 +53,7 @@ jobs:
   generate:
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: '3.9'
@@ -78,36 +78,6 @@ jobs:
           API_DESCRIPTION: ${{ github.event.inputs.api_description }}
           LIBRARY_NAME: ${{ github.event.inputs.library_name }}
           DISTRIBUTION_NAME: ${{ github.event.inputs.distribution_name }}
-      - name: setup docker environment
-        shell: bash
-        run: |
-          set -x
-          # we create a volume pointing to `pwd` (google-cloud-java) that will
-          # be referenced by the container and its children
-          if [[ $(docker volume inspect repo-google-cloud-java) != '[]' ]]; then
-            docker volume rm repo-google-cloud-java
-          fi
-          docker volume create --name "repo-google-cloud-java" --opt "type=none" --opt "device=$(pwd)" --opt "o=bind"
-      - name: generate from configuration
-        id: generation
-        shell: bash
-        run: |
-          set -x
-          repo_volumes="-v repo-google-cloud-java:/workspace/google-cloud-java"
-          echo "::set-output name=repo_volumes::${repo_volumes}"
-          docker run --rm \
-            ${repo_volumes} \
-            -v /tmp:/tmp \
-            -v /var/run/docker.sock:/var/run/docker.sock \
-            -e "RUNNING_IN_DOCKER=true" \
-            -e "REPO_BINDING_VOLUMES=${repo_volumes}" \
-            gcr.io/cloud-devrel-public-resources/java-library-generation:latest \
-            python /src/generate_repo.py generate \
-            --generation-config-yaml=/workspace/google-cloud-java/generation_config.yaml \
-            --repository-path=/workspace/google-cloud-java \
-            --target-library-names=${API_SHORTNAME}
-        env:
-          API_SHORTNAME: ${{ github.event.inputs.api_shortname }}
       - name: Push to branch and create PR
         run: |
           set -x
@@ -133,19 +103,6 @@ jobs:
 
           \`\`\`
           python generation/new_client_hermetic_build/add-new-client-config.py add-new-client ${GENERATION_ARGUMENTS}
-
-          docker run --rm \\
-            ${DOCKER_VOLUMES} \\
-            -v /tmp:/tmp \\
-            -v /var/run/docker.sock:/var/run/docker.sock \\
-            -e \"RUNNING_IN_DOCKER=true\" \\
-            -e \"REPO_BINDING_VOLUMES=${DOCKER_VOLUMES}\" \\
-            gcr.io/cloud-devrel-public-resources/java-library-generation:latest \\
-            python /src/generate_repo.py generate \\
-            --generation-config-yaml=/workspace/google-cloud-java/generation_config.yaml \\
-            --repository-path=/workspace/google-cloud-java \\
-            --target-library-names=${API_SHORTNAME}
-
           \`\`\`"
           gh pr create --title "${commit_message}" --label "owlbot:run" --head "${branch_name}" --body "${pr_body}"
         env:
@@ -154,4 +111,3 @@ jobs:
           GENERATION_ARGUMENTS: ${{ steps.config_generation.outputs.new_library_args }}
           DOCKER_VOLUMES: ${{ steps.generation.outputs.repo_volumes }}
           GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }}
-
diff --git a/.github/workflows/generated_files_sync.yaml b/.github/workflows/generated_files_sync.yaml
index 1c0e25ce125c..f3a48566d93f 100644
--- a/.github/workflows/generated_files_sync.yaml
+++ b/.github/workflows/generated_files_sync.yaml
@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# Github action job to test core java library features on
+# GitHub action job to test core java library features on
 # downstream client libraries before they are released.
 on:
   push:
@@ -20,7 +20,7 @@ on:
   pull_request:
 name: generation diff
 env:
-  library_generation_image_tag: latest
+  library_generation_image_tag: 2.39.0
   repo_volumes: "-v repo-google-cloud-java:/workspace/google-cloud-java"
 jobs:
   root-pom:
diff --git a/.github/workflows/hermetic_library_generation.yaml b/.github/workflows/hermetic_library_generation.yaml
index 7d71749ec671..24c666e9e05e 100644
--- a/.github/workflows/hermetic_library_generation.yaml
+++ b/.github/workflows/hermetic_library_generation.yaml
@@ -1,27 +1,42 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# GitHub action job to test core java library features on
+# downstream client libraries before they are released.
 name: Hermetic library generation upon generation config change through pull requests
 on:
   pull_request:
-    types:
-      - synchronize
     paths:
-      - generation_config.yaml
+    - "generation_config.yaml"
+
 jobs:
   library_generation:
     runs-on: ubuntu-latest
     env:
-      library_generation_image_tag: latest
-      repo_volumes: "-v repo-google-cloud-java:/workspace/google-cloud-java"
+      library_generation_image_tag: 2.39.0
     steps:
     - uses: actions/checkout@v4
       with:
-        ref: ${{ github.head_ref }}
-    - name: get baseline generation config
-      uses: actions/checkout@v4
-      with:
-        ref: ${{ github.base_ref }}
-        path: baseline
-        sparse-checkout: generation_config.yaml
-    - name: diff
+        fetch-depth: 0
+    - name: Generate changed libraries
       shell: bash
       run: |
-        diff generation_config.yaml baseline/generation_config.yaml
+        set -x
+        [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com"
+        [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot"
+        bash generation/hermetic_library_generation.sh \
+          --target_branch ${{ github.base_ref }} \
+          --current_branch ${{ github.head_ref }} \
+          --image_tag "${library_generation_image_tag}"
+      env:
+        GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }}
diff --git a/.github/workflows/update_googleapis_commit.yaml b/.github/workflows/update_googleapis_commit.yaml
new file mode 100644
index 000000000000..3efb1ad5ce54
--- /dev/null
+++ b/.github/workflows/update_googleapis_commit.yaml
@@ -0,0 +1,41 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# GitHub action job to test core java library features on
+# downstream client libraries before they are released.
+name: Update googleapis commit
+on:
+  schedule:
+  - cron: '* 2 * * *'
+  workflow_dispatch:
+
+
+jobs:
+  update-googleapis-commit:
+    runs-on: ubuntu-22.04
+    env:
+      # the branch into which the pull request is merged
+      base_branch: main
+    steps:
+    - uses: actions/checkout@v4
+    - name: Update googleapis commit to latest
+      shell: bash
+      run: |
+        set -x
+        [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com"
+        [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot"
+        bash generation/update_googleapis_commit.sh \
+          --base_branch "${base_branch}"\
+          --repo ${{ github.repository }}
+      env:
+        GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }}
diff --git a/generation/hermetic_library_generation.sh b/generation/hermetic_library_generation.sh
new file mode 100755
index 000000000000..5fe34e47f26d
--- /dev/null
+++ b/generation/hermetic_library_generation.sh
@@ -0,0 +1,111 @@
+#!/bin/bash
+# This script should be run at the root of the repository.
+# This script is used to, when a pull request changes the generation
+# configuration (generation_config.yaml by default):
+# 1. compare generation configurations in the current branch (with which the
+# pull request associated) and target branch (into which the pull request is
+# merged);
+# 2. generate changed libraries using library_generation image;
+# 3. commit the change to the pull request
+# 4. edit the pr body with generated pull request description, if applicable.
+
+# The following commands need to be installed before running the script:
+# 1. git
+# 2. gh
+# 3. docker
+
+# The parameters of this script is:
+# 1. target_branch, the branch into which the pull request is merged.
+# 2. current_branch, the branch with which the pull request is associated.
+# 3. image_tag, the tag of gcr.io/cloud-devrel-public-resources/java-library-generation.
+# 3. [optional] generation_config, the path to the generation configuration,
+# the default value is generation_config.yaml in the repository root.
+while [[ $# -gt 0 ]]; do
+key="$1"
+case "${key}" in
+  --target_branch)
+    target_branch="$2"
+    shift
+    ;;
+  --current_branch)
+    current_branch="$2"
+    shift
+    ;;
+  --image_tag)
+    image_tag="$2"
+    shift
+    ;;
+  --generation_config)
+    generation_config="$2"
+    shift
+    ;;
+  *)
+    echo "Invalid option: [$1]"
+    exit 1
+    ;;
+esac
+shift
+done
+
+if [ -z "${target_branch}" ]; then
+  echo "missing required argument --target_branch"
+  exit 1
+fi
+
+if [ -z "${current_branch}" ]; then
+  echo "missing required argument --current_branch"
+  exit 1
+fi
+
+if [ -z "${image_tag}" ]; then
+  echo "missing required argument --image_tag"
+  exit 1
+fi
+
+if [ -z "${generation_config}" ]; then
+  generation_config=generation_config.yaml
+  echo "Use default generation config: ${generation_config}"
+fi
+
+volume_name="repo"
+workspace_name="/workspace/repo"
+repo_volumes="${volume_name}:${workspace_name}"
+baseline_generation_config="baseline_generation_config.yaml"
+message="chore: generate libraries at $(date)"
+
+git checkout "${target_branch}"
+git checkout "${current_branch}"
+# copy generation configuration from target branch to current branch.
+git show "${target_branch}":"${generation_config}" > "${baseline_generation_config}"
+diff "${generation_config}" "${baseline_generation_config}" || echo "config diff"
+# bind docker volume to include the repository in docker running environment.
+if [[ $(docker volume inspect ${volume_name}) != '[]' ]]; then
+  docker volume rm ${volume_name}
+fi
+docker volume create \
+  --name ${volume_name} \
+  --opt "type=none" \
+  --opt "device=$(pwd)" \
+  --opt "o=bind"
+# run hermetic code generation docker image.
+docker run \
+  --rm \
+  -v "${repo_volumes}" \
+  -v /tmp:/tmp \
+  -v /var/run/docker.sock:/var/run/docker.sock \
+  -e "RUNNING_IN_DOCKER=true" \
+  -e "REPO_BINDING_VOLUMES=-v ${repo_volumes}" \
+  gcr.io/cloud-devrel-public-resources/java-library-generation:"${image_tag}" \
+  python /src/cli/entry_point.py generate \
+  --baseline-generation-config-path="${workspace_name}/${baseline_generation_config}" \
+  --current-generation-config-path="${workspace_name}/${generation_config}" \
+  --repository-path="${workspace_name}"
+# commit the change to the pull request.
+git add java-* pom.xml gapic-libraries-bom/pom.xml versions.txt "${generation_config}"
+git commit --allow-empty -m "${message}"
+git push
+# set pr body if pr_description.txt is generated.
+if [[ -f "pr_description.txt" ]]; then
+  pr_num=$(gh pr list -s open -H "${current_branch}" -q . --json number | jq ".[] | .number")
+  gh pr edit "${pr_num}" --body "$(cat pr_description.txt)"
+fi
diff --git a/generation/update_googleapis_commit.sh b/generation/update_googleapis_commit.sh
new file mode 100644
index 000000000000..365eb0e4db86
--- /dev/null
+++ b/generation/update_googleapis_commit.sh
@@ -0,0 +1,88 @@
+#!/bin/bash
+set -e
+# This script should be run at the root of the repository.
+# This script is used to update googleapis commit to latest in generation
+# configuration at the time of running and create a pull request.
+
+# The following commands need to be installed before running the script:
+# 1. git
+# 2. gh
+
+# The parameters of this script is:
+# 1. base_branch, the base branch of the result pull request.
+# 2. repo, organization/repo-name, e.g., googleapis/google-cloud-java
+# 3. [optional] generation_config, the path to the generation configuration,
+# the default value is generation_config.yaml in the repository root.
+while [[ $# -gt 0 ]]; do
+key="$1"
+case "${key}" in
+  --base_branch)
+    base_branch="$2"
+    shift
+    ;;
+  --repo)
+    repo="$2"
+    shift
+    ;;
+  --generation_config)
+    generation_config="$2"
+    shift
+    ;;
+  *)
+    echo "Invalid option: [$1]"
+    exit 1
+    ;;
+esac
+shift
+done
+
+if [ -z "${base_branch}" ]; then
+  echo "missing required argument --base_branch"
+  exit 1
+fi
+
+if [ -z "${repo}" ]; then
+  echo "missing required argument --repo"
+  exit 1
+fi
+
+if [ -z "${generation_config}" ]; then
+  generation_config="generation_config.yaml"
+  echo "Use default generation config: ${generation_config}"
+fi
+
+current_branch="generate-libraries-${base_branch}"
+title="chore: update googleapis commit at $(date)"
+
+# try to find a open pull request associated with the branch
+pr_num=$(gh pr list -s open -H "${current_branch}" -q . --json number | jq ".[] | .number")
+# create a branch if there's no open pull request associated with the
+# branch; otherwise checkout the pull request.
+if [ -z "${pr_num}" ]; then
+  git checkout -b "${current_branch}"
+else
+  gh pr checkout "${pr_num}"
+fi
+
+mkdir tmp-googleapis
+# use partial clone because only commit history is needed.
+git clone --filter=blob:none https://github.com/googleapis/googleapis.git tmp-googleapis
+pushd tmp-googleapis
+git pull
+latest_commit=$(git rev-parse HEAD)
+popd
+rm -rf tmp-googleapis
+sed -i -e "s/^googleapis_commitish.*$/googleapis_commitish: ${latest_commit}/" "${generation_config}"
+
+
+git add "${generation_config}"
+# use --allow-empty because (rarely) there's no change.
+git commit --allow-empty -m "${title}"
+if [ -z "${pr_num}" ]; then
+  git remote add remote_repo https://cloud-java-bot:"${GH_TOKEN}@github.com/${repo}.git"
+  git fetch -q --unshallow remote_repo
+  git push -f remote_repo "${current_branch}"
+  gh pr create --title "${title}" --head "${current_branch}" --body "${title}" --base "${base_branch}"
+else
+  git push
+fi