chore: add a GitHub Action for generating new clients using the hermetic build scripts

.github/workflows/generate_new_client_hermetic_build.yaml

@@ -0,0 +1,157 @@
+name: Generate new GAPIC client library (Hermetic Build)
+on:
+  workflow_dispatch:
+      # some inputs are ommited due to limit of 10 input arguments
+      inputs:
+        api_shortname:
+          required: true
+          type: string
+          description: "`api_shortname`: Name for the new directory name and (default) artifact name"
+        name_pretty:
+          required: true
+          type: string
+          description: "`name_pretty`: The human-friendly name that appears in README.md"
+        api_description:
+          required: true
+          description: "`api_description`: Description that appears in README.md"
+        proto_path:
+          required: true
+          type: string
+          description: |
+            `proto_path`: Path to proto file from the root of the googleapis repository to the
+            directory that contains the proto files (without the version).
+            For example, to generate the library for 'google/maps/routing/v2',
+            then you specify this value as 'google/maps/routing'
+        product_docs:
+          required: true
+          type: string
+          description: "`product_docs`: Documentation URL that appears in README.md"
+        rest_docs:
+          required: false
+          type: string
+          description: |
+            `rest_docs`: If it exists, link to the REST Documentation for a service
+        rpc_docs:
+          required: false
+          type: string
+          description: |
+            `rpc_docs`: If it exists, link to the RPC Documentation for a service
+        library_name:
+          required: false
+          type: string
+          description: |
+            `library_name`: The directory name of the new library. By default it's
+            java-<api_shortname>
+        distribution_name:
+          required: false
+          type: string
+          description: |
+            `distribution_name`: Maven coordinates of the generated library. By default it's
+            com.google.cloud:google-cloud-<api_shortname>
+
+jobs:
+  generate:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+          cache: 'pip' # caching pip dependencies
+      - name: Install new-client.py dependencies
+        run: pip install --require-hashes -r generation/new_client_hermetic_build/requirements.txt
+      - name: Add entry to generation_config.yaml
+        id: config_generation
+        run: |
+          set -x
+          arguments=$(python generation/new_client_hermetic_build/generate-arguments.py)
+          echo "::set-output name=new_library_args::${arguments}"
+          echo "${arguments}" \
+                | xargs python generation/new_client_hermetic_build/new-client.py add-new-library
+        env:
+          API_SHORTNAME: ${{ github.event.inputs.api_shortname }}
+          NAME_PRETTY: ${{ github.event.inputs.name_pretty }}
+          PROTO_PATH: ${{ github.event.inputs.proto_path }}
+          PRODUCT_DOCS: ${{ github.event.inputs.product_docs }}
+          REST_DOCS: ${{ github.event.inputs.rest_docs }}
+          RPC_DOCS: ${{ github.event.inputs.rpc_docs }}
+          API_DESCRIPTION: ${{ github.event.inputs.api_description }}
+          LIBRARY_NAME: ${{ github.event.inputs.library_name }}
+          DISTRIBUTION_NAME: ${{ github.event.inputs.distribution_name }}
+      - name: setup docker environment
+        shell: bash
+        run: |
+          set -x
+          # we create a volume pointing to `pwd` (google-cloud-java) that will
+          # be referenced by the container and its children
+          if [[ $(docker volume inspect repo-google-cloud-java) != '[]' ]]; then
+            docker volume rm repo-google-cloud-java
+          fi
+          docker volume create --name "repo-google-cloud-java" --opt "type=none" --opt "device=$(pwd)" --opt "o=bind"
+      - name: generate from configuration
+        id: generation
+        shell: bash
+        run: |
+          set -x
+          repo_volumes="-v repo-google-cloud-java:/workspace/google-cloud-java"
+          echo "::set-output name=repo_volumes::${repo_volumes}"
+          docker run --rm \
+            ${repo_volumes} \
+            -v /tmp:/tmp \
+            -v /var/run/docker.sock:/var/run/docker.sock \
+            -e "RUNNING_IN_DOCKER=true" \
+            -e "REPO_BINDING_VOLUMES=${repo_volumes}" \
+            gcr.io/cloud-devrel-public-resources/java-library-generation:latest \
+            python /src/generate_repo.py generate \
+            --generation-config-yaml=/workspace/google-cloud-java/generation_config.yaml \
+            --repository-path=/workspace/google-cloud-java \
+            --target-library-api-shortname=${API_SHORTNAME}
+        env:
+          API_SHORTNAME: ${{ github.event.inputs.api_shortname }}
+      - name: Push to branch and create PR
+        run: |
+          set -x
+          [ -z "`git config user.email`" ] && git config --global user.email "[email protected]"
+          [ -z "`git config user.name`" ] && git config --global user.name "cloud-java-bot"
+
+          # create and push to branch in origin
+          # random_id allows multiple runs of this workflow
+          random_id=$(tr -dc A-Za-z0-9 </dev/urandom | head -c 5; echo)
+          branch_name="new-library/${{ github.event.inputs.api_shortname }}-${random_id}"
+          git checkout -b "${branch_name}"
+          git add --all
+          commit_message="feat: [${API_SHORTNAME}] new module for ${API_SHORTNAME}"
+          git commit -m "${commit_message}"
+          git remote add monorepo https://cloud-java-bot:${GH_TOKEN}@github.com/${{ github.repository }}.git
+          git fetch -q --unshallow monorepo
+          git push -f monorepo "${branch_name}"
+
+          # create PR
+          pr_body="Generated by @${USERNAME} via [generate_new_client_hermetic_build.yaml](https://github.com/googleapis/google-cloud-java/actions/workflows/generate_new_client_hermetic_build.yaml)
+
+          Command used:
+
+          \`\`\`
+          python generation/new_client_hermetic_build/new-client.py add-new-client ${GENERATION_ARGUMENTS}
+
+          docker run --rm \\
+            ${DOCKER_VOLUMES} \\
+            -v /tmp:/tmp \\
+            -v /var/run/docker.sock:/var/run/docker.sock \\
+            -e \"RUNNING_IN_DOCKER=true\" \\
+            -e \"REPO_BINDING_VOLUMES=${DOCKER_VOLUMES}\" \\
+            gcr.io/cloud-devrel-public-resources/java-library-generation:latest \\
+            python /src/generate_repo.py generate \\
+            --generation-config-yaml=/workspace/google-cloud-java/generation_config.yaml \\
+            --repository-path=/workspace/google-cloud-java \\
+            --target-library-api-shortname=${API_SHORTNAME}
+
+          \`\`\`"
+          gh pr create --title "${commit_message}" --label "owlbot:run" --head "${branch_name}" --body "${pr_body}"
+        env:
+          USERNAME: ${{ github.actor }}
+          API_SHORTNAME: ${{ github.event.inputs.api_shortname }}
+          GENERATION_ARGUMENTS: ${{ steps.config_generation.outputs.new_library_args }}
+          DOCKER_VOLUMES: ${{ steps.generation.outputs.repo_volumes }}
+          GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }}
+

generation/new_client_hermetic_build/README.md

@@ -0,0 +1,158 @@
+# New client generation (GitHub Action)
+This new generation workflow enables generation of new libraries by
+ 1. appending a new library to our [generation_config.yaml](https://github.com/googleapis/google-cloud-java/blob/c7429c0eec419c01d4e2fe14d063b9335efb810b/generation_config.yaml).
+ 2. running the hermetic build scripts docker image and
+ generate the newly added library.
+ 3. create a PR with the changes.
+
+
+## Components
+### `new_library_hermetic_build/new-client.py`
+This script takes 10 arguments that map to items in the newly added library that
+goes in `generation_config.yaml`. A new entry will be added to `libraries` with
+the necessary generation configuration.
+
+### `.github/workflows/generate_new_client_hermetic_build.yaml`
+This workflow orchestrates the `new-client.py` script and also uses our docker
+image
+([gcr.io/cloud-devrel-public-resources/java-library-generation](https://pantheon.corp.google.com/gcr/images/cloud-devrel-public-resources/global/java-library-generation))
+to generate a new library. It also creates the pull request.
+
+
+## Execute the Github Action
+
+In order to run the
+[Github Action](https://github.com/googleapis/google-cloud-java/actions/workflows/generate_new_client_hermetic_build.yaml)
+, you need to specify a few parameters.
+These parameters will be available in the Cloud Drop link (a YAML file) included in the buganizer request.
+The example in this README uses AlloyDB's [Cloud Drop](https://github.com/googleapis/googleapis/blob/master/google/cloud/alloydb/v1/alloydb_v1.yaml) file as an example.
+
+### API short name (`api_shortname`)
+
+For convenience of the subsequent commands, define a variable for API short name.
+This value will be used by default to generate the following:
+* `--distribution-name`
+
+The corresponding value in the Cloud Drop page is `api_short_name`.
+
+Example: `alloydb`
+
+> [!IMPORTANT]
+> `api_short_name` is not always unique across client libraries.
+> In the instance that the `api_short_name` is already in use by an existing client library, you will need to determine a unique name.
+> See example under [Advanced Options](#Example with duplicate api_short_name).
+
+### Proto path (`proto_path`)
+
+The script takes "proto path" parameter. This is the path from the internal `google3/third_party/googleapis/stable` root to the
+directory that contains versions (e.g., "v1" or "v2").
+Note that the internal `google3/third_party/googleapis/stable` directory is mirrored externally in https://github.com/googleapis/googleapis/blob/master/.
+
+For example, if the buganizer ticket includes:
+
+> Link to protos: `http://...(omit).../google/cloud/alloydb/v1alpha/alloydb_v1alpha.yaml`.
+
+then the corresponding external mirrored proto is here: https://github.com/googleapis/googleapis/blob/master/google/cloud/alloydb/v1alpha/alloydb_v1alpha.yaml.
+
+Therefore, the "proto path" value we supply to the command is `google/cloud/alloydb`.
+
+We will publish a single module for a service that includes all versions in this path. Any future version must be manually added to
+the configuration yaml (`google-cloud-java/generation_config.yaml`)
+
+### Name pretty (`name_pretty`)
+
+The corresponding value in the Cloud Drop page is `title`.
+
+Example: `AlloyDB API`
+
+### Product Docs (`product_docs`)
+
+The corresponding value in the Cloud Drop page is `documentation_uri`.
+The value must starts with "https://".
+
+Example: `https://cloud.google.com/alloydb/docs`
+
+### REST Docs (`rest_docs`)
+
+The corresponding value in the Cloud Drop page is `rest_reference_documentation_uri`.
+The value must starts with "https://".
+
+Example: `https://cloud.google.com/alloydb/docs/reference/rest`
+
+If the value exists, add it as a flag to the python command below (see [Advanced
+Options](#advanced-options]):
+`--rest-docs="https://cloud.google.com/alloydb/docs/reference/rest" \`
+
+### RPC Docs (`rpc_docs`)
+
+The corresponding value in the Cloud Drop page is `proto_reference_documentation_uri`.
+The value must starts with "https://".
+
+Example: `https://cloud.google.com/speech-to-text/docs/reference/rpc`
+
+If the value exists, add it as a flag to the python command below (see [Advanced
+Options](#advanced-options]):
+`--rpc-docs="https://cloud.google.com/speech-to-text/docs/reference/rpc" \`
+
+### API description (`api_description`)
+
+The corresponding value in the Cloud Drop page is `documentation.summary` or `documentation.overview`.
+If both of those fields are missing, take the description from the product page above. Use the first sentence to keep it concise.
+
+Example:
+``` 
+    AlloyDB for PostgreSQL is an open source-compatible database service that
+    provides a powerful option for migrating, modernizing, or building
+    commercial-grade applications.
+ ```
+
+### Distribution Name (`distribution_name`)
+
+This variable determines the Maven coordinates of the generated library. It
+defaults to `com.google.cloud:google-cloud-{api_shortname}`. This mainly affect
+the values in the generated `pom.xml` files.
+
+### Library Name (`library_name`)
+
+This variable indicates the output folder of the library. For example you can
+have two libraries with `alloydb` (AlloyDB and AlloyDB Connectors)
+as `api_shortname`. In order to avoid both
+libraries going to the default `java-alloydb` folder, we can override this
+behavior by specifying a value like `alloydb-connectors` so the AlloyDB
+Connectors goes to `java-alloydb-connectors`.
+
+## Advanced Options
+
+In case the steps above don't show you how to specify the desired options, you can
+run the `new-client.py` script in your local evironment. The advanced options
+not shown in the section above **cannot be specified in the Github Action**,
+hence the need for a local run (refer to the "Prerequisites
+(for local environment)" section).
+For the explanation of the available parameters, run:
+`python3.9 generation/new_client_hermetic_build/new-client.py generate  --help`.
+
+After you run the script, you will see that the `generation_config.yaml` file
+was modified (or the script exited because the library already existed)
+
+The last step you need is to `cd` into the root of `google-cloud-java` and run
+```
+docker volume create --name "repo-google-cloud-java" --opt "type=none" --opt "device=$(pwd)" --opt "o=bind"
+repo_volumes="-v repo-google-cloud-java:/workspace/google-cloud-java"
+docker run --rm \
+  ${repo_volumes} \
+  -v /tmp:/tmp \
+  -v /var/run/docker.sock:/var/run/docker.sock \
+  -e "RUNNING_IN_DOCKER=true" \
+  -e "REPO_BINDING_VOLUMES=${repo_volumes}" \
+  gcr.io/cloud-devrel-public-resources/java-library-generation:latest \
+  python /src/generate_repo.py generate \
+  --generation-config-yaml=/workspace/google-cloud-java/generation_config.yaml \
+  --repository-path=/workspace/google-cloud-java \
+  --target-library-api-shortname=<your api_shortname>
+
+```
+
+This docker container will run the generation scripts and generate the library
+in your repo. You can create a PR explaining what commands you used (the docker
+command is not as informative as the `new-client.py` call, so make sure at least
+the new-client arguments were listed).

generation/new_client_hermetic_build/generate-arguments.py

@@ -0,0 +1,43 @@
+"""
+Helper script to generate arguments for new-client.py from environment variables
+"""
+import sys
+import os
+
+required_args = [
+  'API_SHORTNAME',
+  'NAME_PRETTY',
+  'API_DESCRIPTION',
+  'PROTO_PATH',
+  'PRODUCT_DOCS',
+]
+
+optional_args = [
+  'REST_DOCS',
+  'RPC_DOCS',
+  'LIBRARY_NAME',
+  'DISTRIBUTION_NAME',
+]
+
+def main() -> None:
+  result = ''
+  queries = [(required_args, True), (optional_args, False)]
+  for (args, is_required) in queries:
+    for arg in args:
+      value = os.getenv(arg)
+      result += __to_python_arg(arg, value, is_required)
+
+  print(result)
+
+def __to_python_arg(arg: str, value: str, is_required: bool) -> str:
+  if value is None or value == '':
+    if is_required:
+      sys.exit(f'required env var {arg} is not set')
+    return ""
+
+  return f"--{arg.lower().replace('_','-')} \"{value}\" "
+
+
+
+if __name__ == "__main__":
+  main()