Merge branch 'main' into update-to-go-1.21

.github/actions/build-downstream/action.yml

@@ -0,0 +1,105 @@
+name: build
+description: Generates given downstream Magic Modules Repository from `repo` input
+inputs:
+  repo:
+    description: "provider repo"
+    required: true
+  token:
+    description: "github token"
+    required: true
+
+runs:
+    using: "composite"
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@036ef458ddccddb148a2b9fb67e95a22fdbf728b # v1.160.0
+        with:
+          ruby-version: '3.1'
+
+      - name: Cache Bundler gems
+        uses: actions/cache@v3
+        with:
+          path: mmv1/vendor/bundle
+          key: ${{ runner.os }}-gems-${{ hashFiles('mmv1/**/Gemfile.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-gems-
+
+      - name: Install Ruby dependencies
+        shell: bash
+        run: |
+          bundle config path mmv1/vendor/bundle
+          bundle install
+        working-directory: mmv1
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: '^1.20'
+
+      # Cache Go modules
+      - name: Cache Go modules
+        uses: actions/cache@v3
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+
+      - run: go install golang.org/x/tools/cmd/goimports@latest
+        shell: bash
+      - name: Build ${{ inputs.repo  }}
+        shell: bash
+        env:
+          BASE_BRANCH: ${{ github.event.pull_request.base.ref || github.ref_name }}
+          GITHUB_TOKEN: ${{ inputs.token }}
+        run: |
+          set -e
+          set -x
+          # Set GOPATH to a directory the runner user has access to
+          export GOPATH=~/go
+
+          function clone_repo() {
+            export OUTPUT_PATH=$GOPATH/src/github.com/$UPSTREAM_OWNER/$GH_REPO
+            echo "OUTPUT_PATH=$OUTPUT_PATH" >> $GITHUB_ENV
+            GITHUB_PATH=https://x-access-token:[email protected]/$UPSTREAM_OWNER/$GH_REPO
+            mkdir -p "$(dirname $OUTPUT_PATH)"
+            git clone $GITHUB_PATH $OUTPUT_PATH --branch $BASE_BRANCH
+          }
+
+          GH_REPO="${{ inputs.repo  }}"
+          if [ "$GH_REPO" == "docs-examples" ] && [ "$BASE_BRANCH" == "main" ]; then
+              BASE_BRANCH="master"
+          fi
+
+          GITHUB_PATH=https://x-access-token:[email protected]/$UPSTREAM_OWNER/$GH_REPO
+
+          if [[ "$GH_REPO" == terraform-provider-google* ]]; then
+            UPSTREAM_OWNER=hashicorp
+            clone_repo
+          if [ "$GH_REPO" == "terraform-provider-google" ]; then
+              export VERSION=ga
+            else
+              export VERSION=beta
+            fi
+            make clean-provider
+            make provider
+          elif [ "$GH_REPO" == "terraform-google-conversion" ]; then
+            UPSTREAM_OWNER=GoogleCloudPlatform
+            clone_repo
+            make clean-tgc
+            make tgc
+          elif [ "$GH_REPO" == "docs-examples" ]; then
+            UPSTREAM_OWNER=terraform-google-modules
+            clone_repo
+            make tf-oics
+          else
+            echo "case not supported"
+            exit 1
+          fi
+
+          (current_dir=$(pwd) && cd $OUTPUT_PATH && zip -r "$current_dir/output.zip" .)

.github/workflows/teamcity-services-diff-check-weekly.yml

@@ -11,57 +11,37 @@ on:
     - cron:  '0 4 * * 1-2'
 
 jobs:
-  terraform-provider-google:
-    uses: ./.github/workflows/build-downstream.yml
-    with:
-      repo: 'terraform-provider-google'
-
-  terraform-provider-google-beta:
-    uses: ./.github/workflows/build-downstream.yml
-    with:
-      repo: 'terraform-provider-google-beta'
-
   teamcity-services-diff-check:
-    needs: [terraform-provider-google, terraform-provider-google-beta]
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v4
-
-      - name: Setup Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: '^1.21'
-
-      - name: Download built artifacts - GA provider
-        uses: actions/download-artifact@v2
-        with:
-          name: artifact-terraform-provider-google
-          path: artifacts
-
-      - name: Unzip the artifacts and delete the zip
-        run: |
-          unzip -o artifacts/output.zip -d ./provider
-          rm artifacts/output.zip
-
-      - name: Download built artifacts - Beta provider
-        uses: actions/download-artifact@v2
-        with:
-          name: artifact-terraform-provider-google-beta
-          path: artifacts
-
-      - name: Unzip the artifacts and delete the zip
-        run: |
-          unzip -o artifacts/output.zip -d ./provider
-          rm artifacts/output.zip
-
-      - name: Check that new services have been added to the TeamCity configuration code
-        run: |
-          # Create lists of service packages in providers
-          ls provider/google/services > tools/teamcity-diff-check/services_ga.txt
-          ls provider/google-beta/services > tools/teamcity-diff-check/services_beta.txt
-
-          # Run tool to compare service packages in the providers vs those listed in TeamCity config files
-          cd tools/teamcity-diff-check
-          go run main.go -service_file=services_ga
-          go run main.go -service_file=services_beta
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      runs-on: ubuntu-22.04
+      steps:
+        - name: Checkout Repository
+          uses: actions/checkout@v4
+        - name: TeamCity Google Provider Generate
+          uses: ./.github/actions/build-downstream
+          with:
+            repo: 'terraform-provider-google'
+            token: '$GITHUB_TOKEN'
+        # The path where GA/Beta providers are generated is grabbed from the OUTPUT_PATH that's set in build_downstream.yaml
+        # export OUTPUT_PATH=$GOPATH/src/github.com/$UPSTREAM_OWNER/$GH_REPO
+        # OUTPUT_PATH changes after each generate (GA/beta)
+        - name: Set GOOGLE_REPO_PATH to path where GA provider was generated
+          run: echo "GOOGLE_REPO_PATH=${{ env.OUTPUT_PATH}}" >> $GITHUB_ENV
+        - name: TeamCity Google Beta Provider Generate
+          uses: ./.github/actions/build-downstream
+          with:
+            repo: 'terraform-provider-google-beta'
+            token: '$GITHUB_TOKEN'
+        - name: Set GOOGLE_BETA_REPO_PATH to path where beta provider was generated
+          run: echo "GOOGLE_BETA_REPO_PATH=${{ env.OUTPUT_PATH}}" >> $GITHUB_ENV
+        - name: Check that new services have been added to the TeamCity configuration code
+          run: |
+              # Create lists of service packages in providers
+              ls ${{env.GOOGLE_REPO_PATH}}/google/services > tools/teamcity-diff-check/services_ga.txt
+              ls ${{env.GOOGLE_BETA_REPO_PATH}}/google-beta/services > tools/teamcity-diff-check/services_beta.txt
+    
+              # Run tool to compare service packages in the providers vs those listed in TeamCity config files
+              cd tools/teamcity-diff-check
+              go run main.go -service_file=services_ga
+              go run main.go -service_file=services_beta

.github/workflows/teamcity-services-diff-check.yml

@@ -9,10 +9,10 @@ on:
       - 'mmv1/third_party/terraform/.teamcity/components/inputs/services_beta.kt'
       - 'mmv1/products/**'
 jobs:
-  check-pr:
+  teamcity-services-diff-check:
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     runs-on: ubuntu-22.04
-    outputs:
-      services: ${{steps.services.outputs.services}}
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
@@ -24,61 +24,37 @@ jobs:
           newServices=$(($(git diff --name-only --diff-filter=A origin/main HEAD | grep -P "mmv1/products/.*/product.yaml" | wc -l)))
           echo "services=$newServices" >> "${GITHUB_OUTPUT}"
           if [ "$newServices" = "0" ];then
-          echo "No new service found." 
+          echo "No new service found."
           fi
-  terraform-provider-google:
-    if: ${{needs.check-pr.outputs.services != '0'}}
-    needs: check-pr
-    uses: ./.github/workflows/build-downstream.yml
-    with:
-      repo: 'terraform-provider-google'
-
-  terraform-provider-google-beta:
-    if: ${{needs.check-pr.outputs.services != '0'}} 
-    needs: check-pr
-    uses: ./.github/workflows/build-downstream.yml
-    with:
-      repo: 'terraform-provider-google-beta'
-
-  teamcity-services-diff-check:
-    needs: [terraform-provider-google, terraform-provider-google-beta]
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v4
-
-      - name: Setup Go
-        uses: actions/setup-go@v4
+      - name: TeamCity Google Provider Generate
+        id: generate
+        if: ${{steps.services.outputs.services != '0'}}
+        uses: ./.github/actions/build-downstream
         with:
-          go-version: '^1.21'
-
-      - name: Download built artifacts - GA provider
-        uses: actions/download-artifact@v2
+          repo: 'terraform-provider-google'
+          token: '$GITHUB_TOKEN'
+        # The path where GA/Beta providers are generated is grabbed from the OUTPUT_PATH that's set in build_downstream.yaml
+        # export OUTPUT_PATH=$GOPATH/src/github.com/$UPSTREAM_OWNER/$GH_REPO
+        # OUTPUT_PATH changes after each generate (GA/beta)
+      - name: Set GOOGLE_REPO_PATH to path where GA provider was generated
+        run: echo "GOOGLE_REPO_PATH=${{ env.OUTPUT_PATH}}" >> $GITHUB_ENV
+      - name: TeamCity Google Beta Provider Generate
+        if: steps.generate.outcome == 'success'
+        uses: ./.github/actions/build-downstream
         with:
-          name: artifact-terraform-provider-google
-          path: artifacts
-
-      - name: Unzip the artifacts and delete the zip
-        run: |
-          unzip -o artifacts/output.zip -d ./provider
-          rm artifacts/output.zip
-
-      - name: Download built artifacts - Beta provider
-        uses: actions/download-artifact@v2
-        with:
-          name: artifact-terraform-provider-google-beta
-          path: artifacts
-
-      - name: Unzip the artifacts and delete the zip
-        run: |
-          unzip -o artifacts/output.zip -d ./provider
-          rm artifacts/output.zip
-
+          repo: 'terraform-provider-google-beta'
+          token: '$GITHUB_TOKEN'
+      - name: Set GOOGLE_BETA_REPO_PATH to path where beta provider was generated
+        run: echo "GOOGLE_BETA_REPO_PATH=${{ env.OUTPUT_PATH}}" >> $GITHUB_ENV
+      - name: Checkout Repository
+        if: steps.generate.outcome == 'success'
+        uses: actions/checkout@v4
       - name: Check that new services have been added to the TeamCity configuration code
+        if: steps.generate.outcome == 'success'
         run: |
           # Create lists of service packages in providers
-          ls provider/google/services > tools/teamcity-diff-check/services_ga.txt
-          ls provider/google-beta/services > tools/teamcity-diff-check/services_beta.txt
+          ls ${{env.GOOGLE_REPO_PATH}}/google/services > tools/teamcity-diff-check/services_ga.txt
+          ls ${{env.GOOGLE_BETA_REPO_PATH}}/google-beta/services > tools/teamcity-diff-check/services_beta.txt
 
           # Run tool to compare service packages in the providers vs those listed in TeamCity config files
           cd tools/teamcity-diff-check

mmv1/compiler.rb

@@ -192,6 +192,9 @@
       )
       resource.validate
       resources.push(resource)
+    rescue StandardError => e
+      Google::LOGGER.error "Failed to compile #{file_path}: #{e}"
+      raise e
     end
 
     if override_dir
@@ -221,6 +224,9 @@
         )
         resource.validate
         resources.push(resource)
+      rescue StandardError => e
+        Google::LOGGER.error "Failed to compile using override #{override_path}: #{e}"
+        raise e
       end
     end
     resources = resources.sort_by(&:name)

mmv1/products/compute/RegionUrlMap.yaml

@@ -130,6 +130,15 @@ examples:
     skip_docs: true
     skip_test: true  # Similar to other samples
     min_version: beta
+  - !ruby/object:Provider::Terraform::Examples
+    name: "region_url_map_path_template_match"
+    primary_resource_id: "urlmap"
+    vars:
+      url_map_name: "urlmap"
+      home_backend_service_name: "home-service"
+      cart_backend_service_name: "cart-service"
+      user_backend_service_name: "user-service"
+      health_check_name: "health-check"
 parameters:
   - !ruby/object:Api::Type::ResourceRef
     name: 'region'
@@ -558,6 +567,18 @@ properties:
                         and anchor supplied with the original URL. For regular expression grammar please
                         see en.cppreference.com/w/cpp/regex/ecmascript  Only one of prefixMatch,
                         fullPathMatch or regexMatch must be specified.
+                    - !ruby/object:Api::Type::String
+                      name: 'pathTemplateMatch'
+                      description: |
+                        For satisfying the matchRule condition, the path of the request
+                        must match the wildcard pattern specified in pathTemplateMatch
+                        after removing any query parameters and anchor that may be part
+                        of the original URL.
+
+                        pathTemplateMatch must be between 1 and 255 characters
+                        (inclusive).  The pattern specified by pathTemplateMatch may
+                        have at most 5 wildcard operators and at most 5 variable
+                        captures in total.
               - !ruby/object:Api::Type::NestedObject
                 name: 'routeAction'
                 description: |
@@ -784,6 +805,23 @@ properties:
                           Prior to forwarding the request to the selected backend service, the matching
                           portion of the request's path is replaced by pathPrefixRewrite. The value must
                           be between 1 and 1024 characters.
+                      - !ruby/object:Api::Type::String
+                        name: 'pathTemplateRewrite'
+                        description: |
+                          Prior to forwarding the request to the selected origin, if the
+                          request matched a pathTemplateMatch, the matching portion of the
+                          request's path is replaced re-written using the pattern specified
+                          by pathTemplateRewrite.
+
+                          pathTemplateRewrite must be between 1 and 255 characters
+                          (inclusive), must start with a '/', and must only use variables
+                          captured by the route's pathTemplate matchers.
+
+                          pathTemplateRewrite may only be used when all of a route's
+                          MatchRules specify pathTemplate.
+
+                          Only one of pathPrefixRewrite and pathTemplateRewrite may be
+                          specified.
                   - !ruby/object:Api::Type::Array
                     name: 'weightedBackendServices'
                     description: |