Merge branch 'GoogleCloudPlatform:main' into stretch-pc

.ci/containers/README.md

@@ -3,17 +3,27 @@ The docker images located in this folder are used by multiple builds for magic m
 
 ## Naming Convention
 
-The images are named with the languages they contain and the images are versioned with tags that indicate the version of each language contained. eg: the image `go-ruby-python` with a tag of `1.11.5-2.6.0-2.7` indicates that the image has `go 1.11.5`, `ruby 2.6.0` and `python 2.7`.
+The images are named according to their use. We have a small number of images that get reused in multiple places, based around sets of requirements shared by different parts of the build pipeline. The images are:
 
-If there are multiple images with the same language version but different libraries (gems), a `v#` is appended to differentiate. eg: `1.11.5-2.6.0-2.7-v6`
+- `gcr.io/graphite-docker-images/bash-plus`
+- `gcr.io/graphite-docker-images/build-environment`
+- `gcr.io/graphite-docker-images/go-plus`
 
 ## Updating a docker image
-The Dockerfile should be updated, then the image rebuilt and pushed to the container registry stored at the `graphite-docker-images` GCP project. To update any of the images:
+
+Before you begin, set up Docker (including configuring it to [authenticate with gcloud](https://cloud.google.com/container-registry/docs/advanced-authentication#gcloud-helper)).
 
 1. Make changes to the Dockerfile
-2. Configure docker to use gcloud auth: `gcloud auth configure-docker`
-3. Build the image: `docker build . --tag gcr.io/graphite-docker-images/go-ruby-python`
-4. Find the new image's id: `docker images`
-5. Add the appropriate tag `docker tag ac37c0af8ce7 gcr.io/graphite-docker-images/go-ruby-python:1.11.5-2.6.0-2.7-v6`
-6. Push the image: `docker push gcr.io/graphite-docker-images/go-ruby-python:1.11.5-2.6.0-2.7-v6`
-7. Check the UI and ensure the new version is available and tagged at `latest`. It must be tagged `latest` for the Kokoro builds to get the correct version.
+2. Build the image with the `testing` tag:
+   ```bash
+   sudo docker build . --tag gcr.io/graphite-docker-images/bash-plus:testing
+   ```
+3. Push the image:
+   ```bash
+   sudo docker push gcr.io/graphite-docker-images/bash-plus:testing
+   ```
+4. Update cloudbuild yaml files to reference the image you just pushed by adding the `:testing` suffix
+5. Update files that will cause the cloudbuild yaml changes (and therefore your changes) to be exercised
+   - Tip: Modifying `mmv1/third_party/terraform/services/compute/metadata.go.erb` will trigger builds for TPG, TPGB, and TGC.
+6. Create a PR with these changes.
+7. Verify that the cloudbuild steps that should use your testing image _are_ using your testing image (in the Execution Details tab for the step.)

.ci/containers/build-environment/Dockerfile

@@ -36,10 +36,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends git openssh-cli
 RUN git config --global user.name "Modular Magician"
 RUN git config --global user.email "[email protected]"
 
-# Set up Github SSH cloning.
-RUN ssh-keyscan github.com >> /known_hosts
-RUN echo "UserKnownHostsFile /known_hosts" >> /etc/ssh/ssh_config
-
 RUN go install golang.org/x/tools/cmd/goimports@d088b475e3360caabc032aaee1dc66351d4e729a
 RUN go install github.com/github/[email protected]+incompatible
 

.ci/gcb-generate-diffs-new.yml

@@ -277,6 +277,8 @@ steps:
       args:
           - 'request-service-reviewers'
           - $_PR_NUMBER
+      env:
+        - COMMIT_SHA=$COMMIT_SHA
 
 # Long timeout to enable waiting on VCR test
 timeout: 20000s

.github/workflows/changelog-checker.yml

@@ -14,11 +14,22 @@ jobs:
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.2
         with:
           path: repo
+
+      - name: Set up Go
+        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+        with:
+          go-version: '^1.21.0'
+
+      - name: Build
+        run: |
+          cd repo/tools/go-changelog/cmd/changelog-pr-body-check
+          go build
+
       - name: Check Changelog
         env:
           GITHUB_OWNER: GoogleCloudPlatform
           GITHUB_REPO: magic-modules
           GITHUB_TOKEN: ${{github.token}}
         run: |
           cd repo/tools/go-changelog/cmd/changelog-pr-body-check
-          go run . ${{github.event.pull_request.number}}
+          ./changelog-pr-body-check ${{github.event.pull_request.number}}

.github/workflows/unit-test-go-changelog.yml

@@ -0,0 +1,29 @@
+name: unit-tests-go-changelog
+
+permissions: read-all
+
+on:
+  pull_request:
+    paths:
+      - 'tools/go-changelog/**'
+
+jobs:
+  test:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.2
+
+      - name: Set up Go
+        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+        with:
+          go-version: '^1.21.0'
+
+      - name: Build
+        run: |
+          cd tools/go-changelog/cmd/changelog-pr-body-check
+          go build
+
+      - name: Test
+        run: |
+          cd tools/go-changelog
+          go test -v ./...

mmv1/api/async.go

@@ -14,19 +14,30 @@
 package api
 
 import (
+	"strings"
+
 	"github.com/GoogleCloudPlatform/magic-modules/mmv1/google"
+	"golang.org/x/exp/slices"
+	"gopkg.in/yaml.v3"
 )
 
 // Base class from which other Async classes can inherit.
 type Async struct {
 	// Embed YamlValidator object
-	google.YamlValidator
+	// google.YamlValidator
 
 	// Describes an operation
 	Operation *Operation
 
 	// The list of methods where operations are used.
 	Actions []string
+
+	// Describes an operation, one of "OpAsync", "PollAsync"
+	Type string
+
+	OpAsync *OpAsync `yaml:",inline"`
+
+	PollAsync *PollAsync `yaml:",inline"`
 }
 
 // def validate
@@ -37,33 +48,36 @@ type Async struct {
 // end
 
 // def allow?(method)
-//   @actions.include?(method.downcase)
-// end
-
-// Base async operation type
-type Operation struct {
-	google.YamlValidator
-
-	// Contains information about an long-running operation, to make
-	// requests for the state of an operation.
-
-	Timeouts *Timeouts
-
-	Result Result
+func (a Async) Allow(method string) bool {
+	return slices.Contains(a.Actions, strings.ToLower(method))
 }
 
-// def validate
-//   check :result, type: Result
-//   check :timeouts, type: Api::Timeouts
-// end
+func (a Async) IsA(asyncType string) bool {
+	return a.Type == asyncType
+}
 
-// Base result class
-type Result struct {
-	google.YamlValidator
+// The main implementation of Operation,
+// corresponding to common GCP Operation resources.
+type Operation struct {
+	Timeouts         *Timeouts
+	OpAsyncOperation `yaml:",inline"`
+}
 
-	// Contains information about the result of an Operation
+// def initialize(path, base_url, wait_ms, timeouts)
+func NewOperation() *Operation {
+	//   super()
+	op := new(Operation)
+	op.Timeouts = NewTimeouts()
+	return op
+}
 
-	ResourceInsideResponse bool `yaml:"resource_inside_response"`
+func NewAsync() *Async {
+	oa := &Async{
+		Actions:   []string{"create", "delete", "update"},
+		Type:      "OpAsync",
+		Operation: NewOperation(),
+	}
+	return oa
 }
 
 // def validate
@@ -73,11 +87,6 @@ type Result struct {
 
 // Represents an asynchronous operation definition
 type OpAsync struct {
-	// TODO: Should embed Async or not?
-	// < Async
-
-	Operation *OpAsyncOperation
-
 	Result OpAsyncResult
 
 	Status OpAsyncStatus
@@ -87,16 +96,6 @@ type OpAsync struct {
 	// If true, include project as an argument to OperationWaitTime.
 	// It is intended for resources that calculate project/region from a selflink field
 	IncludeProject bool `yaml:"include_project"`
-
-	// The list of methods where operations are used.
-	Actions []string
-}
-
-func NewOpAsync() *OpAsync {
-	oa := new(OpAsync)
-	oa.Operation = NewOpAsyncOperation()
-	oa.Actions = []string{"create", "delete", "update"}
-	return oa
 }
 
 // def initialize(operation, result, status, error)
@@ -118,11 +117,7 @@ func NewOpAsync() *OpAsync {
 //   check :include_project, type: :boolean, default: false
 // end
 
-// The main implementation of Operation,
-// corresponding to common GCP Operation resources.
 type OpAsyncOperation struct {
-	// TODO: Should embed Operation or not?
-	// < Async::Operation
 	Kind string
 
 	Path string
@@ -131,20 +126,10 @@ type OpAsyncOperation struct {
 
 	WaitMs int `yaml:"wait_ms"`
 
-	Timeouts *Timeouts
-
 	// Use this if the resource includes the full operation url.
 	FullUrl string `yaml:"full_url"`
 }
 
-// def initialize(path, base_url, wait_ms, timeouts)
-func NewOpAsyncOperation() *OpAsyncOperation {
-	//   super()
-	op := new(OpAsyncOperation)
-	op.Timeouts = NewTimeouts()
-	return op
-}
-
 // def validate
 //   super
 
@@ -160,7 +145,7 @@ func NewOpAsyncOperation() *OpAsyncOperation {
 
 // Represents the results of an Operation request
 type OpAsyncResult struct {
-	Result Result `yaml:",inline"`
+	ResourceInsideResponse bool `yaml:"resource_inside_response"`
 
 	Path string
 }
@@ -180,7 +165,7 @@ type OpAsyncResult struct {
 // Provides information to parse the result response to check operation
 // status
 type OpAsyncStatus struct {
-	google.YamlValidator
+	// google.YamlValidator
 
 	Path string
 
@@ -222,3 +207,57 @@ type OpAsyncError struct {
 //   check :path, type: String
 //   check :message, type: String
 // end
+
+// Async implementation for polling in Terraform
+type PollAsync struct {
+	// Details how to poll for an eventually-consistent resource state.
+
+	// Function to call for checking the Poll response for
+	// creating and updating a resource
+	CheckResponseFuncExistence string `yaml:"check_response_func_existence"`
+
+	// Function to call for checking the Poll response for
+	// deleting a resource
+	CheckResponseFuncAbsence string `yaml:"check_response_func_absence"`
+
+	// Custom code to get a poll response, if needed.
+	// Will default to same logic as Read() to get current resource
+	CustomPollRead string `yaml:"custom_poll_read"`
+
+	// If true, will suppress errors from polling and default to the
+	// result of the final Read()
+	SuppressError bool `yaml:"suppress_error"`
+
+	// Number of times the desired state has to occur continuously
+	// during polling before returning a success
+	TargetOccurrences int `yaml:"target_occurrences"`
+}
+
+func (pa *PollAsync) UnmarshalYAML(n *yaml.Node) error {
+	pa.CheckResponseFuncAbsence = "transport_tpg.PollCheckForAbsence"
+	pa.TargetOccurrences = 1
+
+	type pollAsyncAlias PollAsync
+	aliasObj := (*pollAsyncAlias)(pa)
+
+	err := n.Decode(&aliasObj)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// 	return nil
+// }
+
+//   def validate
+// 	super
+
+// 	check :check_response_func_existence, type: String, required: true
+// 	check :check_response_func_absence, type: String,
+// 										default: 'transport_tpg.PollCheckForAbsence'
+// 	check :custom_poll_read, type: String
+// 	check :suppress_error, type: :boolean, default: false
+// 	check :target_occurrences, type: Integer, default: 1
+//   end

mmv1/api/async.rb

@@ -39,25 +39,11 @@ class Operation < Google::YamlValidator
       # Contains information about an long-running operation, to make
       # requests for the state of an operation.
       attr_reader :timeouts
-      attr_reader :result
 
       def validate
-        check :result, type: Result
         check :timeouts, type: Api::Timeouts
       end
     end
-
-    # Base result class
-    class Result < Google::YamlValidator
-      # Contains information about the result of an Operation
-
-      attr_reader :resource_inside_response
-
-      def validate
-        super
-        check :resource_inside_response, type: :boolean, default: false
-      end
-    end
   end
 
   # Represents an asynchronous operation definition
@@ -127,8 +113,9 @@ def validate
     end
 
     # Represents the results of an Operation request
-    class Result < Async::Result
+    class Result < Google::YamlValidator
       attr_reader :path
+      attr_reader :resource_inside_response
 
       def initialize(path = nil, resource_inside_response = nil)
         super()
@@ -140,6 +127,7 @@ def validate
         super
 
         check :path, type: String
+        check :resource_inside_response, type: :boolean, default: false
       end
     end