Merge branch 'GoogleCloudPlatform:main' into main

.ci/containers/README.md

@@ -24,6 +24,6 @@ Before you begin, set up Docker (including configuring it to [authenticate with
    ```
 4. Update cloudbuild yaml files to reference the image you just pushed by adding the `:testing` suffix
 5. Update files that will cause the cloudbuild yaml changes (and therefore your changes) to be exercised
-   - Tip: Modifying `mmv1/third_party/terraform/services/compute/metadata.go.erb` will trigger builds for TPG, TPGB, and TGC.
+   - Tip: Modifying `mmv1/third_party/terraform/services/compute/metadata.go.tmpl` will trigger builds for TPG, TPGB, and TGC.
 6. Create a PR with these changes.
 7. Verify that the cloudbuild steps that should use your testing image _are_ using your testing image (in the Execution Details tab for the step.)

.ci/gcb-pr-downstream-generation-and-test.yml

@@ -264,7 +264,7 @@ steps:
         - $COMMIT_SHA
         - $BUILD_ID
         - $PROJECT_ID
-        - "22"  # Build step
+        - "23"  # Build step
 
     - name: 'gcr.io/graphite-docker-images/go-plus'
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'

.ci/gcb-vcr-nightly.yml

@@ -2,10 +2,15 @@
 steps:
     - name: 'gcr.io/graphite-docker-images/go-plus'
       id: gcb-vcr-nightly
-      entrypoint: '/workspace/.ci/scripts/go-plus/vcr-cassette-update/vcr_cassette_update.sh'
-      secretEnv: ["GOOGLE_BILLING_ACCOUNT", "GOOGLE_CUST_ID", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION"]
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
+      secretEnv: ["GOOGLE_BILLING_ACCOUNT", "GOOGLE_CUST_ID", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION", "GITHUB_TOKEN_CLASSIC"]
+      env:
+        - "GOOGLE_REGION=us-central1"
+        - "GOOGLE_ZONE=us-central1-a"
+        - "USER=magician"
       args:
-          - $BUILD_ID
+        - 'vcr-cassette-update'
+        - $BUILD_ID
 
 # Long timeout to enable waiting on VCR test
 timeout: 20000s
@@ -39,3 +44,5 @@ availableSecrets:
       env: SA_KEY
     - versionName: projects/673497134629/secrets/ci-test-public-advertised-prefix-description/versions/latest
       env: GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION
+    - versionName: projects/673497134629/secrets/github-classic--repo-workflow/versions/latest
+      env: GITHUB_TOKEN_CLASSIC

.ci/infra/terraform/main.tf

@@ -376,8 +376,6 @@ resource "google_project_service_identity" "sqladmin_sa" {
   service = "sqladmin.googleapis.com"
 }
 
-# TODO: Replace these permissions with bootstrapped permissions
-
 # TestAccComposerEnvironment_fixPyPiPackages
 # TestAccComposerEnvironmentComposer2_private
 # TestAccComposerEnvironment_withEncryptionConfigComposer1
@@ -409,6 +407,24 @@ resource "google_project_iam_member" "compute_agent_secret_accessor" {
   member  = "serviceAccount:${google_project.proj.number}[email protected]"
 }
 
+# TestAccHealthcarePipelineJob_healthcarePipelineJobMappingReconDestExample
+# TestAccHealthcarePipelineJob_healthcarePipelineJobReconciliationExample
+# TestAccHealthcarePipelineJob_healthcarePipelineJobWhistleMappingExample
+resource "google_project_iam_member" "healthcare_agent_storage_object_admin" {
+  project = google_project.proj.project_id
+  role    = "roles/storage.objectAdmin"
+  member  = "serviceAccount:service-${google_project.proj.number}@gcp-sa-healthcare.iam.gserviceaccount.com"
+}
+
+# TestAccHealthcarePipelineJob_healthcarePipelineJobMappingReconDestExample
+# TestAccHealthcarePipelineJob_healthcarePipelineJobReconciliationExample
+# TestAccHealthcarePipelineJob_healthcarePipelineJobWhistleMappingExample
+resource "google_project_iam_member" "healthcare_agent_fhir_resource_editor" {
+  project = google_project.proj.project_id
+  role    = "roles/healthcare.fhirResourceEditor"
+  member  = "serviceAccount:service-${google_project.proj.number}@gcp-sa-healthcare.iam.gserviceaccount.com"
+}
+
 # TestAccVertexAIEndpoint_vertexAiEndpointNetwork
 # TestAccVertexAIFeaturestoreEntitytype_vertexAiFeaturestoreEntitytypeExample
 # TestAccVertexAIFeaturestoreEntitytype_vertexAiFeaturestoreEntitytypeWithBetaFieldsExample
@@ -423,6 +439,21 @@ resource "google_project_iam_member" "aiplatform_agent_encrypter_decrypter" {
   member  = "serviceAccount:service-${google_project.proj.number}@gcp-sa-aiplatform.iam.gserviceaccount.com"
 }
 
+# TestAccComputeInstance_confidentialHyperDiskBootDisk
+resource "google_project_iam_member" "compute_default_sa_encrypter_decrypter" {
+  project = google_project.proj.project_id
+  role    = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
+  member  = "serviceAccount:${google_project.proj.number}[email protected]"
+}
+
+# TestAccComputeInstance_confidentialHyperDiskBootDisk
+resource "google_project_iam_member" "compute_agent_encrypter_decrypter" {
+  project = google_project.proj.project_id
+  role    = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
+  member  = "serviceAccount:service-${google_project.proj.number}@compute-system.iam.gserviceaccount.com"
+}
+
+
 data "google_organization" "org2" {
   organization = var.org2_id
 }

.ci/magician/cmd/check_cassettes.go

@@ -64,7 +64,7 @@ var checkCassettesCmd = &cobra.Command{
 
 		ctlr := source.NewController(env["GOPATH"], "modular-magician", githubToken, rnr)
 
-		vt, err := vcr.NewTester(env, rnr)
+		vt, err := vcr.NewTester(env, "ci-vcr-cassettes", "vcr-check-cassettes", rnr)
 		if err != nil {
 			return fmt.Errorf("error creating VCR tester: %w", err)
 		}
@@ -110,7 +110,10 @@ func execCheckCassettes(commit string, vt *vcr.Tester, ctlr *source.Controller)
 	if err != nil {
 		fmt.Println("Error running VCR: ", err)
 	}
-	if err := vt.UploadLogs("vcr-check-cassettes", "", "", false, false, vcr.Replaying, provider.Beta); err != nil {
+	if err := vt.UploadLogs(vcr.UploadLogsOptions{
+		Mode:    vcr.Replaying,
+		Version: provider.Beta,
+	}); err != nil {
 		return fmt.Errorf("error uploading logs: %w", err)
 	}
 	fmt.Println(len(result.FailedTests), " failed tests: ", result.FailedTests)

.ci/magician/cmd/generate_comment.go

@@ -40,7 +40,7 @@ import (
 )
 
 var (
-	//go:embed DIFF_COMMENT.md
+	//go:embed templates/DIFF_COMMENT.md.tmpl
 	diffComment string
 )
 
@@ -504,9 +504,9 @@ func detectMissingTests(diffProcessorPath, tpgbLocalPath string, rnr ExecRunner)
 }
 
 func formatDiffComment(data diffCommentData) (string, error) {
-	tmpl, err := template.New("DIFF_COMMENT.md").Parse(diffComment)
+	tmpl, err := template.New("DIFF_COMMENT.md.tmpl").Parse(diffComment)
 	if err != nil {
-		panic(fmt.Sprintf("Unable to parse DIFF_COMMENT.md: %s", err))
+		return "", fmt.Errorf("unable to parse template DIFF_COMMENT.md.tmpl: %s", err)
 	}
 	sb := new(strings.Builder)
 	err = tmpl.Execute(sb, data)

.ci/magician/cmd/interfaces.go

@@ -47,6 +47,7 @@ type ExecRunner interface {
 	PushDir(path string) error
 	PopDir() error
 	WriteFile(name, data string) error
+	AppendFile(name, data string) error // Not used (yet).
 	Run(name string, args []string, env map[string]string) (string, error)
 	MustRun(name string, args []string, env map[string]string) string
 }

.ci/magician/cmd/mock_runner_test.go

@@ -113,6 +113,10 @@ func (mr *mockRunner) WriteFile(name, data string) error {
 	return nil
 }
 
+func (mr *mockRunner) AppendFile(name, data string) error {
+	return nil
+}
+
 func (mr *mockRunner) Copy(src, dest string) error {
 	mr.calledMethods["Copy"] = append(mr.calledMethods["Copy"], ParameterList{src, dest})
 	return nil

.ci/magician/cmd/scheduled_pr_reminders.go

@@ -38,13 +38,13 @@ var (
 	// used for flags
 	dryRun bool
 
-	//go:embed SCHEDULED_PR_WAITING_FOR_CONTRIBUTOR.md.tmpl
+	//go:embed templates/SCHEDULED_PR_WAITING_FOR_CONTRIBUTOR.md.tmpl
 	waitingForContributorTemplate string
 
-	//go:embed SCHEDULED_PR_WAITING_FOR_MERGE.md.tmpl
+	//go:embed templates/SCHEDULED_PR_WAITING_FOR_MERGE.md.tmpl
 	waitingForMergeTemplate string
 
-	//go:embed SCHEDULED_PR_WAITING_FOR_REVIEW.md.tmpl
+	//go:embed templates/SCHEDULED_PR_WAITING_FOR_REVIEW.md.tmpl
 	waitingForReviewTemplate string
 )
 

.ci/magician/cmd/templates.go

@@ -0,0 +1,23 @@
+package cmd
+
+import (
+	"fmt"
+)
+
+func color(color, text string) string {
+	if color == "" || text == "" {
+		return text
+	}
+	var emoji string
+	switch color {
+	case "red":
+		emoji = "🔴"
+	case "yellow":
+		emoji = "🟡"
+	case "green":
+		emoji = "🟢"
+	default:
+		return text
+	}
+	return fmt.Sprintf("%s %s", emoji, text)
+}

.ci/magician/cmd/templates/SCHEDULED_PR_WAITING_FOR_CONTRIBUTOR.md.tmpl

@@ -0,0 +1,14 @@
+{{ if lt .SinceDays 30 -}}
+@{{.User}}, this PR is waiting for action from you. If no action is taken, this PR will be closed in
+{{- if eq .SinceDays 10 }} 28 days.{{end}}
+{{- if eq .SinceDays 20 }} 14 days.{{end}}
+{{- if eq .SinceDays 28 }} 2 weekdays.{{end}}
+
+Please address any comments or change requests, or [re-request review](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/requesting-a-pull-request-review) from a core reviewer if no action is required.
+
+![Image showing the re-request review button](https://docs.github.com/assets/cb-28785/mw-1440/images/help/pull_requests/request-re-review.webp)
+
+This notification can be disabled with the `disable-automatic-closure` label.
+{{ else -}}
+@{{.User}}, this PR is being closed due to inactivity.
+{{ end -}}

.ci/magician/cmd/templates/vcr/record_replay.tmpl

@@ -0,0 +1,31 @@
+{{- if gt (len .RecordingResult.PassedTests) 0 -}}
+{{color "green" "Tests passed during RECORDING mode:"}}
+{{range .RecordingResult.PassedTests}}`{{.}}`[[Debug log](https://storage.cloud.google.com/ci-vcr-logs/beta/refs/heads/auto-pr-{{$.PRNumber}}/artifacts/{{$.BuildID}}/recording/{{.}}.log)]
+{{end}}
+
+{{- if gt (len .ReplayingAfterRecordingResult.FailedTests ) 0 -}}
+{{color "red" "Tests failed when rerunning REPLAYING mode:"}}
+{{range .ReplayingAfterRecordingResult.FailedTests}}`{{.}}`[[Error message](https://storage.cloud.google.com/ci-vcr-logs/beta/refs/heads/auto-pr-{{$.PRNumber}}/artifacts/{{$.BuildID}}/build-log/replaying_build_after_recording/{{.}}_replaying_test.log)] [[Debug log](https://storage.cloud.google.com/ci-vcr-logs/beta/refs/heads/auto-pr-{{$.PRNumber}}/artifacts/{{$.BuildID}}/replaying_after_recording/{{.}}.log)]
+{{end}}
+
+Tests failed due to non-determinism or randomness when the VCR replayed the response after the HTTP request was made.
+
+Please fix these to complete your PR. If you believe these test failures to be incorrect or unrelated to your change, or if you have any questions, please raise the concern with your reviewer.
+
+{{else}}
+{{color "green" "No issues found for passed tests after REPLAYING rerun."}}
+{{end}}{{/* end of if gt (len .ReplayingAfterRecordingResult.FailedTests ) 0 */}}
+---
+{{end}}{{/* end of if gt (len .RecordingResult.PassedTests) 0 */}}
+
+{{if gt (len .RecordingResult.FailedTests) 0 -}}
+{{color "red" "Tests failed during RECORDING mode:"}}
+{{range .RecordingResult.FailedTests}}`{{.}}`[[Error message](https://storage.cloud.google.com/ci-vcr-logs/beta/refs/heads/auto-pr-{{$.PRNumber}}/artifacts/{{$.BuildID}}/build-log/recording_build/{{.}}_recording_test.log)] [[Debug log](https://storage.cloud.google.com/ci-vcr-logs/beta/refs/heads/auto-pr-{{$.PRNumber}}/artifacts/{{$.BuildID}}/recording/{{.}}.log)]
+{{end}}
+{{end}} {{- /* end of if gt (len .RecordingResult.FailedTests) 0 */ -}}
+
+{{if .HasTerminatedTests}}{{color "red" "Several tests got terminated during RECORDING mode."}}{{end}}
+{{if .RecordingErr}}{{color "red" "Errors occurred during RECORDING mode. Please fix them to complete your PR."}}{{end}}
+{{if .AllRecordingPassed}}{{color "green" "All tests passed!"}}{{end}}
+
+View the [build log](https://storage.cloud.google.com/ci-vcr-logs/beta/refs/heads/auto-pr-{{.PRNumber}}/artifacts/{{.BuildID}}/build-log/recording_test.log) or the [debug log](https://console.cloud.google.com/storage/browser/ci-vcr-logs/beta/refs/heads/auto-pr-{{.PRNumber}}/artifacts/{{.BuildID}}/recording) for each test

.ci/magician/cmd/vcr_cassettes_update_recording.tmpl → .ci/magician/cmd/templates/vcr/vcr_cassettes_update_recording.tmpl

@@ -5,6 +5,7 @@ The provider crashed while running the VCR tests in RECORDING mode
 {{- else -}}
 #################################
 RECORDING Tests Report
+#################################
 
 {{if .RecordingResult.PassedTests}}
 Tests passed during RECORDING mode:
@@ -18,7 +19,7 @@ Tests failed during RECORDING mode:
 
 {{if .HasTerminatedTests}}Several tests got terminated during RECORDING mode{{end}}
 
-{{if .RecordingErr}}Errors occurred during RECORDING mode.{{end}}
+{{if .RecordingErr}}Errors occurred during RECORDING mode: {{.RecordingErr}}.{{end}}
 
 {{if .AllRecordingPassed}}All tests passed!{{end}}
 

.ci/magician/cmd/vcr_cassettes_update_replaying.tmpl → .ci/magician/cmd/templates/vcr/vcr_cassettes_update_replaying.tmpl

@@ -3,6 +3,8 @@ The provider crashed while running the VCR tests in REPLAYING mode
 {{- else}}
 #################################
 Tests Analytics
+#################################
+
 Total tests: {{add (add (len .ReplayingResult.PassedTests) (len .ReplayingResult.SkippedTests)) (len .ReplayingResult.FailedTests) }}
 Passed tests: {{len .ReplayingResult.PassedTests}}
 Skipped tests: {{len .ReplayingResult.SkippedTests}}
@@ -15,7 +17,7 @@ Affected tests list:
 {{- end}}
 {{if .ReplayingErr}}
 #################################
-Errors occurred during REPLAYING mode.
+Errors occurred during REPLAYING mode: {{.ReplayingErr}}.
 #################################
 {{- end}}
 {{if .AllReplayingPassed}}

.ci/magician/cmd/templates/vcr/without_replay_failed_tests.tmpl

@@ -0,0 +1,7 @@
+{{- if .ReplayingErr -}}
+{{color "red" "Errors occurred during REPLAYING mode. Please fix them to complete your PR."}}
+{{- else -}}
+{{color "green" "All tests passed!"}}
+{{- end}}
+
+View the [build log](https://storage.cloud.google.com/ci-vcr-logs/beta/refs/heads/auto-pr-{{.PRNumber}}/artifacts/{{.BuildID}}/build-log/replaying_test.log)

.ci/magician/cmd/templates_test.go

@@ -0,0 +1,58 @@
+package cmd
+
+import (
+	"testing"
+)
+
+func TestColor(t *testing.T) {
+	cases := []struct {
+		name  string
+		color string
+		text  string
+		want  string
+	}{
+		{
+			name:  "red",
+			color: "red",
+			text:  "Test text",
+			want:  "🔴 Test text",
+		},
+		{
+			name:  "yellow",
+			color: "yellow",
+			text:  "Test text",
+			want:  "🟡 Test text",
+		},
+		{
+			name: "green",
+			color: "green",
+			text: "Test text",
+			want: "🟢 Test text",
+		},
+		{
+			name: "unsupported color",
+			color: "mauve",
+			text: "Test text",
+			want: "Test text",
+		},
+		{
+			name: "empty color",
+			text: "Test text",
+			want: "Test text",
+		},
+		{
+			name:  "empty text",
+			color: "green",
+			want:  "",
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := color(tc.color, tc.text)
+			if got != tc.want {
+				t.Errorf("color(%s, %s) got %s; want %s", tc.color, tc.text, got, tc.want)
+			}
+		})
+	}
+}