Add E2E tests for Cloud Functions and fix perma-diff

modules/cloud-function-v1/bundle.tf

@@ -54,6 +54,7 @@ resource "google_storage_bucket" "bucket" {
       enabled = true
     }
   }
+  force_destroy = var.bucket_config.force_destroy
 }
 
 # compress bundle in a zip archive if it's a folder

modules/cloud-function-v1/variables.tf

@@ -17,8 +17,9 @@
 variable "bucket_config" {
   description = "Enable and configure auto-created bucket. Set fields to null to use defaults."
   type = object({
-    location                  = optional(string)
+    force_destroy             = optional(bool)
     lifecycle_delete_age_days = optional(number)
+    location                  = optional(string)
   })
   default = null
 }
@@ -176,7 +177,7 @@ variable "secrets" {
   description = "Secret Manager secrets. Key is the variable name or mountpoint, volume versions are in version:path format."
   type = map(object({
     is_volume  = bool
-    project_id = number
+    project_id = string
     secret     = string
     versions   = list(string)
   }))

modules/cloud-function-v2/bundle.tf

@@ -54,6 +54,7 @@ resource "google_storage_bucket" "bucket" {
       enabled = true
     }
   }
+  force_destroy = var.bucket_config.force_destroy
 }
 
 # compress bundle in a zip archive if it's a folder

modules/cloud-function-v2/main.tf

@@ -70,7 +70,7 @@ resource "google_cloudfunctions2_function" "function" {
     worker_pool           = var.build_worker_pool
     runtime               = var.function_config.runtime
     entry_point           = var.function_config.entry_point
-    environment_variables = var.environment_variables
+    environment_variables = var.build_environment_variables
     docker_repository     = var.docker_repository_id
     source {
       storage_source {

modules/cloud-function-v2/variables.tf

@@ -17,8 +17,9 @@
 variable "bucket_config" {
   description = "Enable and configure auto-created bucket. Set fields to null to use defaults."
   type = object({
-    location                  = optional(string)
+    force_destroy             = optional(bool)
     lifecycle_delete_age_days = optional(number)
+    location                  = optional(string)
   })
   default = null
 }
@@ -29,6 +30,12 @@ variable "bucket_name" {
   nullable    = false
 }
 
+variable "build_environment_variables" {
+  description = "A set of key/value environment variable pairs available during build time."
+  type        = map(string)
+  default     = {}
+}
+
 variable "build_service_account" {
   description = "Build service account email."
   type        = string
@@ -89,7 +96,9 @@ variable "docker_repository_id" {
 variable "environment_variables" {
   description = "Cloud function environment variables."
   type        = map(string)
-  default     = {}
+  default = {
+    LOG_EXECUTION_ID = "true"
+  }
 }
 
 variable "function_config" {
@@ -165,7 +174,7 @@ variable "secrets" {
   description = "Secret Manager secrets. Key is the variable name or mountpoint, volume versions are in version:path format."
   type = map(object({
     is_volume  = bool
-    project_id = number
+    project_id = string
     secret     = string
     versions   = list(string)
   }))
@@ -198,7 +207,7 @@ variable "trigger_config" {
     })), [])
     service_account_email  = optional(string)
     service_account_create = optional(bool, false)
-    retry_policy           = optional(string)
+    retry_policy           = optional(string, "RETRY_POLICY_DO_NOT_RETRY") # default to avoid permadiff
   })
   default = null
 }

tests/examples_e2e/setup_module/main.tf

@@ -22,19 +22,27 @@ locals {
   services = [
     # trimmed down list of services, to be extended as needed
     "alloydb.googleapis.com",
-    "assuredworkloads.googleapis.com",
+    "analyticshub.googleapis.com",
     "apigee.googleapis.com",
+    "artifactregistry.googleapis.com",
+    "assuredworkloads.googleapis.com",
     "bigquery.googleapis.com",
     "cloudbuild.googleapis.com",
     "cloudfunctions.googleapis.com",
     "cloudkms.googleapis.com",
     "cloudresourcemanager.googleapis.com",
     "compute.googleapis.com",
     "container.googleapis.com",
+    "dataform.googleapis.com",
+    "dataplex.googleapis.com",
     "dataproc.googleapis.com",
     "dns.googleapis.com",
     "eventarc.googleapis.com",
     "iam.googleapis.com",
+    "logging.googleapis.com",
+    "monitoring.googleapis.com",
+    "networkconnectivity.googleapis.com",
+    "pubsub.googleapis.com",
     "run.googleapis.com",
     "secretmanager.googleapis.com",
     "servicenetworking.googleapis.com",

tests/fixtures/cloudbuild-custom-pool.tf

@@ -0,0 +1,26 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+resource "google_cloudbuild_worker_pool" "pool" {
+  project  = var.project_id
+  name     = "custom-pool"
+  location = var.regions.secondary
+  worker_config {
+    disk_size_gb   = 100
+    machine_type   = "e2-standard-4"
+    no_external_ip = false
+  }
+}

tests/fixtures/functions-default-sa-iam-grants.tf

@@ -0,0 +1,34 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+resource "google_project_iam_member" "bucket_default_compute_account_grant" {
+  project    = var.project_id
+  member     = "serviceAccount:${var.project_number}[email protected]"
+  role       = "roles/storage.objectViewer"
+  depends_on = [google_project_iam_member.artifact_writer]
+}
+
+resource "google_project_iam_member" "debugging_grant" {
+  project = var.project_id
+  member  = "serviceAccount:${var.project_number}[email protected]"
+  role    = "roles/logging.logWriter"
+}
+
+resource "google_project_iam_member" "artifact_writer" {
+  project = var.project_id
+  member  = "serviceAccount:${var.project_number}[email protected]"
+  role    = "roles/artifactregistry.createOnPushWriter"
+}

tests/fixtures/secret-credentials.tf

@@ -22,7 +22,10 @@ module "secret-manager" {
   }
   iam = {
     credentials = {
-      "roles/secretmanager.secretAccessor" = ["serviceAccount:${var.project_number}[email protected]"]
+      "roles/secretmanager.secretAccessor" = [
+        "serviceAccount:${var.project_number}[email protected]",
+        "serviceAccount:${var.project_id}@appspot.gserviceaccount.com",
+      ]
     }
   }
   versions = {

tests/modules/cloud_function_v1/examples/bucket-creation.yaml

@@ -14,23 +14,23 @@
 
 values:
   module.cf-http.google_cloudfunctions_function.function:
-    name: my-prefix-test-cf-http
-    project: my-project
-    source_archive_bucket: my-prefix-test-cf-bundles
+    name: test-test-cf-http
+    project: project-id
+    source_archive_bucket: test-bucket
 
   module.cf-http.google_storage_bucket.bucket[0]:
     lifecycle_rule:
       - action:
           - type: Delete
         condition:
           - age: 1
-    name: my-prefix-test-cf-bundles
+    name: test-bucket
 
 counts:
   google_cloudfunctions_function: 1
   google_storage_bucket: 1
   google_storage_bucket_object: 1
   modules: 1
-  resources: 3
+  resources: 6
 
-outputs: {}
+outputs: {}

tests/modules/cloud_function_v1/examples/iam.yaml

@@ -17,12 +17,12 @@ values:
     condition: []
     members:
     - allUsers
-    project: my-project
-    region: europe-west1
+    project: project-id
+    region: europe-west9
     role: roles/cloudfunctions.invoker
 
 counts:
   google_cloudfunctions_function: 1
   google_storage_bucket_object: 1
   modules: 1
-  resources: 3
+  resources: 6

tests/modules/cloud_function_v1/examples/multiple_functions.yaml

@@ -14,12 +14,12 @@
 
 values:
   module.cf-http-one.google_storage_bucket_object.bundle[0]:
-    source: /tmp/bundle-my-project-test-cf-http-one.zip
+    source: /tmp/bundle-project-id-test-cf-http-one.zip
   module.cf-http-two.google_storage_bucket_object.bundle[0]:
-    source: /tmp/bundle-my-project-test-cf-http-two.zip
+    source: /tmp/bundle-project-id-test-cf-http-two.zip
 
 counts:
   google_cloudfunctions_function: 2
   google_storage_bucket_object: 2
   modules: 2
-  resources: 4
+  resources: 7

tests/modules/cloud_function_v1/examples/secrets.yaml

@@ -16,24 +16,19 @@ values:
   module.cf-http.google_cloudfunctions_function.function:
     secret_environment_variables:
       - key: VARIABLE_SECRET
-        project_id: '1234567890'
-        secret: var_secret
+        project_id: "123" # actually, project number
+        # secret: var_secret # known after apply
         version: latest
     secret_volumes:
       - mount_path: /app/secret
-        project_id: '1234567890'
-        secret: path_secret
+        project_id: "123" # actually, project number
+        # secret: path_secret # known after apply
         versions:
-          - path: first
-            version: '1'
-          - path: second
-            version: '2'
-          - path: latest
-            version: latest
+          - {}
 
 
 counts:
   google_cloudfunctions_function: 1
   google_storage_bucket_object: 1
-  modules: 1
-  resources: 2
+  modules: 2
+  resources: 8

tests/modules/cloud_function_v2/assets/sample-function/main.py

@@ -0,0 +1,20 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import functions_framework
+
+
+@functions_framework.http
+def main(request):
+  return "Hello World!"

tools/create_e2e_sandbox.sh

@@ -28,7 +28,7 @@
 #
 set -e
 
-DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && cd .. && pwd )
+DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && cd .. && pwd)
 DEST=$1
 INFRA="${DEST}/infra"
 TIMESTAMP=$(date +%s)
@@ -42,18 +42,18 @@ cp "${SETUP_MODULE}/main.tf" "${SETUP_MODULE}/variables.tf" "${SETUP_MODULE}/e2e
 
 cp "${DIR}/tests/examples/variables.tf" "${DEST}"
 
-if [ ! -f "${INFRA}/randomizer.auto.tfvars" ] ; then
-  echo "suffix=0" > "${INFRA}/randomizer.auto.tfvars"
-  echo "timestamp=${TIMESTAMP}" >> "${INFRA}/randomizer.auto.tfvars"
+if [ ! -f "${INFRA}/randomizer.auto.tfvars" ]; then
+	echo "suffix=0" >"${INFRA}/randomizer.auto.tfvars"
+	echo "timestamp=${TIMESTAMP}" >>"${INFRA}/randomizer.auto.tfvars"
 fi
 
 # TODO correct environment variable prefix
-export | sed -e 's/^declare -x //' | grep '^TFTEST_E2E_' | sed -e 's/^TFTEST_E2E_//' > "${INFRA}/terraform.tfvars"
+export | sed -e 's/^declare -x //' | grep '^TFTEST_E2E_' | sed -e 's/^TFTEST_E2E_//' >"${INFRA}/terraform.tfvars"
 (
-  cd "${INFRA}"
-  terraform init
-  terraform apply -auto-approve
-  ln -sfT "${INFRA}/e2e_tests.tfvars" "${DEST}/e2e_tests.auto.tfvars"
+	cd "${INFRA}"
+	terraform init
+	terraform apply -auto-approve
+	ln -sfT "${INFRA}/e2e_tests.tfvars" "${DEST}/e2e_tests.auto.tfvars"
 )
 
-tocuh "${DEST}/main.tf"
+touch "${DEST}/main.tf"