Support pre-made bundle archives in cloud function modules

blueprints/apigee/apigee-x-foundations/monitoring.tf

@@ -23,7 +23,7 @@ module "instance_monitor_function" {
   bucket_config = {
   }
   bundle_config = {
-    source_dir  = "${path.module}/functions/instance-monitor"
+    path        = "${path.module}/functions/instance-monitor"
     output_path = "bundle.zip"
   }
   function_config = {

blueprints/apigee/bigquery-analytics/main.tf

@@ -164,7 +164,7 @@ module "function_export" {
     lifecycle_delete_age = 1
   }
   bundle_config = {
-    source_dir  = "${path.module}/functions/export"
+    path        = "${path.module}/functions/export"
     output_path = "${path.module}/bundle-export.zip"
     excludes    = null
   }
@@ -200,7 +200,7 @@ module "function_gcs2bq" {
     lifecycle_delete_age = 1
   }
   bundle_config = {
-    source_dir  = "${path.module}/functions/gcs2bq"
+    path        = "${path.module}/functions/gcs2bq"
     output_path = "${path.module}/bundle-gcs2bq.zip"
     excludes    = null
   }

blueprints/cloud-operations/asset-inventory-feed-remediation/main.tf

@@ -85,7 +85,7 @@ module "cf" {
     location = var.region
   }
   bundle_config = {
-    source_dir  = "${path.module}/cf"
+    path        = "${path.module}/cf"
     output_path = var.bundle_path
   }
   service_account = module.service-account.email

blueprints/cloud-operations/compute-quota-monitoring/main.tf

@@ -60,7 +60,7 @@ module "cf" {
     location = var.region
   }
   bundle_config = {
-    source_dir  = "${path.module}/src"
+    path        = "${path.module}/src"
     output_path = var.bundle_path
   }
   service_account_create = true

blueprints/cloud-operations/network-quota-monitoring/deploy-cloud-function/main.tf

@@ -73,7 +73,7 @@ module "cloud-function" {
   }
   build_worker_pool = var.cloud_function_config.build_worker_pool_id
   bundle_config = {
-    source_dir  = var.cloud_function_config.source_dir
+    path        = var.cloud_function_config.source_dir
     output_path = var.cloud_function_config.bundle_path
   }
   environment_variables = (
@@ -145,7 +145,7 @@ module "cloud-function-v2" {
   }
   build_worker_pool = var.cloud_function_config.build_worker_pool_id
   bundle_config = {
-    source_dir  = var.cloud_function_config.source_dir
+    path        = var.cloud_function_config.source_dir
     output_path = var.cloud_function_config.bundle_path
   }
   environment_variables = (

blueprints/cloud-operations/scheduled-asset-inventory-export-bq/main.tf

@@ -94,7 +94,7 @@ module "cf" {
     location = var.region
   }
   bundle_config = {
-    source_dir  = "${path.module}/cf"
+    path        = "${path.module}/cf"
     output_path = var.bundle_path
   }
   service_account = module.service-account.email
@@ -116,7 +116,7 @@ module "cffile" {
     lifecycle_delete_age_days = null
   }
   bundle_config = {
-    source_dir  = "${path.module}/cffile"
+    path        = "${path.module}/cffile"
     output_path = var.bundle_path_cffile
     excludes    = null
   }

blueprints/cloud-operations/unmanaged-instances-healthcheck/main.tf

@@ -117,7 +117,7 @@ module "cf-restarter" {
     location = var.region
   }
   bundle_config = {
-    source_dir  = "${path.module}/function/restarter"
+    path        = "${path.module}/function/restarter"
     output_path = "restarter.zip"
   }
   service_account = module.service-account-restarter.email
@@ -145,7 +145,7 @@ module "cf-healthchecker" {
   region      = var.region
   bucket_name = module.cf-restarter.bucket_name
   bundle_config = {
-    source_dir  = "${path.module}/function/healthchecker"
+    path        = "${path.module}/function/healthchecker"
     output_path = "healthchecker.zip"
   }
   service_account = module.service-account-healthchecker.email

blueprints/networking/private-cloud-function-from-onprem/main.tf

@@ -184,7 +184,7 @@ module "function-hello" {
   bucket_name      = "${var.name}-tf-cf-deploy"
   ingress_settings = "ALLOW_INTERNAL_ONLY"
   bundle_config = {
-    source_dir  = "${path.module}/assets"
+    path        = "${path.module}/assets"
     output_path = "bundle.zip"
   }
   bucket_config = {

modules/cloud-function-v1/README.md

@@ -39,7 +39,7 @@ module "cf-http" {
   name        = "test-cf-http"
   bucket_name = var.bucket
   bundle_config = {
-    source_dir  = "assets/sample-function/"
+    path        = "assets/sample-function/"
     output_path = "bundle.zip"
   }
 }
@@ -58,7 +58,7 @@ module "cf-http" {
   name        = "test-cf-http"
   bucket_name = "test-cf-bundles"
   bundle_config = {
-    source_dir  = "fabric/assets/"
+    path        = "fabric/assets/"
     output_path = "bundle.zip"
   }
   trigger_config = {
@@ -81,7 +81,7 @@ module "cf-http" {
   name        = "test-cf-http"
   bucket_name = "test-cf-bundles"
   bundle_config = {
-    source_dir  = "fabric/assets/"
+    path        = "fabric/assets/"
     output_path = "bundle.zip"
   }
   iam = {
@@ -107,7 +107,7 @@ module "cf-http" {
     lifecycle_delete_age_days = 1
   }
   bundle_config = {
-    source_dir = "fabric/assets/"
+    path = "fabric/assets/"
   }
 }
 # tftest modules=1 resources=3 inventory=bucket-creation.yaml
@@ -125,7 +125,7 @@ module "cf-http" {
   name        = "test-cf-http"
   bucket_name = "test-cf-bundles"
   bundle_config = {
-    source_dir  = "fabric/assets/"
+    path        = "fabric/assets/"
     output_path = "bundle.zip"
   }
   service_account_create = true
@@ -143,7 +143,7 @@ module "cf-http" {
   name        = "test-cf-http"
   bucket_name = "test-cf-bundles"
   bundle_config = {
-    source_dir  = "fabric/assets/"
+    path        = "fabric/assets/"
     output_path = "bundle.zip"
   }
   service_account = "[email protected]"
@@ -153,6 +153,10 @@ module "cf-http" {
 
 ### Custom bundle config
 
+The Cloud Function bundle can be configured via the `bundle_config` variable, so that either a `zip` archive or a source folder can be used.
+
+If a `zip` archive is already available, simply set the archive path in `bundle_config.path`. If a dynamically generated archive is needed, set `bundle_config.path` to the source folder path, then optionally configure the path where the archive will be created, and any exclusions needed in the archive.
+
 In order to help prevent `archive_zip.output_md5` from changing cross platform (e.g. Cloud Build vs your local development environment), you'll have to make sure that the files included in the zip are always the same.
 
 ```hcl
@@ -163,7 +167,7 @@ module "cf-http" {
   name        = "test-cf-http"
   bucket_name = "test-cf-bundles"
   bundle_config = {
-    source_dir  = "fabric/assets"
+    path        = "fabric/assets/"
     output_path = "bundle.zip"
     excludes    = ["__pycache__"]
   }
@@ -184,7 +188,7 @@ module "cf-http" {
   bucket_name       = "test-cf-bundles"
   build_worker_pool = "projects/my-project/locations/europe-west1/workerPools/my_build_worker_pool"
   bundle_config = {
-    source_dir  = "fabric/assets"
+    path        = "fabric/assets/"
     output_path = "bundle.zip"
   }
 }
@@ -203,7 +207,7 @@ module "cf-http-one" {
   name        = "test-cf-http-one"
   bucket_name = "test-cf-bundles"
   bundle_config = {
-    source_dir = "fabric/assets"
+    path = "fabric/assets"
   }
 }
 
@@ -214,17 +218,20 @@ module "cf-http-two" {
   name        = "test-cf-http-two"
   bucket_name = "test-cf-bundles"
   bundle_config = {
-    source_dir = "fabric/assets"
+    path = "fabric/assets"
   }
 }
 # tftest modules=2 resources=4 inventory=multiple_functions.yaml
 ```
 
 ### Mounting secrets from Secret Manager
+
 This provides the latest value of the secret `var_secret` as `VARIABLE_SECRET` environment variable and three values of `path_secret` mounted in filesystem:
-* `/app/secret/first` contains version 1
-* `/app/secret/second` contains version 2
-* `/app/secret/latest` contains latest version of the secret
+
+- `/app/secret/first` contains version 1
+- `/app/secret/second` contains version 2
+- `/app/secret/latest` contains latest version of the secret
+
 ```hcl
 module "cf-http" {
   source      = "./fabric/modules/cloud-function-v1"
@@ -233,7 +240,7 @@ module "cf-http" {
   name        = "test-cf-http"
   bucket_name = "test-cf-bundles"
   bundle_config = {
-    source_dir  = "fabric/assets"
+    path        = "fabric/assets/"
     output_path = "bundle.zip"
   }
   secrets = {
@@ -261,6 +268,7 @@ module "cf-http" {
 ```
 
 ### Using CMEK to encrypt function resources
+
 This encrypt bucket _gcf-sources-*_ with the provided kms key. The repository has to be encrypted with the same kms key.
 
 ```hcl
@@ -271,7 +279,7 @@ module "cf-http" {
   name        = "test-cf-http"
   bucket_name = "test-cf-bundles"
   bundle_config = {
-    source_dir  = "fabric/assets"
+    path        = "fabric/assets/"
     output_path = "bundle.zip"
   }
   kms_key = "projects/my-project/locations/europe-west1/keyRings/mykeyring/cryptoKeys/mykey"
@@ -287,7 +295,7 @@ module "cf-http" {
 | name | description | type | required | default |
 |---|---|:---:|:---:|:---:|
 | [bucket_name](variables.tf#L26) | Name of the bucket that will be used for the function code. It will be created with prefix prepended if bucket_config is not null. | <code>string</code> | ✓ |  |
-| [bundle_config](variables.tf#L44) | Cloud function source folder and generated zip bundle paths. Output path defaults to '/tmp/bundle.zip' if null. | <code title="object&#40;&#123;&#10;  source_dir  &#61; string&#10;  output_path &#61; optional&#40;string&#41;&#10;  excludes    &#61; optional&#40;list&#40;string&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  |
+| [bundle_config](variables.tf#L44) | Cloud function source. If path points to a .zip archive it is uploaded as-is, otherwise an archive is created on the fly. A null output path will use a unique name for the bundle in /tmp. | <code title="object&#40;&#123;&#10;  path        &#61; string&#10;  excludes    &#61; optional&#40;list&#40;string&#41;&#41;&#10;  output_path &#61; optional&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  |
 | [name](variables.tf#L115) | Name used for cloud function and associated resources. | <code>string</code> | ✓ |  |
 | [project_id](variables.tf#L130) | Project id used for all resources. | <code>string</code> | ✓ |  |
 | [region](variables.tf#L135) | Region used for all resources. | <code>string</code> | ✓ |  |

modules/cloud-function-v1/bundle.tf

@@ -0,0 +1,81 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+  bundle = {
+    md5 = try(
+      data.archive_file.bundle[0].output_md5,
+      data.local_file.bundle[0].content_md5
+    )
+    path = try(
+      data.archive_file.bundle[0].output_path,
+      var.bundle_config.path
+    )
+  }
+}
+
+resource "google_storage_bucket" "bucket" {
+  count                       = var.bucket_config == null ? 0 : 1
+  project                     = var.project_id
+  name                        = "${local.prefix}${var.bucket_name}"
+  uniform_bucket_level_access = true
+  location = (
+    var.bucket_config.location == null
+    ? var.region
+    : var.bucket_config.location
+  )
+  labels = var.labels
+  dynamic "lifecycle_rule" {
+    for_each = var.bucket_config.lifecycle_delete_age_days == null ? [] : [""]
+    content {
+      action { type = "Delete" }
+      condition {
+        age        = var.bucket_config.lifecycle_delete_age_days
+        with_state = "ARCHIVED"
+      }
+    }
+  }
+  dynamic "versioning" {
+    for_each = var.bucket_config.lifecycle_delete_age_days == null ? [] : [""]
+    content {
+      enabled = true
+    }
+  }
+}
+
+resource "google_storage_bucket_object" "bundle" {
+  name   = "bundle-${local.bundle.md5}.zip"
+  bucket = local.bucket
+  source = local.bundle.path
+}
+
+data "archive_file" "bundle" {
+  count = (
+    try(fileexists(var.bundle_config.path), null) == null ? 1 : 0
+  )
+  type             = "zip"
+  source_dir       = var.bundle_config.path
+  output_path      = coalesce(var.bundle_config.output_path, "/tmp/bundle-${var.project_id}-${var.name}.zip")
+  output_file_mode = "0644"
+  excludes         = var.bundle_config.excludes
+}
+
+data "local_file" "bundle" {
+  count = (
+    try(fileexists(var.bundle_config.path), null) == null ? 0 : 1
+  )
+  filename = var.bundle_config.path
+}