diff --git a/blueprints/apigee/apigee-x-foundations/monitoring.tf b/blueprints/apigee/apigee-x-foundations/monitoring.tf index 5e633f9798..3102a38c68 100644 --- a/blueprints/apigee/apigee-x-foundations/monitoring.tf +++ b/blueprints/apigee/apigee-x-foundations/monitoring.tf @@ -23,8 +23,7 @@ module "instance_monitor_function" { bucket_config = { } bundle_config = { - path = "${path.module}/functions/instance-monitor" - output_path = "bundle.zip" + path = "${path.module}/functions/instance-monitor" } function_config = { entry_point = "writeMetric" diff --git a/blueprints/apigee/bigquery-analytics/main.tf b/blueprints/apigee/bigquery-analytics/main.tf index f2c31f2108..dfe3bd12fe 100644 --- a/blueprints/apigee/bigquery-analytics/main.tf +++ b/blueprints/apigee/bigquery-analytics/main.tf @@ -164,9 +164,7 @@ module "function_export" { lifecycle_delete_age = 1 } bundle_config = { - path = "${path.module}/functions/export" - output_path = "${path.module}/bundle-export.zip" - excludes = null + path = "${path.module}/functions/export" } function_config = { entry_point = "export" @@ -200,9 +198,7 @@ module "function_gcs2bq" { lifecycle_delete_age = 1 } bundle_config = { - path = "${path.module}/functions/gcs2bq" - output_path = "${path.module}/bundle-gcs2bq.zip" - excludes = null + path = "${path.module}/functions/gcs2bq" } function_config = { entry_point = "gcs2bq" diff --git a/blueprints/cloud-operations/asset-inventory-feed-remediation/main.tf b/blueprints/cloud-operations/asset-inventory-feed-remediation/main.tf index c84f526e30..bbbe4936ab 100644 --- a/blueprints/cloud-operations/asset-inventory-feed-remediation/main.tf +++ b/blueprints/cloud-operations/asset-inventory-feed-remediation/main.tf @@ -85,8 +85,10 @@ module "cf" { location = var.region } bundle_config = { - path = "${path.module}/cf" - output_path = var.bundle_path + path = "${path.module}/cf" + folder_options = { + archive_path = var.bundle_path + } } service_account = module.service-account.email trigger_config = { diff --git a/blueprints/cloud-operations/compute-quota-monitoring/main.tf b/blueprints/cloud-operations/compute-quota-monitoring/main.tf index f81515544c..0136aa3299 100644 --- a/blueprints/cloud-operations/compute-quota-monitoring/main.tf +++ b/blueprints/cloud-operations/compute-quota-monitoring/main.tf @@ -60,8 +60,10 @@ module "cf" { location = var.region } bundle_config = { - path = "${path.module}/src" - output_path = var.bundle_path + path = "${path.module}/src" + folder_options = { + archive_path = var.bundle_path + } } service_account_create = true trigger_config = { diff --git a/blueprints/cloud-operations/network-quota-monitoring/deploy-cloud-function/main.tf b/blueprints/cloud-operations/network-quota-monitoring/deploy-cloud-function/main.tf index c3d8e2d5bc..ba48d1bfb7 100644 --- a/blueprints/cloud-operations/network-quota-monitoring/deploy-cloud-function/main.tf +++ b/blueprints/cloud-operations/network-quota-monitoring/deploy-cloud-function/main.tf @@ -73,8 +73,10 @@ module "cloud-function" { } build_worker_pool = var.cloud_function_config.build_worker_pool_id bundle_config = { - path = var.cloud_function_config.source_dir - output_path = var.cloud_function_config.bundle_path + path = var.cloud_function_config.source_dir + folder_options = { + archive_path = var.cloud_function_config.bundle_path + } } environment_variables = ( var.cloud_function_config.debug != true ? {} : { DEBUG = "1" } @@ -145,8 +147,10 @@ module "cloud-function-v2" { } build_worker_pool = var.cloud_function_config.build_worker_pool_id bundle_config = { - path = var.cloud_function_config.source_dir - output_path = var.cloud_function_config.bundle_path + path = var.cloud_function_config.source_dir + folder_options = { + archive_path = var.cloud_function_config.bundle_path + } } environment_variables = ( var.cloud_function_config.debug != true ? {} : { DEBUG = "1" } diff --git a/blueprints/cloud-operations/scheduled-asset-inventory-export-bq/main.tf b/blueprints/cloud-operations/scheduled-asset-inventory-export-bq/main.tf index 9ee4fbf34f..3c786f5440 100644 --- a/blueprints/cloud-operations/scheduled-asset-inventory-export-bq/main.tf +++ b/blueprints/cloud-operations/scheduled-asset-inventory-export-bq/main.tf @@ -94,8 +94,10 @@ module "cf" { location = var.region } bundle_config = { - path = "${path.module}/cf" - output_path = var.bundle_path + path = "${path.module}/cf" + folder_options = { + archive_path = var.bundle_path + } } service_account = module.service-account.email trigger_config = { @@ -116,9 +118,10 @@ module "cffile" { lifecycle_delete_age_days = null } bundle_config = { - path = "${path.module}/cffile" - output_path = var.bundle_path_cffile - excludes = null + path = "${path.module}/cffile" + folder_options = { + archive_path = var.bundle_path_cffile + } } service_account = module.service-account.email trigger_config = { diff --git a/blueprints/cloud-operations/unmanaged-instances-healthcheck/main.tf b/blueprints/cloud-operations/unmanaged-instances-healthcheck/main.tf index 9cc68a839e..65aae68c96 100644 --- a/blueprints/cloud-operations/unmanaged-instances-healthcheck/main.tf +++ b/blueprints/cloud-operations/unmanaged-instances-healthcheck/main.tf @@ -117,8 +117,7 @@ module "cf-restarter" { location = var.region } bundle_config = { - path = "${path.module}/function/restarter" - output_path = "restarter.zip" + path = "${path.module}/function/restarter" } service_account = module.service-account-restarter.email @@ -145,8 +144,7 @@ module "cf-healthchecker" { region = var.region bucket_name = module.cf-restarter.bucket_name bundle_config = { - path = "${path.module}/function/healthchecker" - output_path = "healthchecker.zip" + path = "${path.module}/function/healthchecker" } service_account = module.service-account-healthchecker.email function_config = { diff --git a/blueprints/networking/private-cloud-function-from-onprem/main.tf b/blueprints/networking/private-cloud-function-from-onprem/main.tf index e14fe77d7f..377801e004 100644 --- a/blueprints/networking/private-cloud-function-from-onprem/main.tf +++ b/blueprints/networking/private-cloud-function-from-onprem/main.tf @@ -184,8 +184,7 @@ module "function-hello" { bucket_name = "${var.name}-tf-cf-deploy" ingress_settings = "ALLOW_INTERNAL_ONLY" bundle_config = { - path = "${path.module}/assets" - output_path = "bundle.zip" + path = "${path.module}/assets" } bucket_config = { location = var.region diff --git a/modules/cloud-function-v1/README.md b/modules/cloud-function-v1/README.md index 76a551803e..5fb264ace4 100644 --- a/modules/cloud-function-v1/README.md +++ b/modules/cloud-function-v1/README.md @@ -1,8 +1,6 @@ # Cloud Function Module (V1) -Cloud Function management, with support for IAM roles and optional bucket creation. - -The GCS object used for deployment uses a hash of the bundle zip contents in its name, which ensures change tracking and avoids recreating the function if the GCS object is deleted and needs recreating. +Cloud Function management, with support for IAM roles, optional bucket creation and bundle via GCS URI, local zip, or local source folder. - [TODO](#todo) @@ -39,8 +37,7 @@ module "cf-http" { name = "test-cf-http" bucket_name = var.bucket bundle_config = { - path = "assets/sample-function/" - output_path = "bundle.zip" + path = "assets/sample-function/" } } # tftest modules=1 resources=2 e2e @@ -58,8 +55,7 @@ module "cf-http" { name = "test-cf-http" bucket_name = "test-cf-bundles" bundle_config = { - path = "fabric/assets/" - output_path = "bundle.zip" + path = "assets/sample-function/" } trigger_config = { event = "google.pubsub.topic.publish" @@ -81,8 +77,7 @@ module "cf-http" { name = "test-cf-http" bucket_name = "test-cf-bundles" bundle_config = { - path = "fabric/assets/" - output_path = "bundle.zip" + path = "assets/sample-function/" } iam = { "roles/cloudfunctions.invoker" = ["allUsers"] @@ -107,7 +102,7 @@ module "cf-http" { lifecycle_delete_age_days = 1 } bundle_config = { - path = "fabric/assets/" + path = "assets/sample-function/" } } # tftest modules=1 resources=3 inventory=bucket-creation.yaml @@ -125,8 +120,7 @@ module "cf-http" { name = "test-cf-http" bucket_name = "test-cf-bundles" bundle_config = { - path = "fabric/assets/" - output_path = "bundle.zip" + path = "assets/sample-function/" } service_account_create = true } @@ -143,8 +137,7 @@ module "cf-http" { name = "test-cf-http" bucket_name = "test-cf-bundles" bundle_config = { - path = "fabric/assets/" - output_path = "bundle.zip" + path = "assets/sample-function/" } service_account = "non-existent@serice.account.email" } @@ -153,11 +146,13 @@ module "cf-http" { ### Custom bundle config -The Cloud Function bundle can be configured via the `bundle_config` variable, so that either a `zip` archive or a source folder can be used. +The Cloud Function bundle can be configured via the `bundle_config` variable. The only mandatory argument is `bundle_config.path` which can point to: -If a `zip` archive is already available, simply set the archive path in `bundle_config.path`. If a dynamically generated archive is needed, set `bundle_config.path` to the source folder path, then optionally configure the path where the archive will be created, and any exclusions needed in the archive. +- a GCS URI of a ZIP archive +- a local path to a ZIP archive +- a local path to a source folder -If you use a folder and dynamic archive bundling, be mindful that the MD5 checksum of the generated `zip` file does not change across environments (e.g. Cloud Build vs your local development environment), by ensuring that the files in the folder are always the same. +When a GCS URI or a local zip file are used, a change in their names will trigger redeployment. When a local source folder is used a ZIP archive will be automatically generated and its internally derived checksum will drive redeployment. You can optionally control its name and exclusions via the attributes in `bundle_config.folder_options`. ```hcl module "cf-http" { @@ -167,9 +162,11 @@ module "cf-http" { name = "test-cf-http" bucket_name = "test-cf-bundles" bundle_config = { - path = "fabric/assets/" - output_path = "bundle.zip" - excludes = ["__pycache__"] + path = "fabric/assets/" + folder_options = { + archive_path = "bundle.zip" + excludes = ["__pycache__"] + } } } # tftest modules=1 resources=2 @@ -188,8 +185,7 @@ module "cf-http" { bucket_name = "test-cf-bundles" build_worker_pool = "projects/my-project/locations/europe-west1/workerPools/my_build_worker_pool" bundle_config = { - path = "fabric/assets/" - output_path = "bundle.zip" + path = "fabric/assets/" } } # tftest modules=1 resources=2 @@ -197,7 +193,7 @@ module "cf-http" { ### Multiple Cloud Functions within project -When deploying multiple functions do not reuse `bundle_config.output_path` between instances as the result is undefined. Default `output_path` creates file in `/tmp` folder using project Id and function name to avoid name conflicts. +When deploying multiple functions do not reuse `bundle_config.archive_path` between instances as the result is undefined. Default `archive_path` creates file in `/tmp` folder using project Id and function name to avoid name conflicts. ```hcl module "cf-http-one" { @@ -207,7 +203,7 @@ module "cf-http-one" { name = "test-cf-http-one" bucket_name = "test-cf-bundles" bundle_config = { - path = "fabric/assets" + path = "fabric/assets/" } } @@ -218,7 +214,7 @@ module "cf-http-two" { name = "test-cf-http-two" bucket_name = "test-cf-bundles" bundle_config = { - path = "fabric/assets" + path = "fabric/assets/" } } # tftest modules=2 resources=4 inventory=multiple_functions.yaml @@ -240,8 +236,7 @@ module "cf-http" { name = "test-cf-http" bucket_name = "test-cf-bundles" bundle_config = { - path = "fabric/assets/" - output_path = "bundle.zip" + path = "fabric/assets/" } secrets = { VARIABLE_SECRET = { @@ -279,8 +274,7 @@ module "cf-http" { name = "test-cf-http" bucket_name = "test-cf-bundles" bundle_config = { - path = "fabric/assets/" - output_path = "bundle.zip" + path = "fabric/assets/" } kms_key = "projects/my-project/locations/europe-west1/keyRings/mykeyring/cryptoKeys/mykey" repository_settings = { @@ -295,29 +289,29 @@ module "cf-http" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| | [bucket_name](variables.tf#L26) | Name of the bucket that will be used for the function code. It will be created with prefix prepended if bucket_config is not null. | string | ✓ | | -| [bundle_config](variables.tf#L44) | Cloud function source. If path points to a .zip archive it is uploaded as-is, otherwise an archive is created on the fly. A null output path will use a unique name for the bundle in /tmp. | object({…}) | ✓ | | -| [name](variables.tf#L127) | Name used for cloud function and associated resources. | string | ✓ | | -| [project_id](variables.tf#L142) | Project id used for all resources. | string | ✓ | | -| [region](variables.tf#L147) | Region used for all resources. | string | ✓ | | +| [bundle_config](variables.tf#L44) | Cloud function source. Path can point to a GCS object URI, or a local path. A local path to a zip archive will generate a GCS object using its basename, a folder will be zipped and the GCS object name inferred when not specified. | object({…}) | ✓ | | +| [name](variables.tf#L139) | Name used for cloud function and associated resources. | string | ✓ | | +| [project_id](variables.tf#L154) | Project id used for all resources. | string | ✓ | | +| [region](variables.tf#L159) | Region used for all resources. | string | ✓ | | | [bucket_config](variables.tf#L17) | Enable and configure auto-created bucket. Set fields to null to use defaults. | object({…}) | | null | | [build_environment_variables](variables.tf#L32) | A set of key/value environment variable pairs available during build time. | map(string) | | {} | | [build_worker_pool](variables.tf#L38) | Build worker pool, in projects//locations//workerPools/ format. | string | | null | -| [description](variables.tf#L65) | Optional description. | string | | "Terraform managed." | -| [environment_variables](variables.tf#L71) | Cloud function environment variables. | map(string) | | {} | -| [function_config](variables.tf#L77) | Cloud function configuration. Defaults to using main as entrypoint, 1 instance with 256MiB of memory, and 180 second timeout. | object({…}) | | {…} | -| [https_security_level](variables.tf#L97) | The security level for the function: Allowed values are SECURE_ALWAYS, SECURE_OPTIONAL. | string | | null | -| [iam](variables.tf#L103) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [ingress_settings](variables.tf#L109) | Control traffic that reaches the cloud function. Allowed values are ALLOW_ALL, ALLOW_INTERNAL_AND_GCLB and ALLOW_INTERNAL_ONLY . | string | | null | -| [kms_key](variables.tf#L115) | Resource name of a KMS crypto key (managed by the user) used to encrypt/decrypt function resources in key id format. If specified, you must also provide an artifact registry repository using the docker_repository field that was created with the same KMS crypto key. | string | | null | -| [labels](variables.tf#L121) | Resource labels. | map(string) | | {} | -| [prefix](variables.tf#L132) | Optional prefix used for resource names. | string | | null | -| [repository_settings](variables.tf#L152) | Docker Registry to use for storing the function's Docker images and specific repository. If kms_key is provided, the repository must have already been encrypted with the key. | object({…}) | | {…} | -| [secrets](variables.tf#L163) | Secret Manager secrets. Key is the variable name or mountpoint, volume versions are in version:path format. | map(object({…})) | | {} | -| [service_account](variables.tf#L175) | Service account email. Unused if service account is auto-created. | string | | null | -| [service_account_create](variables.tf#L181) | Auto-create service account. | bool | | false | -| [trigger_config](variables.tf#L187) | Function trigger configuration. Leave null for HTTP trigger. | object({…}) | | null | -| [vpc_connector](variables.tf#L197) | VPC connector configuration. Set create to 'true' if a new connector needs to be created. | object({…}) | | null | -| [vpc_connector_config](variables.tf#L207) | VPC connector network configuration. Must be provided if new VPC connector is being created. | object({…}) | | null | +| [description](variables.tf#L77) | Optional description. | string | | "Terraform managed." | +| [environment_variables](variables.tf#L83) | Cloud function environment variables. | map(string) | | {} | +| [function_config](variables.tf#L89) | Cloud function configuration. Defaults to using main as entrypoint, 1 instance with 256MiB of memory, and 180 second timeout. | object({…}) | | {…} | +| [https_security_level](variables.tf#L109) | The security level for the function: Allowed values are SECURE_ALWAYS, SECURE_OPTIONAL. | string | | null | +| [iam](variables.tf#L115) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [ingress_settings](variables.tf#L121) | Control traffic that reaches the cloud function. Allowed values are ALLOW_ALL, ALLOW_INTERNAL_AND_GCLB and ALLOW_INTERNAL_ONLY . | string | | null | +| [kms_key](variables.tf#L127) | Resource name of a KMS crypto key (managed by the user) used to encrypt/decrypt function resources in key id format. If specified, you must also provide an artifact registry repository using the docker_repository field that was created with the same KMS crypto key. | string | | null | +| [labels](variables.tf#L133) | Resource labels. | map(string) | | {} | +| [prefix](variables.tf#L144) | Optional prefix used for resource names. | string | | null | +| [repository_settings](variables.tf#L164) | Docker Registry to use for storing the function's Docker images and specific repository. If kms_key is provided, the repository must have already been encrypted with the key. | object({…}) | | {…} | +| [secrets](variables.tf#L175) | Secret Manager secrets. Key is the variable name or mountpoint, volume versions are in version:path format. | map(object({…})) | | {} | +| [service_account](variables.tf#L187) | Service account email. Unused if service account is auto-created. | string | | null | +| [service_account_create](variables.tf#L193) | Auto-create service account. | bool | | false | +| [trigger_config](variables.tf#L199) | Function trigger configuration. Leave null for HTTP trigger. | object({…}) | | null | +| [vpc_connector](variables.tf#L209) | VPC connector configuration. Set create to 'true' if a new connector needs to be created. | object({…}) | | null | +| [vpc_connector_config](variables.tf#L219) | VPC connector network configuration. Must be provided if new VPC connector is being created. | object({…}) | | null | ## Outputs diff --git a/modules/cloud-function-v1/bundle.tf b/modules/cloud-function-v1/bundle.tf index 1d046404ed..43ad5c8ee2 100644 --- a/modules/cloud-function-v1/bundle.tf +++ b/modules/cloud-function-v1/bundle.tf @@ -15,16 +15,16 @@ */ locals { - bundle = { - name = try( - "bundle-${data.archive_file.bundle[0].output_md5}.zip", - basename(var.bundle_config.path) + bundle_type = ( + startswith(var.bundle_config.path, "gs://") + ? "gcs" + : ( + try(fileexists(pathexpand(var.bundle_config.path)), null) != null && + endswith(var.bundle_config.path, ".zip") + ? "local-file" + : "local-folder" ) - path = try( - data.archive_file.bundle[0].output_path, - pathexpand(var.bundle_config.path) - ) - } + ) } resource "google_storage_bucket" "bucket" { @@ -59,20 +59,29 @@ resource "google_storage_bucket" "bucket" { # compress bundle in a zip archive if it's a folder data "archive_file" "bundle" { - count = ( - try(fileexists(pathexpand(var.bundle_config.path)), null) == null ? 1 : 0 + count = local.bundle_type == "local-folder" ? 1 : 0 + type = "zip" + source_dir = pathexpand(var.bundle_config.path) + output_path = ( + var.bundle_config.folder_options.archive_path != null + ? pathexpand(var.bundle_config.folder_options.archive_path) + : "/tmp/bundle-${var.project_id}-${var.name}.zip" ) - type = "zip" - source_dir = pathexpand(var.bundle_config.path) - output_path = coalesce(var.bundle_config.output_path, "/tmp/bundle-${var.project_id}-${var.name}.zip") output_file_mode = "0644" - excludes = var.bundle_config.excludes + excludes = var.bundle_config.folder_options.excludes } # upload to GCS resource "google_storage_bucket_object" "bundle" { - name = local.bundle.name + count = local.bundle_type != "gcs" ? 1 : 0 + name = try( + "bundle-${data.archive_file.bundle[0].output_md5}.zip", + basename(var.bundle_config.path) + ) bucket = local.bucket - source = local.bundle.path + source = try( + data.archive_file.bundle[0].output_path, + pathexpand(var.bundle_config.path) + ) } diff --git a/modules/cloud-function-v1/main.tf b/modules/cloud-function-v1/main.tf index bcc8cffa9c..4638b74a24 100644 --- a/modules/cloud-function-v1/main.tf +++ b/modules/cloud-function-v1/main.tf @@ -51,19 +51,23 @@ resource "google_vpc_access_connector" "connector" { } resource "google_cloudfunctions_function" "function" { - project = var.project_id - region = var.region - name = "${local.prefix}${var.name}" - description = var.description - runtime = var.function_config.runtime - available_memory_mb = var.function_config.memory_mb - max_instances = var.function_config.instance_count - timeout = var.function_config.timeout_seconds - entry_point = var.function_config.entry_point - environment_variables = var.environment_variables - service_account_email = local.service_account_email - source_archive_bucket = local.bucket - source_archive_object = google_storage_bucket_object.bundle.name + project = var.project_id + region = var.region + name = "${local.prefix}${var.name}" + description = var.description + runtime = var.function_config.runtime + available_memory_mb = var.function_config.memory_mb + max_instances = var.function_config.instance_count + timeout = var.function_config.timeout_seconds + entry_point = var.function_config.entry_point + environment_variables = var.environment_variables + service_account_email = local.service_account_email + source_archive_bucket = local.bucket + source_archive_object = ( + local.bundle_type == "gcs" + ? var.bundle_config.path + : google_storage_bucket_object.bundle[0].name + ) labels = var.labels trigger_http = var.trigger_config == null ? true : null https_trigger_security_level = var.https_security_level == null ? "SECURE_ALWAYS" : var.https_security_level diff --git a/modules/cloud-function-v1/variables.tf b/modules/cloud-function-v1/variables.tf index 5dde7ad388..1c7c7e433d 100644 --- a/modules/cloud-function-v1/variables.tf +++ b/modules/cloud-function-v1/variables.tf @@ -42,23 +42,35 @@ variable "build_worker_pool" { } variable "bundle_config" { - description = "Cloud function source. If path points to a .zip archive it is uploaded as-is, otherwise an archive is created on the fly. A null output path will use a unique name for the bundle in /tmp." + description = "Cloud function source. Path can point to a GCS object URI, or a local path. A local path to a zip archive will generate a GCS object using its basename, a folder will be zipped and the GCS object name inferred when not specified." type = object({ - path = string - excludes = optional(list(string)) - output_path = optional(string) + path = string + folder_options = optional(object({ + archive_path = optional(string) + excludes = optional(list(string)) + }), {}) }) + nullable = false validation { condition = ( var.bundle_config.path != null && ( + # GCS object ( - try(fileexists(pathexpand(var.bundle_config.path)), null) != null && + startswith(var.bundle_config.path, "gs://") && endswith(var.bundle_config.path, ".zip") - ) || + ) + || + # local folder length(fileset(pathexpand(var.bundle_config.path), "**/*")) > 0 + || + # local ZIP archive + ( + try(fileexists(pathexpand(var.bundle_config.path)), null) != null && + endswith(var.bundle_config.path, ".zip") + ) ) ) - error_message = "Bundle path must be set to a local folder or zip file." + error_message = "Bundle path must be set to a GCS object URI, a local folder or a local zip file." } } diff --git a/modules/cloud-function-v2/README.md b/modules/cloud-function-v2/README.md index 1e31db6830..bb87dfa22e 100644 --- a/modules/cloud-function-v2/README.md +++ b/modules/cloud-function-v2/README.md @@ -1,8 +1,6 @@ # Cloud Function Module (v2) -Cloud Function management, with support for IAM roles and optional bucket creation. - -The GCS object used for deployment uses a hash of the bundle zip contents in its name, which ensures change tracking and avoids recreating the function if the GCS object is deleted and needs recreating. +Cloud Function management, with support for IAM roles, optional bucket creation and bundle via GCS URI, local zip, or local source folder. - [TODO](#todo) @@ -38,8 +36,7 @@ module "cf-http" { name = "test-cf-http" bucket_name = "test-cf-bundles" bundle_config = { - path = "fabric/assets/" - output_path = "bundle.zip" + path = "fabric/assets/" } } # tftest modules=1 resources=2 @@ -68,8 +65,7 @@ module "cf-http" { name = "test-cf-http" bucket_name = "test-cf-bundles" bundle_config = { - path = "fabric/assets/" - output_path = "bundle.zip" + path = "fabric/assets/" } trigger_config = { event_type = "google.cloud.pubsub.topic.v1.messagePublished" @@ -95,8 +91,7 @@ module "cf-http" { name = "test-cf-http" bucket_name = "test-cf-bundles" bundle_config = { - path = "fabric/assets/" - output_path = "bundle.zip" + path = "fabric/assets/" } iam = { "roles/run.invoker" = ["allUsers"] @@ -139,8 +134,7 @@ module "cf-http" { name = "test-cf-http" bucket_name = "test-cf-bundles" bundle_config = { - path = "fabric/assets/" - output_path = "bundle.zip" + path = "fabric/assets/" } service_account_create = true } @@ -157,8 +151,7 @@ module "cf-http" { name = "test-cf-http" bucket_name = "test-cf-bundles" bundle_config = { - path = "fabric/assets/" - output_path = "bundle.zip" + path = "fabric/assets/" } service_account = "non-existent@serice.account.email" } @@ -167,11 +160,13 @@ module "cf-http" { ### Custom bundle config -The Cloud Function bundle can be configured via the `bundle_config` variable, so that either a `zip` archive or a source folder can be used. +The Cloud Function bundle can be configured via the `bundle_config` variable. The only mandatory argument is `bundle_config.path` which can point to: -If a `zip` archive is already available, simply set the archive path in `bundle_config.path`. If a dynamically generated archive is needed, set `bundle_config.path` to the source folder path, then optionally configure the path where the archive will be created, and any exclusions needed in the archive. +- a GCS URI of a ZIP archive +- a local path to a ZIP archive +- a local path to a source folder -If you use a folder and dynamic archive bundling, be mindful that the MD5 checksum of the generated `zip` file does not change across environments (e.g. Cloud Build vs your local development environment), by ensuring that the files in the folder are always the same. +When a GCS URI or a local zip file are used, a change in their names will trigger redeployment. When a local source folder is used a ZIP archive will be automatically generated and its internally derived checksum will drive redeployment. You can optionally control its name and exclusions via the attributes in `bundle_config.folder_options`. ```hcl module "cf-http" { @@ -181,9 +176,11 @@ module "cf-http" { name = "test-cf-http" bucket_name = "test-cf-bundles" bundle_config = { - path = "fabric/assets/" - output_path = "bundle.zip" - excludes = ["__pycache__"] + path = "fabric/assets/" + folder_options = { + archive_path = "bundle.zip" + excludes = ["__pycache__"] + } } } # tftest modules=1 resources=2 @@ -202,8 +199,7 @@ module "cf-http" { bucket_name = "test-cf-bundles" build_worker_pool = "projects/my-project/locations/europe-west1/workerPools/my_build_worker_pool" bundle_config = { - path = "fabric/assets/" - output_path = "bundle.zip" + path = "fabric/assets/" } } # tftest modules=1 resources=2 @@ -211,7 +207,7 @@ module "cf-http" { ### Multiple Cloud Functions within project -When deploying multiple functions do not reuse `bundle_config.output_path` between instances as the result is undefined. Default `output_path` creates file in `/tmp` folder using project Id and function name to avoid name conflicts. +When deploying multiple functions via local folders do not reuse `bundle_config.archive_path` between instances as the result is undefined. Default `archive_path` creates file in `/tmp` folder using project Id and function name to avoid name conflicts. ```hcl module "cf-http-one" { @@ -254,8 +250,7 @@ module "cf-http" { name = "test-cf-http" bucket_name = "test-cf-bundles" bundle_config = { - path = "fabric/assets/" - output_path = "bundle.zip" + path = "fabric/assets/" } secrets = { VARIABLE_SECRET = { @@ -287,27 +282,27 @@ module "cf-http" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| | [bucket_name](variables.tf#L26) | Name of the bucket that will be used for the function code. It will be created with prefix prepended if bucket_config is not null. | string | ✓ | | -| [bundle_config](variables.tf#L38) | Cloud function source. If path points to a .zip archive it is uploaded as-is, otherwise an archive is created on the fly. A null output path will use a unique name for the bundle in /tmp. | object({…}) | ✓ | | -| [name](variables.tf#L121) | Name used for cloud function and associated resources. | string | ✓ | | -| [project_id](variables.tf#L136) | Project id used for all resources. | string | ✓ | | -| [region](variables.tf#L141) | Region used for all resources. | string | ✓ | | +| [bundle_config](variables.tf#L38) | Cloud function source. Path can point to a GCS object URI, or a local path. A local path to a zip archive will generate a GCS object using its basename, a folder will be zipped and the GCS object name inferred when not specified. | object({…}) | ✓ | | +| [name](variables.tf#L133) | Name used for cloud function and associated resources. | string | ✓ | | +| [project_id](variables.tf#L148) | Project id used for all resources. | string | ✓ | | +| [region](variables.tf#L153) | Region used for all resources. | string | ✓ | | | [bucket_config](variables.tf#L17) | Enable and configure auto-created bucket. Set fields to null to use defaults. | object({…}) | | null | | [build_worker_pool](variables.tf#L32) | Build worker pool, in projects//locations//workerPools/ format. | string | | null | -| [description](variables.tf#L59) | Optional description. | string | | "Terraform managed." | -| [docker_repository_id](variables.tf#L65) | User managed repository created in Artifact Registry. | string | | null | -| [environment_variables](variables.tf#L71) | Cloud function environment variables. | map(string) | | {} | -| [function_config](variables.tf#L77) | Cloud function configuration. Defaults to using main as entrypoint, 1 instance with 256MiB of memory, and 180 second timeout. | object({…}) | | {…} | -| [iam](variables.tf#L97) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [ingress_settings](variables.tf#L103) | Control traffic that reaches the cloud function. Allowed values are ALLOW_ALL, ALLOW_INTERNAL_AND_GCLB and ALLOW_INTERNAL_ONLY . | string | | null | -| [kms_key](variables.tf#L109) | Resource name of a KMS crypto key (managed by the user) used to encrypt/decrypt function resources in key id format. If specified, you must also provide an artifact registry repository using the docker_repository_id field that was created with the same KMS crypto key. | string | | null | -| [labels](variables.tf#L115) | Resource labels. | map(string) | | {} | -| [prefix](variables.tf#L126) | Optional prefix used for resource names. | string | | null | -| [secrets](variables.tf#L146) | Secret Manager secrets. Key is the variable name or mountpoint, volume versions are in version:path format. | map(object({…})) | | {} | -| [service_account](variables.tf#L158) | Service account email. Unused if service account is auto-created. | string | | null | -| [service_account_create](variables.tf#L164) | Auto-create service account. | bool | | false | -| [trigger_config](variables.tf#L170) | Function trigger configuration. Leave null for HTTP trigger. | object({…}) | | null | -| [vpc_connector](variables.tf#L188) | VPC connector configuration. Set create to 'true' if a new connector needs to be created. | object({…}) | | null | -| [vpc_connector_config](variables.tf#L198) | VPC connector network configuration. Must be provided if new VPC connector is being created. | object({…}) | | null | +| [description](variables.tf#L71) | Optional description. | string | | "Terraform managed." | +| [docker_repository_id](variables.tf#L77) | User managed repository created in Artifact Registry. | string | | null | +| [environment_variables](variables.tf#L83) | Cloud function environment variables. | map(string) | | {} | +| [function_config](variables.tf#L89) | Cloud function configuration. Defaults to using main as entrypoint, 1 instance with 256MiB of memory, and 180 second timeout. | object({…}) | | {…} | +| [iam](variables.tf#L109) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [ingress_settings](variables.tf#L115) | Control traffic that reaches the cloud function. Allowed values are ALLOW_ALL, ALLOW_INTERNAL_AND_GCLB and ALLOW_INTERNAL_ONLY . | string | | null | +| [kms_key](variables.tf#L121) | Resource name of a KMS crypto key (managed by the user) used to encrypt/decrypt function resources in key id format. If specified, you must also provide an artifact registry repository using the docker_repository_id field that was created with the same KMS crypto key. | string | | null | +| [labels](variables.tf#L127) | Resource labels. | map(string) | | {} | +| [prefix](variables.tf#L138) | Optional prefix used for resource names. | string | | null | +| [secrets](variables.tf#L158) | Secret Manager secrets. Key is the variable name or mountpoint, volume versions are in version:path format. | map(object({…})) | | {} | +| [service_account](variables.tf#L170) | Service account email. Unused if service account is auto-created. | string | | null | +| [service_account_create](variables.tf#L176) | Auto-create service account. | bool | | false | +| [trigger_config](variables.tf#L182) | Function trigger configuration. Leave null for HTTP trigger. | object({…}) | | null | +| [vpc_connector](variables.tf#L200) | VPC connector configuration. Set create to 'true' if a new connector needs to be created. | object({…}) | | null | +| [vpc_connector_config](variables.tf#L210) | VPC connector network configuration. Must be provided if new VPC connector is being created. | object({…}) | | null | ## Outputs diff --git a/modules/cloud-function-v2/bundle.tf b/modules/cloud-function-v2/bundle.tf index 1d046404ed..43ad5c8ee2 100644 --- a/modules/cloud-function-v2/bundle.tf +++ b/modules/cloud-function-v2/bundle.tf @@ -15,16 +15,16 @@ */ locals { - bundle = { - name = try( - "bundle-${data.archive_file.bundle[0].output_md5}.zip", - basename(var.bundle_config.path) + bundle_type = ( + startswith(var.bundle_config.path, "gs://") + ? "gcs" + : ( + try(fileexists(pathexpand(var.bundle_config.path)), null) != null && + endswith(var.bundle_config.path, ".zip") + ? "local-file" + : "local-folder" ) - path = try( - data.archive_file.bundle[0].output_path, - pathexpand(var.bundle_config.path) - ) - } + ) } resource "google_storage_bucket" "bucket" { @@ -59,20 +59,29 @@ resource "google_storage_bucket" "bucket" { # compress bundle in a zip archive if it's a folder data "archive_file" "bundle" { - count = ( - try(fileexists(pathexpand(var.bundle_config.path)), null) == null ? 1 : 0 + count = local.bundle_type == "local-folder" ? 1 : 0 + type = "zip" + source_dir = pathexpand(var.bundle_config.path) + output_path = ( + var.bundle_config.folder_options.archive_path != null + ? pathexpand(var.bundle_config.folder_options.archive_path) + : "/tmp/bundle-${var.project_id}-${var.name}.zip" ) - type = "zip" - source_dir = pathexpand(var.bundle_config.path) - output_path = coalesce(var.bundle_config.output_path, "/tmp/bundle-${var.project_id}-${var.name}.zip") output_file_mode = "0644" - excludes = var.bundle_config.excludes + excludes = var.bundle_config.folder_options.excludes } # upload to GCS resource "google_storage_bucket_object" "bundle" { - name = local.bundle.name + count = local.bundle_type != "gcs" ? 1 : 0 + name = try( + "bundle-${data.archive_file.bundle[0].output_md5}.zip", + basename(var.bundle_config.path) + ) bucket = local.bucket - source = local.bundle.path + source = try( + data.archive_file.bundle[0].output_path, + pathexpand(var.bundle_config.path) + ) } diff --git a/modules/cloud-function-v2/main.tf b/modules/cloud-function-v2/main.tf index df445ffe17..4ebfcf21c1 100644 --- a/modules/cloud-function-v2/main.tf +++ b/modules/cloud-function-v2/main.tf @@ -74,7 +74,11 @@ resource "google_cloudfunctions2_function" "function" { source { storage_source { bucket = local.bucket - object = google_storage_bucket_object.bundle.name + object = ( + local.bundle_type == "gcs" + ? var.bundle_config.path + : google_storage_bucket_object.bundle[0].name + ) } } } diff --git a/modules/cloud-function-v2/variables.tf b/modules/cloud-function-v2/variables.tf index c37e8d804c..5e41288e3a 100644 --- a/modules/cloud-function-v2/variables.tf +++ b/modules/cloud-function-v2/variables.tf @@ -36,23 +36,35 @@ variable "build_worker_pool" { } variable "bundle_config" { - description = "Cloud function source. If path points to a .zip archive it is uploaded as-is, otherwise an archive is created on the fly. A null output path will use a unique name for the bundle in /tmp." + description = "Cloud function source. Path can point to a GCS object URI, or a local path. A local path to a zip archive will generate a GCS object using its basename, a folder will be zipped and the GCS object name inferred when not specified." type = object({ - path = string - excludes = optional(list(string)) - output_path = optional(string) + path = string + folder_options = optional(object({ + archive_path = optional(string) + excludes = optional(list(string)) + }), {}) }) + nullable = false validation { condition = ( var.bundle_config.path != null && ( + # GCS object ( - try(fileexists(pathexpand(var.bundle_config.path)), null) != null && + startswith(var.bundle_config.path, "gs://") && endswith(var.bundle_config.path, ".zip") - ) || + ) + || + # local folder length(fileset(pathexpand(var.bundle_config.path), "**/*")) > 0 + || + # local ZIP archive + ( + try(fileexists(pathexpand(var.bundle_config.path)), null) != null && + endswith(var.bundle_config.path, ".zip") + ) ) ) - error_message = "Bundle path must be set to a local folder or zip file." + error_message = "Bundle path must be set to a GCS object URI, a local folder or a local zip file." } } diff --git a/tests/modules/cloud_function_v1/examples/multiple_functions.yaml b/tests/modules/cloud_function_v1/examples/multiple_functions.yaml index b9956db5b6..2fa803f7db 100644 --- a/tests/modules/cloud_function_v1/examples/multiple_functions.yaml +++ b/tests/modules/cloud_function_v1/examples/multiple_functions.yaml @@ -13,9 +13,9 @@ # limitations under the License. values: - module.cf-http-one.google_storage_bucket_object.bundle: + module.cf-http-one.google_storage_bucket_object.bundle[0]: source: /tmp/bundle-my-project-test-cf-http-one.zip - module.cf-http-two.google_storage_bucket_object.bundle: + module.cf-http-two.google_storage_bucket_object.bundle[0]: source: /tmp/bundle-my-project-test-cf-http-two.zip counts: diff --git a/tests/modules/cloud_function_v2/examples/iam.yaml b/tests/modules/cloud_function_v2/examples/iam.yaml index 11f656fd1c..08dc5c430d 100644 --- a/tests/modules/cloud_function_v2/examples/iam.yaml +++ b/tests/modules/cloud_function_v2/examples/iam.yaml @@ -22,12 +22,12 @@ values: role: roles/run.invoker service: test-cf-http module.cf-http.google_cloudfunctions2_function.function: {} - module.cf-http.google_storage_bucket_object.bundle: + module.cf-http.google_storage_bucket_object.bundle[0]: bucket: test-cf-bundles customer_encryption: [] detect_md5hash: different hash name: bundle-6f1ece136848fee658e335b05fe2d79d.zip - source: bundle.zip + source: /tmp/bundle-my-project-test-cf-http.zip counts: google_cloud_run_service_iam_binding: 1 diff --git a/tests/modules/cloud_function_v2/examples/multiple_functions.yaml b/tests/modules/cloud_function_v2/examples/multiple_functions.yaml index bcff9c270e..e7cf10d540 100644 --- a/tests/modules/cloud_function_v2/examples/multiple_functions.yaml +++ b/tests/modules/cloud_function_v2/examples/multiple_functions.yaml @@ -13,9 +13,9 @@ # limitations under the License. values: - module.cf-http-one.google_storage_bucket_object.bundle: + module.cf-http-one.google_storage_bucket_object.bundle[0]: source: /tmp/bundle-my-project-test-cf-http-one.zip - module.cf-http-two.google_storage_bucket_object.bundle: + module.cf-http-two.google_storage_bucket_object.bundle[0]: source: /tmp/bundle-my-project-test-cf-http-two.zip counts: