From 98accdb3ad70aff5f272b0c51d5985e32c7c9432 Mon Sep 17 00:00:00 2001 From: luigi-bitonti <93377317+luigi-bitonti@users.noreply.github.com> Date: Fri, 24 Nov 2023 15:47:45 +0100 Subject: [PATCH] Added PSC support to CloudSQL Module (#1874) * Added Feature * Added PSC to CloudSQL module * Added psc to read replica * Changed variables * Updated README * Ran fmt * Removed old variables * Fix README * Fixed blueprints * Fix README * Fixed output * Added more outputs and bug fixes * Changed variable structure * Bug fix * Added PSC example. --- .../cloudsql-multiregion/cloudsql.tf | 16 +- .../third-party-solutions/phpipam/cloudsql.tf | 14 +- .../wordpress/cloudrun/cloudsql.tf | 12 +- modules/cloudsql-instance/README.md | 181 ++++++++++++------ modules/cloudsql-instance/main.tf | 32 +++- modules/cloudsql-instance/outputs.tf | 26 +++ modules/cloudsql-instance/variables.tf | 51 ++--- 7 files changed, 219 insertions(+), 113 deletions(-) diff --git a/blueprints/data-solutions/cloudsql-multiregion/cloudsql.tf b/blueprints/data-solutions/cloudsql-multiregion/cloudsql.tf index 5dab5d6b60..020466e2bd 100644 --- a/blueprints/data-solutions/cloudsql-multiregion/cloudsql.tf +++ b/blueprints/data-solutions/cloudsql-multiregion/cloudsql.tf @@ -17,11 +17,17 @@ module "db" { project_id = module.project.project_id availability_type = var.sql_configuration.availability_type encryption_key_name = var.service_encryption_keys != null ? try(var.service_encryption_keys[var.regions.primary], null) : null - network = local.vpc_self_link - name = "${var.prefix}-db" - region = var.regions.primary - database_version = var.sql_configuration.database_version - tier = var.sql_configuration.tier + network_config = { + connectivity = { + psa_config = { + private_network = local.vpc_self_link + } + } + } + name = "${var.prefix}-db" + region = var.regions.primary + database_version = var.sql_configuration.database_version + tier = var.sql_configuration.tier flags = { "cloudsql.iam_authentication" = "on" } diff --git a/blueprints/third-party-solutions/phpipam/cloudsql.tf b/blueprints/third-party-solutions/phpipam/cloudsql.tf index f3b7a78665..1cf238d7f6 100644 --- a/blueprints/third-party-solutions/phpipam/cloudsql.tf +++ b/blueprints/third-party-solutions/phpipam/cloudsql.tf @@ -22,10 +22,16 @@ module "cloudsql" { database_version = local.cloudsql_conf.database_version deletion_protection = var.deletion_protection databases = [local.cloudsql_conf.db] - network = local.network - prefix = var.prefix - region = var.region - tier = local.cloudsql_conf.tier + network_config = { + connectivity = { + psa_config = { + private_network = local.network + } + } + } + prefix = var.prefix + region = var.region + tier = local.cloudsql_conf.tier users = { "${local.cloudsql_conf.user}" = { password = var.cloudsql_password diff --git a/blueprints/third-party-solutions/wordpress/cloudrun/cloudsql.tf b/blueprints/third-party-solutions/wordpress/cloudrun/cloudsql.tf index 2ebe9e1436..3daa12dad8 100644 --- a/blueprints/third-party-solutions/wordpress/cloudrun/cloudsql.tf +++ b/blueprints/third-party-solutions/wordpress/cloudrun/cloudsql.tf @@ -52,9 +52,15 @@ resource "google_vpc_access_connector" "connector" { # Set up CloudSQL module "cloudsql" { - source = "../../../../modules/cloudsql-instance" - project_id = module.project.project_id - network = module.vpc.self_link + source = "../../../../modules/cloudsql-instance" + project_id = module.project.project_id + network_config = { + connectivity = { + psa_config = { + private_network = module.vpc.self_link + } + } + } name = "${var.prefix}-mysql" region = var.region database_version = local.cloudsql_conf.database_version diff --git a/modules/cloudsql-instance/README.md b/modules/cloudsql-instance/README.md index ce633fa68d..130a780f51 100644 --- a/modules/cloudsql-instance/README.md +++ b/modules/cloudsql-instance/README.md @@ -31,9 +31,15 @@ module "vpc" { } module "db" { - source = "./fabric/modules/cloudsql-instance" - project_id = module.project.project_id - network = module.vpc.self_link + source = "./fabric/modules/cloudsql-instance" + project_id = module.project.project_id + network_config = { + connectivity = { + psa_config = { + private_network = module.vpc.self_link + } + } + } name = "db" region = "europe-west1" database_version = "POSTGRES_13" @@ -46,9 +52,15 @@ module "db" { ```hcl module "db" { - source = "./fabric/modules/cloudsql-instance" - project_id = var.project_id - network = var.vpc.self_link + source = "./fabric/modules/cloudsql-instance" + project_id = var.project_id + network_config = { + connectivity = { + psa_config = { + private_network = var.vpc.self_link + } + } + } prefix = "myprefix" name = "db" region = "europe-west1" @@ -67,9 +79,15 @@ module "db" { ```hcl module "db" { - source = "./fabric/modules/cloudsql-instance" - project_id = var.project_id - network = var.vpc.self_link + source = "./fabric/modules/cloudsql-instance" + project_id = var.project_id + network_config = { + connectivity = { + psa_config = { + private_network = var.vpc.self_link + } + } + } name = "db" region = "europe-west1" database_version = "MYSQL_8_0" @@ -134,30 +152,63 @@ module "db" { source = "./fabric/modules/cloudsql-instance" project_id = module.project.project_id encryption_key_name = module.kms.keys["key-sql"].id - network = var.vpc.self_link - name = "db" - region = var.region - database_version = "POSTGRES_13" - tier = "db-g1-small" + network_config = { + connectivity = { + psa_config = { + private_network = var.vpc.self_link + } + } + } + name = "db" + region = var.region + database_version = "POSTGRES_13" + tier = "db-g1-small" } # tftest modules=3 resources=10 ``` +### Instance with PSC enabled + +```hcl +module "db" { + source = "./fabric/modules/cloudsql-instance" + project_id = var.project_id + network_config = { + connectivity = { + psc_allowed_consumer_projects = ["my-project-id"] + } + } + prefix = "myprefix" + name = "db" + region = "europe-west1" + availability_type = "REGIONAL" + database_version = "POSTGRES_13" + tier = "db-g1-small" +} +# tftest modules=1 resources=1 +``` + ### Enable public IP Use `ipv_enabled` to create instances with a public IP. ```hcl module "db" { - source = "./fabric/modules/cloudsql-instance" - project_id = var.project_id - network = var.vpc.self_link + source = "./fabric/modules/cloudsql-instance" + project_id = var.project_id + network_config = { + connectivity = { + public_ipv4 = true + psa_config = { + private_network = var.vpc.self_link + } + } + } name = "db" region = "europe-west1" tier = "db-g1-small" database_version = "MYSQL_8_0" - ipv4_enabled = true replicas = { replica1 = { region = "europe-west3", encryption_key_name = null } } @@ -171,9 +222,15 @@ Provide `insights_config` (can be just empty `{}`) to enable [Query Insights](ht ```hcl module "db" { - source = "./fabric/modules/cloudsql-instance" - project_id = var.project_id - network = var.vpc.self_link + source = "./fabric/modules/cloudsql-instance" + project_id = var.project_id + network_config = { + connectivity = { + psa_config = { + private_network = var.vpc.self_link + } + } + } name = "db" region = "europe-west1" database_version = "POSTGRES_13" @@ -190,37 +247,33 @@ module "db" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [database_version](variables.tf#L83) | Database type and version to create. | string | ✓ | | -| [name](variables.tf#L167) | Name of primary instance. | string | ✓ | | -| [network](variables.tf#L172) | VPC self link where the instances will be deployed. Private Service Networking must be enabled and configured in this VPC. | string | ✓ | | -| [project_id](variables.tf#L193) | The ID of the project where this instances will be created. | string | ✓ | | -| [region](variables.tf#L198) | Region of the primary instance. | string | ✓ | | -| [tier](variables.tf#L224) | The machine type to use for the instances. | string | ✓ | | +| [database_version](variables.tf#L68) | Database type and version to create. | string | ✓ | | +| [name](variables.tf#L146) | Name of primary instance. | string | ✓ | | +| [network_config](variables.tf#L151) | Network configuration for the instance. Only one between private_network and psc_config can be used. | object({…}) | ✓ | | +| [project_id](variables.tf#L190) | The ID of the project where this instances will be created. | string | ✓ | | +| [region](variables.tf#L195) | Region of the primary instance. | string | ✓ | | +| [tier](variables.tf#L215) | The machine type to use for the instances. | string | ✓ | | | [activation_policy](variables.tf#L16) | This variable specifies when the instance should be active. Can be either ALWAYS, NEVER or ON_DEMAND. Default is ALWAYS. | string | | "ALWAYS" | -| [allocated_ip_ranges](variables.tf#L27) | (Optional)The name of the allocated ip range for the private ip CloudSQL instance. For example: \"google-managed-services-default\". If set, the instance ip will be created in the allocated range. The range name must comply with RFC 1035. Specifically, the name must be 1-63 characters long and match the regular expression a-z?. | object({…}) | | {} | -| [authorized_networks](variables.tf#L36) | Map of NAME=>CIDR_RANGE to allow to connect to the database(s). | map(string) | | null | -| [availability_type](variables.tf#L42) | Availability type for the primary replica. Either `ZONAL` or `REGIONAL`. | string | | "ZONAL" | -| [backup_configuration](variables.tf#L48) | Backup settings for primary instance. Will be automatically enabled if using MySQL with one or more replicas. | object({…}) | | {…} | -| [collation](variables.tf#L71) | The name of server instance collation. | string | | null | -| [connector_enforcement](variables.tf#L77) | Specifies if connections must use Cloud SQL connectors. | string | | null | -| [databases](variables.tf#L88) | Databases to create once the primary instance is created. | list(string) | | null | -| [deletion_protection](variables.tf#L94) | Prevent terraform from deleting instances. | bool | | true | -| [deletion_protection_enabled](variables.tf#L101) | Set Google's deletion protection attribute which applies across all surfaces (UI, API, & Terraform). | bool | | true | -| [disk_autoresize_limit](variables.tf#L108) | The maximum size to which storage capacity can be automatically increased. The default value is 0, which specifies that there is no limit. | number | | 0 | -| [disk_size](variables.tf#L114) | Disk size in GB. Set to null to enable autoresize. | number | | null | -| [disk_type](variables.tf#L120) | The type of data disk: `PD_SSD` or `PD_HDD`. | string | | "PD_SSD" | -| [edition](variables.tf#L126) | The edition of the instance, can be ENTERPRISE or ENTERPRISE_PLUS. | string | | "ENTERPRISE" | -| [encryption_key_name](variables.tf#L132) | The full path to the encryption key used for the CMEK disk encryption of the primary instance. | string | | null | -| [flags](variables.tf#L138) | Map FLAG_NAME=>VALUE for database-specific tuning. | map(string) | | null | -| [insights_config](variables.tf#L144) | Query Insights configuration. Defaults to null which disables Query Insights. | object({…}) | | null | -| [ipv4_enabled](variables.tf#L155) | Add a public IP address to database instance. | bool | | false | -| [labels](variables.tf#L161) | Labels to be attached to all instances. | map(string) | | null | -| [postgres_client_certificates](variables.tf#L177) | Map of cert keys connect to the application(s) using public IP. | list(string) | | null | -| [prefix](variables.tf#L183) | Optional prefix used to generate instance names. | string | | null | -| [replicas](variables.tf#L203) | Map of NAME=> {REGION, KMS_KEY} for additional read replicas. Set to null to disable replica creation. | map(object({…})) | | {} | -| [require_ssl](variables.tf#L212) | Enable SSL connections only. | bool | | null | -| [root_password](variables.tf#L218) | Root password of the Cloud SQL instance. Required for MS SQL Server. | string | | null | -| [users](variables.tf#L229) | Map of users to create in the primary instance (and replicated to other replicas). For MySQL, anything afterr the first `@` (if persent) will be used as the user's host. Set PASSWORD to null if you want to get an autogenerated password. The user types available are: 'BUILT_IN', 'CLOUD_IAM_USER' or 'CLOUD_IAM_SERVICE_ACCOUNT'. | map(object({…})) | | null | +| [availability_type](variables.tf#L27) | Availability type for the primary replica. Either `ZONAL` or `REGIONAL`. | string | | "ZONAL" | +| [backup_configuration](variables.tf#L33) | Backup settings for primary instance. Will be automatically enabled if using MySQL with one or more replicas. | object({…}) | | {…} | +| [collation](variables.tf#L56) | The name of server instance collation. | string | | null | +| [connector_enforcement](variables.tf#L62) | Specifies if connections must use Cloud SQL connectors. | string | | null | +| [databases](variables.tf#L73) | Databases to create once the primary instance is created. | list(string) | | null | +| [deletion_protection](variables.tf#L79) | Prevent terraform from deleting instances. | bool | | true | +| [deletion_protection_enabled](variables.tf#L86) | Set Google's deletion protection attribute which applies across all surfaces (UI, API, & Terraform). | bool | | true | +| [disk_autoresize_limit](variables.tf#L93) | The maximum size to which storage capacity can be automatically increased. The default value is 0, which specifies that there is no limit. | number | | 0 | +| [disk_size](variables.tf#L99) | Disk size in GB. Set to null to enable autoresize. | number | | null | +| [disk_type](variables.tf#L105) | The type of data disk: `PD_SSD` or `PD_HDD`. | string | | "PD_SSD" | +| [edition](variables.tf#L111) | The edition of the instance, can be ENTERPRISE or ENTERPRISE_PLUS. | string | | "ENTERPRISE" | +| [encryption_key_name](variables.tf#L117) | The full path to the encryption key used for the CMEK disk encryption of the primary instance. | string | | null | +| [flags](variables.tf#L123) | Map FLAG_NAME=>VALUE for database-specific tuning. | map(string) | | null | +| [insights_config](variables.tf#L129) | Query Insights configuration. Defaults to null which disables Query Insights. | object({…}) | | null | +| [labels](variables.tf#L140) | Labels to be attached to all instances. | map(string) | | null | +| [postgres_client_certificates](variables.tf#L174) | Map of cert keys connect to the application(s) using public IP. | list(string) | | null | +| [prefix](variables.tf#L180) | Optional prefix used to generate instance names. | string | | null | +| [replicas](variables.tf#L200) | Map of NAME=> {REGION, KMS_KEY} for additional read replicas. Set to null to disable replica creation. | map(object({…})) | | {} | +| [root_password](variables.tf#L209) | Root password of the Cloud SQL instance. Required for MS SQL Server. | string | | null | +| [users](variables.tf#L220) | Map of users to create in the primary instance (and replicated to other replicas). For MySQL, anything afterr the first `@` (if persent) will be used as the user's host. Set PASSWORD to null if you want to get an autogenerated password. The user types available are: 'BUILT_IN', 'CLOUD_IAM_USER' or 'CLOUD_IAM_SERVICE_ACCOUNT'. | map(object({…})) | | null | ## Outputs @@ -228,15 +281,19 @@ module "db" { |---|---|:---:| | [connection_name](outputs.tf#L24) | Connection name of the primary instance. | | | [connection_names](outputs.tf#L29) | Connection names of all instances. | | -| [id](outputs.tf#L37) | Fully qualified primary instance id. | | -| [ids](outputs.tf#L42) | Fully qualified ids of all instances. | | -| [instances](outputs.tf#L50) | Cloud SQL instance resources. | ✓ | -| [ip](outputs.tf#L56) | IP address of the primary instance. | | -| [ips](outputs.tf#L61) | IP addresses of all instances. | | -| [name](outputs.tf#L69) | Name of the primary instance. | | -| [names](outputs.tf#L74) | Names of all instances. | | -| [postgres_client_certificates](outputs.tf#L82) | The CA Certificate used to connect to the SQL Instance via SSL. | ✓ | -| [self_link](outputs.tf#L88) | Self link of the primary instance. | | -| [self_links](outputs.tf#L93) | Self links of all instances. | | -| [user_passwords](outputs.tf#L101) | Map of containing the password of all users created through terraform. | ✓ | +| [dns_name](outputs.tf#L37) | The dns name of the instance. | | +| [dns_names](outputs.tf#L42) | Dns names of all instances. | | +| [id](outputs.tf#L50) | Fully qualified primary instance id. | | +| [ids](outputs.tf#L55) | Fully qualified ids of all instances. | | +| [instances](outputs.tf#L63) | Cloud SQL instance resources. | ✓ | +| [ip](outputs.tf#L69) | IP address of the primary instance. | | +| [ips](outputs.tf#L74) | IP addresses of all instances. | | +| [name](outputs.tf#L82) | Name of the primary instance. | | +| [names](outputs.tf#L87) | Names of all instances. | | +| [postgres_client_certificates](outputs.tf#L95) | The CA Certificate used to connect to the SQL Instance via SSL. | ✓ | +| [psc_service_attachment_link](outputs.tf#L101) | The link to service attachment of PSC instance. | | +| [psc_service_attachment_links](outputs.tf#L106) | Links to service attachment of PSC instances. | | +| [self_link](outputs.tf#L114) | Self link of the primary instance. | | +| [self_links](outputs.tf#L119) | Self links of all instances. | | +| [user_passwords](outputs.tf#L127) | Map of containing the password of all users created through terraform. | ✓ | diff --git a/modules/cloudsql-instance/main.tf b/modules/cloudsql-instance/main.tf index 8b36d91bac..a39469e829 100644 --- a/modules/cloudsql-instance/main.tf +++ b/modules/cloudsql-instance/main.tf @@ -68,18 +68,25 @@ resource "google_sql_database_instance" "primary" { connector_enforcement = var.connector_enforcement ip_configuration { - ipv4_enabled = var.ipv4_enabled - private_network = var.network - allocated_ip_range = var.allocated_ip_ranges.primary - require_ssl = var.require_ssl + ipv4_enabled = var.network_config.connectivity.public_ipv4 + private_network = try(var.network_config.connectivity.psa_config.private_network, null) + allocated_ip_range = try(var.network_config.connectivity.psa_config.allocated_ip_ranges.primary, null) + require_ssl = var.network_config.require_ssl dynamic "authorized_networks" { - for_each = var.authorized_networks != null ? var.authorized_networks : {} + for_each = var.network_config.authorized_networks != null ? var.network_config.authorized_networks : {} iterator = network content { name = network.key value = network.value } } + dynamic "psc_config" { + for_each = var.network_config.connectivity.psc_allowed_consumer_projects != null ? [""] : [] + content { + psc_enabled = true + allowed_consumer_projects = var.network_config.connectivity.psc_allowed_consumer_projects + } + } } dynamic "backup_configuration" { @@ -149,17 +156,24 @@ resource "google_sql_database_instance" "replicas" { activation_policy = var.activation_policy ip_configuration { - ipv4_enabled = var.ipv4_enabled - private_network = var.network - allocated_ip_range = var.allocated_ip_ranges.replica + ipv4_enabled = var.network_config.connectivity.public_ipv4 + private_network = try(var.network_config.connectivity.psa_config.private_network, null) + allocated_ip_range = try(var.network_config.connectivity.psa_config.allocated_ip_ranges.replica, null) dynamic "authorized_networks" { - for_each = var.authorized_networks != null ? var.authorized_networks : {} + for_each = var.network_config.authorized_networks != null ? var.network_config.authorized_networks : {} iterator = network content { name = network.key value = network.value } } + dynamic "psc_config" { + for_each = var.network_config.connectivity.psc_allowed_consumer_projects != null ? [""] : [] + content { + psc_enabled = true + allowed_consumer_projects = var.network_config.connectivity.psc_allowed_consumer_projects + } + } } dynamic "database_flags" { diff --git a/modules/cloudsql-instance/outputs.tf b/modules/cloudsql-instance/outputs.tf index 1859d8453c..c78cfd4baa 100644 --- a/modules/cloudsql-instance/outputs.tf +++ b/modules/cloudsql-instance/outputs.tf @@ -34,6 +34,19 @@ output "connection_names" { } } +output "dns_name" { + description = "The dns name of the instance." + value = google_sql_database_instance.primary.dns_name +} + +output "dns_names" { + description = "Dns names of all instances." + value = { + for id, instance in local._all_instances : + id => instance.dns_name + } +} + output "id" { description = "Fully qualified primary instance id." value = google_sql_database_instance.primary.private_ip_address @@ -85,6 +98,19 @@ output "postgres_client_certificates" { sensitive = true } +output "psc_service_attachment_link" { + description = "The link to service attachment of PSC instance." + value = google_sql_database_instance.primary.psc_service_attachment_link +} + +output "psc_service_attachment_links" { + description = "Links to service attachment of PSC instances." + value = { + for id, instance in local._all_instances : + id => instance.psc_service_attachment_link + } +} + output "self_link" { description = "Self link of the primary instance." value = google_sql_database_instance.primary.self_link diff --git a/modules/cloudsql-instance/variables.tf b/modules/cloudsql-instance/variables.tf index 0bb58cd22f..7ee516bf51 100644 --- a/modules/cloudsql-instance/variables.tf +++ b/modules/cloudsql-instance/variables.tf @@ -24,21 +24,6 @@ variable "activation_policy" { nullable = false } -variable "allocated_ip_ranges" { - description = "(Optional)The name of the allocated ip range for the private ip CloudSQL instance. For example: \"google-managed-services-default\". If set, the instance ip will be created in the allocated range. The range name must comply with RFC 1035. Specifically, the name must be 1-63 characters long and match the regular expression a-z?." - type = object({ - primary = optional(string) - replica = optional(string) - }) - default = {} - nullable = false -} -variable "authorized_networks" { - description = "Map of NAME=>CIDR_RANGE to allow to connect to the database(s)." - type = map(string) - default = null -} - variable "availability_type" { description = "Availability type for the primary replica. Either `ZONAL` or `REGIONAL`." type = string @@ -152,12 +137,6 @@ variable "insights_config" { default = null } -variable "ipv4_enabled" { - description = "Add a public IP address to database instance." - type = bool - default = false -} - variable "labels" { description = "Labels to be attached to all instances." type = map(string) @@ -169,9 +148,27 @@ variable "name" { type = string } -variable "network" { - description = "VPC self link where the instances will be deployed. Private Service Networking must be enabled and configured in this VPC." - type = string +variable "network_config" { + description = "Network configuration for the instance. Only one between private_network and psc_config can be used." + type = object({ + authorized_networks = optional(map(string)) + require_ssl = optional(bool) + connectivity = object({ + public_ipv4 = optional(bool, false) + psa_config = optional(object({ + private_network = string + allocated_ip_ranges = optional(object({ + primary = optional(string) + replica = optional(string) + })) + })) + psc_allowed_consumer_projects = optional(list(string)) + }) + }) + validation { + condition = (var.network_config.connectivity.psa_config != null ? 1 : 0) + (var.network_config.connectivity.psc_allowed_consumer_projects != null ? 1 : 0) < 2 + error_message = "Only one between private network and psc can be specified." + } } variable "postgres_client_certificates" { @@ -209,12 +206,6 @@ variable "replicas" { default = {} } -variable "require_ssl" { - description = "Enable SSL connections only." - type = bool - default = null -} - variable "root_password" { description = "Root password of the Cloud SQL instance. Required for MS SQL Server." type = string