From 67c42707ee939af18919651ec28c6c4845d4912d Mon Sep 17 00:00:00 2001 From: Ludo Date: Thu, 1 Feb 2024 15:48:51 +0100 Subject: [PATCH 1/2] default to shielded nodes in FAST gke stage --- fast/stages/3-gke-multitenant/dev/README.md | 30 +++++++++---------- .../stages/3-gke-multitenant/dev/variables.tf | 1 + 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/fast/stages/3-gke-multitenant/dev/README.md b/fast/stages/3-gke-multitenant/dev/README.md index 2b7bace650..eba77e01bc 100644 --- a/fast/stages/3-gke-multitenant/dev/README.md +++ b/fast/stages/3-gke-multitenant/dev/README.md @@ -216,21 +216,21 @@ Leave all these variables unset (or set to `null`) to disable fleet management. |---|---|:---:|:---:|:---:|:---:| | [automation](variables.tf#L21) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 0-bootstrap | | [billing_account](variables.tf#L29) | Billing account id. If billing account is not part of the same org set `is_org_level` to false. | object({…}) | ✓ | | 0-bootstrap | -| [folder_ids](variables.tf#L174) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | ✓ | | 1-resman | -| [host_project_ids](variables.tf#L189) | Host project for the shared VPC. | object({…}) | ✓ | | 2-networking | -| [prefix](variables.tf#L241) | Prefix used for resources that need unique names. | string | ✓ | | | -| [vpc_self_links](variables.tf#L257) | Self link for the shared VPC. | object({…}) | ✓ | | 2-networking | -| [clusters](variables.tf#L42) | Clusters configuration. Refer to the gke-cluster-standard module for type details. | map(object({…})) | | {} | | -| [fleet_configmanagement_clusters](variables.tf#L111) | Config management features enabled on specific sets of member clusters, in config name => [cluster name] format. | map(list(string)) | | {} | | -| [fleet_configmanagement_templates](variables.tf#L119) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(object({…})) | | {} | | -| [fleet_features](variables.tf#L154) | Enable and configure fleet features. Set to null to disable GKE Hub if fleet workload identity is not used. | object({…}) | | null | | -| [fleet_workload_identity](variables.tf#L167) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool | | false | | -| [group_iam](variables.tf#L182) | Project-level authoritative IAM bindings for groups in {GROUP_EMAIL => [ROLES]} format. Use group emails as keys, list of roles as values. | map(list(string)) | | {} | | -| [iam](variables.tf#L197) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | -| [labels](variables.tf#L204) | Project-level labels. | map(string) | | {} | | -| [nodepools](variables.tf#L210) | Nodepools configuration. Refer to the gke-nodepool module for type details. | map(map(object({…}))) | | {} | | -| [outputs_location](variables.tf#L235) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | -| [project_services](variables.tf#L250) | Additional project services to enable. | list(string) | | [] | | +| [folder_ids](variables.tf#L175) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | ✓ | | 1-resman | +| [host_project_ids](variables.tf#L190) | Host project for the shared VPC. | object({…}) | ✓ | | 2-networking | +| [prefix](variables.tf#L242) | Prefix used for resources that need unique names. | string | ✓ | | | +| [vpc_self_links](variables.tf#L258) | Self link for the shared VPC. | object({…}) | ✓ | | 2-networking | +| [clusters](variables.tf#L42) | Clusters configuration. Refer to the gke-cluster-standard module for type details. | map(object({…})) | | {} | | +| [fleet_configmanagement_clusters](variables.tf#L112) | Config management features enabled on specific sets of member clusters, in config name => [cluster name] format. | map(list(string)) | | {} | | +| [fleet_configmanagement_templates](variables.tf#L120) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(object({…})) | | {} | | +| [fleet_features](variables.tf#L155) | Enable and configure fleet features. Set to null to disable GKE Hub if fleet workload identity is not used. | object({…}) | | null | | +| [fleet_workload_identity](variables.tf#L168) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool | | false | | +| [group_iam](variables.tf#L183) | Project-level authoritative IAM bindings for groups in {GROUP_EMAIL => [ROLES]} format. Use group emails as keys, list of roles as values. | map(list(string)) | | {} | | +| [iam](variables.tf#L198) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | +| [labels](variables.tf#L205) | Project-level labels. | map(string) | | {} | | +| [nodepools](variables.tf#L211) | Nodepools configuration. Refer to the gke-nodepool module for type details. | map(map(object({…}))) | | {} | | +| [outputs_location](variables.tf#L236) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | +| [project_services](variables.tf#L251) | Additional project services to enable. | list(string) | | [] | | ## Outputs diff --git a/fast/stages/3-gke-multitenant/dev/variables.tf b/fast/stages/3-gke-multitenant/dev/variables.tf index 50dff44a63..ce6a58095d 100644 --- a/fast/stages/3-gke-multitenant/dev/variables.tf +++ b/fast/stages/3-gke-multitenant/dev/variables.tf @@ -48,6 +48,7 @@ variable "clusters" { horizontal_pod_autoscaling = true, http_load_balancing = true }) enable_features = optional(any, { + shielded_nodes = true workload_identity = true }) issue_client_certificate = optional(bool, false) From 3634a18058ea1899050f152055a0b927b6d304e5 Mon Sep 17 00:00:00 2001 From: Ludo Date: Thu, 1 Feb 2024 16:00:39 +0100 Subject: [PATCH 2/2] use custom service account in GKE multitenant blueprint --- blueprints/gke/multitenant-fleet/README.md | 36 +++++++++---------- .../gke/multitenant-fleet/gke-nodepools.tf | 36 ++++++++++--------- blueprints/gke/multitenant-fleet/main.tf | 22 ++++++++++++ blueprints/gke/multitenant-fleet/variables.tf | 5 +-- 4 files changed, 61 insertions(+), 38 deletions(-) diff --git a/blueprints/gke/multitenant-fleet/README.md b/blueprints/gke/multitenant-fleet/README.md index 6ce8294a7a..fb959ebf44 100644 --- a/blueprints/gke/multitenant-fleet/README.md +++ b/blueprints/gke/multitenant-fleet/README.md @@ -115,7 +115,7 @@ module "gke-fleet" { vpc_self_link = "projects/prj-host/global/networks/prod-0" } } -# tftest modules=7 resources=27 +# tftest modules=8 resources=33 ``` ## GKE Fleet @@ -218,7 +218,7 @@ module "gke" { vpc_self_link = "projects/prj-host/global/networks/prod-0" } } -# tftest modules=8 resources=38 +# tftest modules=9 resources=44 ``` @@ -230,7 +230,7 @@ module "gke" { | [gke-clusters.tf](./gke-clusters.tf) | GKE clusters. | gke-cluster-standard | | [gke-hub.tf](./gke-hub.tf) | GKE hub configuration. | gke-hub | | [gke-nodepools.tf](./gke-nodepools.tf) | GKE nodepools. | gke-nodepool | -| [main.tf](./main.tf) | Project and usage dataset. | bigquery-dataset · project | +| [main.tf](./main.tf) | Project and usage dataset. | bigquery-dataset · iam-service-account · project | | [outputs.tf](./outputs.tf) | Output variables. | | | [variables.tf](./variables.tf) | Module variables. | | @@ -239,21 +239,21 @@ module "gke" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| | [billing_account_id](variables.tf#L17) | Billing account ID. | string | ✓ | | -| [folder_id](variables.tf#L134) | Folder used for the GKE project in folders/nnnnnnnnnnn format. | string | ✓ | | -| [prefix](variables.tf#L184) | Prefix used for resource names. | string | ✓ | | -| [project_id](variables.tf#L193) | ID of the project that will contain all the clusters. | string | ✓ | | -| [vpc_config](variables.tf#L205) | Shared VPC project and VPC details. | object({…}) | ✓ | | -| [clusters](variables.tf#L22) | Clusters configuration. Refer to the gke-cluster module for type details. | map(object({…})) | | {} | -| [deletion_protection](variables.tf#L92) | Prevent Terraform from destroying data storage resources (storage buckets, GKE clusters, CloudSQL instances) in this blueprint. When this field is set in Terraform state, a terraform destroy or terraform apply that would delete data storage resources will fail. | bool | | false | -| [fleet_configmanagement_clusters](variables.tf#L99) | Config management features enabled on specific sets of member clusters, in config name => [cluster name] format. | map(list(string)) | | {} | -| [fleet_configmanagement_templates](variables.tf#L106) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(any) | | {} | -| [fleet_features](variables.tf#L114) | Enable and configure fleet features. Set to null to disable GKE Hub if fleet workload identity is not used. | object({…}) | | null | -| [fleet_workload_identity](variables.tf#L127) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool | | false | -| [group_iam](variables.tf#L139) | Project-level IAM bindings for groups. Use group emails as keys, list of roles as values. | map(list(string)) | | {} | -| [iam](variables.tf#L146) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [labels](variables.tf#L153) | Project-level labels. | map(string) | | {} | -| [nodepools](variables.tf#L159) | Nodepools configuration. Refer to the gke-nodepool module for type details. | map(map(object({…}))) | | {} | -| [project_services](variables.tf#L198) | Additional project services to enable. | list(string) | | [] | +| [folder_id](variables.tf#L131) | Folder used for the GKE project in folders/nnnnnnnnnnn format. | string | ✓ | | +| [prefix](variables.tf#L181) | Prefix used for resource names. | string | ✓ | | +| [project_id](variables.tf#L190) | ID of the project that will contain all the clusters. | string | ✓ | | +| [vpc_config](variables.tf#L202) | Shared VPC project and VPC details. | object({…}) | ✓ | | +| [clusters](variables.tf#L22) | Clusters configuration. Refer to the gke-cluster module for type details. | map(object({…})) | | {} | +| [deletion_protection](variables.tf#L89) | Prevent Terraform from destroying data storage resources (storage buckets, GKE clusters, CloudSQL instances) in this blueprint. When this field is set in Terraform state, a terraform destroy or terraform apply that would delete data storage resources will fail. | bool | | false | +| [fleet_configmanagement_clusters](variables.tf#L96) | Config management features enabled on specific sets of member clusters, in config name => [cluster name] format. | map(list(string)) | | {} | +| [fleet_configmanagement_templates](variables.tf#L103) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(any) | | {} | +| [fleet_features](variables.tf#L111) | Enable and configure fleet features. Set to null to disable GKE Hub if fleet workload identity is not used. | object({…}) | | null | +| [fleet_workload_identity](variables.tf#L124) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool | | false | +| [group_iam](variables.tf#L136) | Project-level IAM bindings for groups. Use group emails as keys, list of roles as values. | map(list(string)) | | {} | +| [iam](variables.tf#L143) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [labels](variables.tf#L150) | Project-level labels. | map(string) | | {} | +| [nodepools](variables.tf#L156) | Nodepools configuration. Refer to the gke-nodepool module for type details. | map(map(object({…}))) | | {} | +| [project_services](variables.tf#L195) | Additional project services to enable. | list(string) | | [] | ## Outputs diff --git a/blueprints/gke/multitenant-fleet/gke-nodepools.tf b/blueprints/gke/multitenant-fleet/gke-nodepools.tf index fdadf95455..692b52ac72 100644 --- a/blueprints/gke/multitenant-fleet/gke-nodepools.tf +++ b/blueprints/gke/multitenant-fleet/gke-nodepools.tf @@ -29,22 +29,26 @@ locals { } module "gke-nodepool" { - source = "../../../modules/gke-nodepool" - for_each = local.nodepools - name = each.value.name - project_id = module.gke-project-0.project_id - cluster_name = module.gke-cluster[each.value.cluster].name - location = module.gke-cluster[each.value.cluster].location - gke_version = each.value.gke_version - labels = each.value.labels - max_pods_per_node = each.value.max_pods_per_node - node_config = each.value.node_config - node_count = each.value.node_count - node_locations = each.value.node_locations - nodepool_config = each.value.nodepool_config - pod_range = each.value.pod_range - reservation_affinity = each.value.reservation_affinity - service_account = each.value.service_account + source = "../../../modules/gke-nodepool" + for_each = local.nodepools + name = each.value.name + project_id = module.gke-project-0.project_id + cluster_name = module.gke-cluster[each.value.cluster].name + location = module.gke-cluster[each.value.cluster].location + gke_version = each.value.gke_version + labels = each.value.labels + max_pods_per_node = each.value.max_pods_per_node + node_config = each.value.node_config + node_count = each.value.node_count + node_locations = each.value.node_locations + nodepool_config = each.value.nodepool_config + pod_range = each.value.pod_range + reservation_affinity = each.value.reservation_affinity + service_account = ( + each.value.service_account == null + ? { email = module.gke-nodes-service-account.email } + : each.value.service_account + ) sole_tenant_nodegroup = each.value.sole_tenant_nodegroup tags = each.value.tags taints = each.value.taints diff --git a/blueprints/gke/multitenant-fleet/main.tf b/blueprints/gke/multitenant-fleet/main.tf index 240059e66e..dd117fbb72 100644 --- a/blueprints/gke/multitenant-fleet/main.tf +++ b/blueprints/gke/multitenant-fleet/main.tf @@ -16,6 +16,16 @@ # tfdoc:file:description Project and usage dataset. +locals { + gke_nodes_sa_roles = [ + "autoscaling.metricsWriter", + "logging.logWriter", + "monitoring.viewer", + "monitoring.metricWriter", + "stackdriver.resourceMetadata.writer" + ] +} + module "gke-project-0" { source = "../../../modules/project" billing_account = var.billing_account_id @@ -29,6 +39,12 @@ module "gke-project-0" { "serviceAccount:${module.gke-project-0.service_accounts.robots.fleet}" ] } ) + iam_bindings_additive = { + for r in local.gke_nodes_sa_roles : "gke-nodes-sa-${r}" => { + member = module.gke-nodes-service-account.iam_email + role = r + } + } services = concat( [ "anthos.googleapis.com", @@ -71,3 +87,9 @@ module "gke-dataset-resource-usage" { id = "gke_resource_usage" friendly_name = "GKE resource usage." } + +module "gke-nodes-service-account" { + source = "../../../modules/iam-service-account" + project_id = module.gke-project-0.project_id + name = "gke-node-default" +} diff --git a/blueprints/gke/multitenant-fleet/variables.tf b/blueprints/gke/multitenant-fleet/variables.tf index 83c9787594..93886d0bc1 100644 --- a/blueprints/gke/multitenant-fleet/variables.tf +++ b/blueprints/gke/multitenant-fleet/variables.tf @@ -28,6 +28,7 @@ variable "clusters" { horizontal_pod_autoscaling = true, http_load_balancing = true }) enable_features = optional(any, { + shielded_nodes = true workload_identity = true }) issue_client_certificate = optional(bool, false) @@ -49,12 +50,10 @@ variable "clusters" { min_master_version = optional(string) monitoring_config = optional(object({ enable_system_metrics = optional(bool, true) - # (Optional) control plane metrics enable_api_server_metrics = optional(bool, false) enable_controller_manager_metrics = optional(bool, false) enable_scheduler_metrics = optional(bool, false) - # (Optional) kube state metrics enable_daemonset_metrics = optional(bool, false) enable_deployment_metrics = optional(bool, false) @@ -62,11 +61,9 @@ variable "clusters" { enable_pod_metrics = optional(bool, false) enable_statefulset_metrics = optional(bool, false) enable_storage_metrics = optional(bool, false) - # Google Cloud Managed Service for Prometheus enable_managed_prometheus = optional(bool, true) }), {}) - node_locations = optional(list(string)) private_cluster_config = optional(any) release_channel = optional(string)