From 3874c2f68543cb054634b3da8a8694bd843a6f87 Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Wed, 20 Jul 2022 11:17:08 -0400 Subject: [PATCH 1/5] fix: issue with kubernetes_service_account in k8s 1.24 The problem when generating new service accounts, is that the secret containing the SA token is no longer generated automatically since the LegacyServiceAccountTokenNoAutoGeneration feature gate was enabled as true in Kubernetes clusters version 1.24. (references: https://github.com/kubernetes/enhancements/issues/2799 https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/) This is the reported issue for the terraform resource kubernetes_service_account https://github.com/hashicorp/terraform-provider-kubernetes/issues/1724 Alternative changes were made using the terraform resource kubernetes_manifest to manually generate the service accounts along with their secret --- modules/workload-identity/main.tf | 50 ++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/modules/workload-identity/main.tf b/modules/workload-identity/main.tf index 261b62e1c5..3754a0e5a0 100644 --- a/modules/workload-identity/main.tf +++ b/modules/workload-identity/main.tf @@ -22,10 +22,9 @@ locals { gcp_sa_fqn = "serviceAccount:${local.gcp_sa_email}" # This will cause Terraform to block returning outputs until the service account is created - k8s_given_name = var.k8s_sa_name != null ? var.k8s_sa_name : var.name - output_k8s_name = var.use_existing_k8s_sa ? local.k8s_given_name : kubernetes_service_account.main[0].metadata[0].name - output_k8s_namespace = var.use_existing_k8s_sa ? var.namespace : kubernetes_service_account.main[0].metadata[0].namespace - + k8s_given_name = var.k8s_sa_name != null ? var.k8s_sa_name : var.name + output_k8s_name = var.use_existing_k8s_sa ? local.k8s_given_name : kubernetes_manifest.main_sa[0].manifest.metadata.name + output_k8s_namespace = var.use_existing_k8s_sa ? var.namespace : kubernetes_manifest.main_sa[0].manifest.metadata.namespace k8s_sa_project_id = var.k8s_sa_project_id != null ? var.k8s_sa_project_id : var.project_id k8s_sa_gcp_derived_name = "serviceAccount:${local.k8s_sa_project_id}.svc.id.goog[${var.namespace}/${local.output_k8s_name}]" } @@ -45,17 +44,46 @@ resource "google_service_account" "cluster_service_account" { project = var.project_id } -resource "kubernetes_service_account" "main" { +resource "kubernetes_manifest" "main_secret" { count = var.use_existing_k8s_sa ? 0 : 1 + manifest = { + "apiVersion" = "v1" + "kind" = "Secret" + "metadata" = { + "namespace" = var.namespace + "name" = local.k8s_given_name + "annotations" = { + "kubernetes.io/service-account.name" = local.k8s_given_name + } + } - automount_service_account_token = var.automount_service_account_token - metadata { - name = local.k8s_given_name - namespace = var.namespace - annotations = { - "iam.gke.io/gcp-service-account" = local.gcp_sa_email + "type" = "kubernetes.io/service-account-token" + } +} + +resource "kubernetes_manifest" "main_sa" { + count = var.use_existing_k8s_sa ? 0 : 1 + manifest = { + "apiVersion" = "v1" + "kind" = "ServiceAccount" + "metadata" = { + "namespace" = var.namespace + "name" = local.k8s_given_name + "annotations" = { + "iam.gke.io/gcp-service-account" = local.gcp_sa_email + } } + + "automountServiceAccountToken" = var.automount_service_account_token + "secrets" = [ + { + "name" = local.k8s_given_name + } + ] } + depends_on = [ + kubernetes_manifest.main_secret + ] } module "annotate-sa" { From 89af04790e6fbfc2b4c9d51083a3a3fc87bcdd12 Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Wed, 20 Jul 2022 21:22:23 -0400 Subject: [PATCH 2/5] move depends_on --- modules/workload-identity/main.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/workload-identity/main.tf b/modules/workload-identity/main.tf index 3754a0e5a0..693ab78e61 100644 --- a/modules/workload-identity/main.tf +++ b/modules/workload-identity/main.tf @@ -59,6 +59,9 @@ resource "kubernetes_manifest" "main_secret" { "type" = "kubernetes.io/service-account-token" } + depends_on = [ + kubernetes_manifest.main_sa + ] } resource "kubernetes_manifest" "main_sa" { @@ -81,9 +84,6 @@ resource "kubernetes_manifest" "main_sa" { } ] } - depends_on = [ - kubernetes_manifest.main_secret - ] } module "annotate-sa" { From 3dc0a2f52da450e91aec3c2f5c12a83ba72a7f10 Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Sat, 23 Jul 2022 01:07:25 -0400 Subject: [PATCH 3/5] avoid API access during planning time --- modules/workload-identity/README.md | 4 +- modules/workload-identity/main.tf | 106 ++++++++++++++++--------- modules/workload-identity/variables.tf | 6 +- 3 files changed, 72 insertions(+), 44 deletions(-) diff --git a/modules/workload-identity/README.md b/modules/workload-identity/README.md index ec69dbb473..947427c145 100644 --- a/modules/workload-identity/README.md +++ b/modules/workload-identity/README.md @@ -99,12 +99,12 @@ already bear the `"iam.gke.io/gcp-service-account"` annotation. |------|-------------|------|---------|:--------:| | annotate\_k8s\_sa | Annotate the kubernetes service account with 'iam.gke.io/gcp-service-account' annotation. Valid in cases when an existing SA is used. | `bool` | `true` | no | | automount\_service\_account\_token | Enable automatic mounting of the service account token | `bool` | `false` | no | -| cluster\_name | Cluster name. Required if using existing KSA. | `string` | `""` | no | +| cluster\_name | Cluster name. | `string` | `""` | yes | | gcp\_sa\_name | Name for the Google service account; overrides `var.name`. | `string` | `null` | no | | impersonate\_service\_account | An optional service account to impersonate for gcloud commands. If this service account is not specified, the module will use Application Default Credentials. | `string` | `""` | no | | k8s\_sa\_name | Name for the Kubernetes service account; overrides `var.name`. `cluster_name` and `location` must be set when this input is specified. | `string` | `null` | no | | k8s\_sa\_project\_id | GCP project ID of the k8s service account; overrides `var.project_id`. | `string` | `null` | no | -| location | Cluster location (region if regional cluster, zone if zonal cluster). Required if using existing KSA. | `string` | `""` | no | +| location | Cluster location (region if regional cluster, zone if zonal cluster). | `string` | `""` | yes | | name | Name for both service accounts. The GCP SA will be truncated to the first 30 chars if necessary. | `string` | n/a | yes | | namespace | Namespace for the Kubernetes service account | `string` | `"default"` | no | | project\_id | GCP project ID | `string` | n/a | yes | diff --git a/modules/workload-identity/main.tf b/modules/workload-identity/main.tf index 693ab78e61..419784b882 100644 --- a/modules/workload-identity/main.tf +++ b/modules/workload-identity/main.tf @@ -23,8 +23,8 @@ locals { # This will cause Terraform to block returning outputs until the service account is created k8s_given_name = var.k8s_sa_name != null ? var.k8s_sa_name : var.name - output_k8s_name = var.use_existing_k8s_sa ? local.k8s_given_name : kubernetes_manifest.main_sa[0].manifest.metadata.name - output_k8s_namespace = var.use_existing_k8s_sa ? var.namespace : kubernetes_manifest.main_sa[0].manifest.metadata.namespace + output_k8s_name = var.use_existing_k8s_sa ? local.k8s_given_name : data.kubernetes_secret.main.metadata.0.name + output_k8s_namespace = var.use_existing_k8s_sa ? var.namespace : data.kubernetes_secret.main.metadata.0.namespace k8s_sa_project_id = var.k8s_sa_project_id != null ? var.k8s_sa_project_id : var.project_id k8s_sa_gcp_derived_name = "serviceAccount:${local.k8s_sa_project_id}.svc.id.goog[${var.namespace}/${local.output_k8s_name}]" } @@ -44,46 +44,76 @@ resource "google_service_account" "cluster_service_account" { project = var.project_id } -resource "kubernetes_manifest" "main_secret" { +module "service_account" { count = var.use_existing_k8s_sa ? 0 : 1 - manifest = { - "apiVersion" = "v1" - "kind" = "Secret" - "metadata" = { - "namespace" = var.namespace - "name" = local.k8s_given_name - "annotations" = { - "kubernetes.io/service-account.name" = local.k8s_given_name - } - } - - "type" = "kubernetes.io/service-account-token" - } - depends_on = [ - kubernetes_manifest.main_sa - ] + + source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper" + version = "~> 3.1" + enabled = var.use_existing_k8s_sa ? false : true + skip_download = true + cluster_name = var.cluster_name + cluster_location = var.location + project_id = local.k8s_sa_project_id + impersonate_service_account = var.impersonate_service_account + use_existing_context = var.use_existing_context + internal_ip = false + kubectl_create_command = <<-EOT +kubectl apply -f - < Date: Sat, 23 Jul 2022 01:37:50 -0400 Subject: [PATCH 4/5] update variables and README --- modules/workload-identity/README.md | 4 ++-- modules/workload-identity/variables.tf | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/workload-identity/README.md b/modules/workload-identity/README.md index 947427c145..fd0a411433 100644 --- a/modules/workload-identity/README.md +++ b/modules/workload-identity/README.md @@ -99,12 +99,12 @@ already bear the `"iam.gke.io/gcp-service-account"` annotation. |------|-------------|------|---------|:--------:| | annotate\_k8s\_sa | Annotate the kubernetes service account with 'iam.gke.io/gcp-service-account' annotation. Valid in cases when an existing SA is used. | `bool` | `true` | no | | automount\_service\_account\_token | Enable automatic mounting of the service account token | `bool` | `false` | no | -| cluster\_name | Cluster name. | `string` | `""` | yes | +| cluster\_name | Cluster name. | `string` | `""` | no | | gcp\_sa\_name | Name for the Google service account; overrides `var.name`. | `string` | `null` | no | | impersonate\_service\_account | An optional service account to impersonate for gcloud commands. If this service account is not specified, the module will use Application Default Credentials. | `string` | `""` | no | | k8s\_sa\_name | Name for the Kubernetes service account; overrides `var.name`. `cluster_name` and `location` must be set when this input is specified. | `string` | `null` | no | | k8s\_sa\_project\_id | GCP project ID of the k8s service account; overrides `var.project_id`. | `string` | `null` | no | -| location | Cluster location (region if regional cluster, zone if zonal cluster). | `string` | `""` | yes | +| location | Cluster location (region if regional cluster, zone if zonal cluster). | `string` | `""` | no | | name | Name for both service accounts. The GCP SA will be truncated to the first 30 chars if necessary. | `string` | n/a | yes | | namespace | Namespace for the Kubernetes service account | `string` | `"default"` | no | | project\_id | GCP project ID | `string` | n/a | yes | diff --git a/modules/workload-identity/variables.tf b/modules/workload-identity/variables.tf index 0cc8ab78dc..aba06b771a 100644 --- a/modules/workload-identity/variables.tf +++ b/modules/workload-identity/variables.tf @@ -39,11 +39,13 @@ variable "use_existing_gcp_sa" { variable "cluster_name" { description = "Cluster name." type = string + default = "" } variable "location" { description = "Cluster location (region if regional cluster, zone if zonal cluster)." type = string + default = "" } variable "k8s_sa_name" { From 36b89e54f631bf3446facd8bbd94c2d1483a2e56 Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Tue, 26 Jul 2022 11:29:03 -0400 Subject: [PATCH 5/5] fix heredoc --- modules/workload-identity/main.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/workload-identity/main.tf b/modules/workload-identity/main.tf index 419784b882..dd6d44cb30 100644 --- a/modules/workload-identity/main.tf +++ b/modules/workload-identity/main.tf @@ -83,7 +83,7 @@ EOF EOT kubectl_destroy_command = <<-EOT -kubectl delete -f - <