Move K8s service account creation for GKE out of Terraform.

Due to Terraform limitations, the Kubernetes provider may give errors when making changes to the underlying cluster resource. The provider documentation indicates that Kubernetes resources should not be configured in the same root module or apply step as the cluster resources.

src/main/k8s/base.cue

@@ -67,6 +67,13 @@ objects: [ for objectSet in objectSets for object in objectSet {object}]
 #OpenTelemetryPrometheusExporterPort: 8889
 #OpenTelemetryCollectorEndpoint:      "http://default-collector-headless.default.svc:\(#OpenTelemetryReceiverPort)"
 
+// K8s ServiceAccount.
+#ServiceAccount: {
+	apiVersion: "v1"
+	kind:       "ServiceAccount"
+	metadata:   #ObjectMeta
+}
+
 #ResourceQuantity: {
 	cpu?:    string
 	memory?: string
@@ -267,16 +274,18 @@ objects: [ for objectSet in objectSets for object in objectSet {object}]
 
 // K8s ObjectMeta.
 #ObjectMeta: {
-	_component: string
+	_component?: string
 
 	name: string
 	labels: [_=string]:      string
 	annotations: [_=string]: string
 
 	labels: {
-		"app.kubernetes.io/name":      name
-		"app.kubernetes.io/part-of":   #AppName
-		"app.kubernetes.io/component": _component
+		"app.kubernetes.io/name":    name
+		"app.kubernetes.io/part-of": #AppName
+		if (_component != _|_) {
+			"app.kubernetes.io/component": _component
+		}
 	}
 }
 

src/main/k8s/dev/base_gke.cue

@@ -45,6 +45,13 @@ package k8s
 	}
 }
 
+#WorkloadIdentityServiceAccount: #ServiceAccount & {
+	_iamServiceAccountName: string
+
+	let Principal = "\(_iamServiceAccountName)@\(#GCloudConfig.project).iam.gserviceaccount.com"
+	metadata: annotations: "iam.gke.io/gcp-service-account": Principal
+}
+
 #ServiceAccountNodeSelector: {
 	"iam.gke.io/gke-metadata-server-enabled": "true"
 }

src/main/k8s/dev/bigquery_edp_simulator_gke.cue

@@ -14,6 +14,8 @@
 
 package k8s
 
+#SimulatorServiceAccount: "simulator"
+
 _bigQueryConfig: #BigQueryConfig & {
 	dataset: string @tag("bigquery_dataset")
 	table:   string @tag("bigquery_table")
@@ -34,10 +36,18 @@ edp_simulators: {
 		"\(edp.displayName)": {
 			_additional_args: ["--publisher-id=\(edp.publisherId)"] + _bigQueryConfig.flags
 			_imageConfig: repoSuffix: "simulator/bigquery-edp"
+			serviceAccounts: {
+				"\(#SimulatorServiceAccount)": #WorkloadIdentityServiceAccount & {
+					_iamServiceAccountName: "simulator"
+				}
+			}
 			deployment: {
 				_container: {
 					resources: _resourceRequirements
 				}
+				spec: template: spec: #ServiceAccountPodSpec & {
+					serviceAccountName: #SimulatorServiceAccount
+				}
 			}
 		}
 	}

src/main/k8s/dev/duchy_gke.cue

@@ -49,6 +49,7 @@ _duchy_cert_name: "duchies/\(_duchy_name)/certificates/\(_certificateId)"
 
 objectSets: [
 	default_deny_ingress_and_egress,
+	duchy.serviceAccounts,
 	duchy.deployments,
 	duchy.services,
 	duchy.networkPolicies,
@@ -76,6 +77,16 @@ duchy: #SpannerDuchy & {
 	_verbose_grpc_logging:      "false"
 	_duchyMillParallelism:      4
 
+	serviceAccounts: [string]: #WorkloadIdentityServiceAccount
+	serviceAccounts: {
+		"\(#InternalServerServiceAccount)": {
+			_iamServiceAccountName: "\(_duchy_name)-duchy-internal"
+		}
+		"\(#StorageServiceAccount)": {
+			_iamServiceAccountName: "\(_duchy_name)-duchy-storage"
+		}
+	}
+
 	deployments: {
 		"internal-api-server-deployment": {
 			_container: {

src/main/k8s/dev/edp_simulator_gke.cue

@@ -26,10 +26,11 @@ _secret_name:            string @tag("secret_name")
 _kingdomPublicApiTarget: string @tag("kingdom_public_api_target")
 _duchyPublicApiTarget:   string @tag("duchy_public_api_target")
 
-#ServiceAccount: "simulator"
-
-objectSets: [ for edp in edp_simulators {[edp.deployment]}] +
-	[ for edp in edp_simulators {edp.networkPolicies}]
+objectSets: [
+	for edp in edp_simulators {edp.serviceAccounts},
+	for edp in edp_simulators {[edp.deployment]},
+	for edp in edp_simulators {edp.networkPolicies},
+]
 
 _edpConfigs: [...#EdpConfig]
 _edpConfigs: [
@@ -50,10 +51,9 @@ edp_simulators: {
 			_duchy_public_api_target:   _duchyPublicApiTarget
 			_kingdom_public_api_target: _kingdomPublicApiTarget
 			_mc_resource_name:          _mc_name
+
 			deployment: {
-				spec: template: spec: #ServiceAccountPodSpec & #SpotVmPodSpec & {
-					serviceAccountName: #ServiceAccount
-				}
+				spec: template: spec: #SpotVmPodSpec
 			}
 		}
 	}

src/main/k8s/dev/kingdom_gke.cue

@@ -40,6 +40,7 @@ _secret_name: string @tag("secret_name")
 
 objectSets: [
 	default_deny_ingress_and_egress,
+	kingdom.serviceAccounts,
 	kingdom.deployments,
 	kingdom.services,
 	kingdom.networkPolicies,
@@ -53,6 +54,12 @@ kingdom: #Kingdom & {
 	_kingdom_image_pull_policy: "Always"
 	_verboseGrpcServerLogging:  true
 
+	serviceAccounts: {
+		"\(#InternalServerServiceAccount)": #WorkloadIdentityServiceAccount & {
+			_iamServiceAccountName: "kingdom-internal"
+		}
+	}
+
 	deployments: {
 		"gcp-kingdom-data-server": {
 			_container: {

src/main/k8s/dev/reporting_gke.cue

@@ -41,6 +41,7 @@ _reportingMcConfigSecretName: string @tag("mc_config_secret_name")
 
 objectSets: [
 	default_deny_ingress_and_egress,
+	reporting.serviceAccounts,
 	reporting.deployments,
 	reporting.services,
 	reporting.networkPolicies,
@@ -59,6 +60,12 @@ reporting: #Reporting & {
 
 	_verboseGrpcServerLogging: true
 
+	serviceAccounts: {
+		"\(#InternalServerServiceAccount)": #WorkloadIdentityServiceAccount & {
+			_iamServiceAccountName: "reporting-internal"
+		}
+	}
+
 	deployments: {
 		"postgres-reporting-data-server": {
 			_container: resources: #InternalServerResourceRequirements

src/main/k8s/dev/reporting_v2_gke.cue

@@ -41,6 +41,7 @@ _reportingMcConfigSecretName: string @tag("mc_config_secret_name")
 
 objectSets: [
 	default_deny_ingress_and_egress,
+	reporting.serviceAccounts,
 	reporting.deployments,
 	reporting.services,
 	reporting.networkPolicies,
@@ -59,6 +60,12 @@ reporting: #Reporting & {
 
 	_verboseGrpcServerLogging: true
 
+	serviceAccounts: {
+		"\(#InternalServerServiceAccount)": #WorkloadIdentityServiceAccount & {
+			_iamServiceAccountName: "reporting-internal"
+		}
+	}
+
 	deployments: {
 		"postgres-internal-reporting-server": {
 			_container: resources: #InternalServerResourceRequirements

src/main/k8s/duchy.cue

@@ -331,4 +331,8 @@ import ("strings")
 			]
 		}
 	}
+
+	serviceAccounts: [Name=string]: #ServiceAccount & {
+		metadata: name: Name
+	}
 }

src/main/k8s/edp_simulator.cue

@@ -68,4 +68,8 @@ package k8s
 			}
 		}
 	}
+
+	serviceAccounts: [Name=string]: #ServiceAccount & {
+		metadata: name: Name
+	}
 }

src/main/k8s/kingdom.cue

@@ -316,4 +316,8 @@ import ("strings")
 			]
 		}
 	}
+
+	serviceAccounts: [Name=string]: #ServiceAccount & {
+		metadata: name: Name
+	}
 }

src/main/k8s/reporting.cue

@@ -165,4 +165,8 @@ package k8s
 			}
 		}
 	}
+
+	serviceAccounts: [Name=string]: #ServiceAccount & {
+		metadata: name: Name
+	}
 }

src/main/k8s/reporting_v2.cue

@@ -165,4 +165,8 @@ package k8s
 			}
 		}
 	}
+
+	serviceAccounts: [Name=string]: #ServiceAccount & {
+		metadata: name: Name
+	}
 }

src/main/terraform/gcloud/cmms/duchies.tf

@@ -61,50 +61,11 @@ module "storage" {
   location = local.storage_bucket_location
 }
 
-provider "kubernetes" {
-  # Due to the fact that this is using interpolation, the cluster must already exist.
-  # See https://registry.terraform.io/providers/hashicorp/kubernetes/2.20.0/docs
-
-  alias = "aggregator"
-  host  = "https://${data.google_container_cluster.clusters["aggregator"].endpoint}"
-  token = data.google_client_config.default.access_token
-  cluster_ca_certificate = base64decode(
-    data.google_container_cluster.clusters["aggregator"].master_auth[0].cluster_ca_certificate,
-  )
-}
-
-provider "kubernetes" {
-  # Due to the fact that this is using interpolation, the cluster must already exist.
-  # See https://registry.terraform.io/providers/hashicorp/kubernetes/2.20.0/docs
-
-  alias = "worker1"
-  host  = "https://${data.google_container_cluster.clusters["worker1"].endpoint}"
-  token = data.google_client_config.default.access_token
-  cluster_ca_certificate = base64decode(
-    data.google_container_cluster.clusters["worker1"].master_auth[0].cluster_ca_certificate,
-  )
-}
-
-provider "kubernetes" {
-  # Due to the fact that this is using interpolation, the cluster must already exist.
-  # See https://registry.terraform.io/providers/hashicorp/kubernetes/2.20.0/docs
-
-  alias = "worker2"
-  host  = "https://${data.google_container_cluster.clusters["worker2"].endpoint}"
-  token = data.google_client_config.default.access_token
-  cluster_ca_certificate = base64decode(
-    data.google_container_cluster.clusters["worker2"].master_auth[0].cluster_ca_certificate,
-  )
-}
-
 # TODO(hashicorp/terraform#24476): Use a for_each for the Duchy modules once
 # that works with providers.
 
 module "aggregator_duchy" {
   source = "../modules/duchy"
-  providers = {
-    kubernetes = kubernetes.aggregator
-  }
 
   name             = "aggregator"
   database_name    = "aggregator_duchy_computations"
@@ -114,9 +75,6 @@ module "aggregator_duchy" {
 
 module "worker1_duchy" {
   source = "../modules/duchy"
-  providers = {
-    kubernetes = kubernetes.worker1
-  }
 
   name             = "worker1"
   database_name    = "worker1_duchy_computations"
@@ -126,9 +84,6 @@ module "worker1_duchy" {
 
 module "worker2_duchy" {
   source = "../modules/duchy"
-  providers = {
-    kubernetes = kubernetes.worker2
-  }
 
   name             = "worker2"
   database_name    = "worker2_duchy_computations"

src/main/terraform/gcloud/cmms/kingdom.tf

@@ -38,25 +38,8 @@ module "kingdom_default_node_pool" {
   max_node_count  = 3
 }
 
-provider "kubernetes" {
-  # Due to the fact that this is using interpolation, the cluster must already exist.
-  # See https://registry.terraform.io/providers/hashicorp/kubernetes/2.20.0/docs
-
-  alias = "kingdom"
-  host  = "https://${data.google_container_cluster.kingdom.endpoint}"
-  token = data.google_client_config.default.access_token
-  cluster_ca_certificate = base64decode(
-    data.google_container_cluster.kingdom.master_auth[0].cluster_ca_certificate,
-  )
-}
-
 module "kingdom" {
   source = "../modules/kingdom"
 
-  providers = {
-    google     = google
-    kubernetes = kubernetes.kingdom
-  }
-
   spanner_instance = google_spanner_instance.spanner_instance
 }