Grant IAM rights to service identities in host project (#1542)

* [module/project] Grant IAM rights to service identities based on used services in host project
* [blueprints/factories/project-factory] enable granting IAM permissions in host VPC for service identities directly or by specifying services in use

blueprints/factories/project-factory/README.md

@@ -76,7 +76,7 @@ module "projects" {
   service_identities_iam = try(each.value.service_identities_iam, {})
   vpc                    = try(each.value.vpc, null)
 }
-# tftest modules=7 resources=36 inventory=example.yaml
+# tftest modules=7 resources=38 inventory=example.yaml
 ```
 
 ### Projects configuration
@@ -212,6 +212,16 @@ vpc:
   # Host project the project will be service project of
   host_project: fast-prod-net-spoke-0
 
+  # [opt] Services for which set up the IAM in the host project
+  service_iam_grants:
+    - dataproc.googleapis.com
+
+  # [opt] Roles to rant service project service identities in host project
+  service_identity_iam:
+    "roles/compute.networkUser":
+      - cloudservices
+      - container-engine
+
   # [opt] Subnets in the host project where principals will be granted networkUser
   # in region/subnet-name => [principals]
   subnets_iam:
@@ -248,7 +258,7 @@ vpc:
 | [service_identities_iam](variables.tf#L184) | Custom IAM settings for service identities in service => [role] format. | <code>map&#40;list&#40;string&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
 | [service_identities_iam_additive](variables.tf#L191) | Custom additive IAM settings for service identities in service => [role] format. | <code>map&#40;list&#40;string&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
 | [services](variables.tf#L198) | Services to be enabled for the project. | <code>list&#40;string&#41;</code> |  | <code>&#91;&#93;</code> |
-| [vpc](variables.tf#L205) | VPC configuration for the project. | <code title="object&#40;&#123;&#10;  host_project &#61; string&#10;  gke_setup &#61; object&#40;&#123;&#10;    enable_security_admin     &#61; bool&#10;    enable_host_service_agent &#61; bool&#10;  &#125;&#41;&#10;  subnets_iam &#61; map&#40;list&#40;string&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
+| [vpc](variables.tf#L205) | VPC configuration for the project. | <code title="object&#40;&#123;&#10;  host_project &#61; string&#10;  gke_setup &#61; optional&#40;object&#40;&#123;&#10;    enable_security_admin     &#61; optional&#40;bool, false&#41;&#10;    enable_host_service_agent &#61; optional&#40;bool, false&#41;&#10;  &#125;&#41;, &#123;&#125;&#41;&#10;  service_iam_grants   &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10;  service_identity_iam &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10;  subnets_iam          &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code title="&#123;&#10;  host_project &#61; null&#10;&#125;">&#123;&#8230;&#125;</code> |
 
 ## Outputs
 

blueprints/factories/project-factory/main.tf

@@ -15,6 +15,19 @@
  */
 
 locals {
+  _gke_config_service_identity_iam = {
+    "roles/compute.networkUser" = compact([
+      var.vpc.gke_setup.enable_host_service_agent ? "container-engine" : null,
+      local.vpc_cloudservices ? "cloudservices" : null
+    ])
+    "roles/compute.securityAdmin" = compact([
+      var.vpc.gke_setup.enable_security_admin ? "container-engine" : null,
+    ])
+    "roles/container.hostServiceAgentUser" = compact([
+      var.vpc.gke_setup.enable_host_service_agent ? "container-engine" : null
+    ])
+  }
+
   _group_iam = {
     for r in local._group_iam_bindings : r => [
       for k, v in var.group_iam :
@@ -76,10 +89,10 @@ locals {
     ]
   }
   _vpc_subnet_bindings = (
-    local.vpc.subnets_iam == null || local.vpc.host_project == null
+    var.vpc.subnets_iam == null || var.vpc.host_project == null
     ? []
     : flatten([
-      for subnet, members in local.vpc.subnets_iam : [
+      for subnet, members in var.vpc.subnets_iam : [
         for member in members : {
           region = split("/", subnet)[0]
           subnet = split("/", subnet)[1]
@@ -131,19 +144,18 @@ locals {
     coalesce(var.labels, {}), coalesce(try(var.defaults.labels, {}), {})
   )
   services = distinct(concat(var.services, local._services))
-  vpc = coalesce(var.vpc, {
-    host_project = null, gke_setup = null, subnets_iam = null
-  })
   vpc_cloudservices = (
-    local.vpc_gke_service_agent ||
+    var.vpc.gke_setup.enable_host_service_agent ||
     contains(var.services, "compute.googleapis.com")
   )
-  vpc_gke_security_admin = coalesce(
-    try(local.vpc.gke_setup.enable_security_admin, null), false
-  )
-  vpc_gke_service_agent = coalesce(
-    try(local.vpc.gke_setup.enable_host_service_agent, null), false
-  )
+
+  vpc_service_identity_iam = {
+    for role in setunion(keys(local._gke_config_service_identity_iam), keys(var.vpc.service_identity_iam)) :
+    role => setunion(
+      lookup(local._gke_config_service_identity_iam, role, []),
+      lookup(var.vpc.service_identity_iam, role, []),
+    )
+  }
   vpc_subnet_bindings = {
     for binding in local._vpc_subnet_bindings :
     "${binding.subnet}:${binding.member}" => binding
@@ -169,7 +181,7 @@ module "billing-alert" {
 module "dns" {
   source     = "../../../modules/dns"
   for_each   = toset(var.dns_zones)
-  project_id = coalesce(local.vpc.host_project, module.project.project_id)
+  project_id = coalesce(var.vpc.host_project, module.project.project_id)
   name       = each.value
   zone_config = {
     domain = "${each.value}.${var.defaults.environment_dns_zone}"
@@ -194,20 +206,10 @@ module "project" {
   service_encryption_key_ids = var.kms_service_agents
   services                   = local.services
   shared_vpc_service_config = var.vpc == null ? null : {
-    host_project = local.vpc.host_project
+    host_project = var.vpc.host_project
     # these are non-authoritative
-    service_identity_iam = {
-      "roles/compute.networkUser" = compact([
-        local.vpc_gke_service_agent ? "container-engine" : null,
-        local.vpc_cloudservices ? "cloudservices" : null
-      ])
-      "roles/compute.securityAdmin" = compact([
-        local.vpc_gke_security_admin ? "container-engine" : null,
-      ])
-      "roles/container.hostServiceAgentUser" = compact([
-        local.vpc_gke_service_agent ? "container-engine" : null
-      ])
-    }
+    service_identity_iam = local.vpc_service_identity_iam
+    service_iam_grants   = var.vpc.service_iam_grants
   }
 }
 
@@ -221,8 +223,8 @@ module "service-accounts" {
 
 resource "google_compute_subnetwork_iam_member" "default" {
   for_each   = local.vpc_subnet_bindings
-  project    = local.vpc.host_project
-  subnetwork = "projects/${local.vpc.host_project}/regions/${each.value.region}/subnetworks/${each.value.subnet}"
+  project    = var.vpc.host_project
+  subnetwork = "projects/${var.vpc.host_project}/regions/${each.value.region}/subnetworks/${each.value.subnet}"
   region     = each.value.region
   role       = "roles/compute.networkUser"
   member = (

blueprints/factories/project-factory/sample-data/projects/project.yaml

@@ -99,6 +99,16 @@ vpc:
   # Host project the project will be service project of
   host_project: fast-dev-net-spoke-0
 
+  # [opt] Services for which set up the IAM in the host project
+  service_iam_grants:
+    - dataproc.googleapis.com
+
+  # [opt] Roles to rant service project service identities in host project
+  service_identity_iam:
+    "roles/compute.networkUser":
+      - cloudservices
+      - container-engine
+
   # [opt] Subnets in the host project where principals will be granted networkUser
   # in region/subnet-name => [principals]
   subnets_iam:

blueprints/factories/project-factory/variables.tf

@@ -206,11 +206,23 @@ variable "vpc" {
   description = "VPC configuration for the project."
   type = object({
     host_project = string
-    gke_setup = object({
-      enable_security_admin     = bool
-      enable_host_service_agent = bool
-    })
-    subnets_iam = map(list(string))
+    gke_setup = optional(object({
+      enable_security_admin     = optional(bool, false)
+      enable_host_service_agent = optional(bool, false)
+    }), {})
+    service_iam_grants   = optional(list(string), [])
+    service_identity_iam = optional(map(list(string)), {})
+    subnets_iam          = optional(map(list(string)), {})
   })
-  default = null
+  default = {
+    host_project = null
+  }
+  nullable = false
+  validation {
+    condition = var.vpc.host_project != null || (
+      var.vpc.host_project == null && length(var.vpc.gke_setup) == 0 && length(var.vpc.service_iam_grants) == 0 &&
+      length(var.vpc.service_identity_iam) == 0 && length(var.vpc.subnets_iam) == 0
+    )
+    error_message = "host_project is required if providing any additional configuration for vpc"
+  }
 }

modules/project/README.md

@@ -259,6 +259,30 @@ module "service-project" {
 # tftest modules=2 resources=8 inventory=shared-vpc.yaml
 ```
 
+The module allows also granting necessary permissions in host project to service identities by specifying which services will be used in service project in `grant_iam_for_services`.
+```hcl
+module "host-project" {
+  source = "./fabric/modules/project"
+  name   = "my-host-project"
+  shared_vpc_host_config = {
+    enabled = true
+  }
+}
+
+module "service-project" {
+  source = "./fabric/modules/project"
+  name   = "my-service-project"
+  services = [
+    "container.googleapis.com",
+  ]
+  shared_vpc_service_config = {
+    host_project       = module.host-project.project_id
+    service_iam_grants = module.service-project.services
+  }
+}
+# tftest modules=2 resources=9 inventory=shared-vpc-auto-grants.yaml
+```
+
 ## Organization Policies
 
 To manage organization policies, the `orgpolicy.googleapis.com` service should be enabled in the quota project.
@@ -577,7 +601,6 @@ output "compute_robot" {
 ```
 
 <!-- TFDOC OPTS files:1 -->
-
 <!-- BEGIN TFDOC -->
 ## Files
 
@@ -631,9 +654,9 @@ output "compute_robot" {
 | [service_perimeter_standard](variables.tf#L272) | Name of VPC-SC Standard perimeter to add project into. See comment in the variables file for format. | <code>string</code> |  | <code>null</code> |
 | [services](variables.tf#L278) | Service APIs to enable. | <code>list&#40;string&#41;</code> |  | <code>&#91;&#93;</code> |
 | [shared_vpc_host_config](variables.tf#L284) | Configures this project as a Shared VPC host project (mutually exclusive with shared_vpc_service_project). | <code title="object&#40;&#123;&#10;  enabled          &#61; bool&#10;  service_projects &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
-| [shared_vpc_service_config](variables.tf#L293) | Configures this project as a Shared VPC service project (mutually exclusive with shared_vpc_host_config). | <code title="object&#40;&#123;&#10;  host_project         &#61; string&#10;  service_identity_iam &#61; optional&#40;map&#40;list&#40;string&#41;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
-| [skip_delete](variables.tf#L303) | Allows the underlying resources to be destroyed without destroying the project itself. | <code>bool</code> |  | <code>false</code> |
-| [tag_bindings](variables.tf#L309) | Tag bindings for this project, in key => tag value id format. | <code>map&#40;string&#41;</code> |  | <code>null</code> |
+| [shared_vpc_service_config](variables.tf#L293) | Configures this project as a Shared VPC service project (mutually exclusive with shared_vpc_host_config). | <code title="object&#40;&#123;&#10;  host_project         &#61; string&#10;  service_identity_iam &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10;  service_iam_grants   &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code title="&#123;&#10;  host_project &#61; null&#10;&#125;">&#123;&#8230;&#125;</code> |
+| [skip_delete](variables.tf#L315) | Allows the underlying resources to be destroyed without destroying the project itself. | <code>bool</code> |  | <code>false</code> |
+| [tag_bindings](variables.tf#L321) | Tag bindings for this project, in key => tag value id format. | <code>map&#40;string&#41;</code> |  | <code>null</code> |
 
 ## Outputs
 
@@ -645,6 +668,6 @@ output "compute_robot" {
 | [number](outputs.tf#L56) | Project number. |  |
 | [project_id](outputs.tf#L75) | Project id. |  |
 | [service_accounts](outputs.tf#L94) | Product robot service accounts in project. |  |
-| [sink_writer_identities](outputs.tf#L110) | Writer identities created for each sink. |  |
+| [services](outputs.tf#L110) | Service APIs to enabled in the project. |  |
+| [sink_writer_identities](outputs.tf#L119) | Writer identities created for each sink. |  |
 <!-- END TFDOC -->
-

modules/project/outputs.tf

@@ -107,6 +107,15 @@ output "service_accounts" {
   ]
 }
 
+output "services" {
+  description = "Service APIs to enabled in the project."
+  value       = var.services
+  depends_on = [
+    google_project_service.project_services,
+    google_project_service_identity.jit_si,
+  ]
+}
+
 output "sink_writer_identities" {
   description = "Writer identities created for each sink."
   value = {

modules/project/shared-vpc.tf

@@ -17,15 +17,25 @@
 # tfdoc:file:description Shared VPC project-level configuration.
 
 locals {
+  _shared_vpc_agent_config = yamldecode(file("${path.module}/sharedvpc-agent-iam.yaml"))
+  _shared_vpc_agent_config_filtered = [
+    for config in local._shared_vpc_agent_config : config
+    if contains(var.shared_vpc_service_config.service_iam_grants, config.service)
+  ]
+  _shared_vpc_agent_grants = flatten(flatten([
+    for api in local._shared_vpc_agent_config_filtered : [
+      for service, roles in api.agents : [
+        for role in roles : { role = role, service = service }
+      ]
+    ]
+  ]))
+
   # compute the host project IAM bindings for this project's service identities
   _svpc_service_iam = flatten([
-    for role, services in local._svpc_service_identity_iam : [
+    for role, services in var.shared_vpc_service_config.service_identity_iam : [
       for service in services : { role = role, service = service }
     ]
   ])
-  _svpc_service_identity_iam = coalesce(
-    local.svpc_service_config.service_identity_iam, {}
-  )
   svpc_host_config = {
     enabled = coalesce(
       try(var.shared_vpc_host_config.enabled, null), false
@@ -34,11 +44,9 @@ locals {
       try(var.shared_vpc_host_config.service_projects, null), []
     )
   }
-  svpc_service_config = coalesce(var.shared_vpc_service_config, {
-    host_project = null, service_identity_iam = {}
-  })
+
   svpc_service_iam = {
-    for b in local._svpc_service_iam : "${b.role}:${b.service}" => b
+    for b in setunion(local._svpc_service_iam, local._shared_vpc_agent_grants) : "${b.role}:${b.service}" => b
   }
 }
 
@@ -59,7 +67,7 @@ resource "google_compute_shared_vpc_service_project" "service_projects" {
 
 resource "google_compute_shared_vpc_service_project" "shared_vpc_service" {
   provider        = google-beta
-  count           = local.svpc_service_config.host_project != null ? 1 : 0
+  count           = var.shared_vpc_service_config.host_project != null ? 1 : 0
   host_project    = var.shared_vpc_service_config.host_project
   service_project = local.project.project_id
 }