Merge pull request #682 from GoogleCloudPlatform/fast/fast-fundation

FAST: fix CI/CD source repositories in stage 01
GoogleCloudPlatform · Jun 16, 2022 · 974f2ba · 974f2ba
2 parents 58e553c + a09eb39
commit 974f2ba
Show file tree

Hide file tree

Showing 14 changed files with 99 additions and 52 deletions.
diff --git a/fast/stages/00-bootstrap/README.md b/fast/stages/00-bootstrap/README.md
@@ -457,15 +457,15 @@ The remaining configuration is manual, as it regards the repositories themselves
 
 | name | description | sensitive | consumers |
 |---|---|:---:|---|
-| [automation](outputs.tf#L81) | Automation resources. |  |  |
-| [billing_dataset](outputs.tf#L86) | BigQuery dataset prepared for billing export. |  |  |
-| [cicd_repositories](outputs.tf#L91) | CI/CD repository configurations. |  |  |
-| [custom_roles](outputs.tf#L103) | Organization-level custom roles. |  |  |
-| [federated_identity](outputs.tf#L108) | Workload Identity Federation pool and providers. |  |  |
-| [outputs_bucket](outputs.tf#L118) | GCS bucket where generated output files are stored. |  |  |
-| [project_ids](outputs.tf#L123) | Projects created by this stage. |  |  |
-| [providers](outputs.tf#L142) | Terraform provider files for this stage and dependent stages. | ✓ | <code>stage-01</code> |
-| [service_accounts](outputs.tf#L132) | Automation service accounts created by this stage. |  |  |
-| [tfvars](outputs.tf#L151) | Terraform variable files for the following stages. | ✓ |  |
+| [automation](outputs.tf#L82) | Automation resources. |  |  |
+| [billing_dataset](outputs.tf#L87) | BigQuery dataset prepared for billing export. |  |  |
+| [cicd_repositories](outputs.tf#L92) | CI/CD repository configurations. |  |  |
+| [custom_roles](outputs.tf#L104) | Organization-level custom roles. |  |  |
+| [federated_identity](outputs.tf#L109) | Workload Identity Federation pool and providers. |  |  |
+| [outputs_bucket](outputs.tf#L119) | GCS bucket where generated output files are stored. |  |  |
+| [project_ids](outputs.tf#L124) | Projects created by this stage. |  |  |
+| [providers](outputs.tf#L143) | Terraform provider files for this stage and dependent stages. | ✓ | <code>stage-01</code> |
+| [service_accounts](outputs.tf#L133) | Automation service accounts created by this stage. |  |  |
+| [tfvars](outputs.tf#L152) | Terraform variable files for the following stages. | ✓ |  |
 
 <!-- END TFDOC -->
diff --git a/fast/stages/00-bootstrap/automation.tf b/fast/stages/00-bootstrap/automation.tf
@@ -38,12 +38,18 @@ module "automation-project" {
     "roles/owner" = [
       module.automation-tf-bootstrap-sa.iam_email
     ]
+    "roles/cloudbuild.builds.editor" = [
+      module.automation-tf-resman-sa.iam_email
+    ]
     "roles/iam.serviceAccountAdmin" = [
       module.automation-tf-resman-sa.iam_email
     ]
     "roles/iam.workloadIdentityPoolAdmin" = [
       module.automation-tf-resman-sa.iam_email
     ]
+    "roles/source.admin" = [
+      module.automation-tf-resman-sa.iam_email
+    ]
     "roles/storage.admin" = [
       module.automation-tf-resman-sa.iam_email
     ]

diff --git a/fast/stages/00-bootstrap/outputs.tf b/fast/stages/00-bootstrap/outputs.tf
@@ -57,6 +57,7 @@ locals {
       federated_identity_providers = local.wif_providers
       outputs_bucket               = module.automation-tf-output-gcs.name
       project_id                   = module.automation-project.project_id
+      project_number               = module.automation-project.number
     }
     custom_roles = local.custom_roles
   }

diff --git a/fast/stages/01-resman/README.md b/fast/stages/01-resman/README.md
@@ -178,30 +178,30 @@ Due to its simplicity, this stage lends itself easily to customizations: adding
 
 | name | description | type | required | default | producer |
 |---|---|:---:|:---:|:---:|:---:|
-| [automation](variables.tf#L20) | Automation resources created by the bootstrap stage. | <code title="object&#40;&#123;&#10;  outputs_bucket          &#61; string&#10;  project_id              &#61; string&#10;  federated_identity_pool &#61; string&#10;  federated_identity_providers &#61; map&#40;object&#40;&#123;&#10;    issuer           &#61; string&#10;    issuer_uri       &#61; string&#10;    name             &#61; string&#10;    principal_tpl    &#61; string&#10;    principalset_tpl &#61; string&#10;  &#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  | <code>00-bootstrap</code> |
-| [billing_account](variables.tf#L37) | Billing account id and organization id ('nnnnnnnn' or null). | <code title="object&#40;&#123;&#10;  id              &#61; string&#10;  organization_id &#61; number&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  | <code>00-bootstrap</code> |
-| [organization](variables.tf#L140) | Organization details. | <code title="object&#40;&#123;&#10;  domain      &#61; string&#10;  id          &#61; number&#10;  customer_id &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  | <code>00-bootstrap</code> |
-| [prefix](variables.tf#L164) | Prefix used for resources that need unique names. Use 9 characters or less. | <code>string</code> | ✓ |  | <code>00-bootstrap</code> |
-| [cicd_repositories](variables.tf#L46) | CI/CD repository configuration. Identity providers reference keys in the `automation.federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | <code title="object&#40;&#123;&#10;  data_platform_dev &#61; object&#40;&#123;&#10;    branch            &#61; string&#10;    identity_provider &#61; string&#10;    name              &#61; string&#10;    type              &#61; string&#10;  &#125;&#41;&#10;  data_platform_prod &#61; object&#40;&#123;&#10;    branch            &#61; string&#10;    identity_provider &#61; string&#10;    name              &#61; string&#10;    type              &#61; string&#10;  &#125;&#41;&#10;  networking &#61; object&#40;&#123;&#10;    branch            &#61; string&#10;    identity_provider &#61; string&#10;    name              &#61; string&#10;    type              &#61; string&#10;  &#125;&#41;&#10;  project_factory_dev &#61; object&#40;&#123;&#10;    branch            &#61; string&#10;    identity_provider &#61; string&#10;    name              &#61; string&#10;    type              &#61; string&#10;  &#125;&#41;&#10;  project_factory_prod &#61; object&#40;&#123;&#10;    branch            &#61; string&#10;    identity_provider &#61; string&#10;    name              &#61; string&#10;    type              &#61; string&#10;  &#125;&#41;&#10;  security &#61; object&#40;&#123;&#10;    branch            &#61; string&#10;    identity_provider &#61; string&#10;    name              &#61; string&#10;    type              &#61; string&#10;  &#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |  |
-| [custom_roles](variables.tf#L116) | Custom roles defined at the org level, in key => id format. | <code title="object&#40;&#123;&#10;  service_project_network_admin &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> | <code>00-bootstrap</code> |
-| [groups](variables.tf#L125) | Group names to grant organization-level permissions. | <code>map&#40;string&#41;</code> |  | <code title="&#123;&#10;  gcp-billing-admins      &#61; &#34;gcp-billing-admins&#34;,&#10;  gcp-devops              &#61; &#34;gcp-devops&#34;,&#10;  gcp-network-admins      &#61; &#34;gcp-network-admins&#34;&#10;  gcp-organization-admins &#61; &#34;gcp-organization-admins&#34;&#10;  gcp-security-admins     &#61; &#34;gcp-security-admins&#34;&#10;  gcp-support             &#61; &#34;gcp-support&#34;&#10;&#125;">&#123;&#8230;&#125;</code> | <code>00-bootstrap</code> |
-| [organization_policy_configs](variables.tf#L150) | Organization policies customization. | <code title="object&#40;&#123;&#10;  allowed_policy_member_domains &#61; list&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |  |
-| [outputs_location](variables.tf#L158) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable | <code>string</code> |  | <code>null</code> |  |
-| [tag_names](variables.tf#L175) | Customized names for resource management tags. | <code title="object&#40;&#123;&#10;  context     &#61; string&#10;  environment &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code title="&#123;&#10;  context     &#61; &#34;context&#34;&#10;  environment &#61; &#34;environment&#34;&#10;&#125;">&#123;&#8230;&#125;</code> |  |
-| [team_folders](variables.tf#L192) | Team folders to be created. Format is described in a code comment. | <code title="map&#40;object&#40;&#123;&#10;  descriptive_name     &#61; string&#10;  group_iam            &#61; map&#40;list&#40;string&#41;&#41;&#10;  impersonation_groups &#61; list&#40;string&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>null</code> |  |
+| [automation](variables.tf#L20) | Automation resources created by the bootstrap stage. | <code title="object&#40;&#123;&#10;  outputs_bucket          &#61; string&#10;  project_id              &#61; string&#10;  project_number          &#61; string&#10;  federated_identity_pool &#61; string&#10;  federated_identity_providers &#61; map&#40;object&#40;&#123;&#10;    issuer           &#61; string&#10;    issuer_uri       &#61; string&#10;    name             &#61; string&#10;    principal_tpl    &#61; string&#10;    principalset_tpl &#61; string&#10;  &#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  | <code>00-bootstrap</code> |
+| [billing_account](variables.tf#L38) | Billing account id and organization id ('nnnnnnnn' or null). | <code title="object&#40;&#123;&#10;  id              &#61; string&#10;  organization_id &#61; number&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  | <code>00-bootstrap</code> |
+| [organization](variables.tf#L141) | Organization details. | <code title="object&#40;&#123;&#10;  domain      &#61; string&#10;  id          &#61; number&#10;  customer_id &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  | <code>00-bootstrap</code> |
+| [prefix](variables.tf#L165) | Prefix used for resources that need unique names. Use 9 characters or less. | <code>string</code> | ✓ |  | <code>00-bootstrap</code> |
+| [cicd_repositories](variables.tf#L47) | CI/CD repository configuration. Identity providers reference keys in the `automation.federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | <code title="object&#40;&#123;&#10;  data_platform_dev &#61; object&#40;&#123;&#10;    branch            &#61; string&#10;    identity_provider &#61; string&#10;    name              &#61; string&#10;    type              &#61; string&#10;  &#125;&#41;&#10;  data_platform_prod &#61; object&#40;&#123;&#10;    branch            &#61; string&#10;    identity_provider &#61; string&#10;    name              &#61; string&#10;    type              &#61; string&#10;  &#125;&#41;&#10;  networking &#61; object&#40;&#123;&#10;    branch            &#61; string&#10;    identity_provider &#61; string&#10;    name              &#61; string&#10;    type              &#61; string&#10;  &#125;&#41;&#10;  project_factory_dev &#61; object&#40;&#123;&#10;    branch            &#61; string&#10;    identity_provider &#61; string&#10;    name              &#61; string&#10;    type              &#61; string&#10;  &#125;&#41;&#10;  project_factory_prod &#61; object&#40;&#123;&#10;    branch            &#61; string&#10;    identity_provider &#61; string&#10;    name              &#61; string&#10;    type              &#61; string&#10;  &#125;&#41;&#10;  security &#61; object&#40;&#123;&#10;    branch            &#61; string&#10;    identity_provider &#61; string&#10;    name              &#61; string&#10;    type              &#61; string&#10;  &#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |  |
+| [custom_roles](variables.tf#L117) | Custom roles defined at the org level, in key => id format. | <code title="object&#40;&#123;&#10;  service_project_network_admin &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> | <code>00-bootstrap</code> |
+| [groups](variables.tf#L126) | Group names to grant organization-level permissions. | <code>map&#40;string&#41;</code> |  | <code title="&#123;&#10;  gcp-billing-admins      &#61; &#34;gcp-billing-admins&#34;,&#10;  gcp-devops              &#61; &#34;gcp-devops&#34;,&#10;  gcp-network-admins      &#61; &#34;gcp-network-admins&#34;&#10;  gcp-organization-admins &#61; &#34;gcp-organization-admins&#34;&#10;  gcp-security-admins     &#61; &#34;gcp-security-admins&#34;&#10;  gcp-support             &#61; &#34;gcp-support&#34;&#10;&#125;">&#123;&#8230;&#125;</code> | <code>00-bootstrap</code> |
+| [organization_policy_configs](variables.tf#L151) | Organization policies customization. | <code title="object&#40;&#123;&#10;  allowed_policy_member_domains &#61; list&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |  |
+| [outputs_location](variables.tf#L159) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable | <code>string</code> |  | <code>null</code> |  |
+| [tag_names](variables.tf#L176) | Customized names for resource management tags. | <code title="object&#40;&#123;&#10;  context     &#61; string&#10;  environment &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code title="&#123;&#10;  context     &#61; &#34;context&#34;&#10;  environment &#61; &#34;environment&#34;&#10;&#125;">&#123;&#8230;&#125;</code> |  |
+| [team_folders](variables.tf#L193) | Team folders to be created. Format is described in a code comment. | <code title="map&#40;object&#40;&#123;&#10;  descriptive_name     &#61; string&#10;  group_iam            &#61; map&#40;list&#40;string&#41;&#41;&#10;  impersonation_groups &#61; list&#40;string&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>null</code> |  |
 
 ## Outputs
 
 | name | description | sensitive | consumers |
 |---|---|:---:|---|
-| [cicd_repositories](outputs.tf#L143) | WIF configuration for CI/CD repositories. |  |  |
-| [dataplatform](outputs.tf#L155) | Data for the Data Platform stage. |  |  |
-| [networking](outputs.tf#L171) | Data for the networking stage. |  |  |
-| [project_factories](outputs.tf#L180) | Data for the project factories stage. |  |  |
-| [providers](outputs.tf#L196) | Terraform provider files for this stage and dependent stages. | ✓ | <code>02-networking</code> · <code>02-security</code> · <code>03-dataplatform</code> · <code>xx-sandbox</code> · <code>xx-teams</code> |
-| [sandbox](outputs.tf#L203) | Data for the sandbox stage. |  | <code>xx-sandbox</code> |
-| [security](outputs.tf#L213) | Data for the networking stage. |  | <code>02-security</code> |
-| [teams](outputs.tf#L223) | Data for the teams stage. |  |  |
-| [tfvars](outputs.tf#L236) | Terraform variable files for the following stages. | ✓ |  |
+| [cicd_repositories](outputs.tf#L145) | WIF configuration for CI/CD repositories. |  |  |
+| [dataplatform](outputs.tf#L159) | Data for the Data Platform stage. |  |  |
+| [networking](outputs.tf#L175) | Data for the networking stage. |  |  |
+| [project_factories](outputs.tf#L184) | Data for the project factories stage. |  |  |
+| [providers](outputs.tf#L200) | Terraform provider files for this stage and dependent stages. | ✓ | <code>02-networking</code> · <code>02-security</code> · <code>03-dataplatform</code> · <code>xx-sandbox</code> · <code>xx-teams</code> |
+| [sandbox](outputs.tf#L207) | Data for the sandbox stage. |  | <code>xx-sandbox</code> |
+| [security](outputs.tf#L217) | Data for the networking stage. |  | <code>02-security</code> |
+| [teams](outputs.tf#L227) | Data for the teams stage. |  |  |
+| [tfvars](outputs.tf#L240) | Terraform variable files for the following stages. | ✓ |  |
 
 <!-- END TFDOC -->
diff --git a/fast/stages/01-resman/cicd-data-platform.tf b/fast/stages/01-resman/cicd-data-platform.tf
@@ -37,7 +37,7 @@ module "branch-dp-dev-cicd-repo" {
       included_files = [
         "**/*json", "**/*tf", "**/*yaml", ".cloudbuild/workflow.yaml"
       ]
-      service_account = module.branch-dp-dev-sa.iam_email
+      service_account = module.branch-dp-dev-sa-cicd.0.id
       substitutions   = {}
       template = {
         project_id  = null
@@ -47,6 +47,7 @@ module "branch-dp-dev-cicd-repo" {
       }
     }
   }
+  depends_on = [module.branch-dp-dev-sa-cicd]
 }
 
 module "branch-dp-prod-cicd-repo" {
@@ -68,7 +69,7 @@ module "branch-dp-prod-cicd-repo" {
       included_files = [
         "**/*json", "**/*tf", "**/*yaml", ".cloudbuild/workflow.yaml"
       ]
-      service_account = module.branch-dp-prod-sa.iam_email
+      service_account = module.branch-dp-prod-sa-cicd.0.id
       substitutions   = {}
       template = {
         project_id  = null
@@ -78,6 +79,7 @@ module "branch-dp-prod-cicd-repo" {
       }
     }
   }
+  depends_on = [module.branch-dp-prod-sa-cicd]
 }
 
 # SAs used by CI/CD workflows to impersonate automation SAs
@@ -96,7 +98,9 @@ module "branch-dp-dev-sa-cicd" {
   iam = (
     each.value.type == "sourcerepo"
     # used directly from the cloud build trigger for source repos
-    ? {}
+    ? {
+      "roles/iam.serviceAccountUser" = local.automation_resman_sa
+    }
     # impersonated via workload identity federation for external repos
     : {
       "roles/iam.workloadIdentityUser" = [
@@ -135,7 +139,9 @@ module "branch-dp-prod-sa-cicd" {
   iam = (
     each.value.type == "sourcerepo"
     # used directly from the cloud build trigger for source repos
-    ? {}
+    ? {
+      "roles/iam.serviceAccountUser" = local.automation_resman_sa
+    }
     # impersonated via workload identity federation for external repos
     : {
       "roles/iam.workloadIdentityUser" = [

diff --git a/fast/stages/01-resman/cicd-networking.tf b/fast/stages/01-resman/cicd-networking.tf
@@ -35,7 +35,7 @@ module "branch-network-cicd-repo" {
     fast-02-networking = {
       filename        = ".cloudbuild/workflow.yaml"
       included_files  = ["**/*tf", ".cloudbuild/workflow.yaml"]
-      service_account = module.branch-network-sa.id
+      service_account = module.branch-network-sa-cicd.0.id
       substitutions   = {}
       template = {
         project_id  = null
@@ -45,6 +45,7 @@ module "branch-network-cicd-repo" {
       }
     }
   }
+  depends_on = [module.branch-network-sa-cicd]
 }
 
 # SA used by CI/CD workflows to impersonate automation SAs
@@ -63,7 +64,9 @@ module "branch-network-sa-cicd" {
   iam = (
     each.value.type == "sourcerepo"
     # used directly from the cloud build trigger for source repos
-    ? {}
+    ? {
+      "roles/iam.serviceAccountUser" = local.automation_resman_sa
+    }
     # impersonated via workload identity federation for external repos
     : {
       "roles/iam.workloadIdentityUser" = [

diff --git a/fast/stages/01-resman/cicd-security.tf b/fast/stages/01-resman/cicd-security.tf
@@ -35,7 +35,7 @@ module "branch-security-cicd-repo" {
     fast-02-security = {
       filename        = ".cloudbuild/workflow.yaml"
       included_files  = ["**/*tf", ".cloudbuild/workflow.yaml"]
-      service_account = module.branch-security-sa.id
+      service_account = module.branch-security-sa-cicd.0.id
       substitutions   = {}
       template = {
         project_id  = null
@@ -45,6 +45,7 @@ module "branch-security-cicd-repo" {
       }
     }
   }
+  depends_on = [module.branch-security-sa-cicd]
 }
 
 # SA used by CI/CD workflows to impersonate automation SAs
@@ -63,7 +64,9 @@ module "branch-security-sa-cicd" {
   iam = (
     each.value.type == "sourcerepo"
     # used directly from the cloud build trigger for source repos
-    ? {}
+    ? {
+      "roles/iam.serviceAccountUser" = local.automation_resman_sa
+    }
     # impersonated via workload identity federation for external repos
     : {
       "roles/iam.workloadIdentityUser" = [