GoogleCloudPlatform · rosmo · Jun 21, 2022 · Jun 21, 2022
diff --git a/fast/assets/templates/workflow-gitlab.yaml b/fast/assets/templates/workflow-gitlab.yaml
@@ -15,6 +15,15 @@
 default:
   image:
     name: registry.gitlab.com/gitlab-org/terraform-images/releases/1.1
+  before_script:
+    - |
+      ssh-agent -a $SSH_AUTH_SOCK > /dev/null
+      echo "$CICD_MODULES_KEY" | base64 -d | tr -d '\r' | ssh-add - > /dev/null
+      mkdir -p ~/.ssh
+      ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts
+      ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts
+      cd "$${TF_ROOT}"
+      cp -R .tf-setup/. .
 
 variables:
   FAST_OUTPUTS_BUCKET: ${outputs_bucket}
@@ -43,6 +52,7 @@ cache:
 # Configure GCP Auth with Access Token
 gcp-auth:
   stage: gcp-auth
+  before_script: []
   script:
     - |
       PAYLOAD="$(cat <<EOF
@@ -83,6 +93,7 @@ gcp-auth:
 # Downloading from bucket into cache
 tf-setup:
   stage: tf-setup
+  before_script: []
   script:
     - |
       mkdir -p .tf-setup
@@ -104,13 +115,6 @@ tf-init:
   stage: tf-init
   script:
     - |
-      ssh-agent -a $SSH_AUTH_SOCK > /dev/null
-      echo "$CICD_MODULES_KEY" | tr -d '\r' | ssh-add - > /dev/null
-      mkdir -p ~/.ssh
-      ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts
-      ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts
-      cd "$${TF_ROOT}"
-      cp -R .tf-setup/. .
       gitlab-terraform init
   dependencies:
     - gcp-auth
@@ -120,13 +124,6 @@ tf-validate:
   stage: tf-validate
   script:
     - |
-      ssh-agent -a $SSH_AUTH_SOCK > /dev/null
-      echo "$CICD_MODULES_KEY" | tr -d '\r' | ssh-add - > /dev/null
-      mkdir -p ~/.ssh
-      ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts
-      ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts
-      cd "$${TF_ROOT}"
-      cp -R .tf-setup/. .
       gitlab-terraform validate
   dependencies:
     - gcp-auth
@@ -136,13 +133,6 @@ tf-plan:
   stage: tf-plan
   script:
   - |
-    ssh-agent -a $SSH_AUTH_SOCK > /dev/null
-    echo "$CICD_MODULES_KEY" | tr -d '\r' | ssh-add - > /dev/null
-    mkdir -p ~/.ssh
-    ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts
-    ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts
-    cd "$${TF_ROOT}"
-    cp -R .tf-setup/. .
     gitlab-terraform plan
     gitlab-terraform plan-json
   dependencies:

diff --git a/fast/stages/00-bootstrap/README.md b/fast/stages/00-bootstrap/README.md
@@ -358,10 +358,20 @@ federated_identity_providers = {
   github-sample = {
     attribute_condition = "attribute.repository_owner==\"my-github-org\""
     issuer              = "github"
+    custom_settings     = null
   }
   gitlab-sample = {
     attribute_condition = "attribute.namespace_path==\"my-gitlab-org\""
     issuer              = "gitlab"
+    custom_settings     = null
+  }
+  gitlab-ce-sample = {
+    attribute_condition = "attribute.namespace_path==\"my-gitlab-org\""
+    issuer              = "gitlab"
+    custom_settings     = {
+      issuer_uri          = "https://gitlab.fast.example.com"
+      allowed_audiences   = ["https://gitlab.fast.example.com"]
+    }
   }
 }
 ```
@@ -382,6 +392,12 @@ cicd_repositories = {
     name              = "my-gh-org/fast-bootstrap"
     type              = "github"
   }
+  cicd = {
+    branch            = null
+    identity_provider = "github-sample"
+    name              = "my-gh-org/fast-cicd"
+    type              = "github"
+  }
   resman = {
     branch            = "main"
     identity_provider = "github-sample"
@@ -395,6 +411,8 @@ The `type` attribute can be set to one of the supported repository types: `githu
 
 Once the stage is applied the generated output files will contain pre-configured workflow files for each repository, that will use Workload Identity Federation via a dedicated service account for each repository to impersonate the automation service account for the stage.
 
+You can use Terraform to automate creation of the repositories using the `00-cicd` stage.
+
 The remaining configuration is manual, as it regards the repositories themselves:
 
 - create a repository for modules
@@ -403,7 +421,7 @@ The remaining configuration is manual, as it regards the repositories themselves
     - for GitHub
       - create a key pair
       - create a [deploy key](https://docs.github.com/en/developers/overview/managing-deploy-keys#deploy-keys) in the modules repository with the public key
-      - create a `CICD_MODULES_KEY` secret with the private key in each of the repositories that need to access modules
+      - create a `CICD_MODULES_KEY` secret with the private key in each of the repositories that need to access modules (for Gitlab, please Base64 encode the private key for masking)
     - for Gitlab
       - TODO
     - for Source Repositories
@@ -443,31 +461,31 @@ The remaining configuration is manual, as it regards the repositories themselves
 | name | description | type | required | default | producer |
 |---|---|:---:|:---:|:---:|:---:|
 | [billing_account](variables.tf#L17) | Billing account id and organization id ('nnnnnnnn' or null). | <code title="object&#40;&#123;&#10;  id              &#61; string&#10;  organization_id &#61; number&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  |  |
-| [organization](variables.tf#L152) | Organization details. | <code title="object&#40;&#123;&#10;  domain      &#61; string&#10;  id          &#61; number&#10;  customer_id &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  |  |
-| [prefix](variables.tf#L167) | Prefix used for resources that need unique names. Use 9 characters or less. | <code>string</code> | ✓ |  |  |
+| [organization](variables.tf#L162) | Organization details. | <code title="object&#40;&#123;&#10;  domain      &#61; string&#10;  id          &#61; number&#10;  customer_id &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  |  |
+| [prefix](variables.tf#L177) | Prefix used for resources that need unique names. Use 9 characters or less. | <code>string</code> | ✓ |  |  |
 | [bootstrap_user](variables.tf#L25) | Email of the nominal user running this stage for the first time. | <code>string</code> |  | <code>null</code> |  |
-| [cicd_repositories](variables.tf#L31) | CI/CD repository configuration. Identity providers reference keys in the `federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | <code title="object&#40;&#123;&#10;  bootstrap &#61; object&#40;&#123;&#10;    branch            &#61; string&#10;    identity_provider &#61; string&#10;    name              &#61; string&#10;    type              &#61; string&#10;  &#125;&#41;&#10;  resman &#61; object&#40;&#123;&#10;    branch            &#61; string&#10;    identity_provider &#61; string&#10;    name              &#61; string&#10;    type              &#61; string&#10;  &#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |  |
-| [custom_role_names](variables.tf#L77) | Names of custom roles defined at the org level. | <code title="object&#40;&#123;&#10;  organization_iam_admin        &#61; string&#10;  service_project_network_admin &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code title="&#123;&#10;  organization_iam_admin        &#61; &#34;organizationIamAdmin&#34;&#10;  service_project_network_admin &#61; &#34;serviceProjectNetworkAdmin&#34;&#10;&#125;">&#123;&#8230;&#125;</code> |  |
-| [federated_identity_providers](variables.tf#L89) | Workload Identity Federation pools. The `cicd_repositories` variable references keys here. | <code title="map&#40;object&#40;&#123;&#10;  attribute_condition &#61; string&#10;  issuer              &#61; string&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |  |
-| [groups](variables.tf#L99) | Group names to grant organization-level permissions. | <code>map&#40;string&#41;</code> |  | <code title="&#123;&#10;  gcp-billing-admins      &#61; &#34;gcp-billing-admins&#34;,&#10;  gcp-devops              &#61; &#34;gcp-devops&#34;,&#10;  gcp-network-admins      &#61; &#34;gcp-network-admins&#34;&#10;  gcp-organization-admins &#61; &#34;gcp-organization-admins&#34;&#10;  gcp-security-admins     &#61; &#34;gcp-security-admins&#34;&#10;  gcp-support             &#61; &#34;gcp-support&#34;&#10;&#125;">&#123;&#8230;&#125;</code> |  |
-| [iam](variables.tf#L113) | Organization-level custom IAM settings in role => [principal] format. | <code>map&#40;list&#40;string&#41;&#41;</code> |  | <code>&#123;&#125;</code> |  |
-| [iam_additive](variables.tf#L119) | Organization-level custom IAM settings in role => [principal] format for non-authoritative bindings. | <code>map&#40;list&#40;string&#41;&#41;</code> |  | <code>&#123;&#125;</code> |  |
-| [log_sinks](variables.tf#L127) | Org-level log sinks, in name => {type, filter} format. | <code title="map&#40;object&#40;&#123;&#10;  filter &#61; string&#10;  type   &#61; string&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code title="&#123;&#10;  audit-logs &#61; &#123;&#10;    filter &#61; &#34;logName:&#92;&#34;&#47;logs&#47;cloudaudit.googleapis.com&#37;2Factivity&#92;&#34; OR logName:&#92;&#34;&#47;logs&#47;cloudaudit.googleapis.com&#37;2Fsystem_event&#92;&#34;&#34;&#10;    type   &#61; &#34;bigquery&#34;&#10;  &#125;&#10;  vpc-sc &#61; &#123;&#10;    filter &#61; &#34;protoPayload.metadata.&#64;type&#61;&#92;&#34;type.googleapis.com&#47;google.cloud.audit.VpcServiceControlAuditMetadata&#92;&#34;&#34;&#10;    type   &#61; &#34;bigquery&#34;&#10;  &#125;&#10;&#125;">&#123;&#8230;&#125;</code> |  |
-| [outputs_location](variables.tf#L161) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable | <code>string</code> |  | <code>null</code> |  |
+| [cicd_repositories](variables.tf#L31) | CI/CD repository configuration. Identity providers reference keys in the `federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | <code title="object&#40;&#123;&#10;  bootstrap &#61; object&#40;&#123;&#10;    branch            &#61; string&#10;    identity_provider &#61; string&#10;    name              &#61; string&#10;    type              &#61; string&#10;  &#125;&#41;&#10;  cicd &#61; object&#40;&#123;&#10;    branch            &#61; string&#10;    identity_provider &#61; string&#10;    name              &#61; string&#10;    type              &#61; string&#10;  &#125;&#41;&#10;  resman &#61; object&#40;&#123;&#10;    branch            &#61; string&#10;    identity_provider &#61; string&#10;    name              &#61; string&#10;    type              &#61; string&#10;  &#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |  |
+| [custom_role_names](variables.tf#L83) | Names of custom roles defined at the org level. | <code title="object&#40;&#123;&#10;  organization_iam_admin        &#61; string&#10;  service_project_network_admin &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code title="&#123;&#10;  organization_iam_admin        &#61; &#34;organizationIamAdmin&#34;&#10;  service_project_network_admin &#61; &#34;serviceProjectNetworkAdmin&#34;&#10;&#125;">&#123;&#8230;&#125;</code> |  |
+| [federated_identity_providers](variables.tf#L95) | Workload Identity Federation pools. The `cicd_repositories` variable references keys here. | <code title="map&#40;object&#40;&#123;&#10;  attribute_condition &#61; string&#10;  issuer              &#61; string&#10;  custom_settings &#61; object&#40;&#123;&#10;    issuer_uri        &#61; string&#10;    allowed_audiences &#61; list&#40;string&#41;&#10;  &#125;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |  |
+| [groups](variables.tf#L109) | Group names to grant organization-level permissions. | <code>map&#40;string&#41;</code> |  | <code title="&#123;&#10;  gcp-billing-admins      &#61; &#34;gcp-billing-admins&#34;,&#10;  gcp-devops              &#61; &#34;gcp-devops&#34;,&#10;  gcp-network-admins      &#61; &#34;gcp-network-admins&#34;&#10;  gcp-organization-admins &#61; &#34;gcp-organization-admins&#34;&#10;  gcp-security-admins     &#61; &#34;gcp-security-admins&#34;&#10;  gcp-support             &#61; &#34;gcp-support&#34;&#10;&#125;">&#123;&#8230;&#125;</code> |  |
+| [iam](variables.tf#L123) | Organization-level custom IAM settings in role => [principal] format. | <code>map&#40;list&#40;string&#41;&#41;</code> |  | <code>&#123;&#125;</code> |  |
+| [iam_additive](variables.tf#L129) | Organization-level custom IAM settings in role => [principal] format for non-authoritative bindings. | <code>map&#40;list&#40;string&#41;&#41;</code> |  | <code>&#123;&#125;</code> |  |
+| [log_sinks](variables.tf#L137) | Org-level log sinks, in name => {type, filter} format. | <code title="map&#40;object&#40;&#123;&#10;  filter &#61; string&#10;  type   &#61; string&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code title="&#123;&#10;  audit-logs &#61; &#123;&#10;    filter &#61; &#34;logName:&#92;&#34;&#47;logs&#47;cloudaudit.googleapis.com&#37;2Factivity&#92;&#34; OR logName:&#92;&#34;&#47;logs&#47;cloudaudit.googleapis.com&#37;2Fsystem_event&#92;&#34;&#34;&#10;    type   &#61; &#34;bigquery&#34;&#10;  &#125;&#10;  vpc-sc &#61; &#123;&#10;    filter &#61; &#34;protoPayload.metadata.&#64;type&#61;&#92;&#34;type.googleapis.com&#47;google.cloud.audit.VpcServiceControlAuditMetadata&#92;&#34;&#34;&#10;    type   &#61; &#34;bigquery&#34;&#10;  &#125;&#10;&#125;">&#123;&#8230;&#125;</code> |  |
+| [outputs_location](variables.tf#L171) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable | <code>string</code> |  | <code>null</code> |  |
 
 ## Outputs
 
 | name | description | sensitive | consumers |
 |---|---|:---:|---|
-| [automation](outputs.tf#L82) | Automation resources. |  |  |
-| [billing_dataset](outputs.tf#L87) | BigQuery dataset prepared for billing export. |  |  |
-| [cicd_repositories](outputs.tf#L92) | CI/CD repository configurations. |  |  |
-| [custom_roles](outputs.tf#L104) | Organization-level custom roles. |  |  |
-| [federated_identity](outputs.tf#L109) | Workload Identity Federation pool and providers. |  |  |
-| [outputs_bucket](outputs.tf#L119) | GCS bucket where generated output files are stored. |  |  |
-| [project_ids](outputs.tf#L124) | Projects created by this stage. |  |  |
-| [providers](outputs.tf#L143) | Terraform provider files for this stage and dependent stages. | ✓ | <code>stage-01</code> |
-| [service_accounts](outputs.tf#L133) | Automation service accounts created by this stage. |  |  |
-| [tfvars](outputs.tf#L152) | Terraform variable files for the following stages. | ✓ |  |
+| [automation](outputs.tf#L87) | Automation resources. |  |  |
+| [billing_dataset](outputs.tf#L92) | BigQuery dataset prepared for billing export. |  |  |
+| [cicd_repositories](outputs.tf#L97) | CI/CD repository configurations. |  |  |
+| [custom_roles](outputs.tf#L109) | Organization-level custom roles. |  |  |
+| [federated_identity](outputs.tf#L114) | Workload Identity Federation pool and providers. |  |  |
+| [outputs_bucket](outputs.tf#L124) | GCS bucket where generated output files are stored. |  |  |
+| [project_ids](outputs.tf#L129) | Projects created by this stage. |  |  |
+| [providers](outputs.tf#L149) | Terraform provider files for this stage and dependent stages. | ✓ | <code>stage-01</code> |
+| [service_accounts](outputs.tf#L138) | Automation service accounts created by this stage. |  |  |
+| [tfvars](outputs.tf#L158) | Terraform variable files for the following stages. | ✓ |  |
 
 <!-- END TFDOC -->
diff --git a/fast/stages/00-bootstrap/automation.tf b/fast/stages/00-bootstrap/automation.tf
@@ -121,6 +121,37 @@ module "automation-tf-bootstrap-sa" {
 
 # resource hierarchy stage's bucket and service account
 
+module "automation-tf-cicd-gcs" {
+  source     = "../../../modules/gcs"
+  project_id = module.automation-project.project_id
+  name       = "iac-core-cicd-0"
+  prefix     = local.prefix
+  versioning = true
+  iam = {
+    "roles/storage.objectAdmin" = [module.automation-tf-cicd-provisioning-sa.iam_email]
+  }
+  depends_on = [module.organization]
+}
+
+module "automation-tf-cicd-provisioning-sa" {
+  source      = "../../../modules/iam-service-account"
+  project_id  = module.automation-project.project_id
+  name        = "cicd-0"
+  description = "Terraform stage 1 CICD service account."
+  prefix      = local.prefix
+  # allow SA used by CI/CD workflow to impersonate this SA
+  iam = {
+    "roles/iam.serviceAccountTokenCreator" = compact([
+      try(module.automation-tf-cicd-sa["cicd"].iam_email, null)
+    ])
+  }
+  iam_storage_roles = {
+    (module.automation-tf-output-gcs.name) = ["roles/storage.admin"]
+  }
+}
+
+# resource hierarchy stage's bucket and service account
+
 module "automation-tf-resman-gcs" {
   source     = "../../../modules/gcs"
   project_id = module.automation-project.project_id

diff --git a/fast/stages/00-bootstrap/cicd.tf b/fast/stages/00-bootstrap/cicd.tf
@@ -23,20 +23,25 @@ locals {
       v != null
       &&
       (
-        v.type == "sourcerepo"
+        try(v.type, null) == "sourcerepo"
         ||
-        contains(keys(local.identity_providers), coalesce(v.identity_provider, ":"))
+        contains(keys(local.identity_providers), coalesce(try(v.identity_provider, null), ":"))
       )
       &&
-      fileexists("${path.module}/templates/workflow-${v.type}.yaml")
+      fileexists(format("${path.module}/templates/workflow-%s.yaml", try(v.type, "")))
     )
   }
   cicd_workflow_providers = {
     bootstrap = "00-bootstrap-providers.tf"
+    cicd      = "00-cicd-providers.tf"
     resman    = "01-resman-providers.tf"
   }
   cicd_workflow_var_files = {
     bootstrap = []
+    cicd = [
+      "00-bootstrap.auto.tfvars.json",
+      "globals.auto.tfvars.json"
+    ]
     resman = [
       "00-bootstrap.auto.tfvars.json",
       "globals.auto.tfvars.json"

diff --git a/fast/stages/00-bootstrap/identity-providers.tf b/fast/stages/00-bootstrap/identity-providers.tf
@@ -19,7 +19,8 @@
 locals {
   identity_providers = {
     for k, v in var.federated_identity_providers : k => merge(
-      v, lookup(local.identity_providers_defs, v.issuer, {})
+      v, lookup(local.identity_providers_defs, v.issuer, {}),
+      { for kk, vv in lookup(v, "custom_settings", {}) : kk => vv if vv != null }
     )
   }
   identity_providers_defs = {

diff --git a/fast/stages/00-bootstrap/outputs.tf b/fast/stages/00-bootstrap/outputs.tf
@@ -43,6 +43,11 @@ locals {
       name   = "bootstrap"
       sa     = module.automation-tf-bootstrap-sa.email
     })
+    "00-cicd" = templatefile(local._tpl_providers, {
+      bucket = module.automation-tf-cicd-gcs.name
+      name   = "cicd"
+      sa     = module.automation-tf-cicd-provisioning-sa.email
+    })
     "01-resman" = templatefile(local._tpl_providers, {
       bucket = module.automation-tf-resman-gcs.name
       name   = "resman"
@@ -134,6 +139,7 @@ output "service_accounts" {
   description = "Automation service accounts created by this stage."
   value = {
     bootstrap = module.automation-tf-bootstrap-sa.email
+    cicd      = module.automation-tf-cicd-provisioning-sa.email
     resman    = module.automation-tf-resman-sa.email
   }
 }

diff --git a/fast/stages/00-bootstrap/variables.tf b/fast/stages/00-bootstrap/variables.tf
@@ -37,6 +37,12 @@ variable "cicd_repositories" {
       name              = string
       type              = string
     })
+    cicd = object({
+      branch            = string
+      identity_provider = string
+      name              = string
+      type              = string
+    })
     resman = object({
       branch            = string
       identity_provider = string
@@ -91,6 +97,10 @@ variable "federated_identity_providers" {
   type = map(object({
     attribute_condition = string
     issuer              = string
+    custom_settings = object({
+      issuer_uri        = string
+      allowed_audiences = list(string)
+    })
   }))
   default  = {}
   nullable = false