diff --git a/fast/assets/templates/workflow-github.yaml b/fast/assets/templates/workflow-github.yaml index 913ecf5114..cfeacf20a9 100644 --- a/fast/assets/templates/workflow-github.yaml +++ b/fast/assets/templates/workflow-github.yaml @@ -98,7 +98,7 @@ jobs: name: Terraform plan continue-on-error: true run: | - terraform plan -out ../plan.out -no-color + terraform plan -input=false -out ../plan.out -no-color - id: tf-apply if: github.event.pull_request.merged == true diff --git a/fast/assets/templates/workflow-gitlab.yaml b/fast/assets/templates/workflow-gitlab.yaml index 101f5bed22..986d57602e 100644 --- a/fast/assets/templates/workflow-gitlab.yaml +++ b/fast/assets/templates/workflow-gitlab.yaml @@ -14,7 +14,7 @@ default: image: - name: registry.gitlab.com/gitlab-org/terraform-images/releases/1.1 + name: registry.gitlab.com/gitlab-org/terraform-images/releases/1.1 variables: FAST_OUTPUTS_BUCKET: ${outputs_bucket} @@ -116,7 +116,7 @@ tf-init: - gcp-auth # Terraform Validate -tf-validate: +tf-validate: stage: tf-validate script: - | @@ -134,17 +134,17 @@ tf-validate: # Terraform Plan tf-plan: stage: tf-plan - script: - - | - ssh-agent -a $SSH_AUTH_SOCK > /dev/null - echo "$CICD_MODULES_KEY" | tr -d '\r' | ssh-add - > /dev/null - mkdir -p ~/.ssh - ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts - ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts - cd "$${TF_ROOT}" - cp -R .tf-setup/. . - gitlab-terraform plan - gitlab-terraform plan-json + script: + - | + ssh-agent -a $SSH_AUTH_SOCK > /dev/null + echo "$CICD_MODULES_KEY" | tr -d '\r' | ssh-add - > /dev/null + mkdir -p ~/.ssh + ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts + ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts + cd "$${TF_ROOT}" + cp -R .tf-setup/. . + gitlab-terraform plan + gitlab-terraform plan-json dependencies: - gcp-auth artifacts: diff --git a/fast/assets/templates/workflow-sourcerepo.yaml b/fast/assets/templates/workflow-sourcerepo.yaml new file mode 100644 index 0000000000..7f6f08ff04 --- /dev/null +++ b/fast/assets/templates/workflow-sourcerepo.yaml @@ -0,0 +1,98 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: + - name: alpine:3 + id: tf-download + entrypoint: sh + args: + - -eEuo + - pipefail + - -c + - |- + mkdir -p /builder/home/.local/bin + wget https://releases.hashicorp.com/terraform/$${_TF_VERSION}/terraform_$${_TF_VERSION}_linux_amd64.zip + unzip terraform_$${_TF_VERSION}_linux_amd64.zip -d /builder/home/.local/bin + rm terraform_$${_TF_VERSION}_linux_amd64.zip + chmod 755 /builder/home/.local/bin/terraform + - name: alpine:3 + id: tf-check-format + entrypoint: sh + args: + - -eEuo + - pipefail + - -c + - |- + terraform fmt -recursive -check /workspace/ + - name: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine + id: tf-files + entrypoint: bash + args: + - -eEuo + - pipefail + - -c + - |- + /google-cloud-sdk/bin/gsutil cp \ + gs://$${_FAST_OUTPUTS_BUCKET}/providers/$${_TF_PROVIDERS_FILE} ./ + /google-cloud-sdk/bin/gsutil cp -r \ + gs://$${_FAST_OUTPUTS_BUCKET}/tfvars ./ + for f in $${_TF_VAR_FILES}; do + ln -s tfvars/$f ./ + done + - name: alpine:3 + id: tf-init + entrypoint: sh + args: + - -eEuo + - pipefail + - -c + - |- + terraform init -no-color + - name: alpine:3 + id: tf-check-validate + entrypoint: sh + args: + - -eEuo + - pipefail + - -c + - |- + terraform validate -no-color + - name: alpine:3 + id: tf-plan + entrypoint: sh + args: + - -eEuo + - pipefail + - -c + - |- + terraform plan -no-color -input=false -out plan.out + # store artifact and ask for approval here if needed + - name: alpine:3 + id: tf-apply + entrypoint: sh + args: + - -eEuo + - pipefail + - -c + - |- + terraform apply -no-color -input=false -auto-approve plan.out +options: + env: + - PATH=/usr/local/bin:/usr/bin:/bin:/builder/home/.local/bin + logging: CLOUD_LOGGING_ONLY +substitutions: + _FAST_OUTPUTS_BUCKET: ${outputs_bucket} + _TF_PROVIDERS_FILE: ${tf_providers_file} + _TF_VAR_FILES: ${tf_var_files == [] ? "''" : join("\n ", tf_var_files)} + _TF_VERSION: 1.1.7 diff --git a/fast/stages/00-bootstrap/README.md b/fast/stages/00-bootstrap/README.md index 1b702f6973..8698e284dd 100644 --- a/fast/stages/00-bootstrap/README.md +++ b/fast/stages/00-bootstrap/README.md @@ -385,6 +385,8 @@ cicd_repositories = { } ``` +The `type` attribute can be set to one of the supported repository types: `github`, `gitlab`, or `sourcerepo`. + Once the stage is applied the generated output files will contain pre-configured workflow files for each repository, that will use Workload Identity Federation via a dedicated service account for each repository to impersonate the automation service account for the stage. The remaining configuration is manual, as it regards the repositories themselves: @@ -396,6 +398,10 @@ The remaining configuration is manual, as it regards the repositories themselves - create a key pair - create a [deploy key](https://docs.github.com/en/developers/overview/managing-deploy-keys#deploy-keys) in the modules repository with the public key - create a `CICD_MODULES_KEY` secret with the private key in each of the repositories that need to access modules + - for Gitlab + - TODO + - for Source Repositories + - assign the reader role to the CI/CD service accounts - create one repository for each stage - clone and populate them with the stage source - edit the modules source to match your modules repository @@ -405,6 +411,7 @@ The remaining configuration is manual, as it regards the repositories themselves - copy the generated workflow file for the stage from the GCS output files bucket or from the local clone if enabled - for GitHub, place it in a `.github/workflows` folder in the repository root - for Gitlab, rename it to `.gitlab-ci.yml` and place it in the repository root + - for Source Repositories, place it in `.cloudbuild/workflow.yaml` @@ -415,7 +422,7 @@ The remaining configuration is manual, as it regards the repositories themselves |---|---|---|---| | [automation.tf](./automation.tf) | Automation project and resources. | gcs · iam-service-account · project | | | [billing.tf](./billing.tf) | Billing export project and dataset. | bigquery-dataset · organization · project | google_billing_account_iam_member · google_organization_iam_binding | -| [cicd.tf](./cicd.tf) | Workload Identity Federation configurations for CI/CD. | iam-service-account | | +| [cicd.tf](./cicd.tf) | Workload Identity Federation configurations for CI/CD. | iam-service-account · source-repository | | | [identity-providers.tf](./identity-providers.tf) | Workload Identity Federation provider definitions. | | google_iam_workload_identity_pool · google_iam_workload_identity_pool_provider | | [log-export.tf](./log-export.tf) | Audit log project and sink. | bigquery-dataset · gcs · logging-bucket · project · pubsub | | | [main.tf](./main.tf) | Module-level locals and resources. | | | @@ -430,31 +437,31 @@ The remaining configuration is manual, as it regards the repositories themselves | name | description | type | required | default | producer | |---|---|:---:|:---:|:---:|:---:| | [billing_account](variables.tf#L17) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | ✓ | | | -| [organization](variables.tf#L146) | Organization details. | object({…}) | ✓ | | | -| [prefix](variables.tf#L161) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | | +| [organization](variables.tf#L152) | Organization details. | object({…}) | ✓ | | | +| [prefix](variables.tf#L167) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | | | [bootstrap_user](variables.tf#L25) | Email of the nominal user running this stage for the first time. | string | | null | | | [cicd_repositories](variables.tf#L31) | CI/CD repository configuration. Identity providers reference keys in the `federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | object({…}) | | null | | -| [custom_role_names](variables.tf#L71) | Names of custom roles defined at the org level. | object({…}) | | {…} | | -| [federated_identity_providers](variables.tf#L83) | Workload Identity Federation pools. The `cicd_repositories` variable references keys here. | map(object({…})) | | {} | | -| [groups](variables.tf#L93) | Group names to grant organization-level permissions. | map(string) | | {…} | | -| [iam](variables.tf#L107) | Organization-level custom IAM settings in role => [principal] format. | map(list(string)) | | {} | | -| [iam_additive](variables.tf#L113) | Organization-level custom IAM settings in role => [principal] format for non-authoritative bindings. | map(list(string)) | | {} | | -| [log_sinks](variables.tf#L121) | Org-level log sinks, in name => {type, filter} format. | map(object({…})) | | {…} | | -| [outputs_location](variables.tf#L155) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable | string | | null | | +| [custom_role_names](variables.tf#L77) | Names of custom roles defined at the org level. | object({…}) | | {…} | | +| [federated_identity_providers](variables.tf#L89) | Workload Identity Federation pools. The `cicd_repositories` variable references keys here. | map(object({…})) | | {} | | +| [groups](variables.tf#L99) | Group names to grant organization-level permissions. | map(string) | | {…} | | +| [iam](variables.tf#L113) | Organization-level custom IAM settings in role => [principal] format. | map(list(string)) | | {} | | +| [iam_additive](variables.tf#L119) | Organization-level custom IAM settings in role => [principal] format for non-authoritative bindings. | map(list(string)) | | {} | | +| [log_sinks](variables.tf#L127) | Org-level log sinks, in name => {type, filter} format. | map(object({…})) | | {…} | | +| [outputs_location](variables.tf#L161) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable | string | | null | | ## Outputs | name | description | sensitive | consumers | |---|---|:---:|---| -| [automation](outputs.tf#L93) | Automation resources. | | | -| [billing_dataset](outputs.tf#L98) | BigQuery dataset prepared for billing export. | | | -| [cicd_repositories](outputs.tf#L103) | CI/CD repository configurations. | | | -| [custom_roles](outputs.tf#L115) | Organization-level custom roles. | | | -| [federated_identity](outputs.tf#L120) | Workload Identity Federation pool and providers. | | | -| [outputs_bucket](outputs.tf#L130) | GCS bucket where generated output files are stored. | | | -| [project_ids](outputs.tf#L135) | Projects created by this stage. | | | -| [providers](outputs.tf#L154) | Terraform provider files for this stage and dependent stages. | ✓ | stage-01 | -| [service_accounts](outputs.tf#L144) | Automation service accounts created by this stage. | | | -| [tfvars](outputs.tf#L163) | Terraform variable files for the following stages. | ✓ | | +| [automation](outputs.tf#L81) | Automation resources. | | | +| [billing_dataset](outputs.tf#L86) | BigQuery dataset prepared for billing export. | | | +| [cicd_repositories](outputs.tf#L91) | CI/CD repository configurations. | | | +| [custom_roles](outputs.tf#L103) | Organization-level custom roles. | | | +| [federated_identity](outputs.tf#L108) | Workload Identity Federation pool and providers. | | | +| [outputs_bucket](outputs.tf#L118) | GCS bucket where generated output files are stored. | | | +| [project_ids](outputs.tf#L123) | Projects created by this stage. | | | +| [providers](outputs.tf#L142) | Terraform provider files for this stage and dependent stages. | ✓ | stage-01 | +| [service_accounts](outputs.tf#L132) | Automation service accounts created by this stage. | | | +| [tfvars](outputs.tf#L151) | Terraform variable files for the following stages. | ✓ | | diff --git a/fast/stages/00-bootstrap/automation.tf b/fast/stages/00-bootstrap/automation.tf index 1caaf94ca7..0874fc4ff6 100644 --- a/fast/stages/00-bootstrap/automation.tf +++ b/fast/stages/00-bootstrap/automation.tf @@ -55,6 +55,7 @@ module "automation-project" { "bigquerystorage.googleapis.com", "billingbudgets.googleapis.com", "cloudbilling.googleapis.com", + "cloudbuild.googleapis.com", "cloudkms.googleapis.com", "cloudresourcemanager.googleapis.com", "container.googleapis.com", @@ -65,6 +66,7 @@ module "automation-project" { "pubsub.googleapis.com", "servicenetworking.googleapis.com", "serviceusage.googleapis.com", + "sourcerepo.googleapis.com", "stackdriver.googleapis.com", "storage-component.googleapis.com", "storage.googleapis.com", @@ -72,7 +74,7 @@ module "automation-project" { ] } -# outputt files bucket +# output files bucket module "automation-tf-output-gcs" { source = "../../../modules/gcs" @@ -100,6 +102,7 @@ module "automation-tf-bootstrap-sa" { name = "bootstrap-0" description = "Terraform organization bootstrap service account." prefix = local.prefix + # allow SA used by CI/CD workflow to impersonate this SA iam = { "roles/iam.serviceAccountTokenCreator" = compact([ try(module.automation-tf-cicd-sa["bootstrap"].iam_email, null) @@ -130,6 +133,7 @@ module "automation-tf-resman-sa" { name = "resman-0" description = "Terraform stage 1 resman service account." prefix = local.prefix + # allow SA used by CI/CD workflow to impersonate this SA iam = { "roles/iam.serviceAccountTokenCreator" = compact([ try(module.automation-tf-cicd-sa["resman"].iam_email, null) diff --git a/fast/stages/00-bootstrap/cicd.tf b/fast/stages/00-bootstrap/cicd.tf index f4c8c6c289..b4032c7665 100644 --- a/fast/stages/00-bootstrap/cicd.tf +++ b/fast/stages/00-bootstrap/cicd.tf @@ -17,45 +17,101 @@ # tfdoc:file:description Workload Identity Federation configurations for CI/CD. locals { - # TODO: map null provider to Cloud Build once we add support for it cicd_repositories = { for k, v in coalesce(var.cicd_repositories, {}) : k => v if( v != null && - contains(keys(local.identity_providers), v.identity_provider) + ( + v.type == "sourcerepo" + || + contains(keys(local.identity_providers), coalesce(v.identity_provider, ":")) + ) && fileexists("${path.module}/templates/workflow-${v.type}.yaml") ) } - cicd_service_accounts = { - for k, v in module.automation-tf-cicd-sa : - k => v.iam_email + cicd_workflow_providers = { + bootstrap = "00-bootstrap-providers.tf" + resman = "01-resman-providers.tf" + } + cicd_workflow_var_files = { + bootstrap = [] + resman = [ + "00-bootstrap.auto.tfvars.json", + "globals.auto.tfvars.json" + ] } } +# source repository + +module "automation-tf-cicd-repo" { + source = "../../../modules/source-repository" + for_each = { + for k, v in local.cicd_repositories : k => v if v.type == "sourcerepo" + } + project_id = module.automation-project.project_id + name = each.value.name + iam = { + "roles/source.admin" = [ + each.key == "bootstrap" + ? module.automation-tf-bootstrap-sa.iam_email + : module.automation-tf-resman-sa.iam_email + ] + "roles/source.reader" = [ + module.automation-tf-cicd-sa[each.key].iam_email + ] + } + triggers = { + "fast-00-${each.key}" = { + filename = ".cloudbuild/workflow.yaml" + included_files = ["**/*tf", ".cloudbuild/workflow.yaml"] + service_account = module.automation-tf-cicd-sa[each.key].id + substitutions = {} + template = { + project_id = null + branch_name = each.value.branch + repo_name = each.value.name + tag_name = null + } + } + } +} + +# SAs used by CI/CD workflows to impersonate automation SAs + module "automation-tf-cicd-sa" { source = "../../../modules/iam-service-account" for_each = local.cicd_repositories project_id = module.automation-project.project_id name = "${each.key}-1" - description = "Terraform CI/CD stage 1 ${each.key} service account." + description = "Terraform CI/CD ${each.key} service account." prefix = local.prefix - iam = { - "roles/iam.workloadIdentityUser" = [ - each.value.branch == null - ? format( - local.identity_providers_defs[each.value.type].principalset_tpl, - google_iam_workload_identity_pool.default.0.name, - each.value.name - ) - : format( - local.identity_providers_defs[each.value.type].principal_tpl, - google_iam_workload_identity_pool.default.0.name, - each.value.name, - each.value.branch - ) - ] + iam = ( + each.value.type == "sourcerepo" + # used directly from the cloud build trigger for source repos + ? {} + # impersonated via workload identity federation for external repos + : { + "roles/iam.workloadIdentityUser" = [ + each.value.branch == null + ? format( + local.identity_providers_defs[each.value.type].principalset_tpl, + google_iam_workload_identity_pool.default.0.name, + each.value.name + ) + : format( + local.identity_providers_defs[each.value.type].principal_tpl, + google_iam_workload_identity_pool.default.0.name, + each.value.name, + each.value.branch + ) + ] + } + ) + iam_project_roles = { + (module.automation-project.project_id) = ["roles/logging.logWriter"] } iam_storage_roles = { (module.automation-tf-output-gcs.name) = ["roles/storage.objectViewer"] diff --git a/fast/stages/00-bootstrap/identity-providers.tf b/fast/stages/00-bootstrap/identity-providers.tf index f202bb09a2..31bf5d2cf2 100644 --- a/fast/stages/00-bootstrap/identity-providers.tf +++ b/fast/stages/00-bootstrap/identity-providers.tf @@ -35,6 +35,7 @@ locals { principal_tpl = "principal://iam.googleapis.com/%s/subject/repo:%s:ref:refs/heads/%s" principalset_tpl = "principalSet://iam.googleapis.com/%s/attribute.repository/%s" } + # https://docs.gitlab.com/ee/ci/cloud_services/index.html#how-it-works gitlab = { attribute_mapping = { "google.subject" = "assertion.sub" diff --git a/fast/stages/00-bootstrap/outputs.tf b/fast/stages/00-bootstrap/outputs.tf index a60342e68d..cfb2460b87 100644 --- a/fast/stages/00-bootstrap/outputs.tf +++ b/fast/stages/00-bootstrap/outputs.tf @@ -15,34 +15,22 @@ */ locals { - _cicd_workflow_attrs = { - bootstrap = { - service_account = try( - module.automation-tf-cicd-sa["bootstrap"].email, null - ) - tf_providers_file = "00-bootstrap-providers.tf" - tf_var_files = [] - } - resman = { - service_account = try( - module.automation-tf-cicd-sa["resman"].email, null - ) - tf_providers_file = "01-resman-providers.tf" - tf_var_files = [ - "00-bootstrap.auto.tfvars.json", - "globals.auto.tfvars.json" - ] - } - } _tpl_providers = "${path.module}/templates/providers.tf.tpl" + # render CI/CD workflow templates cicd_workflows = { for k, v in local.cicd_repositories : k => templatefile( - "${path.module}/templates/workflow-${v.type}.yaml", - merge(local._cicd_workflow_attrs[k], { - identity_provider = local.wif_providers[v["identity_provider"]].name - outputs_bucket = module.automation-tf-output-gcs.name + "${path.module}/templates/workflow-${v.type}.yaml", { + identity_provider = try( + local.wif_providers[v["identity_provider"]].name, "" + ) + outputs_bucket = module.automation-tf-output-gcs.name + service_account = try( + module.automation-tf-cicd-sa[k].email, "" + ) stage_name = k - }) + tf_providers_file = local.cicd_workflow_providers[k] + tf_var_files = local.cicd_workflow_var_files[k] + } ) } custom_roles = { @@ -106,8 +94,8 @@ output "cicd_repositories" { for k, v in local.cicd_repositories : k => { branch = v.branch name = v.name - provider = local.wif_providers[v.identity_provider].name - service_account = module.automation-tf-cicd-sa[k].email + provider = try(local.wif_providers[v.identity_provider].name, null) + service_account = try(module.automation-tf-cicd-sa[k].email, null) } } } diff --git a/fast/stages/00-bootstrap/variables.tf b/fast/stages/00-bootstrap/variables.tf index bb55393f46..a08fedb1c5 100644 --- a/fast/stages/00-bootstrap/variables.tf +++ b/fast/stages/00-bootstrap/variables.tf @@ -29,7 +29,6 @@ variable "bootstrap_user" { } variable "cicd_repositories" { - # TODO: edit description once we add support for Cloud Build (null provider) description = "CI/CD repository configuration. Identity providers reference keys in the `federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed." type = object({ bootstrap = object({ @@ -46,25 +45,32 @@ variable "cicd_repositories" { }) }) default = null + validation { + condition = alltrue([ + for k, v in coalesce(var.cicd_repositories, {}) : + v == null || try(v.name, null) != null + ]) + error_message = "Non-null repositories need a non-null name." + } validation { condition = alltrue([ for k, v in coalesce(var.cicd_repositories, {}) : v == null || ( - try(v.name, null) != null - && try(v.identity_provider, null) != null + || + try(v.type, null) == "sourcerepo" ) ]) - error_message = "Non-null repositories need non-null name and providers." + error_message = "Non-null repositories need a non-null provider unless type is 'sourcerepo'." } validation { condition = alltrue([ for k, v in coalesce(var.cicd_repositories, {}) : v == null || ( - contains(["gitlab", "github"], coalesce(try(v.type, null), "null")) + contains(["github", "gitlab", "sourcerepo"], coalesce(try(v.type, null), "null")) ) ]) - error_message = "Invalid repository type, supported types: 'github' or 'gitlab'." + error_message = "Invalid repository type, supported types: 'github' 'gitlab' or 'sourcerepo'." } } diff --git a/fast/stages/01-resman/README.md b/fast/stages/01-resman/README.md index e3f07c678c..7151f45e79 100644 --- a/fast/stages/01-resman/README.md +++ b/fast/stages/01-resman/README.md @@ -163,6 +163,10 @@ Due to its simplicity, this stage lends itself easily to customizations: adding | [branch-sandbox.tf](./branch-sandbox.tf) | Sandbox stage resources. | folder · gcs · iam-service-account | | | [branch-security.tf](./branch-security.tf) | Security stage resources. | folder · gcs · iam-service-account | | | [branch-teams.tf](./branch-teams.tf) | Team stage resources. | folder · gcs · iam-service-account | | +| [cicd-data-platform.tf](./cicd-data-platform.tf) | CI/CD resources for the data platform branch. | iam-service-account · source-repository | | +| [cicd-networking.tf](./cicd-networking.tf) | CI/CD resources for the networking branch. | iam-service-account · source-repository | | +| [cicd-security.tf](./cicd-security.tf) | CI/CD resources for the security branch. | iam-service-account · source-repository | | +| [cicd-teams.tf](./cicd-teams.tf) | CI/CD resources for the teams branch. | iam-service-account · source-repository | | | [main.tf](./main.tf) | Module-level locals and resources. | | | | [organization.tf](./organization.tf) | Organization policies. | organization | google_organization_iam_member | | [outputs-files.tf](./outputs-files.tf) | Output files persistence to local filesystem. | | local_file | @@ -176,28 +180,28 @@ Due to its simplicity, this stage lends itself easily to customizations: adding |---|---|:---:|:---:|:---:|:---:| | [automation](variables.tf#L20) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 00-bootstrap | | [billing_account](variables.tf#L37) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | ✓ | | 00-bootstrap | -| [organization](variables.tf#L133) | Organization details. | object({…}) | ✓ | | 00-bootstrap | -| [prefix](variables.tf#L157) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 00-bootstrap | +| [organization](variables.tf#L140) | Organization details. | object({…}) | ✓ | | 00-bootstrap | +| [prefix](variables.tf#L164) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 00-bootstrap | | [cicd_repositories](variables.tf#L46) | CI/CD repository configuration. Identity providers reference keys in the `automation.federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | object({…}) | | null | | -| [custom_roles](variables.tf#L109) | Custom roles defined at the org level, in key => id format. | object({…}) | | null | 00-bootstrap | -| [groups](variables.tf#L118) | Group names to grant organization-level permissions. | map(string) | | {…} | 00-bootstrap | -| [organization_policy_configs](variables.tf#L143) | Organization policies customization. | object({…}) | | null | | -| [outputs_location](variables.tf#L151) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable | string | | null | | -| [tag_names](variables.tf#L168) | Customized names for resource management tags. | object({…}) | | {…} | | -| [team_folders](variables.tf#L185) | Team folders to be created. Format is described in a code comment. | map(object({…})) | | null | | +| [custom_roles](variables.tf#L116) | Custom roles defined at the org level, in key => id format. | object({…}) | | null | 00-bootstrap | +| [groups](variables.tf#L125) | Group names to grant organization-level permissions. | map(string) | | {…} | 00-bootstrap | +| [organization_policy_configs](variables.tf#L150) | Organization policies customization. | object({…}) | | null | | +| [outputs_location](variables.tf#L158) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable | string | | null | | +| [tag_names](variables.tf#L175) | Customized names for resource management tags. | object({…}) | | {…} | | +| [team_folders](variables.tf#L192) | Team folders to be created. Format is described in a code comment. | map(object({…})) | | null | | ## Outputs | name | description | sensitive | consumers | |---|---|:---:|---| -| [cicd_repositories](outputs.tf#L157) | WIF configuration for CI/CD repositories. | | | -| [dataplatform](outputs.tf#L169) | Data for the Data Platform stage. | | | -| [networking](outputs.tf#L185) | Data for the networking stage. | | | -| [project_factories](outputs.tf#L194) | Data for the project factories stage. | | | -| [providers](outputs.tf#L210) | Terraform provider files for this stage and dependent stages. | ✓ | 02-networking · 02-security · 03-dataplatform · xx-sandbox · xx-teams | -| [sandbox](outputs.tf#L217) | Data for the sandbox stage. | | xx-sandbox | -| [security](outputs.tf#L227) | Data for the networking stage. | | 02-security | -| [teams](outputs.tf#L237) | Data for the teams stage. | | | -| [tfvars](outputs.tf#L250) | Terraform variable files for the following stages. | ✓ | | +| [cicd_repositories](outputs.tf#L143) | WIF configuration for CI/CD repositories. | | | +| [dataplatform](outputs.tf#L155) | Data for the Data Platform stage. | | | +| [networking](outputs.tf#L171) | Data for the networking stage. | | | +| [project_factories](outputs.tf#L180) | Data for the project factories stage. | | | +| [providers](outputs.tf#L196) | Terraform provider files for this stage and dependent stages. | ✓ | 02-networking · 02-security · 03-dataplatform · xx-sandbox · xx-teams | +| [sandbox](outputs.tf#L203) | Data for the sandbox stage. | | xx-sandbox | +| [security](outputs.tf#L213) | Data for the networking stage. | | 02-security | +| [teams](outputs.tf#L223) | Data for the teams stage. | | | +| [tfvars](outputs.tf#L236) | Terraform variable files for the following stages. | ✓ | | diff --git a/fast/stages/01-resman/branch-data-platform.tf b/fast/stages/01-resman/branch-data-platform.tf index c5e186ae77..84e6d81ee2 100644 --- a/fast/stages/01-resman/branch-data-platform.tf +++ b/fast/stages/01-resman/branch-data-platform.tf @@ -122,67 +122,3 @@ module "branch-dp-prod-gcs" { "roles/storage.objectAdmin" = [module.branch-dp-prod-sa.iam_email] } } - -# ci/cd service accounts - -module "branch-dp-dev-sa-cicd" { - source = "../../../modules/iam-service-account" - for_each = ( - lookup(local.cicd_repositories, "dp_dev", null) == null - ? {} - : { 0 = local.cicd_repositories.dp_dev } - ) - project_id = var.automation.project_id - name = "dev-resman-dp-1" - description = "Terraform CI/CD data platform development service account." - prefix = var.prefix - iam = { - "roles/iam.workloadIdentityUser" = [ - each.value.branch == null - ? format( - local.identity_providers[each.value.identity_provider].principalset_tpl, - each.value.name - ) - : format( - local.identity_providers[each.value.identity_provider].principal_tpl, - each.value.name, - each.value.branch - ) - ] - } - iam_storage_roles = { - (var.automation.outputs_bucket) = ["roles/storage.objectViewer"] - } -} - -module "branch-dp-prod-sa-cicd" { - source = "../../../modules/iam-service-account" - for_each = ( - lookup(local.cicd_repositories, "dp_prod", null) == null - ? {} - : { 0 = local.cicd_repositories.dp_prod } - ) - project_id = var.automation.project_id - name = "prod-resman-dp-1" - description = "Terraform CI/CD data platform production service account." - prefix = var.prefix - iam = { - "roles/iam.workloadIdentityUser" = [ - each.value.branch == null - ? format( - local.identity_providers[each.value.identity_provider].principalset_tpl, - var.automation.federated_identity_pool, - each.value.name - ) - : format( - local.identity_providers[each.value.identity_provider].principal_tpl, - var.automation.federated_identity_pool, - each.value.name, - each.value.branch - ) - ] - } - iam_storage_roles = { - (var.automation.outputs_bucket) = ["roles/storage.objectViewer"] - } -} diff --git a/fast/stages/01-resman/branch-networking.tf b/fast/stages/01-resman/branch-networking.tf index 5cf3c6e085..8757e7f894 100644 --- a/fast/stages/01-resman/branch-networking.tf +++ b/fast/stages/01-resman/branch-networking.tf @@ -107,37 +107,3 @@ module "branch-network-gcs" { "roles/storage.objectAdmin" = [module.branch-network-sa.iam_email] } } - -# ci/cd service account - -module "branch-network-sa-cicd" { - source = "../../../modules/iam-service-account" - for_each = ( - lookup(local.cicd_repositories, "networking", null) == null - ? {} - : { 0 = local.cicd_repositories.networking } - ) - project_id = var.automation.project_id - name = "prod-resman-net-1" - description = "Terraform CI/CD stage 2 networking service account." - prefix = var.prefix - iam = { - "roles/iam.workloadIdentityUser" = [ - each.value.branch == null - ? format( - local.identity_providers[each.value.identity_provider].principalset_tpl, - var.automation.federated_identity_pool, - each.value.name - ) - : format( - local.identity_providers[each.value.identity_provider].principal_tpl, - var.automation.federated_identity_pool, - each.value.name, - each.value.branch - ) - ] - } - iam_storage_roles = { - (var.automation.outputs_bucket) = ["roles/storage.objectViewer"] - } -} diff --git a/fast/stages/01-resman/branch-security.tf b/fast/stages/01-resman/branch-security.tf index c206730464..1965bf08e8 100644 --- a/fast/stages/01-resman/branch-security.tf +++ b/fast/stages/01-resman/branch-security.tf @@ -74,37 +74,3 @@ module "branch-security-gcs" { "roles/storage.objectAdmin" = [module.branch-security-sa.iam_email] } } - -# ci/cd service account - -module "branch-security-sa-cicd" { - source = "../../../modules/iam-service-account" - for_each = ( - lookup(local.cicd_repositories, "security", null) == null - ? {} - : { 0 = local.cicd_repositories.security } - ) - project_id = var.automation.project_id - name = "prod-resman-sec-1" - description = "Terraform CI/CD stage 2 security service account." - prefix = var.prefix - iam = { - "roles/iam.workloadIdentityUser" = [ - each.value.branch == null - ? format( - local.identity_providers[each.value.identity_provider].principalset_tpl, - var.automation.federated_identity_pool, - each.value.name - ) - : format( - local.identity_providers[each.value.identity_provider].principal_tpl, - var.automation.federated_identity_pool, - each.value.name, - each.value.branch - ) - ] - } - iam_storage_roles = { - (var.automation.outputs_bucket) = ["roles/storage.objectViewer"] - } -} diff --git a/fast/stages/01-resman/branch-teams.tf b/fast/stages/01-resman/branch-teams.tf index 124301d523..465255be63 100644 --- a/fast/stages/01-resman/branch-teams.tf +++ b/fast/stages/01-resman/branch-teams.tf @@ -132,7 +132,7 @@ module "branch-teams-dev-pf-sa" { prefix = var.prefix iam = { "roles/iam.serviceAccountTokenCreator" = compact([ - try(module.branch-pf-dev-sa-cicd.0.iam_email, null) + try(module.branch-teams-dev-pf-sa-cicd.0.iam_email, null) ]) } iam_storage_roles = { @@ -149,7 +149,7 @@ module "branch-teams-prod-pf-sa" { prefix = var.prefix iam = { "roles/iam.serviceAccountTokenCreator" = compact([ - try(module.branch-pf-prod-sa-cicd.0.iam_email, null) + try(module.branch-teams-prod-pf-sa-cicd.0.iam_email, null) ]) } iam_storage_roles = { @@ -180,67 +180,3 @@ module "branch-teams-prod-pf-gcs" { "roles/storage.objectAdmin" = [module.branch-teams-prod-pf-sa.iam_email] } } - -# project factory per-team environment CI/CD service accounts - -module "branch-pf-dev-sa-cicd" { - source = "../../../modules/iam-service-account" - for_each = ( - lookup(local.cicd_repositories, "pf_dev", null) == null - ? {} - : { 0 = local.cicd_repositories.pf_dev } - ) - project_id = var.automation.project_id - name = "dev-resman-pf-1" - description = "Terraform CI/CD project factory development service account." - prefix = var.prefix - iam = { - "roles/iam.workloadIdentityUser" = [ - each.value.branch == null - ? format( - local.identity_providers[each.value.identity_provider].principalset_tpl, - each.value.name - ) - : format( - local.identity_providers[each.value.identity_provider].principal_tpl, - each.value.name, - each.value.branch - ) - ] - } - iam_storage_roles = { - (var.automation.outputs_bucket) = ["roles/storage.objectViewer"] - } -} - -module "branch-pf-prod-sa-cicd" { - source = "../../../modules/iam-service-account" - for_each = ( - lookup(local.cicd_repositories, "pf_prod", null) == null - ? {} - : { 0 = local.cicd_repositories.pf_prod } - ) - project_id = var.automation.project_id - name = "prod-resman-pf-1" - description = "Terraform CI/CD project factory production service account." - prefix = var.prefix - iam = { - "roles/iam.workloadIdentityUser" = [ - each.value.branch == null - ? format( - local.identity_providers[each.value.identity_provider].principalset_tpl, - var.automation.federated_identity_pool, - each.value.name - ) - : format( - local.identity_providers[each.value.identity_provider].principal_tpl, - var.automation.federated_identity_pool, - each.value.name, - each.value.branch - ) - ] - } - iam_storage_roles = { - (var.automation.outputs_bucket) = ["roles/storage.objectViewer"] - } -} diff --git a/fast/stages/01-resman/cicd-data-platform.tf b/fast/stages/01-resman/cicd-data-platform.tf new file mode 100644 index 0000000000..e62a022086 --- /dev/null +++ b/fast/stages/01-resman/cicd-data-platform.tf @@ -0,0 +1,163 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description CI/CD resources for the data platform branch. + +# source repositories + +module "branch-dp-dev-cicd-repo" { + source = "../../../modules/source-repository" + for_each = ( + try(local.cicd_repositories.data_platform_dev.type, null) == "sourcerepo" + ? { 0 = local.cicd_repositories.data_platform_dev } + : {} + ) + project_id = var.automation.project_id + name = each.value.name + iam = { + "roles/source.admin" = [module.branch-dp-dev-sa.iam_email] + "roles/source.reader" = [module.branch-dp-dev-sa-cicd.0.iam_email] + } + triggers = { + fast-03-dp-dev = { + filename = ".cloudbuild/workflow.yaml" + included_files = [ + "**/*json", "**/*tf", "**/*yaml", ".cloudbuild/workflow.yaml" + ] + service_account = module.branch-dp-dev-sa.iam_email + substitutions = {} + template = { + project_id = null + branch_name = each.value.branch + repo_name = each.value.name + tag_name = null + } + } + } +} + +module "branch-dp-prod-cicd-repo" { + source = "../../../modules/source-repository" + for_each = ( + try(local.cicd_repositories.data_platform_prod.type, null) == "sourcerepo" + ? { 0 = local.cicd_repositories.data_platform_prod } + : {} + ) + project_id = var.automation.project_id + name = each.value.name + iam = { + "roles/source.admin" = [module.branch-dp-prod-sa.iam_email] + "roles/source.reader" = [module.branch-dp-prod-sa-cicd.0.iam_email] + } + triggers = { + fast-03-dp-prod = { + filename = ".cloudbuild/workflow.yaml" + included_files = [ + "**/*json", "**/*tf", "**/*yaml", ".cloudbuild/workflow.yaml" + ] + service_account = module.branch-dp-prod-sa.iam_email + substitutions = {} + template = { + project_id = null + branch_name = each.value.branch + repo_name = each.value.name + tag_name = null + } + } + } +} + +# SAs used by CI/CD workflows to impersonate automation SAs + +module "branch-dp-dev-sa-cicd" { + source = "../../../modules/iam-service-account" + for_each = ( + try(local.cicd_repositories.data_platform_dev.name, null) != null + ? { 0 = local.cicd_repositories.data_platform_dev } + : {} + ) + project_id = var.automation.project_id + name = "dev-resman-dp-1" + description = "Terraform CI/CD data platform development service account." + prefix = var.prefix + iam = ( + each.value.type == "sourcerepo" + # used directly from the cloud build trigger for source repos + ? {} + # impersonated via workload identity federation for external repos + : { + "roles/iam.workloadIdentityUser" = [ + each.value.branch == null + ? format( + local.identity_providers[each.value.identity_provider].principalset_tpl, + each.value.name + ) + : format( + local.identity_providers[each.value.identity_provider].principal_tpl, + each.value.name, + each.value.branch + ) + ] + } + ) + iam_project_roles = { + (var.automation.project_id) = ["roles/logging.logWriter"] + } + iam_storage_roles = { + (var.automation.outputs_bucket) = ["roles/storage.objectViewer"] + } +} + +module "branch-dp-prod-sa-cicd" { + source = "../../../modules/iam-service-account" + for_each = ( + try(local.cicd_repositories.data_platform_prod.name, null) != null + ? { 0 = local.cicd_repositories.data_platform_prod } + : {} + ) + project_id = var.automation.project_id + name = "prod-resman-dp-1" + description = "Terraform CI/CD data platform production service account." + prefix = var.prefix + iam = ( + each.value.type == "sourcerepo" + # used directly from the cloud build trigger for source repos + ? {} + # impersonated via workload identity federation for external repos + : { + "roles/iam.workloadIdentityUser" = [ + each.value.branch == null + ? format( + local.identity_providers[each.value.identity_provider].principalset_tpl, + var.automation.federated_identity_pool, + each.value.name + ) + : format( + local.identity_providers[each.value.identity_provider].principal_tpl, + var.automation.federated_identity_pool, + each.value.name, + each.value.branch + ) + ] + } + ) + iam_project_roles = { + (var.automation.project_id) = ["roles/logging.logWriter"] + } + iam_storage_roles = { + (var.automation.outputs_bucket) = ["roles/storage.objectViewer"] + } +} diff --git a/fast/stages/01-resman/cicd-networking.tf b/fast/stages/01-resman/cicd-networking.tf new file mode 100644 index 0000000000..541d8bda06 --- /dev/null +++ b/fast/stages/01-resman/cicd-networking.tf @@ -0,0 +1,91 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description CI/CD resources for the networking branch. + +# source repository + +module "branch-network-cicd-repo" { + source = "../../../modules/source-repository" + for_each = ( + try(local.cicd_repositories.networking.type, null) == "sourcerepo" + ? { 0 = local.cicd_repositories.networking } + : {} + ) + project_id = var.automation.project_id + name = each.value.name + iam = { + "roles/source.admin" = [module.branch-network-sa.iam_email] + "roles/source.reader" = [module.branch-network-sa-cicd.0.iam_email] + } + triggers = { + fast-02-networking = { + filename = ".cloudbuild/workflow.yaml" + included_files = ["**/*tf", ".cloudbuild/workflow.yaml"] + service_account = module.branch-network-sa.id + substitutions = {} + template = { + project_id = null + branch_name = each.value.branch + repo_name = each.value.name + tag_name = null + } + } + } +} + +# SA used by CI/CD workflows to impersonate automation SAs + +module "branch-network-sa-cicd" { + source = "../../../modules/iam-service-account" + for_each = ( + try(local.cicd_repositories.networking.name, null) != null + ? { 0 = local.cicd_repositories.networking } + : {} + ) + project_id = var.automation.project_id + name = "prod-resman-net-1" + description = "Terraform CI/CD stage 2 networking service account." + prefix = var.prefix + iam = ( + each.value.type == "sourcerepo" + # used directly from the cloud build trigger for source repos + ? {} + # impersonated via workload identity federation for external repos + : { + "roles/iam.workloadIdentityUser" = [ + each.value.branch == null + ? format( + local.identity_providers[each.value.identity_provider].principalset_tpl, + var.automation.federated_identity_pool, + each.value.name + ) + : format( + local.identity_providers[each.value.identity_provider].principal_tpl, + var.automation.federated_identity_pool, + each.value.name, + each.value.branch + ) + ] + } + ) + iam_project_roles = { + (var.automation.project_id) = ["roles/logging.logWriter"] + } + iam_storage_roles = { + (var.automation.outputs_bucket) = ["roles/storage.objectViewer"] + } +} diff --git a/fast/stages/01-resman/cicd-security.tf b/fast/stages/01-resman/cicd-security.tf new file mode 100644 index 0000000000..d6b0b86917 --- /dev/null +++ b/fast/stages/01-resman/cicd-security.tf @@ -0,0 +1,91 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description CI/CD resources for the security branch. + +# source repository + +module "branch-security-cicd-repo" { + source = "../../../modules/source-repository" + for_each = ( + try(local.cicd_repositories.security.type, null) == "sourcerepo" + ? { 0 = local.cicd_repositories.security } + : {} + ) + project_id = var.automation.project_id + name = each.value.name + iam = { + "roles/source.admin" = [module.branch-security-sa.iam_email] + "roles/source.reader" = [module.branch-security-sa-cicd.0.iam_email] + } + triggers = { + fast-02-security = { + filename = ".cloudbuild/workflow.yaml" + included_files = ["**/*tf", ".cloudbuild/workflow.yaml"] + service_account = module.branch-security-sa.id + substitutions = {} + template = { + project_id = null + branch_name = each.value.branch + repo_name = each.value.name + tag_name = null + } + } + } +} + +# SA used by CI/CD workflows to impersonate automation SAs + +module "branch-security-sa-cicd" { + source = "../../../modules/iam-service-account" + for_each = ( + try(local.cicd_repositories.security.name, null) != null + ? { 0 = local.cicd_repositories.security } + : {} + ) + project_id = var.automation.project_id + name = "prod-resman-sec-1" + description = "Terraform CI/CD stage 2 security service account." + prefix = var.prefix + iam = ( + each.value.type == "sourcerepo" + # used directly from the cloud build trigger for source repos + ? {} + # impersonated via workload identity federation for external repos + : { + "roles/iam.workloadIdentityUser" = [ + each.value.branch == null + ? format( + local.identity_providers[each.value.identity_provider].principalset_tpl, + var.automation.federated_identity_pool, + each.value.name + ) + : format( + local.identity_providers[each.value.identity_provider].principal_tpl, + var.automation.federated_identity_pool, + each.value.name, + each.value.branch + ) + ] + } + ) + iam_project_roles = { + (var.automation.project_id) = ["roles/logging.logWriter"] + } + iam_storage_roles = { + (var.automation.outputs_bucket) = ["roles/storage.objectViewer"] + } +} diff --git a/fast/stages/01-resman/cicd-teams.tf b/fast/stages/01-resman/cicd-teams.tf new file mode 100644 index 0000000000..2766e301e0 --- /dev/null +++ b/fast/stages/01-resman/cicd-teams.tf @@ -0,0 +1,163 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description CI/CD resources for the teams branch. + +# source repositories + +module "branch-teams-dev-pf-cicd-repo" { + source = "../../../modules/source-repository" + for_each = ( + try(local.cicd_repositories.project_factory_dev.type, null) == "sourcerepo" + ? { 0 = local.cicd_repositories.project_factory_dev } + : {} + ) + project_id = var.automation.project_id + name = each.value.name + iam = { + "roles/source.admin" = [module.branch-teams-dev-pf-sa.iam_email] + "roles/source.reader" = [module.branch-teams-dev-pf-sa-cicd.0.iam_email] + } + triggers = { + fast-03-pf-dev = { + filename = ".cloudbuild/workflow.yaml" + included_files = [ + "**/*json", "**/*tf", "**/*yaml", ".cloudbuild/workflow.yaml" + ] + service_account = module.branch-teams-dev-pf-sa.iam_email + substitutions = {} + template = { + project_id = null + branch_name = each.value.branch + repo_name = each.value.name + tag_name = null + } + } + } +} + +module "branch-teams-prod-pf-cicd-repo" { + source = "../../../modules/source-repository" + for_each = ( + try(local.cicd_repositories.project_factory_prod.type, null) == "sourcerepo" + ? { 0 = local.cicd_repositories.project_factory_prod } + : {} + ) + project_id = var.automation.project_id + name = each.value.name + iam = { + "roles/source.admin" = [module.branch-teams-prod-pf-sa.iam_email] + "roles/source.reader" = [module.branch-teams-prod-pf-sa-cicd.0.iam_email] + } + triggers = { + fast-03-pf-prod = { + filename = ".cloudbuild/workflow.yaml" + included_files = [ + "**/*json", "**/*tf", "**/*yaml", ".cloudbuild/workflow.yaml" + ] + service_account = module.branch-teams-prod-pf-sa.iam_email + substitutions = {} + template = { + project_id = null + branch_name = each.value.branch + repo_name = each.value.name + tag_name = null + } + } + } +} + +# SAs used by CI/CD workflows to impersonate automation SAs + +module "branch-teams-dev-pf-sa-cicd" { + source = "../../../modules/iam-service-account" + for_each = ( + try(local.cicd_repositories.project_factory_dev.name, null) != null + ? { 0 = local.cicd_repositories.project_factory_dev } + : {} + ) + project_id = var.automation.project_id + name = "dev-pf-resman-pf-1" + description = "Terraform CI/CD project factory development service account." + prefix = var.prefix + iam = ( + each.value.type == "sourcerepo" + # used directly from the cloud build trigger for source repos + ? {} + # impersonated via workload identity federation for external repos + : { + "roles/iam.workloadIdentityUser" = [ + each.value.branch == null + ? format( + local.identity_providers[each.value.identity_provider].principalset_tpl, + each.value.name + ) + : format( + local.identity_providers[each.value.identity_provider].principal_tpl, + each.value.name, + each.value.branch + ) + ] + } + ) + iam_project_roles = { + (var.automation.project_id) = ["roles/logging.logWriter"] + } + iam_storage_roles = { + (var.automation.outputs_bucket) = ["roles/storage.objectViewer"] + } +} + +module "branch-teams-prod-pf-sa-cicd" { + source = "../../../modules/iam-service-account" + for_each = ( + try(local.cicd_repositories.project_factory_prod.name, null) != null + ? { 0 = local.cicd_repositories.project_factory_prod } + : {} + ) + project_id = var.automation.project_id + name = "prod-pf-resman-pf-1" + description = "Terraform CI/CD project factory production service account." + prefix = var.prefix + iam = ( + each.value.type == "sourcerepo" + # used directly from the cloud build trigger for source repos + ? {} + # impersonated via workload identity federation for external repos + : { + "roles/iam.workloadIdentityUser" = [ + each.value.branch == null + ? format( + local.identity_providers[each.value.identity_provider].principalset_tpl, + var.automation.federated_identity_pool, + each.value.name + ) + : format( + local.identity_providers[each.value.identity_provider].principal_tpl, + var.automation.federated_identity_pool, + each.value.name, + each.value.branch + ) + ] + } + ) + iam_project_roles = { + (var.automation.project_id) = ["roles/logging.logWriter"] + } + iam_storage_roles = { + (var.automation.outputs_bucket) = ["roles/storage.objectViewer"] + } +} diff --git a/fast/stages/01-resman/main.tf b/fast/stages/01-resman/main.tf index a0c58dc23a..6cefbd25d9 100644 --- a/fast/stages/01-resman/main.tf +++ b/fast/stages/01-resman/main.tf @@ -24,11 +24,33 @@ locals { if( v != null && - contains(keys(local.identity_providers), try(v.identity_provider, "")) + ( + try(v.type, null) == "sourcerepo" + || + contains( + keys(local.identity_providers), + coalesce(try(v.identity_provider, null), ":") + ) + ) && fileexists("${path.module}/templates/workflow-${try(v.type, "")}.yaml") ) } + cicd_workflow_var_files = { + stage_2 = [ + "00-bootstrap.auto.tfvars.json", + "01-resman.auto.tfvars.json", + "globals.auto.tfvars.json" + ] + stage_3 = [ + "00-bootstrap.auto.tfvars.json", + "01-resman.auto.tfvars.json", + "globals.auto.tfvars.json", + "02-networking.auto.tfvars.json", + "02-security.auto.tfvars.json" + ] + } + custom_roles = coalesce(var.custom_roles, {}) groups = { for k, v in var.groups : diff --git a/fast/stages/01-resman/outputs.tf b/fast/stages/01-resman/outputs.tf index aefdf9e5a9..f91a843d81 100644 --- a/fast/stages/01-resman/outputs.tf +++ b/fast/stages/01-resman/outputs.tf @@ -15,51 +15,37 @@ */ locals { - _cicd_tf_var_files = { - stage_2 = [ - "00-bootstrap.auto.tfvars.json", - "01-resman.auto.tfvars.json", - "globals.auto.tfvars.json" - ] - stage_3 = [ - "00-bootstrap.auto.tfvars.json", - "01-resman.auto.tfvars.json", - "globals.auto.tfvars.json", - "02-networking.auto.tfvars.json", - "02-security.auto.tfvars.json" - ] - } _tpl_providers = "${path.module}/templates/providers.tf.tpl" cicd_workflow_attrs = { data_platform_dev = { service_account = try(module.branch-dp-dev-sa-cicd.0.email, null) tf_providers_file = "03-data-platform-dev-providers.tf" - tf_var_files = local._cicd_tf_var_files.stage_3 + tf_var_files = local.cicd_workflow_var_files.stage_3 } data_platform_prod = { service_account = try(module.branch-dp-prod-sa-cicd.0.email, null) tf_providers_file = "03-data-platform-prod-providers.tf" - tf_var_files = local._cicd_tf_var_files.stage_3 + tf_var_files = local.cicd_workflow_var_files.stage_3 } networking = { service_account = try(module.branch-network-sa-cicd.0.email, null) tf_providers_file = "02-networking-providers.tf" - tf_var_files = local._cicd_tf_var_files.stage_2 + tf_var_files = local.cicd_workflow_var_files.stage_2 } project_factory_dev = { - service_account = try(module.branch-pf-dev-sa-cicd.0.email, null) + service_account = try(module.branch-teams-dev-pf-sa-cicd.0.email, null) tf_providers_file = "03-project-factory-dev-providers.tf" - tf_var_files = local._cicd_tf_var_files.stage_3 + tf_var_files = local.cicd_workflow_var_files.stage_3 } project_factory_prod = { - service_account = try(module.branch-pf-prod-sa-cicd.0.email, null) + service_account = try(module.branch-teams-prod-pf-sa-cicd.0.email, null) tf_providers_file = "03-project-factory-prod-providers.tf" - tf_var_files = local._cicd_tf_var_files.stage_3 + tf_var_files = local.cicd_workflow_var_files.stage_3 } security = { service_account = try(module.branch-security-sa-cicd.0.email, null) tf_providers_file = "02-security-providers.tf" - tf_var_files = local._cicd_tf_var_files.stage_2 + tf_var_files = local.cicd_workflow_var_files.stage_2 } } cicd_workflows = { diff --git a/fast/stages/01-resman/variables.tf b/fast/stages/01-resman/variables.tf index a9b101e6da..0f2bc5b9aa 100644 --- a/fast/stages/01-resman/variables.tf +++ b/fast/stages/01-resman/variables.tf @@ -84,25 +84,32 @@ variable "cicd_repositories" { }) }) default = null + validation { + condition = alltrue([ + for k, v in coalesce(var.cicd_repositories, {}) : + v == null || try(v.name, null) != null + ]) + error_message = "Non-null repositories need a non-null name." + } validation { condition = alltrue([ for k, v in coalesce(var.cicd_repositories, {}) : v == null || ( - try(v.name, null) != null - && try(v.identity_provider, null) != null + || + try(v.type, null) == "sourcerepo" ) ]) - error_message = "Non-null repositories need non-null name and providers." + error_message = "Non-null repositories need a non-null provider unless type is 'sourcerepo'." } validation { condition = alltrue([ for k, v in coalesce(var.cicd_repositories, {}) : v == null || ( - contains(["gitlab", "github"], coalesce(try(v.type, null), "null")) + contains(["github", "gitlab", "sourcerepo"], coalesce(try(v.type, null), "null")) ) ]) - error_message = "Invalid repository type, supported types: 'github' or 'gitlab'." + error_message = "Invalid repository type, supported types: 'github' 'gitlab' or 'sourcerepo'." } } diff --git a/modules/iam-service-account/README.md b/modules/iam-service-account/README.md index bd2240b8a3..ab6b1882c8 100644 --- a/modules/iam-service-account/README.md +++ b/modules/iam-service-account/README.md @@ -63,9 +63,10 @@ module "myproject-default-service-accounts" { |---|---|:---:| | [email](outputs.tf#L17) | Service account email. | | | [iam_email](outputs.tf#L25) | IAM-format service account email. | | -| [key](outputs.tf#L33) | Service account key. | ✓ | -| [name](outputs.tf#L39) | Service account id. | | -| [service_account](outputs.tf#L44) | Service account resource. | | -| [service_account_credentials](outputs.tf#L49) | Service account json credential templates for uploaded public keys data. | | +| [id](outputs.tf#L33) | Service account id. | | +| [key](outputs.tf#L38) | Service account key. | ✓ | +| [name](outputs.tf#L44) | Service account name. | | +| [service_account](outputs.tf#L49) | Service account resource. | | +| [service_account_credentials](outputs.tf#L54) | Service account json credential templates for uploaded public keys data. | | diff --git a/modules/iam-service-account/outputs.tf b/modules/iam-service-account/outputs.tf index 8234ed96c7..4f0e0aa522 100644 --- a/modules/iam-service-account/outputs.tf +++ b/modules/iam-service-account/outputs.tf @@ -30,6 +30,11 @@ output "iam_email" { ] } +output "id" { + description = "Service account id." + value = local.service_account.id +} + output "key" { description = "Service account key." sensitive = true @@ -37,7 +42,7 @@ output "key" { } output "name" { - description = "Service account id." + description = "Service account name." value = local.service_account.name } diff --git a/modules/source-repository/README.md b/modules/source-repository/README.md index 48f29aa14a..2075b89a72 100644 --- a/modules/source-repository/README.md +++ b/modules/source-repository/README.md @@ -1,11 +1,10 @@ # Google Cloud Source Repository Module -This module allows managing a single Cloud Source Repository, including IAM bindings. - +This module allows managing a single Cloud Source Repository, including IAM bindings and basic Cloud Build triggers. ## Examples -### Simple repository with IAM +### Repository with IAM ```hcl module "repo" { @@ -18,21 +17,64 @@ module "repo" { } # tftest modules=1 resources=2 ``` + +### Repository with Cloud Build trigger + +```hcl +module "repo" { + source = "./modules/source-repository" + project_id = "my-project" + name = "my-repo" + triggers = { + foo = { + filename = "ci/workflow-foo.yaml" + included_files = ["**/*tf"] + service_account = null + substitutions = { + BAR = 1 + } + template = { + branch_name = "main" + project_id = null + tag_name = null + } + } + } +} +# tftest modules=1 resources=2 +``` + + +## Files + +| name | description | resources | +|---|---|---| +| [iam.tf](./iam.tf) | IAM resources. | google_sourcerepo_repository_iam_binding · google_sourcerepo_repository_iam_member | +| [main.tf](./main.tf) | Module-level locals and resources. | google_cloudbuild_trigger · google_sourcerepo_repository | +| [outputs.tf](./outputs.tf) | Module outputs. | | +| [variables.tf](./variables.tf) | Module variables. | | +| [versions.tf](./versions.tf) | Version pins. | | + ## Variables | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [name](variables.tf#L23) | Repository name. | string | ✓ | | -| [project_id](variables.tf#L28) | Project used for resources. | string | ✓ | | -| [iam](variables.tf#L17) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [name](variables.tf#L44) | Repository name. | string | ✓ | | +| [project_id](variables.tf#L49) | Project used for resources. | string | ✓ | | +| [group_iam](variables.tf#L17) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | +| [iam](variables.tf#L24) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [iam_additive](variables.tf#L31) | IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [iam_additive_members](variables.tf#L38) | IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values. | map(list(string)) | | {} | +| [triggers](variables.tf#L54) | Cloud Build triggers. | map(object({…})) | | {} | ## Outputs | name | description | sensitive | |---|---|:---:| | [id](outputs.tf#L17) | Repository id. | | -| [url](outputs.tf#L22) | Repository URL. | | +| [name](outputs.tf#L22) | Repository name. | | +| [url](outputs.tf#L27) | Repository URL. | | diff --git a/modules/source-repository/iam.tf b/modules/source-repository/iam.tf new file mode 100644 index 0000000000..e5c3ec499e --- /dev/null +++ b/modules/source-repository/iam.tf @@ -0,0 +1,67 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description IAM resources. + +locals { + _group_iam_roles = distinct(flatten(values(var.group_iam))) + _group_iam = { + for r in local._group_iam_roles : r => [ + for k, v in var.group_iam : "group:${k}" if try(index(v, r), null) != null + ] + } + _iam_additive_pairs = flatten([ + for role, members in var.iam_additive : [ + for member in members : { role = role, member = member } + ] + ]) + _iam_additive_member_pairs = flatten([ + for member, roles in var.iam_additive_members : [ + for role in roles : { role = role, member = member } + ] + ]) + iam = { + for role in distinct(concat(keys(var.iam), keys(local._group_iam))) : + role => concat( + try(var.iam[role], []), + try(local._group_iam[role], []) + ) + } + iam_additive = { + for pair in concat(local._iam_additive_pairs, local._iam_additive_member_pairs) : + "${pair.role}-${pair.member}" => pair + } +} + +resource "google_sourcerepo_repository_iam_binding" "authoritative" { + for_each = local.iam + project = var.project_id + repository = google_sourcerepo_repository.default.name + role = each.key + members = each.value +} + +resource "google_sourcerepo_repository_iam_member" "additive" { + for_each = ( + length(var.iam_additive) + length(var.iam_additive_members) > 0 + ? local.iam_additive + : {} + ) + project = var.project_id + repository = google_sourcerepo_repository.default.name + role = each.value.role + member = each.value.member +} diff --git a/modules/source-repository/main.tf b/modules/source-repository/main.tf index c4057d76d3..d74b7e6c87 100644 --- a/modules/source-repository/main.tf +++ b/modules/source-repository/main.tf @@ -19,14 +19,18 @@ resource "google_sourcerepo_repository" "default" { name = var.name } -resource "google_sourcerepo_repository_iam_binding" "default" { - for_each = var.iam - project = var.project_id - repository = google_sourcerepo_repository.default.name - role = each.key - members = each.value - - depends_on = [ - google_sourcerepo_repository.default - ] +resource "google_cloudbuild_trigger" "default" { + for_each = coalesce(var.triggers, {}) + project = var.project_id + name = each.key + filename = each.value.filename + included_files = each.value.included_files + service_account = each.value.service_account + substitutions = each.value.substitutions + trigger_template { + project_id = try(each.value.template.project_id, var.project_id) + branch_name = try(each.value.template.branch_name, null) + repo_name = google_sourcerepo_repository.default.name + tag_name = try(each.value.template.tag_name, null) + } } diff --git a/modules/source-repository/outputs.tf b/modules/source-repository/outputs.tf index d1a4b25e9d..be55307a6d 100644 --- a/modules/source-repository/outputs.tf +++ b/modules/source-repository/outputs.tf @@ -19,6 +19,11 @@ output "id" { value = google_sourcerepo_repository.default.id } +output "name" { + description = "Repository name." + value = google_sourcerepo_repository.default.name +} + output "url" { description = "Repository URL." value = google_sourcerepo_repository.default.url diff --git a/modules/source-repository/variables.tf b/modules/source-repository/variables.tf index e592f35892..587b0f6d2b 100644 --- a/modules/source-repository/variables.tf +++ b/modules/source-repository/variables.tf @@ -14,10 +14,31 @@ * limitations under the License. */ +variable "group_iam" { + description = "Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable." + type = map(list(string)) + default = {} + nullable = false +} + variable "iam" { description = "IAM bindings in {ROLE => [MEMBERS]} format." type = map(list(string)) default = {} + nullable = false +} + +variable "iam_additive" { + description = "IAM additive bindings in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = {} + nullable = false +} + +variable "iam_additive_members" { + description = "IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values." + type = map(list(string)) + default = {} } variable "name" { @@ -29,3 +50,20 @@ variable "project_id" { description = "Project used for resources." type = string } + +variable "triggers" { + description = "Cloud Build triggers." + type = map(object({ + filename = string + included_files = list(string) + service_account = string + substitutions = map(string) + template = object({ + branch_name = string + project_id = string + tag_name = string + }) + })) + default = {} + nullable = false +} diff --git a/tests/modules/source_repository/fixture/main.tf b/tests/modules/source_repository/fixture/main.tf index 00dd7bd349..122556bacf 100644 --- a/tests/modules/source_repository/fixture/main.tf +++ b/tests/modules/source_repository/fixture/main.tf @@ -14,9 +14,52 @@ * limitations under the License. */ +variable "group_iam" { + type = any + default = {} +} + +variable "iam" { + type = any + default = {} + nullable = false +} + +variable "iam_additive" { + type = any + default = {} + nullable = false +} + +variable "iam_additive_members" { + type = any + default = {} +} + +variable "name" { + description = "Repository name." + type = string + default = "test" +} + +variable "project_id" { + description = "Project used for resources." + type = string + default = "test" +} + +variable "triggers" { + type = any + default = null +} + module "test" { - source = "../../../../modules/source-repository" - project_id = var.project_id - name = var.name - iam = var.iam + source = "../../../../modules/source-repository" + project_id = var.project_id + name = var.name + group_iam = var.group_iam + iam = var.iam + iam_additive = var.iam_additive + iam_additive_members = var.iam_additive_members + triggers = var.triggers } diff --git a/tests/modules/source_repository/fixture/variables.tf b/tests/modules/source_repository/fixture/variables.tf deleted file mode 100644 index a1c540a5bc..0000000000 --- a/tests/modules/source_repository/fixture/variables.tf +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - type = string - default = "test-project" -} - -variable "iam" { - type = map(list(string)) - default = { - "roles/source.reader" = ["foo@example.org"] - } -} - -variable "name" { - type = string - default = "test" -} diff --git a/tests/modules/source_repository/test_plan.py b/tests/modules/source_repository/test_plan.py index 8713a8957d..83b27ee2fb 100644 --- a/tests/modules/source_repository/test_plan.py +++ b/tests/modules/source_repository/test_plan.py @@ -12,23 +12,49 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - -@pytest.fixture -def resources(plan_runner): +def test_resource_count(plan_runner): + 'Test number of resources created.' _, resources = plan_runner() - return resources + assert len(resources) == 1 -def test_resource_count(resources): - "Test number of resources created." - assert len(resources) == 2 +def test_iam(plan_runner): + 'Test IAM binding resources.' + group_iam = '{"fooers@example.org"=["roles/owner"]}' + iam = '''{ + "roles/editor" = ["user:a@example.org", "user:b@example.org"] + "roles/owner" = ["user:c@example.org"] + }''' + _, resources = plan_runner(group_iam=group_iam, iam=iam) + bindings = { + r['values']['role']: r['values']['members'] + for r in resources + if r['type'] == 'google_sourcerepo_repository_iam_binding' + } + assert bindings == { + 'roles/editor': ['user:a@example.org', 'user:b@example.org'], + 'roles/owner': ['group:fooers@example.org', 'user:c@example.org'] + } -def test_iam(resources): - "Test IAM binding resources." - bindings = [r['values'] for r in resources if r['type'] - == 'google_sourcerepo_repository_iam_binding'] - assert len(bindings) == 1 - assert bindings[0]['role'] == 'roles/source.reader' +def test_triggers(plan_runner): + 'Test trigger resources.' + triggers = '''{ + foo = { + filename = "ci/foo.yaml" + included_files = ["**/*yaml"] + service_account = null + substitutions = null + template = { + branch_name = null + project_id = null + tag_name = "foo" + } + } + }''' + _, resources = plan_runner(triggers=triggers) + triggers = [ + r['index'] for r in resources if r['type'] == 'google_cloudbuild_trigger' + ] + assert triggers == ['foo'] \ No newline at end of file