From 09e91ca56e7851c3934247b80c64b227e9007caf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Taneli=20Lepp=C3=A4?= Date: Tue, 21 Jun 2022 10:45:27 +0200 Subject: [PATCH] Allow creating repositories in Gitlab via Terraform. --- fast/stages/00-bootstrap/README.md | 58 +++-- fast/stages/00-bootstrap/automation.tf | 31 +++ fast/stages/00-bootstrap/cicd.tf | 11 +- .../stages/00-bootstrap/identity-providers.tf | 3 +- fast/stages/00-bootstrap/outputs.tf | 6 + fast/stages/00-bootstrap/variables.tf | 10 + fast/stages/00-cicd/README.md | 208 ++++++++++++++++++ fast/stages/00-cicd/cicd.tf | 34 +++ fast/stages/00-cicd/github.tf | 46 ++++ fast/stages/00-cicd/gitlab.tf | 57 +++++ fast/stages/00-cicd/main.tf | 16 ++ fast/stages/00-cicd/outputs-files.tf | 24 ++ fast/stages/00-cicd/outputs-gcs.tf | 23 ++ fast/stages/00-cicd/outputs.tf | 34 +++ fast/stages/00-cicd/variables.tf | 145 ++++++++++++ fast/stages/00-cicd/versions.tf | 27 +++ 16 files changed, 709 insertions(+), 24 deletions(-) create mode 100644 fast/stages/00-cicd/README.md create mode 100644 fast/stages/00-cicd/cicd.tf create mode 100644 fast/stages/00-cicd/github.tf create mode 100644 fast/stages/00-cicd/gitlab.tf create mode 100644 fast/stages/00-cicd/main.tf create mode 100644 fast/stages/00-cicd/outputs-files.tf create mode 100644 fast/stages/00-cicd/outputs-gcs.tf create mode 100644 fast/stages/00-cicd/outputs.tf create mode 100644 fast/stages/00-cicd/variables.tf create mode 100644 fast/stages/00-cicd/versions.tf diff --git a/fast/stages/00-bootstrap/README.md b/fast/stages/00-bootstrap/README.md index 5ef1bfb4e9..2e53c8a52d 100644 --- a/fast/stages/00-bootstrap/README.md +++ b/fast/stages/00-bootstrap/README.md @@ -358,10 +358,20 @@ federated_identity_providers = { github-sample = { attribute_condition = "attribute.repository_owner==\"my-github-org\"" issuer = "github" + custom_settings = null } gitlab-sample = { attribute_condition = "attribute.namespace_path==\"my-gitlab-org\"" issuer = "gitlab" + custom_settings = null + } + gitlab-ce-sample = { + attribute_condition = "attribute.namespace_path==\"my-gitlab-org\"" + issuer = "gitlab" + custom_settings = { + issuer_uri = "https://gitlab.fast.example.com" + allowed_audiences = ["https://gitlab.fast.example.com"] + } } } ``` @@ -382,6 +392,12 @@ cicd_repositories = { name = "my-gh-org/fast-bootstrap" type = "github" } + cicd = { + branch = null + identity_provider = "github-sample" + name = "my-gh-org/fast-cicd" + type = "github" + } resman = { branch = "main" identity_provider = "github-sample" @@ -395,6 +411,8 @@ The `type` attribute can be set to one of the supported repository types: `githu Once the stage is applied the generated output files will contain pre-configured workflow files for each repository, that will use Workload Identity Federation via a dedicated service account for each repository to impersonate the automation service account for the stage. +You can use Terraform to automate creation of the repositories using the `00-cicd` stage. + The remaining configuration is manual, as it regards the repositories themselves: - create a repository for modules @@ -443,31 +461,31 @@ The remaining configuration is manual, as it regards the repositories themselves | name | description | type | required | default | producer | |---|---|:---:|:---:|:---:|:---:| | [billing_account](variables.tf#L17) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | ✓ | | | -| [organization](variables.tf#L152) | Organization details. | object({…}) | ✓ | | | -| [prefix](variables.tf#L167) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | | +| [organization](variables.tf#L162) | Organization details. | object({…}) | ✓ | | | +| [prefix](variables.tf#L177) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | | | [bootstrap_user](variables.tf#L25) | Email of the nominal user running this stage for the first time. | string | | null | | -| [cicd_repositories](variables.tf#L31) | CI/CD repository configuration. Identity providers reference keys in the `federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | object({…}) | | null | | -| [custom_role_names](variables.tf#L77) | Names of custom roles defined at the org level. | object({…}) | | {…} | | -| [federated_identity_providers](variables.tf#L89) | Workload Identity Federation pools. The `cicd_repositories` variable references keys here. | map(object({…})) | | {} | | -| [groups](variables.tf#L99) | Group names to grant organization-level permissions. | map(string) | | {…} | | -| [iam](variables.tf#L113) | Organization-level custom IAM settings in role => [principal] format. | map(list(string)) | | {} | | -| [iam_additive](variables.tf#L119) | Organization-level custom IAM settings in role => [principal] format for non-authoritative bindings. | map(list(string)) | | {} | | -| [log_sinks](variables.tf#L127) | Org-level log sinks, in name => {type, filter} format. | map(object({…})) | | {…} | | -| [outputs_location](variables.tf#L161) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable | string | | null | | +| [cicd_repositories](variables.tf#L31) | CI/CD repository configuration. Identity providers reference keys in the `federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | object({…}) | | null | | +| [custom_role_names](variables.tf#L83) | Names of custom roles defined at the org level. | object({…}) | | {…} | | +| [federated_identity_providers](variables.tf#L95) | Workload Identity Federation pools. The `cicd_repositories` variable references keys here. | map(object({…})) | | {} | | +| [groups](variables.tf#L109) | Group names to grant organization-level permissions. | map(string) | | {…} | | +| [iam](variables.tf#L123) | Organization-level custom IAM settings in role => [principal] format. | map(list(string)) | | {} | | +| [iam_additive](variables.tf#L129) | Organization-level custom IAM settings in role => [principal] format for non-authoritative bindings. | map(list(string)) | | {} | | +| [log_sinks](variables.tf#L137) | Org-level log sinks, in name => {type, filter} format. | map(object({…})) | | {…} | | +| [outputs_location](variables.tf#L171) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable | string | | null | | ## Outputs | name | description | sensitive | consumers | |---|---|:---:|---| -| [automation](outputs.tf#L82) | Automation resources. | | | -| [billing_dataset](outputs.tf#L87) | BigQuery dataset prepared for billing export. | | | -| [cicd_repositories](outputs.tf#L92) | CI/CD repository configurations. | | | -| [custom_roles](outputs.tf#L104) | Organization-level custom roles. | | | -| [federated_identity](outputs.tf#L109) | Workload Identity Federation pool and providers. | | | -| [outputs_bucket](outputs.tf#L119) | GCS bucket where generated output files are stored. | | | -| [project_ids](outputs.tf#L124) | Projects created by this stage. | | | -| [providers](outputs.tf#L143) | Terraform provider files for this stage and dependent stages. | ✓ | stage-01 | -| [service_accounts](outputs.tf#L133) | Automation service accounts created by this stage. | | | -| [tfvars](outputs.tf#L152) | Terraform variable files for the following stages. | ✓ | | +| [automation](outputs.tf#L87) | Automation resources. | | | +| [billing_dataset](outputs.tf#L92) | BigQuery dataset prepared for billing export. | | | +| [cicd_repositories](outputs.tf#L97) | CI/CD repository configurations. | | | +| [custom_roles](outputs.tf#L109) | Organization-level custom roles. | | | +| [federated_identity](outputs.tf#L114) | Workload Identity Federation pool and providers. | | | +| [outputs_bucket](outputs.tf#L124) | GCS bucket where generated output files are stored. | | | +| [project_ids](outputs.tf#L129) | Projects created by this stage. | | | +| [providers](outputs.tf#L149) | Terraform provider files for this stage and dependent stages. | ✓ | stage-01 | +| [service_accounts](outputs.tf#L138) | Automation service accounts created by this stage. | | | +| [tfvars](outputs.tf#L158) | Terraform variable files for the following stages. | ✓ | | diff --git a/fast/stages/00-bootstrap/automation.tf b/fast/stages/00-bootstrap/automation.tf index bba34d06f4..59da5afcef 100644 --- a/fast/stages/00-bootstrap/automation.tf +++ b/fast/stages/00-bootstrap/automation.tf @@ -121,6 +121,37 @@ module "automation-tf-bootstrap-sa" { # resource hierarchy stage's bucket and service account +module "automation-tf-cicd-gcs" { + source = "../../../modules/gcs" + project_id = module.automation-project.project_id + name = "iac-core-cicd-0" + prefix = local.prefix + versioning = true + iam = { + "roles/storage.objectAdmin" = [module.automation-tf-cicd-provisioning-sa.iam_email] + } + depends_on = [module.organization] +} + +module "automation-tf-cicd-provisioning-sa" { + source = "../../../modules/iam-service-account" + project_id = module.automation-project.project_id + name = "cicd-0" + description = "Terraform stage 1 CICD service account." + prefix = local.prefix + # allow SA used by CI/CD workflow to impersonate this SA + iam = { + "roles/iam.serviceAccountTokenCreator" = compact([ + try(module.automation-tf-cicd-sa["cicd"].iam_email, null) + ]) + } + iam_storage_roles = { + (module.automation-tf-output-gcs.name) = ["roles/storage.admin"] + } +} + +# resource hierarchy stage's bucket and service account + module "automation-tf-resman-gcs" { source = "../../../modules/gcs" project_id = module.automation-project.project_id diff --git a/fast/stages/00-bootstrap/cicd.tf b/fast/stages/00-bootstrap/cicd.tf index b4032c7665..fba8f24816 100644 --- a/fast/stages/00-bootstrap/cicd.tf +++ b/fast/stages/00-bootstrap/cicd.tf @@ -23,20 +23,25 @@ locals { v != null && ( - v.type == "sourcerepo" + try(v.type, null) == "sourcerepo" || - contains(keys(local.identity_providers), coalesce(v.identity_provider, ":")) + contains(keys(local.identity_providers), coalesce(try(v.identity_provider, null), ":")) ) && - fileexists("${path.module}/templates/workflow-${v.type}.yaml") + fileexists(format("${path.module}/templates/workflow-%s.yaml", try(v.type, ""))) ) } cicd_workflow_providers = { bootstrap = "00-bootstrap-providers.tf" + cicd = "00-cicd-providers.tf" resman = "01-resman-providers.tf" } cicd_workflow_var_files = { bootstrap = [] + cicd = [ + "00-bootstrap.auto.tfvars.json", + "globals.auto.tfvars.json" + ] resman = [ "00-bootstrap.auto.tfvars.json", "globals.auto.tfvars.json" diff --git a/fast/stages/00-bootstrap/identity-providers.tf b/fast/stages/00-bootstrap/identity-providers.tf index 7d4f2c6abd..7d2500c30d 100644 --- a/fast/stages/00-bootstrap/identity-providers.tf +++ b/fast/stages/00-bootstrap/identity-providers.tf @@ -19,7 +19,8 @@ locals { identity_providers = { for k, v in var.federated_identity_providers : k => merge( - v, lookup(local.identity_providers_defs, v.issuer, {}) + v, lookup(local.identity_providers_defs, v.issuer, {}), + { for kk, vv in lookup(v, "custom_settings", {}) : kk => vv if vv != null } ) } identity_providers_defs = { diff --git a/fast/stages/00-bootstrap/outputs.tf b/fast/stages/00-bootstrap/outputs.tf index 9104586510..eceb5f8202 100644 --- a/fast/stages/00-bootstrap/outputs.tf +++ b/fast/stages/00-bootstrap/outputs.tf @@ -43,6 +43,11 @@ locals { name = "bootstrap" sa = module.automation-tf-bootstrap-sa.email }) + "00-cicd" = templatefile(local._tpl_providers, { + bucket = module.automation-tf-cicd-gcs.name + name = "cicd" + sa = module.automation-tf-cicd-provisioning-sa.email + }) "01-resman" = templatefile(local._tpl_providers, { bucket = module.automation-tf-resman-gcs.name name = "resman" @@ -134,6 +139,7 @@ output "service_accounts" { description = "Automation service accounts created by this stage." value = { bootstrap = module.automation-tf-bootstrap-sa.email + cicd = module.automation-tf-cicd-provisioning-sa.email resman = module.automation-tf-resman-sa.email } } diff --git a/fast/stages/00-bootstrap/variables.tf b/fast/stages/00-bootstrap/variables.tf index a08fedb1c5..f2f1edcd57 100644 --- a/fast/stages/00-bootstrap/variables.tf +++ b/fast/stages/00-bootstrap/variables.tf @@ -37,6 +37,12 @@ variable "cicd_repositories" { name = string type = string }) + cicd = object({ + branch = string + identity_provider = string + name = string + type = string + }) resman = object({ branch = string identity_provider = string @@ -91,6 +97,10 @@ variable "federated_identity_providers" { type = map(object({ attribute_condition = string issuer = string + custom_settings = object({ + issuer_uri = string + allowed_audiences = list(string) + }) })) default = {} nullable = false diff --git a/fast/stages/00-cicd/README.md b/fast/stages/00-cicd/README.md new file mode 100644 index 0000000000..6061067272 --- /dev/null +++ b/fast/stages/00-cicd/README.md @@ -0,0 +1,208 @@ +# CI/CD bootstrap + +The primary purpose of this stage is to set up your CI/CD project structure automatically, with most of the +necessary configuration to run the pipelines out of the box. + +## How to run this stage + +This stage is meant to be executed after the [bootstrap](../00-bootstrap) stage has run, as it leverages the automation service account and bucket created there. +The entire stage is optional, you may also choose to create your repositories manually. + +### Providers configuration + +The default way of making sure you have the right permissions, is to use the identity of the service account pre-created for this stage during bootstrap, and that you are a member of the group that can impersonate it via provider-level configuration (`gcp-devops` or `organization-admins`). + +To simplify setup, the previous stage pre-configures a valid providers file in its output, and optionally writes it to a local file if the `outputs_location` variable is set to a valid path. + +If you have set a valid value for `outputs_location` in the bootstrap stage (see the [bootstrap stage README](../00-bootstrap/#output-files-and-cross-stage-variables) for more details), simply link the relevant `providers.tf` file from this stage's folder in the path you specified: + +```bash +# `outputs_location` is set to `~/fast-config` +ln -s ~/fast-config/providers/00-cicd-providers.tf . +``` + +If you have not configured `outputs_location` in bootstrap, you can derive the providers file from that stage's outputs: + +```bash +cd ../00-bootstrap +terraform output -json providers | jq -r '.["00-cicd"]' \ + > ../00-cicd/providers.tf +``` + +If you want to continue to rely on `outputs_location` logic, create a `terraform.tfvars` file and configure it as described [here](../00-bootstrap/#output-files-and-cross-stage-variables). + +### Variable configuration + +There are two broad sets of variables you will need to fill in: + +- variables shared by other stages (org id, billing account id, etc.), or derived from a resource managed by a different stage (folder id, automation project id, etc.) +- variables specific to resources managed by this stage + +To avoid the tedious job of filling in the first group of variable with values derived from other stages' outputs, the same mechanism used above for the provider configuration can be used to leverage pre-configured `.tfvars` files. + +If you configured a valid path for `outputs_location` in the bootstrap stage, simply link the relevant `terraform-*.auto.tfvars.json` files from the outputs folder. For this stage, you need the `.tfvars` file compiled manually for the bootstrap stage, and the one generated by it: + +```bash +# `outputs_location` is set to `~/fast-config` +ln -s ~/fast-config/tfvars/00-bootstrap.auto.tfvars.json . +# also copy the tfvars file used for the bootstrap stage +cp ../00-bootstrap/terraform.tfvars . +``` + +A second set of variables is specific to this stage, they are all optional so if you need to customize them, create an extra `terraform.tfvars` file or add them to the file copied from bootstrap. + +Refer to the [Variables](#variables) table at the bottom of this document, for a full list of variables, their origin (e.g. a stage or specific to this one), and descriptions explaining their meaning. The sections below also describe some of the possible customizations. + +### CI/CD systems + +#### Gitlab + +To configure Gitlab, add the following variable: + +```hcl +gitlab = { + url = "https://gitlab.com" # Or self-hosted URL + project_visibility = "private" + shared_runners_enabled = true +} +``` + +Also set `GITLAB_TOKEN` to a token that has appropriate permissions. + +#### GitHub + +To configure GitHub, add the following variable: + +```hcl +github = { + url = null # Or GitHub Enterprise base URL + visibility = "private" +} +``` + +Also set `GITHUB_TOKEN` to a token that has appropriate permissions. + +### CI/CD repositories + +While the other stages create the necessary supporting structure for their CI/CD pipelines, like service accounts +and such, the `00-cicd` stage creates all the repositories in your CI/CD system through automation. Its +configuration is essentially a combination of all the `cicd_repositories` variables of the other stages +plus additional CI/CD system specific configuration information. + +This is an example of configuring the repositories in this stage. + +```hcl +cicd_repositories = { + bootstrap = { + branch = null + identity_provider = "github-sample" + name = "my-gh-org/fast-bootstrap" + description = "Google Cloud bootstrapping" + type = "github" + create = true + create_group = true + } + cicd = { + branch = null + identity_provider = "github-sample" + name = "my-gh-org/fast-cicd" + description = "Fabric FAST CI/CD setup" + type = "github" + create = true + create_group = true + } + resman = { + branch = "main" + identity_provider = "github-sample" + name = "my-gh-org/fast-resman" + description = "Google Cloud resource management" + type = "github" + create = true + create_group = true + } + networking = { + branch = "main" + identity_provider = "github-sample" + name = "my-gh-org/fast-networking" + description = "Google Cloud networking setup" + type = "github" + create = true + create_group = true + } + security = { + branch = "main" + identity_provider = "github-sample" + description = "Google Cloud security settings" + name = "my-gh-org/fast-security" + type = "github" + create = true + create_group = true + } + data-platform = { + branch = "main" + identity_provider = "github-sample" + name = "my-gh-org/fast-data-platform" + description = "Google Cloud data platform" + type = "github" + create = true + create_group = true + } + project-factory = { + branch = "main" + identity_provider = "github-sample" + name = "my-gh-org/fast-project-factory" + description = "Google Cloud project factory" + type = "github" + create = true + create_group = true + } +} +``` + +The `type` attribute can be set to one of the supported repository types: `github` or `gitlab`. + +Once the stage is applied the generated output files will contain pre-configured workflow files for each repository, that will use Workload Identity Federation via a dedicated service account for each repository to impersonate the automation service account for the stage. + + +Once done, you can run this stage: + +```bash +terraform init +terraform apply +``` + + + + +## Files + +| name | description | resources | +|---|---|---| +| [cicd.tf](./cicd.tf) | None | | +| [github.tf](./github.tf) | None | github_repository | +| [gitlab.tf](./gitlab.tf) | None | gitlab_group · gitlab_project | +| [main.tf](./main.tf) | Module-level locals and resources. | | +| [outputs-files.tf](./outputs-files.tf) | Output files persistence to local filesystem. | local_file | +| [outputs-gcs.tf](./outputs-gcs.tf) | Output files persistence to automation GCS bucket. | google_storage_bucket_object | +| [outputs.tf](./outputs.tf) | Module outputs. | | +| [variables.tf](./variables.tf) | Module variables. | | +| [versions.tf](./versions.tf) | Version pins. | | + +## Variables + +| name | description | type | required | default | producer | +|---|---|:---:|:---:|:---:|:---:| +| [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 00-bootstrap | +| [cicd_repositories](variables.tf#L35) | CI/CD repository configuration. Identity providers reference keys in the `federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | object({…}) | | null | | +| [custom_roles](variables.tf#L132) | Custom roles defined at the org level, in key => id format. | object({…}) | | null | 00-bootstrap | +| [github](variables.tf#L120) | GitHub settings | object({…}) | | {…} | | +| [gitlab](variables.tf#L106) | Gitlab settings | object({…}) | | {…} | | +| [outputs_location](variables.tf#L141) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable | string | | null | | + +## Outputs + +| name | description | sensitive | consumers | +|---|---|:---:|---| +| [tfvars](outputs.tf#L30) | Terraform variable files for the following stages. | ✓ | | + + diff --git a/fast/stages/00-cicd/cicd.tf b/fast/stages/00-cicd/cicd.tf new file mode 100644 index 0000000000..cd6a3ae847 --- /dev/null +++ b/fast/stages/00-cicd/cicd.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + supported_cicd_systems = ["gitlab", "github", "sourcerepo"] + cicd_repositories = { + for k, v in coalesce(var.cicd_repositories, {}) : k => merge(v, + { group = join("/", slice(split("/", v.name), 0, length(split("/", v.name)) - 1)) }, + { name = element(split("/", v.name), length(split("/", v.name)) - 1) }, + { create_group = try(v.create_group, true) }) + if( + v != null + && + contains(local.supported_cicd_systems, try(v.type, "")) + ) + } + cicd_repositories_by_system = { for system in local.supported_cicd_systems : system => { + for k, v in local.cicd_repositories : k => v if v.type == system + } + } +} diff --git a/fast/stages/00-cicd/github.tf b/fast/stages/00-cicd/github.tf new file mode 100644 index 0000000000..9a8d390aa4 --- /dev/null +++ b/fast/stages/00-cicd/github.tf @@ -0,0 +1,46 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + github_groups = distinct([for k, v in local.cicd_repositories_by_system["github"] : v.group]) +} + +provider "github" { + base_url = var.github.url + owner = local.github_groups[0] +} + +data "github_organization" "organization" { + for_each = toset(local.github_groups) + name = each.value +} + +data "github_repository" "repositories" { + for_each = { for name, repo in local.cicd_repositories_by_system["github"] : name => repo if !try(repo.create, true) } + full_name = format("%s/%s", each.value.group, each.value.name) +} + +resource "github_repository" "repositories" { + for_each = { for name, repo in local.cicd_repositories_by_system["github"] : name => repo if try(repo.create, true) } + + name = each.value.name + description = each.value.description + + visibility = var.github.visibility +} + + + diff --git a/fast/stages/00-cicd/gitlab.tf b/fast/stages/00-cicd/gitlab.tf new file mode 100644 index 0000000000..f072df6634 --- /dev/null +++ b/fast/stages/00-cicd/gitlab.tf @@ -0,0 +1,57 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + gitlab_create_groups = distinct([for k, v in local.cicd_repositories_by_system["gitlab"] : v.group if try(v.create_group, false)]) + gitlab_existing_groups = distinct([for k, v in local.cicd_repositories_by_system["gitlab"] : v.group if !try(v.create_group, false)]) +} + +provider "gitlab" { + base_url = var.gitlab.url +} + +data "gitlab_group" "group" { + for_each = toset(local.gitlab_existing_groups) + full_path = each.value +} + +data "gitlab_project" "projects" { + for_each = { for name, repo in local.cicd_repositories_by_system["gitlab"] : name => repo if !try(repo.create, true) } + id = format("%s/%s", each.value.group, each.value.name) +} + +resource "gitlab_group" "group" { + for_each = toset(local.gitlab_create_groups) + + name = each.value + path = each.value + description = "Cloud Foundation Fabric FAST: github.com/GoogleCloudPlatform/cloud-foundation-fabric/tree/master/fast/" +} + +resource "gitlab_project" "projects" { + for_each = { for name, repo in local.cicd_repositories_by_system["gitlab"] : name => repo if try(repo.create, true) } + + name = each.value.name + namespace_id = each.value.create_group ? gitlab_group.group[each.value.group].id : data.gitlab_group.group[each.value.group].id + description = each.value.description + + visibility_level = var.gitlab.project_visibility + shared_runners_enabled = var.gitlab.shared_runners_enabled + auto_devops_enabled = false +} + + + diff --git a/fast/stages/00-cicd/main.tf b/fast/stages/00-cicd/main.tf new file mode 100644 index 0000000000..3ea74b550c --- /dev/null +++ b/fast/stages/00-cicd/main.tf @@ -0,0 +1,16 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + diff --git a/fast/stages/00-cicd/outputs-files.tf b/fast/stages/00-cicd/outputs-files.tf new file mode 100644 index 0000000000..6c201fc4c7 --- /dev/null +++ b/fast/stages/00-cicd/outputs-files.tf @@ -0,0 +1,24 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Output files persistence to local filesystem. + +resource "local_file" "tfvars" { + for_each = var.outputs_location == null ? {} : { 1 = 1 } + file_permission = "0644" + filename = "${pathexpand(var.outputs_location)}/tfvars/00-cicd.auto.tfvars.json" + content = jsonencode(local.tfvars) +} diff --git a/fast/stages/00-cicd/outputs-gcs.tf b/fast/stages/00-cicd/outputs-gcs.tf new file mode 100644 index 0000000000..3a03ae4d71 --- /dev/null +++ b/fast/stages/00-cicd/outputs-gcs.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Output files persistence to automation GCS bucket. + +resource "google_storage_bucket_object" "tfvars" { + bucket = var.automation.outputs_bucket + name = "tfvars/00-bootstrap.auto.tfvars.json" + content = jsonencode(local.tfvars) +} diff --git a/fast/stages/00-cicd/outputs.tf b/fast/stages/00-cicd/outputs.tf new file mode 100644 index 0000000000..67341ba16c --- /dev/null +++ b/fast/stages/00-cicd/outputs.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + gitlab_cicd_https = { for k, v in local.cicd_repositories_by_system["gitlab"] : k => v.create ? gitlab_project.projects[k].http_url_to_repo : data.gitlab_project.projects[k].http_url_to_repo } + gitlab_cicd_ssh = { for k, v in local.cicd_repositories_by_system["gitlab"] : k => v.create ? gitlab_project.projects[k].ssh_url_to_repo : data.gitlab_project.projects[k].ssh_url_to_repo } + github_cicd_https = { for k, v in local.cicd_repositories_by_system["github"] : k => v.create ? github_repository.repositories[k].http_clone_url : data.github_repository.repositories[k].http_clone_url } + github_cicd_ssh = { for k, v in local.cicd_repositories_by_system["github"] : k => v.create ? github_repository.repositories[k].git_clone_url : data.github_repository.repositories[k].git_clone_url } + + tfvars = { + cicd_repositories = merge(local.cicd_repositories_by_system["gitlab"], local.cicd_repositories_by_system["github"]) + cicd_ssh_urls = merge(local.gitlab_cicd_ssh, local.github_cicd_ssh) + cicd_https_urls = merge(local.gitlab_cicd_https, local.gitlab_cicd_https) + } +} + +output "tfvars" { + description = "Terraform variable files for the following stages." + sensitive = true + value = local.tfvars +} diff --git a/fast/stages/00-cicd/variables.tf b/fast/stages/00-cicd/variables.tf new file mode 100644 index 0000000000..c5cc5f695c --- /dev/null +++ b/fast/stages/00-cicd/variables.tf @@ -0,0 +1,145 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "automation" { + # tfdoc:variable:source 00-bootstrap + description = "Automation resources created by the bootstrap stage." + type = object({ + outputs_bucket = string + project_id = string + project_number = string + federated_identity_pool = string + federated_identity_providers = map(object({ + issuer = string + issuer_uri = string + name = string + principal_tpl = string + principalset_tpl = string + })) + }) +} + +variable "cicd_repositories" { + description = "CI/CD repository configuration. Identity providers reference keys in the `federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed." + type = object({ + bootstrap = object({ + branch = string + name = string + description = string + type = string + create = bool + create_group = bool + }) + resman = object({ + branch = string + name = string + description = string + type = string + create = bool + create_group = bool + }) + networking = object({ + branch = string + name = string + description = string + type = string + create = bool + create_group = bool + }) + security = object({ + branch = string + name = string + description = string + type = string + create = bool + create_group = bool + }) + data-platform = object({ + branch = string + name = string + description = string + type = string + create = bool + create_group = bool + }) + project-factory = object({ + branch = string + name = string + description = string + type = string + create = bool + create_group = bool + }) + }) + default = null + validation { + condition = alltrue([ + for k, v in coalesce(var.cicd_repositories, {}) : + v == null || try(v.name, null) != null + ]) + error_message = "Non-null repositories need a non-null name." + } + validation { + condition = alltrue([ + for k, v in coalesce(var.cicd_repositories, {}) : + v == null || ( + contains(["github", "gitlab", "sourcerepo"], coalesce(try(v.type, null), "null")) + ) + ]) + error_message = "Invalid repository type, supported types: 'github' 'gitlab' or 'sourcerepo'." + } +} + +variable "gitlab" { + description = "Gitlab settings" + type = object({ + url = string + project_visibility = string + shared_runners_enabled = bool + }) + default = { + url = "https://gitlab.com" + project_visibility = "private" + shared_runners_enabled = true + } +} + +variable "github" { + description = "GitHub settings" + type = object({ + url = string + visibility = string + }) + default = { + url = null + visibility = "private" + } +} + +variable "custom_roles" { + # tfdoc:variable:source 00-bootstrap + description = "Custom roles defined at the org level, in key => id format." + type = object({ + service_project_network_admin = string + }) + default = null +} + +variable "outputs_location" { + description = "Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable" + type = string + default = null +} diff --git a/fast/stages/00-cicd/versions.tf b/fast/stages/00-cicd/versions.tf new file mode 100644 index 0000000000..cbd1f00b79 --- /dev/null +++ b/fast/stages/00-cicd/versions.tf @@ -0,0 +1,27 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.1.0" + required_providers { + gitlab = { + source = "gitlabhq/gitlab" + version = ">= 3.15.0" + } + github = { + source = "integrations/github" + version = ">= 4.26.0" + } + } +}