From 270300f9f1dfb056d26d73e24962030331be9a66 Mon Sep 17 00:00:00 2001 From: Andrew Peabody Date: Tue, 31 Jan 2023 20:22:35 -0800 Subject: [PATCH] feat: add support for policy bundles and metrics SA (#1529) --- examples/simple_zonal_with_acm/README.md | 10 +- examples/simple_zonal_with_acm/acm.tf | 4 + examples/simple_zonal_with_acm/variables.tf | 3 + examples/simple_zonal_with_acm/versions.tf | 7 +- modules/acm/README.md | 7 +- modules/acm/creds.tf | 93 ++++++++++++++++++- modules/acm/feature.tf | 22 +++-- modules/acm/outputs.tf | 7 +- modules/acm/policy_bundles.tf | 29 ++++++ modules/acm/variables.tf | 25 +++++ test/integration/simple_zonal/controls/acm.rb | 9 +- 11 files changed, 198 insertions(+), 18 deletions(-) create mode 100644 modules/acm/policy_bundles.tf diff --git a/examples/simple_zonal_with_acm/README.md b/examples/simple_zonal_with_acm/README.md index 2607c45b8b..9c8c4fe2e5 100644 --- a/examples/simple_zonal_with_acm/README.md +++ b/examples/simple_zonal_with_acm/README.md @@ -1,6 +1,6 @@ # Simple Zonal Cluster -This example illustrates how to create a simple cluster and install [Anthos Config Management](https://cloud.google.com/anthos-config-management/docs/). +This example illustrates how to create a simple cluster and install [Anthos Config Management](https://cloud.google.com/anthos-config-management/docs/)'s [Config Sync](https://cloud.google.com/anthos-config-management/docs/config-sync-overview) and [Policy Controller](https://cloud.google.com/anthos-config-management/docs/concepts/policy-controller) with the [Policy Essentials v2022 policy bundle](https://cloud.google.com/anthos-config-management/docs/how-to/using-policy-essentials-v2022). It incorporates the standard cluster module and the [ACM install module](../../modules/acm). @@ -27,13 +27,19 @@ After applying the Terraform configuration, you can run the following commands t kubectl describe ns shipping-dev ``` +4. You can also use `kubectl` to view any policy violations on the cluster: + + ``` + kubectl get constraint -l policycontroller.gke.io/bundleName=policy-essentials-v2022 -o json | jq -cC '.items[]| [.metadata.name,.status.totalViolations]' + ``` + ## Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | cluster\_name\_suffix | A suffix to append to the default cluster name | `string` | `""` | no | -| project\_id | The project ID to host the cluster in | `any` | n/a | yes | +| project\_id | The project ID to host the cluster in | `string` | n/a | yes | | region | The region to host the cluster in | `string` | `"us-central1"` | no | | zone | The zone to host the cluster in | `string` | `"us-central1-a"` | no | diff --git a/examples/simple_zonal_with_acm/acm.tf b/examples/simple_zonal_with_acm/acm.tf index 67ce5b5969..c72df471e0 100644 --- a/examples/simple_zonal_with_acm/acm.tf +++ b/examples/simple_zonal_with_acm/acm.tf @@ -25,4 +25,8 @@ module "acm" { policy_dir = "foo-corp" secret_type = "ssh" + + policy_bundles = ["https://github.com/GoogleCloudPlatform/acm-policy-controller-library/bundles/policy-essentials-v2022#e4094aacb91a35b0219f6f4cf6a31580e85b3c28"] + + create_metrics_gcp_sa = true } diff --git a/examples/simple_zonal_with_acm/variables.tf b/examples/simple_zonal_with_acm/variables.tf index c02931ccd9..722b32e674 100644 --- a/examples/simple_zonal_with_acm/variables.tf +++ b/examples/simple_zonal_with_acm/variables.tf @@ -16,15 +16,18 @@ variable "project_id" { description = "The project ID to host the cluster in" + type = string } variable "cluster_name_suffix" { description = "A suffix to append to the default cluster name" + type = string default = "" } variable "region" { description = "The region to host the cluster in" + type = string default = "us-central1" } diff --git a/examples/simple_zonal_with_acm/versions.tf b/examples/simple_zonal_with_acm/versions.tf index e8fbb1aadd..591f742201 100644 --- a/examples/simple_zonal_with_acm/versions.tf +++ b/examples/simple_zonal_with_acm/versions.tf @@ -21,7 +21,12 @@ terraform { version = "~> 4.0" } kubernetes = { - source = "hashicorp/kubernetes" + source = "hashicorp/kubernetes" + version = "~> 2.10" + } + random = { + source = "hashicorp/random" + version = ">= 2.1" } } required_version = ">= 0.13" diff --git a/modules/acm/README.md b/modules/acm/README.md index 03bfccb198..228d3e3631 100644 --- a/modules/acm/README.md +++ b/modules/acm/README.md @@ -67,7 +67,9 @@ data "google_client_config" "default" {} | cluster\_membership\_id | The cluster membership ID. If unset, one will be autogenerated. | `string` | `""` | no | | cluster\_name | GCP cluster Name used to reach cluster and which becomes the cluster name in the Config Sync kubernetes custom resource. | `string` | n/a | yes | | configmanagement\_version | Version of ACM. | `string` | `""` | no | +| create\_metrics\_gcp\_sa | Create a Google service account for ACM metrics writing | `bool` | `false` | no | | create\_ssh\_key | Controls whether a key will be generated for Git authentication | `bool` | `true` | no | +| enable\_config\_sync | Whether to enable the ACM Config Sync on the cluster | `bool` | `true` | no | | enable\_fleet\_feature | Whether to enable the ACM feature on the fleet. | `bool` | `true` | no | | enable\_fleet\_registration | Whether to create a new membership. | `bool` | `true` | no | | enable\_log\_denies | Whether to enable logging of all denies and dryrun failures for ACM Policy Controller. | `bool` | `false` | no | @@ -77,19 +79,22 @@ data "google_client_config" "default" {} | https\_proxy | URL for the HTTPS proxy to be used when communicating with the Git repo. | `string` | `null` | no | | install\_template\_library | Whether to install the default Policy Controller template library | `bool` | `true` | no | | location | GCP location used to reach cluster. | `string` | n/a | yes | +| metrics\_gcp\_sa\_name | The name of the Google service account for ACM metrics writing | `string` | `"acm-metrics-writer"` | no | +| policy\_bundles | A list of Policy Controller policy bundles git urls (example: https://github.com/GoogleCloudPlatform/acm-policy-controller-library.git/bundles/policy-essentials-v2022) to install on the cluster. | `list(string)` | `[]` | no | | policy\_dir | Subfolder containing configs in ACM Git repo. If un-set, uses Config Management default. | `string` | `""` | no | | project\_id | GCP project\_id used to reach cluster. | `string` | n/a | yes | | secret\_type | git authentication secret type, is passed through to ConfigManagement spec.git.secretType. Overriden to value 'ssh' if `create_ssh_key` is true | `string` | `"ssh"` | no | | source\_format | Configures a non-hierarchical repo if set to 'unstructured'. Uses [ACM defaults](https://cloud.google.com/anthos-config-management/docs/how-to/installing#configuring-config-management-operator) when unset. | `string` | `""` | no | | ssh\_auth\_key | Key for Git authentication. Overrides 'create\_ssh\_key' variable. Can be set using 'file(path/to/file)'-function. | `string` | `null` | no | | sync\_branch | ACM repo Git branch. If un-set, uses Config Management default. | `string` | `""` | no | -| sync\_repo | ACM Git repo address | `string` | n/a | yes | +| sync\_repo | ACM Git repo address | `string` | `""` | no | | sync\_revision | ACM repo Git revision. If un-set, uses Config Management default. | `string` | `""` | no | ## Outputs | Name | Description | |------|-------------| +| acm\_metrics\_writer\_sa | The ACM metrics writer Service Account | | configmanagement\_version | Version of ACM installed. | | git\_creds\_public | Public key of SSH keypair to allow the Anthos Config Management Operator to authenticate to your Git repository. | | wait | An output to use when you want to depend on cmd finishing | diff --git a/modules/acm/creds.tf b/modules/acm/creds.tf index 45a6748949..b4b2f96c98 100644 --- a/modules/acm/creds.tf +++ b/modules/acm/creds.tf @@ -14,6 +14,11 @@ * limitations under the License. */ +locals { + # GCP service account ids must be <= 30 chars matching regex ^[a-z](?:[-a-z0-9]{4,28}[a-z0-9])$ + service_account_name = trimsuffix(substr(var.metrics_gcp_sa_name, 0, 30), "-") +} + resource "tls_private_key" "k8sop_creds" { count = var.create_ssh_key ? 1 : 0 algorithm = "RSA" @@ -22,10 +27,92 @@ resource "tls_private_key" "k8sop_creds" { # Wait for the ACM operator to create the namespace resource "time_sleep" "wait_acm" { - count = (var.create_ssh_key == true || var.ssh_auth_key != null) ? 1 : 0 + count = (var.create_ssh_key == true || var.ssh_auth_key != null || var.enable_policy_controller || var.enable_config_sync) ? 1 : 0 + depends_on = [google_gke_hub_feature_membership.main] + + create_duration = "300s" +} + +resource "google_service_account_iam_binding" "config-management-monitoring-iam" { + count = var.enable_config_sync && var.create_metrics_gcp_sa ? 1 : 0 + service_account_id = google_service_account.acm_metrics_writer_sa[0].name + role = "roles/iam.workloadIdentityUser" + + members = ["serviceAccount:${var.project_id}.svc.id.goog[config-management-monitoring/default]"] + depends_on = [google_gke_hub_feature_membership.main] +} + +resource "google_service_account_iam_binding" "gatekeeper-system-iam" { + count = var.enable_policy_controller && var.create_metrics_gcp_sa ? 1 : 0 + service_account_id = google_service_account.acm_metrics_writer_sa[0].name + role = "roles/iam.workloadIdentityUser" + + members = ["serviceAccount:${var.project_id}.svc.id.goog[gatekeeper-system/gatekeeper-admin]"] + + depends_on = [google_gke_hub_feature_membership.main] +} + +module "annotate-sa-config-management-monitoring" { + source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper" + version = "~> 3.1" + + count = var.enable_config_sync && var.create_metrics_gcp_sa ? 1 : 0 + skip_download = true + cluster_name = var.cluster_name + cluster_location = var.location + project_id = var.project_id + + kubectl_create_command = "kubectl annotate --overwrite sa -n config-management-monitoring default iam.gke.io/gcp-service-account=${google_service_account.acm_metrics_writer_sa[0].email}" + kubectl_destroy_command = "kubectl annotate sa -n config-management-monitoring default iam.gke.io/gcp-service-account-" + + module_depends_on = time_sleep.wait_acm +} + +module "annotate-sa-gatekeeper-system" { + source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper" + version = "~> 3.1" + + count = var.enable_policy_controller && var.create_metrics_gcp_sa ? 1 : 0 + skip_download = true + cluster_name = var.cluster_name + cluster_location = var.location + project_id = var.project_id + + kubectl_create_command = "kubectl annotate --overwrite sa -n gatekeeper-system gatekeeper-admin iam.gke.io/gcp-service-account=${google_service_account.acm_metrics_writer_sa[0].email}" + kubectl_destroy_command = "kubectl annotate sa -n gatekeeper-system gatekeeper-admin iam.gke.io/gcp-service-account-" + + module_depends_on = time_sleep.wait_acm +} + +module "annotate-sa-gatekeeper-system-restart" { + source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper" + version = "~> 3.1" + + count = var.enable_policy_controller && var.create_metrics_gcp_sa ? 1 : 0 + skip_download = true + cluster_name = var.cluster_name + cluster_location = var.location + project_id = var.project_id + + kubectl_create_command = "kubectl rollout restart deployment gatekeeper-controller-manager -n gatekeeper-system" + kubectl_destroy_command = "" + + module_depends_on = module.annotate-sa-gatekeeper-system +} + +resource "google_service_account" "acm_metrics_writer_sa" { + count = var.create_metrics_gcp_sa ? 1 : 0 + + display_name = "ACM Metrics Writer SA" + account_id = local.service_account_name + project = var.project_id +} - create_duration = "60s" +resource "google_project_iam_member" "acm_metrics_writer_sa_role" { + project = var.project_id + role = "roles/monitoring.metricWriter" + member = "serviceAccount:${google_service_account.acm_metrics_writer_sa[0].email}" } resource "kubernetes_secret_v1" "creds" { @@ -38,6 +125,6 @@ resource "kubernetes_secret_v1" "creds" { } data = { - "${local.k8sop_creds_secret_key}" = local.private_key + (local.k8sop_creds_secret_key) = local.private_key } } diff --git a/modules/acm/feature.tf b/modules/acm/feature.tf index 3bb977388a..101bac31ee 100644 --- a/modules/acm/feature.tf +++ b/modules/acm/feature.tf @@ -38,16 +38,20 @@ resource "google_gke_hub_feature_membership" "main" { configmanagement { version = var.configmanagement_version - config_sync { - source_format = var.source_format != "" ? var.source_format : null + dynamic "config_sync" { + for_each = var.enable_config_sync ? [{ enabled = true }] : [] - git { - sync_repo = var.sync_repo - policy_dir = var.policy_dir != "" ? var.policy_dir : null - sync_branch = var.sync_branch != "" ? var.sync_branch : null - sync_rev = var.sync_revision != "" ? var.sync_revision : null - secret_type = var.secret_type - https_proxy = var.https_proxy + content { + source_format = var.source_format != "" ? var.source_format : null + + git { + sync_repo = var.sync_repo + policy_dir = var.policy_dir != "" ? var.policy_dir : null + sync_branch = var.sync_branch != "" ? var.sync_branch : null + sync_rev = var.sync_revision != "" ? var.sync_revision : null + secret_type = var.secret_type + https_proxy = var.https_proxy + } } } diff --git a/modules/acm/outputs.tf b/modules/acm/outputs.tf index 063d5f171f..3e9594cee5 100644 --- a/modules/acm/outputs.tf +++ b/modules/acm/outputs.tf @@ -16,7 +16,7 @@ output "git_creds_public" { description = "Public key of SSH keypair to allow the Anthos Config Management Operator to authenticate to your Git repository." - value = var.create_ssh_key ? coalesce(tls_private_key.k8sop_creds.*.public_key_openssh...) : null + value = var.create_ssh_key ? coalesce(tls_private_key.k8sop_creds[*].public_key_openssh...) : null } output "configmanagement_version" { @@ -31,3 +31,8 @@ output "wait" { google_gke_hub_feature_membership.main ] } + +output "acm_metrics_writer_sa" { + description = "The ACM metrics writer Service Account" + value = google_service_account.acm_metrics_writer_sa[0].email +} diff --git a/modules/acm/policy_bundles.tf b/modules/acm/policy_bundles.tf new file mode 100644 index 0000000000..b03987e867 --- /dev/null +++ b/modules/acm/policy_bundles.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "policy_bundles" { + source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper" + version = "~> 3.1" + + for_each = toset(var.policy_bundles) + project_id = var.project_id + cluster_name = var.cluster_name + cluster_location = var.location + kubectl_create_command = "kubectl apply -k ${each.key}" + kubectl_destroy_command = "kubectl delete -k ${each.key}" + + module_depends_on = [time_sleep.wait_acm] +} diff --git a/modules/acm/variables.tf b/modules/acm/variables.tf index f9bf8a41f5..dfb4f6d2fd 100644 --- a/modules/acm/variables.tf +++ b/modules/acm/variables.tf @@ -57,6 +57,7 @@ variable "configmanagement_version" { variable "sync_repo" { description = "ACM Git repo address" type = string + default = "" } variable "sync_branch" { @@ -108,6 +109,12 @@ variable "ssh_auth_key" { default = null } +variable "enable_config_sync" { + description = "Whether to enable the ACM Config Sync on the cluster" + type = bool + default = true +} + # Policy Controller config variable "enable_policy_controller" { description = "Whether to enable the ACM Policy Controller on the cluster" @@ -139,3 +146,21 @@ variable "enable_referential_rules" { type = bool default = true } + +variable "policy_bundles" { + description = "A list of Policy Controller policy bundles git urls (example: https://github.com/GoogleCloudPlatform/acm-policy-controller-library.git/bundles/policy-essentials-v2022) to install on the cluster." + type = list(string) + default = [] +} + +variable "create_metrics_gcp_sa" { + description = "Create a Google service account for ACM metrics writing" + type = bool + default = false +} + +variable "metrics_gcp_sa_name" { + description = "The name of the Google service account for ACM metrics writing" + type = string + default = "acm-metrics-writer" +} diff --git a/test/integration/simple_zonal/controls/acm.rb b/test/integration/simple_zonal/controls/acm.rb index 45663f176d..225f2bf963 100644 --- a/test/integration/simple_zonal/controls/acm.rb +++ b/test/integration/simple_zonal/controls/acm.rb @@ -1,4 +1,4 @@ -# Copyright 2019 Google LLC +# Copyright 2019-2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -48,5 +48,12 @@ expect(namespace).not_to be nil end end + + describe "gatekeeper-system namespace" do + let(:namespace) { client.get_namespace("gatekeeper-system") } + it "should exist" do + expect(namespace).not_to be nil + end + end end end