From e8bf0a644a49f21d688e284915429ffdd0fff02d Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Wed, 18 Dec 2024 16:17:43 +0100 Subject: [PATCH] Add logs scopes to project --- modules/project/README.md | 79 +++++++++++++++----- modules/project/logging.tf | 22 +++++- modules/project/variables-observability.tf | 85 ++++++++++++++++++++++ modules/project/variables.tf | 60 --------------- 4 files changed, 167 insertions(+), 79 deletions(-) create mode 100644 modules/project/variables-observability.tf diff --git a/modules/project/README.md b/modules/project/README.md index 62bb6fea6d..44e45a0961 100644 --- a/modules/project/README.md +++ b/modules/project/README.md @@ -18,6 +18,7 @@ This module implements the creation and management of one GCP project including - [Dry-Run Mode](#dry-run-mode) - [Log Sinks](#log-sinks) - [Data Access Logs](#data-access-logs) +- [Log Scopes](#log-scopes) - [Cloud KMS Encryption Keys](#cloud-kms-encryption-keys) - [Tags](#tags) - [Tag Bindings](#tag-bindings) @@ -720,6 +721,46 @@ module "project" { } # tftest modules=1 resources=3 inventory=logging-data-access.yaml e2e ``` +## Log Scopes + +```hcl +module "bucket" { + source = "./fabric/modules/logging-bucket" + parent_type = "project" + parent = "other-project" + id = "mybucket" + views = { + view1 = { + filter = "LOG_ID(\"stdout\")" + iam = { + "roles/logging.viewAccessor" = ["user:user@example.com"] + } + } + } +} + +module "project" { + source = "./fabric/modules/project" + billing_account = var.billing_account_id + prefix = var.prefix + parent = var.folder_id + name = "logscope" + services = [ + "logging.googleapis.com", + ] + log_scopes = { + scope = { + description = "My log scope" + resource_names = [ + "project1", + "project2", + module.bucket.view_ids["_AllLogs"], + ] + } + } +} +# tftest modules=2 resources=6 inventory=log-scopes.yaml +``` ## Cloud KMS Encryption Keys @@ -1368,7 +1409,7 @@ module "bucket" { |---|---|---| | [cmek.tf](./cmek.tf) | Service Agent IAM Bindings for CMEK | google_kms_crypto_key_iam_member | | [iam.tf](./iam.tf) | IAM bindings. | google_project_iam_binding · google_project_iam_custom_role · google_project_iam_member | -| [logging.tf](./logging.tf) | Log sinks and supporting resources. | google_bigquery_dataset_iam_member · google_logging_project_exclusion · google_logging_project_sink · google_project_iam_audit_config · google_project_iam_member · google_pubsub_topic_iam_member · google_storage_bucket_iam_member | +| [logging.tf](./logging.tf) | Log sinks and supporting resources. | google_bigquery_dataset_iam_member · google_logging_log_scope · google_logging_project_exclusion · google_logging_project_sink · google_project_iam_audit_config · google_project_iam_member · google_pubsub_topic_iam_member · google_storage_bucket_iam_member | | [main.tf](./main.tf) | Module-level locals and resources. | google_compute_project_metadata_item · google_essential_contacts_contact · google_monitoring_monitored_project · google_project · google_project_service · google_resource_manager_lien | | [organization-policies.tf](./organization-policies.tf) | Project-level organization policies. | google_org_policy_policy | | [outputs.tf](./outputs.tf) | Module outputs. | | @@ -1377,6 +1418,7 @@ module "bucket" { | [shared-vpc.tf](./shared-vpc.tf) | Shared VPC project-level configuration. | google_compute_shared_vpc_host_project · google_compute_shared_vpc_service_project · google_compute_subnetwork_iam_member · google_project_iam_member | | [tags.tf](./tags.tf) | None | google_tags_tag_binding · google_tags_tag_key · google_tags_tag_key_iam_binding · google_tags_tag_key_iam_member · google_tags_tag_value · google_tags_tag_value_iam_binding · google_tags_tag_value_iam_member | | [variables-iam.tf](./variables-iam.tf) | None | | +| [variables-observability.tf](./variables-observability.tf) | None | | | [variables-quotas.tf](./variables-quotas.tf) | None | | | [variables-tags.tf](./variables-tags.tf) | None | | | [variables.tf](./variables.tf) | Module variables. | | @@ -1387,7 +1429,7 @@ module "bucket" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [name](variables.tf#L165) | Project name and id suffix. | string | ✓ | | +| [name](variables.tf#L105) | Project name and id suffix. | string | ✓ | | | [auto_create_network](variables.tf#L17) | Whether to create the default network for the project. | bool | | false | | [billing_account](variables.tf#L23) | Billing account id. | string | | null | | [compute_metadata](variables.tf#L29) | Optional compute metadata key/values. Only usable if compute API has been enabled. | map(string) | | {} | @@ -1403,26 +1445,27 @@ module "bucket" { | [iam_by_principals](variables-iam.tf#L54) | Authoritative IAM binding in {PRINCIPAL => [ROLES]} format. Principals need to be statically defined to avoid cycle errors. Merged internally with the `iam` variable. | map(list(string)) | | {} | | [labels](variables.tf#L92) | Resource labels. | map(string) | | {} | | [lien_reason](variables.tf#L99) | If non-empty, creates a project lien with this description. | string | | null | -| [logging_data_access](variables.tf#L105) | Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services. | map(map(list(string))) | | {} | -| [logging_exclusions](variables.tf#L120) | Logging exclusions for this project in the form {NAME -> FILTER}. | map(string) | | {} | -| [logging_sinks](variables.tf#L127) | Logging sinks to create for this project. | map(object({…})) | | {} | -| [metric_scopes](variables.tf#L158) | List of projects that will act as metric scopes for this project. | list(string) | | [] | +| [log_scopes](variables-observability.tf#L39) | Log scopes under this project. | map(object({…})) | | {} | +| [logging_data_access](variables-observability.tf#L17) | Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services. | map(map(list(string))) | | {} | +| [logging_exclusions](variables-observability.tf#L32) | Logging exclusions for this project in the form {NAME -> FILTER}. | map(string) | | {} | +| [logging_sinks](variables-observability.tf#L49) | Logging sinks to create for this project. | map(object({…})) | | {} | +| [metric_scopes](variables-observability.tf#L80) | List of projects that will act as metric scopes for this project. | list(string) | | [] | | [network_tags](variables-tags.tf#L17) | Network tags by key name. If `id` is provided, key creation is skipped. The `iam` attribute behaves like the similarly named one at module level. | map(object({…})) | | {} | -| [org_policies](variables.tf#L170) | Organization policies applied to this project keyed by policy name. | map(object({…})) | | {} | -| [parent](variables.tf#L197) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | string | | null | -| [prefix](variables.tf#L207) | Optional prefix used to generate project id and name. | string | | null | -| [project_create](variables.tf#L217) | Create project. When set to false, uses a data source to reference existing project. | bool | | true | +| [org_policies](variables.tf#L110) | Organization policies applied to this project keyed by policy name. | map(object({…})) | | {} | +| [parent](variables.tf#L137) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | string | | null | +| [prefix](variables.tf#L147) | Optional prefix used to generate project id and name. | string | | null | +| [project_create](variables.tf#L157) | Create project. When set to false, uses a data source to reference existing project. | bool | | true | | [quotas](variables-quotas.tf#L17) | Service quota configuration. | map(object({…})) | | {} | -| [service_agents_config](variables.tf#L223) | Automatic service agent configuration options. | object({…}) | | {} | -| [service_config](variables.tf#L234) | Configure service API activation. | object({…}) | | {…} | -| [service_encryption_key_ids](variables.tf#L246) | Service Agents to be granted encryption/decryption permissions over Cloud KMS encryption keys. Format {SERVICE_AGENT => [KEY_ID]}. | map(list(string)) | | {} | -| [services](variables.tf#L253) | Service APIs to enable. | list(string) | | [] | -| [shared_vpc_host_config](variables.tf#L259) | Configures this project as a Shared VPC host project (mutually exclusive with shared_vpc_service_project). | object({…}) | | null | -| [shared_vpc_service_config](variables.tf#L268) | Configures this project as a Shared VPC service project (mutually exclusive with shared_vpc_host_config). | object({…}) | | {…} | -| [skip_delete](variables.tf#L296) | Deprecated. Use deletion_policy. | bool | | null | +| [service_agents_config](variables.tf#L163) | Automatic service agent configuration options. | object({…}) | | {} | +| [service_config](variables.tf#L174) | Configure service API activation. | object({…}) | | {…} | +| [service_encryption_key_ids](variables.tf#L186) | Service Agents to be granted encryption/decryption permissions over Cloud KMS encryption keys. Format {SERVICE_AGENT => [KEY_ID]}. | map(list(string)) | | {} | +| [services](variables.tf#L193) | Service APIs to enable. | list(string) | | [] | +| [shared_vpc_host_config](variables.tf#L199) | Configures this project as a Shared VPC host project (mutually exclusive with shared_vpc_service_project). | object({…}) | | null | +| [shared_vpc_service_config](variables.tf#L208) | Configures this project as a Shared VPC service project (mutually exclusive with shared_vpc_host_config). | object({…}) | | {…} | +| [skip_delete](variables.tf#L236) | Deprecated. Use deletion_policy. | bool | | null | | [tag_bindings](variables-tags.tf#L81) | Tag bindings for this project, in key => tag value id format. | map(string) | | null | | [tags](variables-tags.tf#L88) | Tags by key name. If `id` is provided, key or value creation is skipped. The `iam` attribute behaves like the similarly named one at module level. | map(object({…})) | | {} | -| [vpc_sc](variables.tf#L308) | VPC-SC configuration for the project, use when `ignore_changes` for resources is set in the VPC-SC module. | object({…}) | | null | +| [vpc_sc](variables.tf#L248) | VPC-SC configuration for the project, use when `ignore_changes` for resources is set in the VPC-SC module. | object({…}) | | null | ## Outputs diff --git a/modules/project/logging.tf b/modules/project/logging.tf index b4f808891a..fba9634fdd 100644 --- a/modules/project/logging.tf +++ b/modules/project/logging.tf @@ -1,5 +1,5 @@ /** - * Copyright 2022 Google LLC + * Copyright 2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,6 +32,17 @@ locals { name => sink if sink.iam && sink.type == type } } + + log_scopes = { + for k, v in var.log_scopes : + k => merge(v, { + # process all resource_names to allow bare project ids + resource_names = [ + for r in v.resource_names : + startswith(r, "projects/") ? r : "projects/${r}" + ] + }) + } } resource "google_project_iam_audit_config" "default" { @@ -132,3 +143,12 @@ resource "google_logging_project_exclusion" "logging-exclusion" { description = "${each.key} (Terraform-managed)." filter = each.value } + +resource "google_logging_log_scope" "log-scopes" { + for_each = local.log_scopes + parent = "projects/${local.project.project_id}" + location = "global" + name = each.key + resource_names = each.value.resource_names + description = each.value.description +} diff --git a/modules/project/variables-observability.tf b/modules/project/variables-observability.tf new file mode 100644 index 0000000000..4b45b6ef4f --- /dev/null +++ b/modules/project/variables-observability.tf @@ -0,0 +1,85 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "logging_data_access" { + description = "Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services." + type = map(map(list(string))) + nullable = false + default = {} + validation { + condition = alltrue(flatten([ + for k, v in var.logging_data_access : [ + for kk, vv in v : contains(["DATA_READ", "DATA_WRITE", "ADMIN_READ"], kk) + ] + ])) + error_message = "Log type keys for each service can only be one of 'DATA_READ', 'DATA_WRITE', 'ADMIN_READ'." + } +} + +variable "logging_exclusions" { + description = "Logging exclusions for this project in the form {NAME -> FILTER}." + type = map(string) + default = {} + nullable = false +} + +variable "log_scopes" { + description = "Log scopes under this project." + type = map(object({ + description = optional(string) + resource_names = list(string) + })) + nullable = false + default = {} +} + +variable "logging_sinks" { + description = "Logging sinks to create for this project." + type = map(object({ + bq_partitioned_table = optional(bool, false) + description = optional(string) + destination = string + disabled = optional(bool, false) + exclusions = optional(map(string), {}) + filter = optional(string) + iam = optional(bool, true) + type = string + unique_writer = optional(bool, true) + })) + default = {} + nullable = false + validation { + condition = alltrue([ + for k, v in var.logging_sinks : + contains(["bigquery", "logging", "project", "pubsub", "storage"], v.type) + ]) + error_message = "Type must be one of 'bigquery', 'logging', 'project', 'pubsub', 'storage'." + } + validation { + condition = alltrue([ + for k, v in var.logging_sinks : + v.bq_partitioned_table != true || v.type == "bigquery" + ]) + error_message = "Can only set bq_partitioned_table when type is `bigquery`." + } +} + +variable "metric_scopes" { + description = "List of projects that will act as metric scopes for this project." + type = list(string) + default = [] + nullable = false +} diff --git a/modules/project/variables.tf b/modules/project/variables.tf index 88f59763a0..245c24c55c 100644 --- a/modules/project/variables.tf +++ b/modules/project/variables.tf @@ -102,66 +102,6 @@ variable "lien_reason" { default = null } -variable "logging_data_access" { - description = "Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services." - type = map(map(list(string))) - nullable = false - default = {} - validation { - condition = alltrue(flatten([ - for k, v in var.logging_data_access : [ - for kk, vv in v : contains(["DATA_READ", "DATA_WRITE", "ADMIN_READ"], kk) - ] - ])) - error_message = "Log type keys for each service can only be one of 'DATA_READ', 'DATA_WRITE', 'ADMIN_READ'." - } -} - -variable "logging_exclusions" { - description = "Logging exclusions for this project in the form {NAME -> FILTER}." - type = map(string) - default = {} - nullable = false -} - -variable "logging_sinks" { - description = "Logging sinks to create for this project." - type = map(object({ - bq_partitioned_table = optional(bool, false) - description = optional(string) - destination = string - disabled = optional(bool, false) - exclusions = optional(map(string), {}) - filter = optional(string) - iam = optional(bool, true) - type = string - unique_writer = optional(bool, true) - })) - default = {} - nullable = false - validation { - condition = alltrue([ - for k, v in var.logging_sinks : - contains(["bigquery", "logging", "project", "pubsub", "storage"], v.type) - ]) - error_message = "Type must be one of 'bigquery', 'logging', 'project', 'pubsub', 'storage'." - } - validation { - condition = alltrue([ - for k, v in var.logging_sinks : - v.bq_partitioned_table != true || v.type == "bigquery" - ]) - error_message = "Can only set bq_partitioned_table when type is `bigquery`." - } -} - -variable "metric_scopes" { - description = "List of projects that will act as metric scopes for this project." - type = list(string) - default = [] - nullable = false -} - variable "name" { description = "Project name and id suffix." type = string