From a3e7010e37c47984dace6d0c8b9d692e3a706857 Mon Sep 17 00:00:00 2001 From: Pablo Moncada Date: Wed, 23 Sep 2020 11:19:25 +0200 Subject: [PATCH 1/5] Add exclusions option for resource_logging_project_sink Added as definedd in https://cloud.google.com/logging/docs/reference/v2/rest/v2/projects.sinks#LogExclusion Log entries that match any of the exclusion filters will not be exported. If a log entry is matched by both filter and one of exclusion_filters it will not be exported. --- google/resource_logging_project_sink_test.go | 52 +++++++++++++ google/resource_logging_sink.go | 82 ++++++++++++++++++++ 2 files changed, 134 insertions(+) diff --git a/google/resource_logging_project_sink_test.go b/google/resource_logging_project_sink_test.go index 32f7401ed22..14f9428f98b 100644 --- a/google/resource_logging_project_sink_test.go +++ b/google/resource_logging_project_sink_test.go @@ -117,6 +117,29 @@ func TestAccLoggingProjectSink_heredoc(t *testing.T) { }) } +func TestAccLoggingProjectSink_loggingbucket(t *testing.T) { + t.Parallel() + + sinkName := "tf-test-sink-" + randString(t, 10) + logBucketID := "tf-test-logbucket-" + randString(t, 10) + + vcrTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckLoggingProjectSinkDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccLoggingProjectSink_loggingbucket(sinkName, getTestProjectFromEnv(), logBucketID), + }, + { + ResourceName: "google_logging_project_sink.loggingbucket", + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + func testAccCheckLoggingProjectSinkDestroyProducer(t *testing.T) func(s *terraform.State) error { return func(s *terraform.State) error { config := googleProviderConfig(t) @@ -248,3 +271,32 @@ resource "google_bigquery_dataset" "logging_sink" { } `, sinkName, getTestProjectFromEnv(), getTestProjectFromEnv(), bqDatasetID) } + +func testAccLoggingProjectSink_loggingbucket(name, project, logBucketID string) string { + return fmt.Sprintf(` +resource "google_logging_project_sink" "loggingbucket" { + name = "%s" + project = "%s" + destination = "logging.googleapis.com/projects/%s/locations/global/buckets/${google_logging_project_bucket_config.logbucket.id}" + exclusions { + name = "ex1" + description = "test" + filter = "resource.type = k8s_container" + } + + exclusions { + name = "ex2" + description = "test-2" + filter = "resource.type = k8s_container" + } + + unique_writer_identity = false +} + +resource "google_logging_project_bucket_config" "logbucket" { + location = "global" + retention_days = 30 + bucket_id = "%s" +} +`, name, project, project, logBucketID) +} diff --git a/google/resource_logging_sink.go b/google/resource_logging_sink.go index 6c5741e152e..f8d58b35fb9 100644 --- a/google/resource_logging_sink.go +++ b/google/resource_logging_sink.go @@ -2,6 +2,7 @@ package google import ( "fmt" + "strconv" "strings" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" @@ -30,6 +31,38 @@ func resourceLoggingSinkSchema() map[string]*schema.Schema { Description: `The filter to apply when exporting logs. Only log entries that match the filter are exported.`, }, + "exclusions": { + Type: schema.TypeList, + Optional: true, + Computed: true, + Description: `Log entries that match any of the exclusion filters will not be exported. If a log entry is matched by both filter and one of exclusion_filters it will not be exported.`, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "name": { + Type: schema.TypeString, + Required: true, + Description: `A client-assigned identifier, such as "load-balancer-exclusion". Identifiers are limited to 100 characters and can include only letters, digits, underscores, hyphens, and periods. First character has to be alphanumeric.`, + }, + "description": { + Type: schema.TypeString, + Optional: true, + Description: `A description of this exclusion.`, + }, + "filter": { + Type: schema.TypeString, + Required: true, + Description: `An advanced logs filter that matches the log entries to be excluded. By using the sample function, you can exclude less than 100% of the matching log entries`, + }, + "disabled": { + Type: schema.TypeString, + Optional: true, + Default: "false", + Description: `If set to True, then this exclusion is disabled and it does not exclude any log entries`, + }, + }, + }, + }, + "writer_identity": { Type: schema.TypeString, Computed: true, @@ -66,6 +99,7 @@ func expandResourceLoggingSink(d *schema.ResourceData, resourceType, resourceId Name: d.Get("name").(string), Destination: d.Get("destination").(string), Filter: d.Get("filter").(string), + Exclusions: expandLoggingSinkExclusions(d.Get("exclusions")), BigqueryOptions: expandLoggingSinkBigqueryOptions(d.Get("bigquery_options")), } return id, &sink @@ -84,6 +118,9 @@ func flattenResourceLoggingSink(d *schema.ResourceData, sink *logging.LogSink) e if err := d.Set("writer_identity", sink.WriterIdentity); err != nil { return fmt.Errorf("Error setting writer_identity: %s", err) } + if err := d.Set("exclusions", flattenLoggingSinkExclusion(sink.Exclusions)); err != nil { + return fmt.Errorf("Error setting exclusions: %s", err) + } if err := d.Set("bigquery_options", flattenLoggingSinkBigqueryOptions(sink.BigqueryOptions)); err != nil { return fmt.Errorf("Error setting bigquery_options: %s", err) } @@ -107,6 +144,10 @@ func expandResourceLoggingSinkForUpdate(d *schema.ResourceData) (sink *logging.L if d.HasChange("filter") { updateFields = append(updateFields, "filter") } + if d.HasChange("exclusions") { + sink.Exclusions = expandLoggingSinkExclusions(d.Get("exclusions")) + updateFields = append(updateFields, "exclusions") + } if d.HasChange("bigquery_options") { sink.BigqueryOptions = expandLoggingSinkBigqueryOptions(d.Get("bigquery_options")) updateFields = append(updateFields, "bigqueryOptions") @@ -141,6 +182,47 @@ func flattenLoggingSinkBigqueryOptions(o *logging.BigQueryOptions) []map[string] return []map[string]interface{}{oMap} } +func expandLoggingSinkExclusions(v interface{}) []*logging.LogExclusion { + if v == nil { + return nil + } + exclusions := v.([]interface{}) + if len(exclusions) == 0 { + return nil + } + results := make([]*logging.LogExclusion, 0, len(exclusions)) + for _, e := range exclusions { + exclusion := e.(map[string]interface{}) + disabled, _ := strconv.ParseBool(exclusion["disabled"].(string)) + results = append(results, &logging.LogExclusion{ + Name: exclusion["name"].(string), + Description: exclusion["description"].(string), + Filter: exclusion["filter"].(string), + Disabled: disabled, + }) + } + return results +} + +func flattenLoggingSinkExclusion(exclusions []*logging.LogExclusion) []map[string]interface{} { + if exclusions == nil { + return nil + } + flattenedExclusions := make([]map[string]interface{}, 0, len(exclusions)) + for _, e := range exclusions { + flattenedExclusion := map[string]interface{}{ + "name": e.Name, + "description": e.Description, + "filter": e.Filter, + "disabled": strconv.FormatBool(e.Disabled), + } + flattenedExclusions = append(flattenedExclusions, flattenedExclusion) + + } + + return flattenedExclusions +} + func resourceLoggingSinkImportState(sinkType string) schema.StateFunc { return func(d *schema.ResourceData, meta interface{}) ([]*schema.ResourceData, error) { loggingSinkId, err := parseLoggingSinkId(d.Id()) From 780d0a23ad9c8457b7fdb633702831b6af20453e Mon Sep 17 00:00:00 2001 From: Pablo Moncada Date: Tue, 29 Sep 2020 14:53:17 +0200 Subject: [PATCH 2/5] Change 'disabled' type to TypeBool This will avoid unnecessary conversions and it is also defined as a boolean in the API client. --- google/resource_logging_sink.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/google/resource_logging_sink.go b/google/resource_logging_sink.go index f8d58b35fb9..a20d8f4fc8c 100644 --- a/google/resource_logging_sink.go +++ b/google/resource_logging_sink.go @@ -54,9 +54,9 @@ func resourceLoggingSinkSchema() map[string]*schema.Schema { Description: `An advanced logs filter that matches the log entries to be excluded. By using the sample function, you can exclude less than 100% of the matching log entries`, }, "disabled": { - Type: schema.TypeString, + Type: schema.TypeBool, Optional: true, - Default: "false", + Default: false, Description: `If set to True, then this exclusion is disabled and it does not exclude any log entries`, }, }, @@ -193,7 +193,7 @@ func expandLoggingSinkExclusions(v interface{}) []*logging.LogExclusion { results := make([]*logging.LogExclusion, 0, len(exclusions)) for _, e := range exclusions { exclusion := e.(map[string]interface{}) - disabled, _ := strconv.ParseBool(exclusion["disabled"].(string)) + disabled, _ := exclusion["disabled"].(bool) results = append(results, &logging.LogExclusion{ Name: exclusion["name"].(string), Description: exclusion["description"].(string), From 1d670aefef7014afa24bc970f1219899265ed403 Mon Sep 17 00:00:00 2001 From: Pablo Moncada Date: Fri, 2 Oct 2020 10:42:16 +0200 Subject: [PATCH 3/5] Remove bool parsing from 'disabled' and remove 'Compute' option --- google/resource_logging_sink.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/google/resource_logging_sink.go b/google/resource_logging_sink.go index a20d8f4fc8c..428c7bcdf6b 100644 --- a/google/resource_logging_sink.go +++ b/google/resource_logging_sink.go @@ -2,7 +2,6 @@ package google import ( "fmt" - "strconv" "strings" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" @@ -34,7 +33,6 @@ func resourceLoggingSinkSchema() map[string]*schema.Schema { "exclusions": { Type: schema.TypeList, Optional: true, - Computed: true, Description: `Log entries that match any of the exclusion filters will not be exported. If a log entry is matched by both filter and one of exclusion_filters it will not be exported.`, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ @@ -193,12 +191,11 @@ func expandLoggingSinkExclusions(v interface{}) []*logging.LogExclusion { results := make([]*logging.LogExclusion, 0, len(exclusions)) for _, e := range exclusions { exclusion := e.(map[string]interface{}) - disabled, _ := exclusion["disabled"].(bool) results = append(results, &logging.LogExclusion{ Name: exclusion["name"].(string), Description: exclusion["description"].(string), Filter: exclusion["filter"].(string), - Disabled: disabled, + Disabled: exclusion["disabled"].(bool), }) } return results @@ -214,7 +211,7 @@ func flattenLoggingSinkExclusion(exclusions []*logging.LogExclusion) []map[strin "name": e.Name, "description": e.Description, "filter": e.Filter, - "disabled": strconv.FormatBool(e.Disabled), + "disabled": e.Disabled, } flattenedExclusions = append(flattenedExclusions, flattenedExclusion) From 4e408bbd9657c203c37b3b9957eedb03a415e315 Mon Sep 17 00:00:00 2001 From: Pablo Moncada Date: Fri, 2 Oct 2020 10:45:30 +0200 Subject: [PATCH 4/5] Remove test resource google_logging_project_bucket_config This test resource does not createa a logging bucket, it just modifies its config, therefore it is not suitable for the test. For this test, the sink with exclusions will be tested directly with the _Default log bucket as we want to test the exclusions, which is enough. --- google/resource_logging_project_sink_test.go | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/google/resource_logging_project_sink_test.go b/google/resource_logging_project_sink_test.go index 14f9428f98b..c56f3792c0f 100644 --- a/google/resource_logging_project_sink_test.go +++ b/google/resource_logging_project_sink_test.go @@ -121,7 +121,6 @@ func TestAccLoggingProjectSink_loggingbucket(t *testing.T) { t.Parallel() sinkName := "tf-test-sink-" + randString(t, 10) - logBucketID := "tf-test-logbucket-" + randString(t, 10) vcrTest(t, resource.TestCase{ PreCheck: func() { testAccPreCheck(t) }, @@ -129,7 +128,7 @@ func TestAccLoggingProjectSink_loggingbucket(t *testing.T) { CheckDestroy: testAccCheckLoggingProjectSinkDestroyProducer(t), Steps: []resource.TestStep{ { - Config: testAccLoggingProjectSink_loggingbucket(sinkName, getTestProjectFromEnv(), logBucketID), + Config: testAccLoggingProjectSink_loggingbucket(sinkName, getTestProjectFromEnv()), }, { ResourceName: "google_logging_project_sink.loggingbucket", @@ -272,12 +271,12 @@ resource "google_bigquery_dataset" "logging_sink" { `, sinkName, getTestProjectFromEnv(), getTestProjectFromEnv(), bqDatasetID) } -func testAccLoggingProjectSink_loggingbucket(name, project, logBucketID string) string { +func testAccLoggingProjectSink_loggingbucket(name, project string) string { return fmt.Sprintf(` resource "google_logging_project_sink" "loggingbucket" { name = "%s" project = "%s" - destination = "logging.googleapis.com/projects/%s/locations/global/buckets/${google_logging_project_bucket_config.logbucket.id}" + destination = "logging.googleapis.com/projects/%s/locations/global/buckets/_Default" exclusions { name = "ex1" description = "test" @@ -290,13 +289,8 @@ resource "google_logging_project_sink" "loggingbucket" { filter = "resource.type = k8s_container" } - unique_writer_identity = false + unique_writer_identity = true } -resource "google_logging_project_bucket_config" "logbucket" { - location = "global" - retention_days = 30 - bucket_id = "%s" -} -`, name, project, project, logBucketID) +`, name, project, project) } From 08bae48b5ee287fcde5405e178502d1ac03de93d Mon Sep 17 00:00:00 2001 From: Pablo Moncada Date: Tue, 6 Oct 2020 09:23:57 +0200 Subject: [PATCH 5/5] Added documentation and examples for log_sink exclusions --- .../docs/r/logging_project_sink.html.markdown | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/website/docs/r/logging_project_sink.html.markdown b/website/docs/r/logging_project_sink.html.markdown index 2e40a8092b4..df0259bd740 100644 --- a/website/docs/r/logging_project_sink.html.markdown +++ b/website/docs/r/logging_project_sink.html.markdown @@ -86,6 +86,30 @@ resource "google_project_iam_binding" "log-writer" { } ``` +The following example uses `exclusions` to filter logs that will not be exported. In this example logs are exported to a [log bucket](https://cloud.google.com/logging/docs/buckets) and there are 2 exclusions configured + +```hcl +resource "google_logging_project_sink" "log-bucket" { + name = "my-logging-sink" + destination = "logging.googleapis.com/projects/my-project/locations/global/buckets/_Default" + + exclusions { + name = "nsexcllusion1" + description = "Exclude logs from namespace-1 in k8s" + filter = "resource.type = k8s_container resource.labels.namespace_name=\"namespace-1\" " + } + + exclusions { + name = "nsexcllusion2" + description = "Exclude logs from namespace-2 in k8s" + filter = "resource.type = k8s_container resource.labels.namespace_name=\"namespace-2\" " + } + + unique_writer_identity = true +``` + + + ## Argument Reference The following arguments are supported: @@ -115,6 +139,8 @@ The following arguments are supported: * `bigquery_options` - (Optional) Options that affect sinks exporting data to BigQuery. Structure documented below. +* `exclusions` - (Optional) Log entries that match any of the exclusion filters will not be exported. If a log entry is matched by both filter and one of exclusion_filters it will not be exported. Can be repeated multiple times for multiple exclusions. Structure is documented below. + The `bigquery_options` block supports: * `use_partitioned_tables` - (Required) Whether to use [BigQuery's partition tables](https://cloud.google.com/bigquery/docs/partitioned-tables). @@ -122,6 +148,14 @@ The `bigquery_options` block supports: tables the date suffix is no longer present and [special query syntax](https://cloud.google.com/bigquery/docs/querying-partitioned-tables) has to be used instead. In both cases, tables are sharded based on UTC timezone. +The `exclusions` block support: + +* `name` - (Required) A client-assigned identifier, such as `load-balancer-exclusion`. Identifiers are limited to 100 characters and can include only letters, digits, underscores, hyphens, and periods. First character has to be alphanumeric. +* `description` - (Optional) A description of this exclusion. +* `filter` - (Required) An advanced logs filter that matches the log entries to be excluded. By using the sample function, you can exclude less than 100% of the matching log entries. See [Advanced Log Filters](https://cloud.google.com/logging/docs/view/advanced_filters) for information on how to + write a filter. +* `disabled` - (Optional) If set to True, then this exclusion is disabled and it does not exclude any log entries. + ## Attributes Reference In addition to the arguments listed above, the following computed attributes are