From c9ea2cf5b00cd60ba431383c8852e51c4996feae Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 22 Sep 2020 16:55:49 +0200 Subject: [PATCH] Add cloudfoundry tags to events (#21177) --- CHANGELOG.next.asciidoc | 1 + x-pack/libbeat/common/cloudfoundry/events.go | 9 ++++++++ .../common/cloudfoundry/events_test.go | 18 +++++++++++++++ .../_meta/fields.yml | 6 +++++ .../cloudfoundry/container/_meta/data.json | 22 +++++++++++-------- .../container/container_integration_test.go | 1 + .../cloudfoundry/counter/_meta/data.json | 18 ++++++++++----- .../counter/counter_integration_test.go | 1 + .../module/cloudfoundry/mtest/config.go | 6 +++++ .../module/cloudfoundry/value/_meta/data.json | 17 ++++++++------ .../value/value_integration_test.go | 1 + 11 files changed, 78 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 417e4ccc0e2..aee7bcc0f57 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -433,6 +433,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added experimental dataset `fortinet/fortimanager`. {pull}20820[20820] - Added experimental dataset `juniper/netscreen`. {pull}20820[20820] - Added experimental dataset `sophos/utm`. {pull}20820[20820] +- Add Cloud Foundry tags in related events. {pull}21177[21177] *Auditbeat* diff --git a/x-pack/libbeat/common/cloudfoundry/events.go b/x-pack/libbeat/common/cloudfoundry/events.go index 3006c472ccd..3a0f1756b6e 100644 --- a/x-pack/libbeat/common/cloudfoundry/events.go +++ b/x-pack/libbeat/common/cloudfoundry/events.go @@ -494,10 +494,19 @@ func baseMap(evt Event) common.MapStr { "cloudfoundry": common.MapStr{ "type": evt.String(), "envelope": envelopMap(evt), + "tags": dedotedTags(evt.Tags()), }, } } +func dedotedTags(tags map[string]string) common.MapStr { + result := common.MapStr{} + for name, value := range tags { + result[common.DeDot(name)] = value + } + return result +} + func baseMapWithApp(evt EventWithAppID) common.MapStr { base := baseMap(evt) appID := evt.AppGuid() diff --git a/x-pack/libbeat/common/cloudfoundry/events_test.go b/x-pack/libbeat/common/cloudfoundry/events_test.go index 91145589969..7dfd9bdcbd7 100644 --- a/x-pack/libbeat/common/cloudfoundry/events_test.go +++ b/x-pack/libbeat/common/cloudfoundry/events_test.go @@ -86,6 +86,9 @@ func TestEventTypeHttpAccess(t *testing.T) { "app": common.MapStr{ "id": "f47ac10b-58cc-4372-a567-0e02b2c3d479", }, + "tags": common.MapStr{ + "tag": "value", + }, }, "http": common.MapStr{ "response": common.MapStr{ @@ -160,6 +163,9 @@ func TestEventTypeLog(t *testing.T) { "app": common.MapStr{ "id": "f47ac10b-58cc-4372-a567-0e02b2c3d479", }, + "tags": common.MapStr{ + "tag": "value", + }, }, "message": "log message", "stream": "stdout", @@ -207,6 +213,9 @@ func TestEventCounter(t *testing.T) { "job": "job", "index": "index", }, + "tags": common.MapStr{ + "tag": "value", + }, }, }, evt.ToFields()) } @@ -252,6 +261,9 @@ func TestEventValueMetric(t *testing.T) { "job": "job", "index": "index", }, + "tags": common.MapStr{ + "tag": "value", + }, }, }, evt.ToFields()) } @@ -315,6 +327,9 @@ func TestEventContainerMetric(t *testing.T) { "app": common.MapStr{ "id": "f47ac10b-58cc-4372-a567-0e02b2c3d479", }, + "tags": common.MapStr{ + "tag": "value", + }, }, }, evt.ToFields()) } @@ -358,6 +373,9 @@ func TestEventError(t *testing.T) { "job": "job", "index": "index", }, + "tags": common.MapStr{ + "tag": "value", + }, }, "message": "message", "code": int32(100), diff --git a/x-pack/libbeat/processors/add_cloudfoundry_metadata/_meta/fields.yml b/x-pack/libbeat/processors/add_cloudfoundry_metadata/_meta/fields.yml index 0ac4a813d50..33d63e362f8 100644 --- a/x-pack/libbeat/processors/add_cloudfoundry_metadata/_meta/fields.yml +++ b/x-pack/libbeat/processors/add_cloudfoundry_metadata/_meta/fields.yml @@ -38,6 +38,12 @@ description: > Cloud Foundry organization name. + - name: tags.* + type: object + object_type: keyword + description: > + Cloud Foundry tags. + - name: envelope type: group fields: diff --git a/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json b/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json index 389b0bc54f9..16a4f1ef128 100644 --- a/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json +++ b/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json @@ -2,23 +2,27 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "cloudfoundry": { "app": { - "id": "c0874969-2811-4d80-837a-301fd485a7bf" + "id": "3ce55e14-de73-49af-836d-adc93f3fee39" }, "container": { - "cpu.pct": 0.45456369228159, - "disk.bytes": 21970944, - "disk.quota.bytes": 1073741824, - "instance_index": 1, - "memory.bytes": 14101162, - "memory.quota.bytes": 67108864 + "cpu.pct": 0.19431789913648675, + "disk.bytes": 16678912, + "disk.quota.bytes": 33554432, + "instance_index": 0, + "memory.bytes": 8529920, + "memory.quota.bytes": 33554432 }, "envelope": { "deployment": "cf-6b7aee31c8d07637ad78", - "index": "8895cf68-2e4f-4c50-9842-e73febd65c2d", - "ip": "192.168.16.31", + "index": "c2bcf5d6-7ff9-4876-890f-6f8fc6c58668", + "ip": "192.168.16.51", "job": "diego_cell", "origin": "rep" }, + "tags": { + "product": "Pivotal Application Service", + "source_id": "3ce55e14-de73-49af-836d-adc93f3fee39" + }, "type": "container" }, "service": { diff --git a/x-pack/metricbeat/module/cloudfoundry/container/container_integration_test.go b/x-pack/metricbeat/module/cloudfoundry/container/container_integration_test.go index e871a5823fc..605f7145f64 100644 --- a/x-pack/metricbeat/module/cloudfoundry/container/container_integration_test.go +++ b/x-pack/metricbeat/module/cloudfoundry/container/container_integration_test.go @@ -49,5 +49,6 @@ func TestData(t *testing.T) { require.NotEmpty(t, events) beatEvent := mbtest.StandardizeEvent(ms, events[0]) + mtest.CleanFields(beatEvent) mbtest.WriteEventToDataJSON(t, beatEvent, "") } diff --git a/x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json b/x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json index b9104346eac..5889e632b6b 100644 --- a/x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json +++ b/x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json @@ -3,15 +3,21 @@ "cloudfoundry": { "counter": { "delta": 0, - "name": "sinks.dropped", - "total": 89547093 + "name": "dropped", + "total": 0 }, "envelope": { "deployment": "cf-6b7aee31c8d07637ad78", - "index": "9adedbd0-3af3-47af-9707-99016050b123", - "ip": "192.168.16.54", - "job": "doppler", - "origin": "loggregator.doppler" + "index": "995eb273-f871-4fea-a834-dbc0a4a72efc", + "ip": "192.168.16.37", + "job": "syslog_adapter", + "origin": "loggregator.metron" + }, + "tags": { + "direction": "ingress", + "metric_version": "2.0", + "product": "Pivotal Application Service", + "source_id": "metron" }, "type": "counter" }, diff --git a/x-pack/metricbeat/module/cloudfoundry/counter/counter_integration_test.go b/x-pack/metricbeat/module/cloudfoundry/counter/counter_integration_test.go index 44cb4935e70..f7c13b1a603 100644 --- a/x-pack/metricbeat/module/cloudfoundry/counter/counter_integration_test.go +++ b/x-pack/metricbeat/module/cloudfoundry/counter/counter_integration_test.go @@ -49,5 +49,6 @@ func TestData(t *testing.T) { require.NotEmpty(t, events) beatEvent := mbtest.StandardizeEvent(ms, events[0]) + mtest.CleanFields(beatEvent) mbtest.WriteEventToDataJSON(t, beatEvent, "") } diff --git a/x-pack/metricbeat/module/cloudfoundry/mtest/config.go b/x-pack/metricbeat/module/cloudfoundry/mtest/config.go index c0756c57cca..8f98d528f58 100644 --- a/x-pack/metricbeat/module/cloudfoundry/mtest/config.go +++ b/x-pack/metricbeat/module/cloudfoundry/mtest/config.go @@ -7,6 +7,7 @@ package mtest import ( "testing" + "github.com/elastic/beats/v7/libbeat/beat" cftest "github.com/elastic/beats/v7/x-pack/libbeat/common/cloudfoundry/test" ) @@ -19,3 +20,8 @@ func GetConfig(t *testing.T, metricset string) map[string]interface{} { return config } + +// CleanFields removes fields that can contain data of real deployments +func CleanFields(e beat.Event) { + e.Fields.Delete("cloudfoundry.tags.system_domain") +} diff --git a/x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json b/x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json index feeb2ce72ba..1d17070a200 100644 --- a/x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json +++ b/x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json @@ -3,16 +3,19 @@ "cloudfoundry": { "envelope": { "deployment": "cf-6b7aee31c8d07637ad78", - "index": "439bb519-3e68-4d08-a973-8f013f364d61", - "ip": "192.168.16.18", - "job": "cloud_controller", - "origin": "routing_api" + "index": "dffd1799-d03a-405a-9309-3fcce23f536f", + "ip": "192.168.16.15", + "job": "diego_database", + "origin": "silk-controller" + }, + "tags": { + "source_id": "silk-controller" }, "type": "value", "value": { - "name": "memoryStats.lastGCPauseTimeNS", - "unit": "count", - "value": 77499 + "name": "LeasesIndexRequestTime", + "unit": "ms", + "value": 0.681265 } }, "service": { diff --git a/x-pack/metricbeat/module/cloudfoundry/value/value_integration_test.go b/x-pack/metricbeat/module/cloudfoundry/value/value_integration_test.go index 610a0a8e029..af815a3d4b7 100644 --- a/x-pack/metricbeat/module/cloudfoundry/value/value_integration_test.go +++ b/x-pack/metricbeat/module/cloudfoundry/value/value_integration_test.go @@ -49,5 +49,6 @@ func TestData(t *testing.T) { require.NotEmpty(t, events) beatEvent := mbtest.StandardizeEvent(ms, events[0]) + mtest.CleanFields(beatEvent) mbtest.WriteEventToDataJSON(t, beatEvent, "") }