From b995e2ecb31e1c0ae6b475a11eb492f57aed8b00 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Wed, 22 Jul 2020 09:22:02 -0400 Subject: [PATCH] [Elastic Agent] Add event.dataset to all events (#20076) * Inject add_fields event.dataset along side dataset. * Add event.dataset to all published events. * Add changelog. (cherry picked from commit c4010d94e8f8b91d431889fd12eb3cf546214a9c) --- x-pack/elastic-agent/CHANGELOG.asciidoc | 1 + .../pkg/agent/operation/monitoring.go | 24 +++++++++++++++++++ .../testdata/constraints_config-filebeat.yml | 4 ++++ .../testdata/enabled_output_true-filebeat.yml | 4 ++++ .../testdata/enabled_true-filebeat.yml | 4 ++++ .../testdata/single_config-filebeat.yml | 8 +++++++ .../testdata/single_config-metricbeat.yml | 12 ++++++++++ .../pkg/agent/transpiler/rules.go | 9 ++++++- 8 files changed, 65 insertions(+), 1 deletion(-) diff --git a/x-pack/elastic-agent/CHANGELOG.asciidoc b/x-pack/elastic-agent/CHANGELOG.asciidoc index 6e6e337191d..f62ab1759e4 100644 --- a/x-pack/elastic-agent/CHANGELOG.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.asciidoc @@ -91,3 +91,4 @@ - Add --insecure option to enroll command {pull}19900[19900] - Will retry to enroll if the server return a 429. {pull}19918[19811] - Add --staging option to enroll command {pull}20026[20026] +- Add `event.dataset` to all events {pull}20076[20076] diff --git a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go index bf03f4f34a5..62372cc3f54 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go +++ b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go @@ -198,6 +198,14 @@ func (o *Operator) getMonitoringFilebeatConfig(output interface{}) (map[string]i }, }, }, + { + "add_fields": map[string]interface{}{ + "target": "event", + "fields": map[string]interface{}{ + "dataset": "elastic.agent", + }, + }, + }, }, }, } @@ -224,6 +232,14 @@ func (o *Operator) getMonitoringFilebeatConfig(output interface{}) (map[string]i }, }, }, + { + "add_fields": map[string]interface{}{ + "target": "event", + "fields": map[string]interface{}{ + "dataset": fmt.Sprintf("elastic.agent.%s", name), + }, + }, + }, }, }) } @@ -266,6 +282,14 @@ func (o *Operator) getMonitoringMetricbeatConfig(output interface{}) (map[string }, }, }, + { + "add_fields": map[string]interface{}{ + "target": "event", + "fields": map[string]interface{}{ + "dataset": fmt.Sprintf("elastic.agent.%s", name), + }, + }, + }, }, }) } diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config-filebeat.yml index 31e7b27eafd..15f6b71a953 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config-filebeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config-filebeat.yml @@ -12,6 +12,10 @@ filebeat: type: logs name: generic namespace: default + - add_fields: + target: "event" + fields: + dataset: generic output: elasticsearch: hosts: diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml index 97b9e529bc6..c2e8c0d26ec 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml @@ -12,6 +12,10 @@ filebeat: type: logs name: generic namespace: default + - add_fields: + target: "event" + fields: + dataset: generic output: elasticsearch: enabled: true diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml index 080303e6d19..1da1c701d81 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml @@ -13,6 +13,10 @@ filebeat: type: logs name: generic namespace: default + - add_fields: + target: "event" + fields: + dataset: generic output: elasticsearch: hosts: diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml index 25b7af4e40a..0fb1a4356b5 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml @@ -14,6 +14,10 @@ filebeat: type: logs name: generic namespace: default + - add_fields: + target: "event" + fields: + dataset: generic - type: log paths: - /var/log/hello3.log @@ -28,6 +32,10 @@ filebeat: type: testtype name: generic namespace: default + - add_fields: + target: "event" + fields: + dataset: generic output: elasticsearch: hosts: diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml index 2e5e070dfb1..67a3815e4a7 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml @@ -11,6 +11,10 @@ metricbeat: type: metrics name: docker.status namespace: default + - add_fields: + target: "event" + fields: + dataset: docker.status - module: docker metricsets: [info] index: metrics-generic-default @@ -22,6 +26,10 @@ metricbeat: type: metrics name: generic namespace: default + - add_fields: + target: "event" + fields: + dataset: generic - module: apache metricsets: [info] index: metrics-generic-testing @@ -36,6 +44,10 @@ metricbeat: type: metrics name: generic namespace: testing + - add_fields: + target: "event" + fields: + dataset: generic output: elasticsearch: diff --git a/x-pack/elastic-agent/pkg/agent/transpiler/rules.go b/x-pack/elastic-agent/pkg/agent/transpiler/rules.go index 69dd59a459f..fe98386a150 100644 --- a/x-pack/elastic-agent/pkg/agent/transpiler/rules.go +++ b/x-pack/elastic-agent/pkg/agent/transpiler/rules.go @@ -639,9 +639,16 @@ func (r *InjectStreamProcessorRule) Apply(ast *AST) error { &Key{name: "namespace", value: &StrVal{value: namespace}}, &Key{name: "name", value: &StrVal{value: dataset}}, }}}) - addFieldsMap := &Dict{value: []Node{&Key{"add_fields", processorMap}}} processorsList.value = mergeStrategy(r.OnConflict).InjectItem(processorsList.value, addFieldsMap) + + processorMap = &Dict{value: make([]Node, 0)} + processorMap.value = append(processorMap.value, &Key{name: "target", value: &StrVal{value: "event"}}) + processorMap.value = append(processorMap.value, &Key{name: "fields", value: &Dict{value: []Node{ + &Key{name: "dataset", value: &StrVal{value: dataset}}, + }}}) + addFieldsMap = &Dict{value: []Node{&Key{"add_fields", processorMap}}} + processorsList.value = mergeStrategy(r.OnConflict).InjectItem(processorsList.value, addFieldsMap) } }