From 87d0d57d02b3d53b01a0e4dcb6d3e7699811af76 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Thu, 4 Jun 2020 12:39:18 +0200 Subject: [PATCH 1/5] moved from stream to dataset --- .../agent/program/testdata/constraints_config-filebeat.yml | 2 +- .../agent/program/testdata/enabled_output_true-filebeat.yml | 2 +- .../pkg/agent/program/testdata/enabled_true-filebeat.yml | 2 +- .../pkg/agent/program/testdata/single_config-filebeat.yml | 2 +- .../pkg/agent/program/testdata/single_config-metricbeat.yml | 6 +++--- x-pack/elastic-agent/pkg/agent/transpiler/rules.go | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config-filebeat.yml index a620d40de2e..9cc342f6458 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config-filebeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config-filebeat.yml @@ -8,7 +8,7 @@ filebeat: index: logs-generic-default processors: - add_fields: - target: "stream" + target: "dataset" fields: type: logs dataset: generic diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml index 71c95920829..b33292c0d86 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml @@ -8,7 +8,7 @@ filebeat: index: logs-generic-default processors: - add_fields: - target: "stream" + target: "dataset" fields: type: logs dataset: generic diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml index 176df2f7675..dfa177efc22 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml @@ -9,7 +9,7 @@ filebeat: index: logs-generic-default processors: - add_fields: - target: "stream" + target: "dataset" fields: type: logs dataset: generic diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml index 442457c5dc1..5153c661ee0 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml @@ -10,7 +10,7 @@ filebeat: var: value processors: - add_fields: - target: "stream" + target: "dataset" fields: type: logs dataset: generic diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml index af734fb28d8..f936bf99a35 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml @@ -6,7 +6,7 @@ metricbeat: hosts: ["http://127.0.0.1:8080"] processors: - add_fields: - target: "stream" + target: "dataset" fields: type: metrics dataset: docker.status @@ -17,7 +17,7 @@ metricbeat: hosts: ["http://127.0.0.1:8080"] processors: - add_fields: - target: "stream" + target: "dataset" fields: type: metrics dataset: generic @@ -31,7 +31,7 @@ metricbeat: fields: should_be: first - add_fields: - target: "stream" + target: "dataset" fields: type: metrics dataset: generic diff --git a/x-pack/elastic-agent/pkg/agent/transpiler/rules.go b/x-pack/elastic-agent/pkg/agent/transpiler/rules.go index 4554912e066..96fb65139e2 100644 --- a/x-pack/elastic-agent/pkg/agent/transpiler/rules.go +++ b/x-pack/elastic-agent/pkg/agent/transpiler/rules.go @@ -628,7 +628,7 @@ func (r *InjectStreamProcessorRule) Apply(ast *AST) error { } processorMap := &Dict{value: make([]Node, 0)} - processorMap.value = append(processorMap.value, &Key{name: "target", value: &StrVal{value: "stream"}}) + processorMap.value = append(processorMap.value, &Key{name: "target", value: &StrVal{value: "dataset"}}) processorMap.value = append(processorMap.value, &Key{name: "fields", value: &Dict{value: []Node{ &Key{name: "type", value: &StrVal{value: r.Type}}, &Key{name: "namespace", value: &StrVal{value: namespace}}, From e149f3bfc34b85100159fe18f363d284ff3086fb Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Thu, 4 Jun 2020 12:42:29 +0200 Subject: [PATCH 2/5] ch --- x-pack/elastic-agent/CHANGELOG.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/elastic-agent/CHANGELOG.asciidoc b/x-pack/elastic-agent/CHANGELOG.asciidoc index c6d6c7af732..e287cc3b35b 100644 --- a/x-pack/elastic-agent/CHANGELOG.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.asciidoc @@ -68,3 +68,4 @@ - Use shorter hash for application differentiator {pull}18770[18770] - When not port are specified and the https is used fallback to 443 {pull}18844[18844] - Agent verifies packages before using them {pull}18876[18876] +- Change stream.* to dataset.* fields {pull}18967[18967] From 89e02ce7b2378cd19e5e0daeca8a4d1398d7e368 Mon Sep 17 00:00:00 2001 From: beats-jenkins Date: Tue, 9 Jun 2020 11:31:47 +0200 Subject: [PATCH 3/5] update mappings --- .../agent/program/testdata/constraints_config-filebeat.yml | 2 +- .../agent/program/testdata/enabled_output_true-filebeat.yml | 2 +- .../pkg/agent/program/testdata/enabled_true-filebeat.yml | 2 +- .../pkg/agent/program/testdata/single_config-filebeat.yml | 2 +- .../pkg/agent/program/testdata/single_config-metricbeat.yml | 6 +++--- x-pack/elastic-agent/pkg/agent/transpiler/rules.go | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config-filebeat.yml index 9cc342f6458..1117593777e 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config-filebeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config-filebeat.yml @@ -11,7 +11,7 @@ filebeat: target: "dataset" fields: type: logs - dataset: generic + name: generic namespace: default output: elasticsearch: diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml index b33292c0d86..4f0d79dfb10 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml @@ -11,7 +11,7 @@ filebeat: target: "dataset" fields: type: logs - dataset: generic + name: generic namespace: default output: elasticsearch: diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml index dfa177efc22..ae969a8fb8f 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml @@ -12,7 +12,7 @@ filebeat: target: "dataset" fields: type: logs - dataset: generic + name: generic namespace: default output: elasticsearch: diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml index 5153c661ee0..e06d41e4f62 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml @@ -13,7 +13,7 @@ filebeat: target: "dataset" fields: type: logs - dataset: generic + name: generic namespace: default output: elasticsearch: diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml index f936bf99a35..27779c88ffe 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml @@ -9,7 +9,7 @@ metricbeat: target: "dataset" fields: type: metrics - dataset: docker.status + name: docker.status namespace: default - module: docker metricsets: [info] @@ -20,7 +20,7 @@ metricbeat: target: "dataset" fields: type: metrics - dataset: generic + name: generic namespace: default - module: apache metricsets: [info] @@ -34,7 +34,7 @@ metricbeat: target: "dataset" fields: type: metrics - dataset: generic + name: generic namespace: testing output: elasticsearch: diff --git a/x-pack/elastic-agent/pkg/agent/transpiler/rules.go b/x-pack/elastic-agent/pkg/agent/transpiler/rules.go index 96fb65139e2..9f80b8df45c 100644 --- a/x-pack/elastic-agent/pkg/agent/transpiler/rules.go +++ b/x-pack/elastic-agent/pkg/agent/transpiler/rules.go @@ -632,7 +632,7 @@ func (r *InjectStreamProcessorRule) Apply(ast *AST) error { processorMap.value = append(processorMap.value, &Key{name: "fields", value: &Dict{value: []Node{ &Key{name: "type", value: &StrVal{value: r.Type}}, &Key{name: "namespace", value: &StrVal{value: namespace}}, - &Key{name: "dataset", value: &StrVal{value: dataset}}, + &Key{name: "name", value: &StrVal{value: dataset}}, }}}) addFieldsMap := &Dict{value: []Node{&Key{"add_fields", processorMap}}} From 6aff6bf31606c9e6aac61b0f0671d9f4a9955ae6 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Tue, 9 Jun 2020 12:08:19 +0200 Subject: [PATCH 4/5] added backward comp and rename dataset to name --- .../testdata/constraints_config-filebeat.yml | 6 ++++++ .../testdata/enabled_output_true-filebeat.yml | 6 ++++++ .../testdata/enabled_true-filebeat.yml | 6 ++++++ .../testdata/single_config-filebeat.yml | 6 ++++++ .../testdata/single_config-metricbeat.yml | 19 +++++++++++++++++++ .../pkg/agent/transpiler/rules.go | 16 +++++++++++++++- 6 files changed, 58 insertions(+), 1 deletion(-) diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config-filebeat.yml index 9cc342f6458..d4a38b09c6b 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config-filebeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config-filebeat.yml @@ -9,6 +9,12 @@ filebeat: processors: - add_fields: target: "dataset" + fields: + type: logs + name: generic + namespace: default + - add_fields: + target: "stream" fields: type: logs dataset: generic diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml index b33292c0d86..bc79c306334 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml @@ -9,6 +9,12 @@ filebeat: processors: - add_fields: target: "dataset" + fields: + type: logs + name: generic + namespace: default + - add_fields: + target: "stream" fields: type: logs dataset: generic diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml index dfa177efc22..9ad59d63f6f 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml @@ -10,6 +10,12 @@ filebeat: processors: - add_fields: target: "dataset" + fields: + type: logs + name: generic + namespace: default + - add_fields: + target: "stream" fields: type: logs dataset: generic diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml index 5153c661ee0..59346260ca2 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml @@ -11,6 +11,12 @@ filebeat: processors: - add_fields: target: "dataset" + fields: + type: logs + name: generic + namespace: default + - add_fields: + target: "stream" fields: type: logs dataset: generic diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml index f936bf99a35..df6ac491bb8 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml @@ -7,6 +7,12 @@ metricbeat: processors: - add_fields: target: "dataset" + fields: + type: metrics + name: docker.status + namespace: default + - add_fields: + target: "stream" fields: type: metrics dataset: docker.status @@ -18,6 +24,12 @@ metricbeat: processors: - add_fields: target: "dataset" + fields: + type: metrics + name: generic + namespace: default + - add_fields: + target: "stream" fields: type: metrics dataset: generic @@ -32,10 +44,17 @@ metricbeat: should_be: first - add_fields: target: "dataset" + fields: + type: metrics + name: generic + namespace: testing + - add_fields: + target: "stream" fields: type: metrics dataset: generic namespace: testing + output: elasticsearch: hosts: [127.0.0.1:9200, 127.0.0.1:9300] diff --git a/x-pack/elastic-agent/pkg/agent/transpiler/rules.go b/x-pack/elastic-agent/pkg/agent/transpiler/rules.go index 96fb65139e2..2e3fb60b6fa 100644 --- a/x-pack/elastic-agent/pkg/agent/transpiler/rules.go +++ b/x-pack/elastic-agent/pkg/agent/transpiler/rules.go @@ -632,11 +632,25 @@ func (r *InjectStreamProcessorRule) Apply(ast *AST) error { processorMap.value = append(processorMap.value, &Key{name: "fields", value: &Dict{value: []Node{ &Key{name: "type", value: &StrVal{value: r.Type}}, &Key{name: "namespace", value: &StrVal{value: namespace}}, - &Key{name: "dataset", value: &StrVal{value: dataset}}, + &Key{name: "name", value: &StrVal{value: dataset}}, }}}) addFieldsMap := &Dict{value: []Node{&Key{"add_fields", processorMap}}} processorsList.value = mergeStrategy(r.OnConflict).InjectItem(processorsList.value, addFieldsMap) + + // add this for backwards compatibility remove later + streamProcessorMap := &Dict{value: make([]Node, 0)} + streamProcessorMap.value = append(streamProcessorMap.value, &Key{name: "target", value: &StrVal{value: "stream"}}) + streamProcessorMap.value = append(streamProcessorMap.value, &Key{name: "fields", value: &Dict{value: []Node{ + &Key{name: "type", value: &StrVal{value: r.Type}}, + &Key{name: "namespace", value: &StrVal{value: namespace}}, + &Key{name: "dataset", value: &StrVal{value: dataset}}, + }}}) + + streamAddFieldsMap := &Dict{value: []Node{&Key{"add_fields", streamProcessorMap}}} + + processorsList.value = mergeStrategy(r.OnConflict).InjectItem(processorsList.value, streamAddFieldsMap) + // end of backward compatibility section } } } From 0c214207ea346027416de5bfbae74a85e185cab6 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Tue, 9 Jun 2020 12:46:27 +0200 Subject: [PATCH 5/5] monitoring --- .../pkg/agent/operation/monitoring.go | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go index de852f7b13a..1cbf29152a5 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go +++ b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go @@ -180,7 +180,17 @@ func (o *Operator) getMonitoringFilebeatConfig(output interface{}) (map[string]i "paths": paths, "index": "logs-agent-default", "processors": []map[string]interface{}{ - map[string]interface{}{ + { + "add_fields": map[string]interface{}{ + "target": "dataset", + "fields": map[string]interface{}{ + "type": "logs", + "name": "agent", + "namespace": "default", + }, + }, + }, + { "add_fields": map[string]interface{}{ "target": "stream", "fields": map[string]interface{}{ @@ -220,7 +230,17 @@ func (o *Operator) getMonitoringMetricbeatConfig(output interface{}) (map[string "hosts": hosts, "index": "metrics-agent-default", "processors": []map[string]interface{}{ - map[string]interface{}{ + { + "add_fields": map[string]interface{}{ + "target": "dataset", + "fields": map[string]interface{}{ + "type": "metrics", + "name": "agent", + "namespace": "default", + }, + }, + }, + { "add_fields": map[string]interface{}{ "target": "stream", "fields": map[string]interface{}{