From a1f986062760260f139bc5bef1f325387b387454 Mon Sep 17 00:00:00 2001 From: Andrew Cholakian Date: Tue, 30 Mar 2021 13:47:58 -0500 Subject: [PATCH] [Heartbeat] Merge synthetic root fields into events (#24770) Fixes #24768 This allows synthetics to drive more field names without requiring heartbeat updates. Any fields in root_fields get merged into the event root. This also improves the testing in this area of the code, which was somewhat lean (and really was only tested in larger functional tests run elsewhere) --- CHANGELOG.next.asciidoc | 1 + .../monitors/browser/synthexec/synthtypes.go | 23 +++- .../browser/synthexec/synthtypes_test.go | 122 ++++++++++++++++++ .../sample-synthetics-config/heartbeat.yml | 6 - 4 files changed, 143 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 00c72c61985..7b816aaafe7 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -520,6 +520,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Bundle synthetics deps with heartbeat docker image. {pull}23274[23274] - Handle datastreams for fleet. {pull}24223[24223] - Add --sandbox option for browser monitor. {pull}24172[24172] +- Support additional 'root' fields from synthetics. {pull}24770[24770] *Heartbeat* diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go index 4d293d86d72..40cf2e06242 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go @@ -27,18 +27,35 @@ type SynthEvent struct { Error *SynthError `json:"error"` URL string `json:"url"` Status string `json:"status"` + RootFields common.MapStr `json:"root_fields"` index int } func (se SynthEvent) ToMap() (m common.MapStr) { // We don't add @timestamp to the map string since that's specially handled in beat.Event - m = common.MapStr{ + // Use the root fields as a base, and layer additional, stricter, fields on top + if se.RootFields != nil { + m = se.RootFields + // We handle url specially since it can be passed as a string, + // but expanded to match ECS + if urlStr, ok := m["url"].(string); ok { + if se.URL == "" { + se.URL = urlStr + } + } + } else { + m = common.MapStr{} + } + + m.DeepUpdate(common.MapStr{ "synthetics": common.MapStr{ "type": se.Type, "package_version": se.PackageVersion, - "payload": se.Payload, "index": se.index, }, + }) + if len(se.Payload) > 0 { + m.Put("synthetics.payload", se.Payload) } if se.Blob != "" { m.Put("synthetics.blob", se.Blob) @@ -61,7 +78,7 @@ func (se SynthEvent) ToMap() (m common.MapStr) { if e != nil { logp.Warn("Could not parse synthetics URL '%s': %s", se.URL, e.Error()) } else { - m["url"] = wrappers.URLFields(u) + m.Put("url", wrappers.URLFields(u)) } } diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes_test.go b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes_test.go index 775c5380137..daa2a710900 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes_test.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes_test.go @@ -5,9 +5,17 @@ package synthexec import ( + "encoding/json" + "net/url" "testing" "time" + "github.com/elastic/beats/v7/heartbeat/monitors/wrappers" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/go-lookslike" + "github.com/elastic/go-lookslike/testslike" + "github.com/stretchr/testify/require" ) @@ -15,3 +23,117 @@ func TestSynthEventTimestamp(t *testing.T) { se := SynthEvent{TimestampEpochMicros: 1000} // 1ms require.Equal(t, time.Unix(0, int64(time.Millisecond)), se.Timestamp()) } + +func TestToMap(t *testing.T) { + testUrl, _ := url.Parse("http://testurl") + + type testCase struct { + name string + source common.MapStr + expected common.MapStr + } + + testCases := []testCase{ + { + "root fields with URL", + common.MapStr{ + "type": "journey/start", + "package_version": "1.2.3", + "root_fields": map[string]interface{}{ + "synthetics": map[string]interface{}{ + "nested": "v1", + }, + "truly_at_root": "v2", + }, + "url": testUrl.String(), + }, + common.MapStr{ + "synthetics": common.MapStr{ + "type": "journey/start", + "package_version": "1.2.3", + "nested": "v1", + }, + "url": wrappers.URLFields(testUrl), + "truly_at_root": "v2", + }, + }, + { + "root fields, step metadata", + common.MapStr{ + "type": "step/start", + "package_version": "1.2.3", + "journey": common.MapStr{"name": "MyJourney", "id": "MyJourney"}, + "step": common.MapStr{"name": "MyStep", "status": "success", "index": 42}, + "root_fields": map[string]interface{}{ + "synthetics": map[string]interface{}{ + "nested": "v1", + }, + "truly_at_root": "v2", + }, + }, + common.MapStr{ + "synthetics": common.MapStr{ + "type": "step/start", + "package_version": "1.2.3", + "nested": "v1", + "journey": common.MapStr{"name": "MyJourney", "id": "MyJourney"}, + "step": common.MapStr{"name": "MyStep", "status": "success", "index": 42}, + }, + "truly_at_root": "v2", + }, + }, + { + "weird error, and blob, no URL", + common.MapStr{ + "type": "someType", + "package_version": "1.2.3", + "journey": common.MapStr{"name": "MyJourney", "id": "MyJourney"}, + "step": common.MapStr{"name": "MyStep", "index": 42, "status": "down"}, + "error": common.MapStr{ + "name": "MyErrorName", + "message": "MyErrorMessage", + "stack": "MyErrorStack", + }, + "blob": "ablob", + "blob_mime": "application/weird", + }, + common.MapStr{ + "synthetics": common.MapStr{ + "type": "someType", + "package_version": "1.2.3", + "journey": common.MapStr{"name": "MyJourney", "id": "MyJourney"}, + "step": common.MapStr{"name": "MyStep", "index": 42, "status": "down"}, + "error": common.MapStr{ + "name": "MyErrorName", + "message": "MyErrorMessage", + "stack": "MyErrorStack", + }, + "blob": "ablob", + "blob_mime": "application/weird", + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Actually marshal to JSON and back to test the struct tags for deserialization from JSON + jsonBytes, err := json.Marshal(tc.source) + require.NoError(t, err) + se := &SynthEvent{} + err = json.Unmarshal(jsonBytes, se) + require.NoError(t, err) + + m := se.ToMap() + + // Index will always be zero in thee tests, so helpfully include it + llvalidator := lookslike.Strict(lookslike.Compose( + lookslike.MustCompile(tc.expected), + lookslike.MustCompile(common.MapStr{"synthetics": common.MapStr{"index": 0}}), + )) + + // Test that even deep maps merge correctly + testslike.Test(t, llvalidator, m) + }) + } +} diff --git a/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml b/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml index 74fc2f4d885..79e18851c9b 100644 --- a/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml +++ b/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml @@ -9,8 +9,6 @@ heartbeat.monitors: enabled: true id: todos-suite name: Todos Suite - data_stream: - namespace: myns source: local: path: "/home/andrewvc/projects/synthetics/examples/todos/" @@ -21,14 +19,10 @@ heartbeat.monitors: urls: http://www.google.com schedule: "@every 15s" name: Simple HTTP - data_stream: - namespace: myns - type: browser enabled: false id: my-monitor name: My Monitor - data_stream: - namespace: myns source: inline: script: