[7.x] model: move Event, Session, and URL up to APMEvent (backport #5872

) (#5893) * model: move Event, Session, and URL up to APMEvent (#5872) Also, remove Transaction.Page and Error.Page. Page details have been duplicated in ECS fields since 7.9.0, and we don't reference the page fields in the UI. (cherry picked from commit 792c7ec) # Conflicts: # changelogs/head.asciidoc * Delete head.asciidoc Co-authored-by: Andrew Wilkins <[email protected]>
elastic · Aug 5, 2021 · 92c8987 · 92c8987
1 parent 88ac6bc
commit 92c8987
Show file tree

Hide file tree

Showing 61 changed files with 325 additions and 497 deletions.
diff --git a/apmpackage/apm/docs/README.md b/apmpackage/apm/docs/README.md
@@ -416,7 +416,9 @@ Traces are written to `traces-apm.*` indices.
   "transaction": {
     "id": "945254c567a5417e"
   },
-  "url.original": "http://localhost:8000"
+  "url": {
+    "original": "http://localhost:8000"
+  }
 }
 ```
 

diff --git a/beater/test_approved_es_documents/TestPublishIntegrationErrors.approved.json b/beater/test_approved_es_documents/TestPublishIntegrationErrors.approved.json
@@ -214,10 +214,6 @@
                             }
                         }
                     ]
-                },
-                "page": {
-                    "referer": "http://localhost:8000/test/e2e/",
-                    "url": "http://localhost:8000/test/e2e/general-usecase/"
                 }
             },
             "host": {

diff --git a/beater/test_approved_es_documents/TestPublishIntegrationEvents.approved.json b/beater/test_approved_es_documents/TestPublishIntegrationEvents.approved.json
@@ -447,7 +447,9 @@
             "transaction": {
                 "id": "1234567890987654"
             },
-            "url.original": "http://localhost:8000"
+            "url": {
+                "original": "http://localhost:8000"
+            }
         },
         {
             "@timestamp": "2019-10-21T11:30:44.929Z",

diff --git a/beater/test_approved_es_documents/TestPublishIntegrationSpans.approved.json b/beater/test_approved_es_documents/TestPublishIntegrationSpans.approved.json
@@ -745,7 +745,9 @@
             "trace": {
                 "id": "abcdef0123456789abcdef9876543210"
             },
-            "url.original": "http://localhost:8000",
+            "url": {
+                "original": "http://localhost:8000"
+            },
             "user": {
                 "domain": "ldap://abc",
                 "email": "[email protected]",

diff --git a/beater/test_approved_es_documents/TestPublishIntegrationTransactions.approved.json b/beater/test_approved_es_documents/TestPublishIntegrationTransactions.approved.json
@@ -323,10 +323,6 @@
                 },
                 "id": "4340a8e0df1906ecbfa9",
                 "name": "GET /api/types",
-                "page": {
-                    "referer": "http://localhost:8000/test/e2e/",
-                    "url": "http://localhost:8000/test/e2e/general-usecase/"
-                },
                 "result": "success",
                 "sampled": true,
                 "span_count": {

diff --git a/docs/data/elasticsearch/generated/errors.json b/docs/data/elasticsearch/generated/errors.json
@@ -263,10 +263,6 @@
                         }
                     }
                 ]
-            },
-            "page": {
-                "referer": "http://localhost:8000/test/e2e/",
-                "url": "http://localhost:8000/test/e2e/general-usecase/"
             }
         },
         "event": {

diff --git a/docs/data/elasticsearch/generated/spans.json b/docs/data/elasticsearch/generated/spans.json
@@ -115,7 +115,9 @@
         "transaction": {
             "id": "945254c567a5417e"
         },
-        "url.original": "http://localhost:8000"
+        "url": {
+            "original": "http://localhost:8000"
+        }
     },
     {
         "@timestamp": "2017-05-30T18:53:42.281Z",

diff --git a/docs/data/elasticsearch/generated/transactions.json b/docs/data/elasticsearch/generated/transactions.json
@@ -467,10 +467,6 @@
                 }
             },
             "name": "GET /api/types",
-            "page": {
-                "referer": "http://localhost:8000/test/e2e/",
-                "url": "http://localhost:8000/test/e2e/general-usecase/"
-            },
             "result": "success",
             "sampled": true,
             "span_count": {

diff --git a/model/apmevent.go b/model/apmevent.go
@@ -36,6 +36,7 @@ type APMEvent struct {
 	// in standalone mode.
 	DataStream DataStream
 
+	Event      Event
 	Agent      Agent
 	Container  Container
 	Kubernetes Kubernetes
@@ -47,6 +48,8 @@ type APMEvent struct {
 	Client     Client
 	Cloud      Cloud
 	Network    Network
+	Session    Session
+	URL        URL
 
 	// Timestamp holds the event timestamp.
 	Timestamp time.Time
@@ -67,7 +70,7 @@ func (e *APMEvent) appendBeatEvent(ctx context.Context, out []beat.Event) []beat
 	case e.Transaction != nil:
 		event.Fields = e.Transaction.fields()
 	case e.Span != nil:
-		event.Fields = e.Span.fields()
+		event.Fields = e.Span.fields(e)
 	case e.Metricset != nil:
 		event.Fields = e.Metricset.fields()
 	case e.Error != nil:
@@ -85,7 +88,7 @@ func (e *APMEvent) appendBeatEvent(ctx context.Context, out []beat.Event) []beat
 		event.Fields["timestamp"] = utility.TimeAsMicros(e.Timestamp)
 	}
 
-	// Set fields common to all events.
+	// Set top-level field sets.
 	fields := (*mapStr)(&event.Fields)
 	event.Timestamp = e.Timestamp
 	e.DataStream.setFields(fields)
@@ -108,5 +111,8 @@ func (e *APMEvent) appendBeatEvent(ctx context.Context, out []beat.Event) []beat
 	fields.maybeSetMapStr("cloud", e.Cloud.fields())
 	fields.maybeSetMapStr("network", e.Network.fields())
 	fields.maybeSetMapStr("labels", sanitizeLabels(e.Labels))
+	fields.maybeSetMapStr("event", e.Event.fields())
+	fields.maybeSetMapStr("url", e.URL.fields())
+	fields.maybeSetMapStr("session", e.Session.fields())
 	return append(out, event)
 }
diff --git a/model/apmevent_test.go b/model/apmevent_test.go
@@ -37,6 +37,7 @@ func TestAPMEventFields(t *testing.T) {
 	uid := "12321"
 	mail := "[email protected]"
 	agentName := "elastic-node"
+	outcome := "success"
 
 	for _, test := range []struct {
 		input  APMEvent
@@ -61,6 +62,9 @@ func TestAPMEventFields(t *testing.T) {
 				Client:      Client{Domain: "client.domain"},
 				Process:     Process{Pid: pid},
 				User:        User{ID: uid, Email: mail},
+				Event:       Event{Outcome: outcome},
+				Session:     Session{ID: "session_id"},
+				URL:         URL{Original: "url"},
 				Labels:      common.MapStr{"a": "b", "c": 123},
 				Transaction: &Transaction{},
 				Timestamp:   time.Date(2019, 1, 3, 15, 17, 4, 908.596*1e6, time.FixedZone("+0100", 3600)),
@@ -75,16 +79,18 @@ func TestAPMEventFields(t *testing.T) {
 					"name": "myservice",
 					"node": common.MapStr{"name": serviceNodeName},
 				},
-				"user":   common.MapStr{"id": "12321", "email": "[email protected]"},
-				"client": common.MapStr{"domain": "client.domain"},
-				"source": common.MapStr{"domain": "client.domain"},
+				"user":    common.MapStr{"id": "12321", "email": "[email protected]"},
+				"client":  common.MapStr{"domain": "client.domain"},
+				"source":  common.MapStr{"domain": "client.domain"},
+				"event":   common.MapStr{"outcome": outcome},
+				"session": common.MapStr{"id": "session_id"},
+				"url":     common.MapStr{"original": "url"},
 				"labels": common.MapStr{
 					"a": "b",
 					"c": 123,
 				},
 
 				// fields related to APMEvent.Transaction
-				"event": common.MapStr{"outcome": ""},
 				"processor": common.MapStr{
 					"name":  "transaction",
 					"event": "transaction",

diff --git a/model/context.go b/model/context.go
@@ -21,26 +21,6 @@ import (
 	"github.com/elastic/beats/v7/libbeat/common"
 )
 
-// Page consists of URL and referer
-type Page struct {
-	URL     *URL
-	Referer string
-}
-
-// Fields returns common.MapStr holding transformed data for attribute page.
-func (page *Page) Fields() common.MapStr {
-	if page == nil {
-		return nil
-	}
-	var fields mapStr
-	if page.URL != nil {
-		// Remove in 8.0
-		fields.set("url", page.URL.Original)
-	}
-	fields.maybeSetString("referer", page.Referer)
-	return common.MapStr(fields)
-}
-
 // customFields transforms in, returning a copy with sanitized keys,
 // suitable for storing as "custom" in transaction and error documents.
 func customFields(in common.MapStr) common.MapStr {

diff --git a/model/error.go b/model/error.go
@@ -44,9 +44,7 @@ type Error struct {
 
 	GroupingKey string
 	Culprit     string
-	Page        *Page
 	HTTP        *HTTP
-	URL         *URL
 	Custom      common.MapStr
 
 	Exception *Exception
@@ -92,7 +90,6 @@ func (e *Error) fields() common.MapStr {
 	if e.HTTP != nil {
 		fields.maybeSetMapStr("http", e.HTTP.transactionTopLevelFields())
 	}
-	fields.maybeSetMapStr("url", e.URL.Fields())
 	if e.Experimental != nil {
 		fields.set("experimental", e.Experimental)
 	}
@@ -113,7 +110,6 @@ func (e *Error) fields() common.MapStr {
 
 	var errorFields mapStr
 	errorFields.maybeSetString("id", e.ID)
-	errorFields.maybeSetMapStr("page", e.Page.Fields())
 	exceptionChain := flattenExceptionTree(e.Exception)
 	if exception := e.exceptionFields(exceptionChain); len(exception) > 0 {
 		errorFields.set("exception", exception)

diff --git a/model/error_test.go b/model/error_test.go
@@ -193,38 +193,3 @@ func TestEventFields(t *testing.T) {
 		})
 	}
 }
-
-func TestErrorTransformPage(t *testing.T) {
-	id := "123"
-	urlExample := "http://example.com/path"
-
-	tests := []struct {
-		Error  Error
-		Output common.MapStr
-		Msg    string
-	}{
-		{
-			Error: Error{
-				ID:  id,
-				URL: ParseURL("https://localhost:8200/", "", ""),
-				Page: &Page{
-					URL: ParseURL(urlExample, "", ""),
-				},
-			},
-			Output: common.MapStr{
-				"domain":   "localhost",
-				"full":     "https://localhost:8200/",
-				"original": "https://localhost:8200/",
-				"path":     "/",
-				"port":     8200,
-				"scheme":   "https",
-			},
-			Msg: "With Page URL and Request URL",
-		},
-	}
-
-	for idx, test := range tests {
-		fields := test.Error.fields()
-		assert.Equal(t, test.Output, fields["url"], fmt.Sprintf("Failed at idx %v; %s", idx, test.Msg))
-	}
-}
diff --git a/model/event.go b/model/event.go
@@ -0,0 +1,36 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package model
+
+import (
+	"github.com/elastic/beats/v7/libbeat/common"
+)
+
+// Event holds information about an event, in ECS terms.
+//
+// https://www.elastic.co/guide/en/ecs/current/ecs-event.html
+type Event struct {
+	// Outcome holds the event outcome: "success", "failure", or "unknown".
+	Outcome string
+}
+
+func (e *Event) fields() common.MapStr {
+	var fields mapStr
+	fields.maybeSetString("outcome", e.Outcome)
+	return common.MapStr(fields)
+}
diff --git a/model/metricset.go b/model/metricset.go
@@ -50,10 +50,6 @@ const (
 
 // Metricset describes a set of metrics and associated metadata.
 type Metricset struct {
-	// Event holds information about the event category with which the
-	// metrics are associated.
-	Event MetricsetEventCategorization
-
 	// Transaction holds information about the transaction group with
 	// which the metrics are associated.
 	Transaction MetricsetTransaction
@@ -115,16 +111,6 @@ type MetricsetSample struct {
 	Counts []int64
 }
 
-// MetricsetEventCategorization holds ECS Event Categorization fields
-// for inclusion in metrics. Typically these fields will have been
-// included in the metric aggregation logic.
-//
-// See https://www.elastic.co/guide/en/ecs/current/ecs-category-field-values-reference.html
-type MetricsetEventCategorization struct {
-	// Outcome holds the event outcome: "success", "failure", or "unknown".
-	Outcome string
-}
-
 // MetricsetTransaction provides enough information to connect a metricset to the related kind of transactions.
 type MetricsetTransaction struct {
 	// Name holds the transaction name: "GET /foo", etc.
@@ -160,7 +146,6 @@ func (me *Metricset) fields() common.MapStr {
 	var fields mapStr
 	fields.set("processor", metricsetProcessorEntry)
 
-	fields.maybeSetMapStr(metricsetEventKey, me.Event.fields())
 	fields.maybeSetMapStr(metricsetTransactionKey, me.Transaction.fields())
 	fields.maybeSetMapStr(metricsetSpanKey, me.Span.fields())
 	if me.TimeseriesInstanceID != "" {
@@ -184,12 +169,6 @@ func (me *Metricset) fields() common.MapStr {
 	return common.MapStr(fields)
 }
 
-func (e *MetricsetEventCategorization) fields() common.MapStr {
-	var fields mapStr
-	fields.maybeSetString("outcome", e.Outcome)
-	return common.MapStr(fields)
-}
-
 func (t *MetricsetTransaction) fields() common.MapStr {
 	var fields mapStr
 	fields.maybeSetString("type", t.Type)

diff --git a/model/metricset_test.go b/model/metricset_test.go
@@ -38,8 +38,6 @@ func TestMetricset(t *testing.T) {
 
 		spType    = "db"
 		spSubtype = "sql"
-
-		eventOutcome = "success"
 	)
 
 	tests := []struct {
@@ -96,7 +94,6 @@ func TestMetricset(t *testing.T) {
 		},
 		{
 			Metricset: &Metricset{
-				Event: MetricsetEventCategorization{Outcome: eventOutcome},
 				Transaction: MetricsetTransaction{
 					Type:   trType,
 					Name:   trName,
@@ -116,7 +113,6 @@ func TestMetricset(t *testing.T) {
 			},
 			Output: common.MapStr{
 				"processor":  common.MapStr{"event": "metric", "name": "metric"},
-				"event":      common.MapStr{"outcome": eventOutcome},
 				"timeseries": common.MapStr{"instance": "foo"},
 				"transaction": common.MapStr{
 					"type":   trType,