Add transaction.experience.longtask metrics fields (#4230)

Update the schema and fields for RUM to report
metrics about longtasks per transaction.

beater/test_approved_es_documents/TestPublishIntegrationTransactions.approved.json

@@ -470,6 +470,11 @@
                 "experience": {
                     "cls": 1,
                     "fid": 2,
+                    "longtask": {
+                        "count": 3,
+                        "max": 1,
+                        "sum": 2.5
+                    },
                     "tbt": 3.4
                 },
                 "id": "cdef4340a8e0df19",

changelogs/head.asciidoc

@@ -35,3 +35,4 @@ https://github.com/elastic/apm-server/compare/7.9\...master[View commits]
 * Set event.outcome for transactions and spans based on http.status_code {pull}4165[4165]
 * Add mapping for `system.process.cgroup.*` metrics {pull}4176[4176]
 * Use transaction.sample_rate to calculate transaction metrics {pull}4212[4212]
+* Add longtask metric fields to transaction.experience {pull}4230[4230]

docs/data/intake-api/generated/rum_v3_events.ndjson

@@ -1,3 +1,3 @@
 {"m": {"se": {"n": "apm-a-rum-test-e2e-general-usecase","ve": "0.0.1","en": "prod","a": {"n": "js-base","ve": "4.8.1"},"ru": {"n": "v8","ve": "8.0"},"la": {"n": "javascript","ve": "6"},"fw": {"n": "angular","ve": "2"}},"u": {"id": 123,"em": "[email protected]","un": "John Doe"},"l": {"testTagKey": "testTagValue"}}}
-{"x": {"id": "ec2e280be8345240","tid": "286ac3ad697892c406528f13c82e0ce1","pid": "1ef08ac234fca23b455d9e27c660f1ab","n": "general-usecase-initial-p-load","t": "p-load","d": 295,"me": [{"sa": {"xdc": {"v": 1},"xds": {"v": 295},"xbc": {"v": 1}}},{"y": {"t": "Request"},"sa": {"ysc": {"v": 1},"yss": {"v": 1}}},{"y": {"t": "Response"},"sa": {"ysc": {"v": 1},"yss": {"v": 1}}}],"y": [{"id": "bbd8bcc3be14d814","n": "Requesting and receiving the document","t": "hard-navigation","su": "browser-timing","s": 4,"d": 2},{"id": "fc546e87a90a774f","n": "Parsing the document, executing sy. scripts","t": "hard-navigation","su": "browser-timing","s": 14,"d": 106},{"id": "fb8f717930697299","n": "http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js","t": "rc","su": "script","s": 22.53499999642372,"d": 35.060000023804605,"c": {"h": {"url": "http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js?token=REDACTED","r": {"ts": 677175,"ebs": 676864,"dbs": 676864}},"dt": {"se": {"n": "http://localhost:8000","rc": "localhost:8000","t": "rc"},"ad": "localhost","po": 8000}}},{"id": "9b80535c4403c9fb","n": "OpenTracing y","t": "cu","s": 96.92999999970198,"d": 198.07000000029802},{"id": "5ecb8ee030749715","n": "GET /test/e2e/common/data.json","t": "external","su": "h","sy": true,"s": 98.94000005442649,"d": 6.72499998472631,"c": {"h": {"mt": "GET","url": "http://localhost:8000/test/e2e/common/data.json?test=hamid","sc": 200},"dt": {"se": {"n": "http://localhost:8000","rc": "localhost:8000","t": "external"},"ad": "localhost","po": 8000}}},{"id": "27f45fd274f976d4","n": "POST http://localhost:8003/data","t": "external","su": "h","sy": true,"s": 106.52000003028661,"d": 11.584999971091747,"c": {"h": {"mt": "POST","url": "http://localhost:8003/data","sc": 200},"dt": {"se": {"n": "http://localhost:8003","rc": "localhost:8003","t": "external"},"ad": "localhost","po": 8003}}},{"id": "a3c043330bc2015e","pi": 0,"n": "POST http://localhost:8003/fetch","t": "external","su": "h","ac": "action","sy": false,"s": 119.93500008247793,"d": 15.949999913573265,"c": {"h": {"mt": "POST","url": "http://localhost:8003/fetch","sc": 200},"dt": {"se": {"n": "http://localhost:8003","rc": "localhost:8003","t": "external"},"ad": "localhost","po": 8003}}},{"id": "bc7665dc25629379","st": [{"ap": "http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js?token=secret","f": "test/e2e/general-usecase/app.e2e-bundle.min.js?token=secret","fn": "generateError","li": 7662,"co": 9},{"ap": "http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js?token=secret","f": "test/e2e/general-usecase/app.e2e-bundle.min.js?token=secret","fn": "<anonymous>","li": 7666,"co": 3}],"n": "Fire \"DOMContentLoaded\" event","t": "hard-navigation","su": "browser-timing","s": 120,"d": 2,"o":"success"}],"c": {"p": {"rf": "http://localhost:8000/test/e2e/","url": "http://localhost:8000/test/e2e/general-usecase/"},"r": {"sc": 200,"ts": 983,"ebs": 690,"dbs": 690,"he": {"Content-Type": "application/json"}},"q": {"he": {"Accept": "application/json"},"hve": "1.1","mt": "GET"},"u": {"id": "uId","un": "un","em": "em"},"cu": {"testContext": "testContext"},"g": {"testTagKey": "testTagValue"}},"k": {"a": {"lp": 131.03000004775822,"fb": 5,"di": 120,"dc": 138,"ds": 100,"de": 110,"fp": 70.82500003930181},"nt": {"fs": 0,"ls": 0,"le": 0,"cs": 0,"ce": 0,"qs": 4,"rs": 5,"re": 6,"dl": 14,"di": 120,"ds": 120,"de": 122,"dc": 138,"es": 138,"ee": 138}},"yc": {"sd": 8,"dd": 1},"sm": true,"exp":{"cls":1,"fid":2.0,"tbt":3.4,"ignored":5,"also":"ignored"}}}
+{"x": {"id": "ec2e280be8345240","tid": "286ac3ad697892c406528f13c82e0ce1","pid": "1ef08ac234fca23b455d9e27c660f1ab","n": "general-usecase-initial-p-load","t": "p-load","d": 295,"me": [{"sa": {"xdc": {"v": 1},"xds": {"v": 295},"xbc": {"v": 1}}},{"y": {"t": "Request"},"sa": {"ysc": {"v": 1},"yss": {"v": 1}}},{"y": {"t": "Response"},"sa": {"ysc": {"v": 1},"yss": {"v": 1}}}],"y": [{"id": "bbd8bcc3be14d814","n": "Requesting and receiving the document","t": "hard-navigation","su": "browser-timing","s": 4,"d": 2},{"id": "fc546e87a90a774f","n": "Parsing the document, executing sy. scripts","t": "hard-navigation","su": "browser-timing","s": 14,"d": 106},{"id": "fb8f717930697299","n": "http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js","t": "rc","su": "script","s": 22.53499999642372,"d": 35.060000023804605,"c": {"h": {"url": "http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js?token=REDACTED","r": {"ts": 677175,"ebs": 676864,"dbs": 676864}},"dt": {"se": {"n": "http://localhost:8000","rc": "localhost:8000","t": "rc"},"ad": "localhost","po": 8000}}},{"id": "9b80535c4403c9fb","n": "OpenTracing y","t": "cu","s": 96.92999999970198,"d": 198.07000000029802},{"id": "5ecb8ee030749715","n": "GET /test/e2e/common/data.json","t": "external","su": "h","sy": true,"s": 98.94000005442649,"d": 6.72499998472631,"c": {"h": {"mt": "GET","url": "http://localhost:8000/test/e2e/common/data.json?test=hamid","sc": 200},"dt": {"se": {"n": "http://localhost:8000","rc": "localhost:8000","t": "external"},"ad": "localhost","po": 8000}}},{"id": "27f45fd274f976d4","n": "POST http://localhost:8003/data","t": "external","su": "h","sy": true,"s": 106.52000003028661,"d": 11.584999971091747,"c": {"h": {"mt": "POST","url": "http://localhost:8003/data","sc": 200},"dt": {"se": {"n": "http://localhost:8003","rc": "localhost:8003","t": "external"},"ad": "localhost","po": 8003}}},{"id": "a3c043330bc2015e","pi": 0,"n": "POST http://localhost:8003/fetch","t": "external","su": "h","ac": "action","sy": false,"s": 119.93500008247793,"d": 15.949999913573265,"c": {"h": {"mt": "POST","url": "http://localhost:8003/fetch","sc": 200},"dt": {"se": {"n": "http://localhost:8003","rc": "localhost:8003","t": "external"},"ad": "localhost","po": 8003}}},{"id": "bc7665dc25629379","st": [{"ap": "http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js?token=secret","f": "test/e2e/general-usecase/app.e2e-bundle.min.js?token=secret","fn": "generateError","li": 7662,"co": 9},{"ap": "http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js?token=secret","f": "test/e2e/general-usecase/app.e2e-bundle.min.js?token=secret","fn": "<anonymous>","li": 7666,"co": 3}],"n": "Fire \"DOMContentLoaded\" event","t": "hard-navigation","su": "browser-timing","s": 120,"d": 2,"o":"success"}],"c": {"p": {"rf": "http://localhost:8000/test/e2e/","url": "http://localhost:8000/test/e2e/general-usecase/"},"r": {"sc": 200,"ts": 983,"ebs": 690,"dbs": 690,"he": {"Content-Type": "application/json"}},"q": {"he": {"Accept": "application/json"},"hve": "1.1","mt": "GET"},"u": {"id": "uId","un": "un","em": "em"},"cu": {"testContext": "testContext"},"g": {"testTagKey": "testTagValue"}},"k": {"a": {"lp": 131.03000004775822,"fb": 5,"di": 120,"dc": 138,"ds": 100,"de": 110,"fp": 70.82500003930181},"nt": {"fs": 0,"ls": 0,"le": 0,"cs": 0,"ce": 0,"qs": 4,"rs": 5,"re": 6,"dl": 14,"di": 120,"ds": 120,"de": 122,"dc": 138,"es": 138,"ee": 138}},"yc": {"sd": 8,"dd": 1},"sm": true,"exp":{"cls":1,"fid":2.0,"tbt":3.4,"ignored":5,"also":"ignored","lt":{"count":3,"sum":2.5,"max":1}}}}
 {"me": {"y": {"t": "Processing","su": "subtype"},"sa": {"ysc": {"v": 1},"yss": {"v": 124}},"g": {"tag1": "value1"}}}

docs/fields.asciidoc

@@ -1767,6 +1767,39 @@ type: scaled_float
 
 --
 
+[float]
+=== longtask
+
+Longtask duration/count metrics
+
+
+*`transaction.experience.longtask.count`*::
++
+--
+The total number of of longtasks
+
+type: long
+
+--
+
+*`transaction.experience.longtask.sum`*::
++
+--
+The sum of longtask durations
+
+type: scaled_float
+
+--
+
+*`transaction.experience.longtask.max`*::
++
+--
+The max longtask duration
+
+type: scaled_float
+
+--
+
 
 *`transaction.span_count.dropped`*::
 +

docs/spec/transactions/rum_experience.json

@@ -0,0 +1,45 @@
+{
+    "$id": "docs/spec/rum_experience.json",
+    "title": "RUM Experience Metrics",
+    "description": "Metrics for measuring real user (browser) experience",
+    "type": ["object", "null"],
+    "properties": {
+        "cls": {
+            "type": ["number", "null"],
+            "description": "The Cumulative Layout Shift metric",
+            "minimum": 0
+        },
+        "tbt": {
+            "type": ["number", "null"],
+            "description": "The Total Blocking Time metric",
+            "minimum": 0
+        },
+        "fid": {
+            "type": ["number", "null"],
+            "description": "The First Input Delay metric",
+            "minimum": 0
+        },
+        "longtask": {
+            "type": ["object", "null"],
+            "description": "Longtask duration/count metrics",
+            "properties": {
+                "count": {
+                    "type": ["integer"],
+                    "description": "The total number of of longtasks",
+                    "minimum": 0
+                },
+                "sum": {
+                    "type": ["number"],
+                    "description": "The sum of longtask durations",
+                    "minimum": 0
+                },
+                "max": {
+                    "type": ["number"],
+                    "description": "The max longtask duration",
+                    "minimum": 0
+                }
+            },
+            "required": ["count", "sum", "max"]
+        }
+    }
+}

docs/spec/transactions/rum_v3_experience.json

@@ -0,0 +1,45 @@
+{
+    "$id": "docs/spec/rum_experience.json",
+    "title": "RUM Experience Metrics",
+    "description": "Metrics for measuring real user (browser) experience",
+    "type": ["object", "null"],
+    "properties": {
+        "cls": {
+            "type": ["number", "null"],
+            "description": "The Cumulative Layout Shift metric",
+            "minimum": 0
+        },
+        "tbt": {
+            "type": ["number", "null"],
+            "description": "The Total Blocking Time metric",
+            "minimum": 0
+        },
+        "fid": {
+            "type": ["number", "null"],
+            "description": "The First Input Delay metric",
+            "minimum": 0
+        },
+        "lt": {
+            "type": ["object", "null"],
+            "description": "Longtask duration/count metrics",
+            "properties": {
+                "count": {
+                    "type": ["integer"],
+                    "description": "The total number of of longtasks",
+                    "minimum": 0
+                },
+                "sum": {
+                    "type": ["number"],
+                    "description": "The sum of longtask durations",
+                    "minimum": 0
+                },
+                "max": {
+                    "type": ["number"],
+                    "description": "The max longtask duration",
+                    "minimum": 0
+                }
+            },
+            "required": ["count", "sum", "max"]
+        }
+    }
+}

docs/spec/transactions/rum_v3_transaction.json

@@ -103,7 +103,7 @@
                     "description": "Transactions that are 'sampled' will include all available information. Transactions that are not sampled will not have 'spans' or 'context'. Defaults to true."
                 },
                 "exp": {
-                    "$ref": "../rum_experience.json"
+                    "$ref": "rum_v3_experience.json"
                 }
             },
             "required": [

docs/spec/transactions/transaction.json

@@ -75,7 +75,7 @@
                     "description": "Transactions that are 'sampled' will include all available information. Transactions that are not sampled will not have 'spans' or 'context'. Defaults to true."
                 },
                 "experience": {
-                    "$ref": "../rum_experience.json"
+                    "$ref": "rum_experience.json"
                 }
             },
             "required": ["id", "trace_id", "span_count", "duration", "type"]

model/experience.go

@@ -34,6 +34,22 @@ type UserExperience struct {
 	// TotalBlockingTime holds the Total Blocking Time (TBT) metric value,
 	// or a negative value if TBT is unknown. See https://web.dev/tbt/
 	TotalBlockingTime float64
+
+	// Longtask holds longtask metrics. If Longtask.Count is negative,
+	// then Longtask is considered unset. See https://www.w3.org/TR/longtasks/
+	Longtask LongtaskMetrics
+}
+
+// LongtaskMetrics holds metrics related to RUM longtasks.
+type LongtaskMetrics struct {
+	// Count holds the number of longtasks, or a negative value if unknown.
+	Count int
+
+	// Sum holds the sum of longtask durations.
+	Sum float64
+
+	// Max holds the maximum longtask duration.
+	Max float64
 }
 
 func (u *UserExperience) Fields() common.MapStr {
@@ -50,5 +66,12 @@ func (u *UserExperience) Fields() common.MapStr {
 	if u.TotalBlockingTime >= 0 {
 		fields.set("tbt", u.TotalBlockingTime)
 	}
+	if u.Longtask.Count >= 0 {
+		fields.set("longtask", common.MapStr{
+			"count": u.Longtask.Count,
+			"sum":   u.Longtask.Sum,
+			"max":   u.Longtask.Max,
+		})
+	}
 	return common.MapStr(fields)
 }

model/experience_test.go

@@ -37,18 +37,29 @@ func TestUserExperienceFields(t *testing.T) {
 			CumulativeLayoutShift: -1,
 			FirstInputDelay:       -1,
 			TotalBlockingTime:     -1,
+			Longtask:              LongtaskMetrics{Count: -1},
 		},
 		Expected: nil,
 	}, {
 		Input: &UserExperience{
 			CumulativeLayoutShift: 1,
 			FirstInputDelay:       2.3,
 			TotalBlockingTime:     4.56,
+			Longtask: LongtaskMetrics{
+				Count: 3,
+				Sum:   2,
+				Max:   1,
+			},
 		},
 		Expected: common.MapStr{
 			"cls": 1.0,
 			"fid": 2.3,
 			"tbt": 4.56,
+			"longtask": common.MapStr{
+				"count": 3,
+				"sum":   2.0,
+				"max":   1.0,
+			},
 		},
 	}}
 	for _, test := range tests {

model/modeldecoder/experience.go

@@ -19,9 +19,10 @@ package modeldecoder
 
 import (
 	"github.com/elastic/apm-server/model"
+	"github.com/elastic/apm-server/model/modeldecoder/field"
 )
 
-func decodeUserExperience(input map[string]interface{}, out *model.UserExperience) {
+func decodeUserExperience(input map[string]interface{}, fieldName field.MapperFunc, out *model.UserExperience) {
 	if input == nil {
 		return
 	}
@@ -35,4 +36,11 @@ func decodeUserExperience(input map[string]interface{}, out *model.UserExperienc
 	if !decodeFloat64(input, "tbt", &out.TotalBlockingTime) {
 		out.TotalBlockingTime = -1
 	}
+
+	out.Longtask.Count = -1
+	if input := getObject(input, fieldName("longtask")); input != nil {
+		decodeInt(input, "count", &out.Longtask.Count)
+		decodeFloat64(input, "sum", &out.Longtask.Sum)
+		decodeFloat64(input, "max", &out.Longtask.Max)
+	}
 }

model/modeldecoder/field/rum_v3_mapping.go

@@ -70,6 +70,7 @@ var rumV3Mapping = map[string]string{
 	"loadEventStart":              "es",
 	"log":                         "log",
 	"logger_name":                 "ln",
+	"longtask":                    "lt",
 	"marks":                       "k",
 	"message":                     "mg",
 	"metadata":                    "m",
@@ -135,14 +136,24 @@ func init() {
 	}
 }
 
-func Mapper(shortFieldNames bool) func(string) string {
+// MapperFunc is the type of a function that maps from one field
+// name to another.
+type MapperFunc func(string) string
+
+// Mapper returns a MapperFunc that maps from long to short names
+// if shortFieldNames is true, and otherwise returns the identity
+// function.
+func Mapper(shortFieldNames bool) MapperFunc {
 	if shortFieldNames {
 		return rumV3Mapper
 	}
 	return identityMapper
 }
 
-func InverseMapper(shortFieldNames bool) func(string) string {
+// InverseMapper returns a MapperFunc that maps from short to long
+// names if shortFieldNames is true, and otherwise returns the identity
+// function.
+func InverseMapper(shortFieldNames bool) MapperFunc {
 	if shortFieldNames {
 		return rumV3InverseMapper
 	}

model/modeldecoder/generator/generator.go

@@ -468,34 +468,18 @@ if val.%s.Val != "" && !%s.MatchString(val.%s.Val){
 						return fmt.Errorf("unhandled tag rule '%s' for '%s'", rule, flattenedName)
 					}
 				}
-			case g.nullableInt:
+			case g.nullableInt, g.nullableFloat64:
 				for _, rule := range sortedRules {
 					val := parts[rule]
 					switch rule {
 					case ruleRequired:
 						ruleNullableRequired(&g.buf, f.name, flattenedName)
-					case ruleMax:
+					case ruleMin, ruleMax:
 						fmt.Fprintf(&g.buf, `
-if val.%s.Val > %s{
+if val.%s.Val %s %s {
 	return fmt.Errorf("validation rule '%s(%s)' violated for '%s'")
 }
-`[1:], f.name, val, rule, val, flattenedName)
-					default:
-						return fmt.Errorf("unhandled tag rule '%s' for '%s'", rule, flattenedName)
-					}
-				}
-			case g.nullableFloat64:
-				for _, rule := range sortedRules {
-					val := parts[rule]
-					switch rule {
-					case ruleRequired:
-						ruleNullableRequired(&g.buf, f.name, flattenedName)
-					case ruleMin:
-						fmt.Fprintf(&g.buf, `
-if val.%s.Val < %s{
-	return fmt.Errorf("validation rule '%s(%s)' violated for '%s'")
-}
-`[1:], f.name, val, rule, val, flattenedName)
+`[1:], f.name, ruleMinMaxOperator(rule), val, rule, val, flattenedName)
 					default:
 						return fmt.Errorf("unhandled tag rule '%s' for '%s'", rule, flattenedName)
 					}
@@ -700,3 +684,14 @@ func validationTag(structTag reflect.StructTag) (map[string]string, error) {
 	}
 	return m, nil
 }
+
+func ruleMinMaxOperator(rule string) string {
+	switch rule {
+	case ruleMin:
+		return "<"
+	case ruleMax:
+		return ">"
+	default:
+		panic("unexpected rule: " + rule)
+	}
+}