add ephemeral_id attribute to agent

_meta/fields.common.yml

@@ -94,7 +94,7 @@
             type: object
             enabled: false
             description: >
-              The canonicalized headers of the monitored HTTP request.
+              The canonical headers of the monitored HTTP request.
 
        - name: response
          type: group
@@ -116,7 +116,7 @@
            type: object
            enabled: false
            description: >
-             The canonicalized headers of the monitored HTTP response.
+             The canonical headers of the monitored HTTP response.
 
     - name: labels
       type: object
@@ -248,6 +248,12 @@
             Version of the agent used.
           overwrite: true
 
+        - name: ephemeral_id
+          type: keyword
+          description: >
+            The Ephemeral ID identifies a running process.
+          overwrite: true
+
     - name: container
       type: group
       dynamic: false

beater/test_approved_es_documents/TestPublishIntegrationErrors.approved.json

@@ -8,6 +8,7 @@
             },
             "@timestamp": "2017-05-09T15:04:05.999Z",
             "agent": {
+                "ephemeral_id": "abcdef123",
                 "name": "elastic-node",
                 "version": "3.14.0"
             },
@@ -314,6 +315,7 @@
             },
             "@timestamp": "2018-08-09T14:59:05.999Z",
             "agent": {
+                "ephemeral_id": "abcdef123",
                 "name": "elastic-node",
                 "version": "3.14.0"
             },
@@ -405,6 +407,7 @@
             },
             "@timestamp": "2019-01-09T21:40:53.000Z",
             "agent": {
+                "ephemeral_id": "justanid",
                 "name": "elastic-ruby",
                 "version": "2.1.3"
             },
@@ -502,6 +505,7 @@
             },
             "@timestamp": "2018-08-09T15:04:05.999Z",
             "agent": {
+                "ephemeral_id": "abcdef123",
                 "name": "elastic-node",
                 "version": "3.14.0"
             },

beater/test_approved_es_documents/TestPublishIntegrationSpans.approved.json

@@ -226,6 +226,7 @@
             },
             "@timestamp": "2019-01-09T21:40:53.000Z",
             "agent": {
+                "ephemeral_id": "justanid",
                 "name": "elastic-ruby",
                 "version": "2.2"
             },

beater/test_approved_es_documents/TestPublishIntegrationTransactions.approved.json

@@ -299,6 +299,7 @@
             },
             "@timestamp": "2018-07-30T18:53:42.281Z",
             "agent": {
+                "ephemeral_id": "justanid",
                 "name": "elastic-ruby",
                 "version": "2.2"
             },

changelogs/head.asciidoc

@@ -12,6 +12,7 @@
 - Add Idle timeout to server config {pull}2122[2122].
 - Upgrade Go to 1.12.4 {pull}2132[2132].
 - Add geoip processing to the default ingest pipeline {pull}2177[2177].
+- Add ephemeral_id attribute in the metadata {pull}2179[2179].
 
 [float]
 ==== Removed

docs/data/elasticsearch/generated/errors.json

@@ -3,6 +3,7 @@
         {
             "@timestamp": "2017-05-09T15:04:05.999999Z",
             "agent": {
+                "ephemeral_id": "abcdef123",
                 "name": "elastic-node",
                 "version": "3.14.0"
             },
@@ -293,6 +294,7 @@
         {
             "@timestamp": "2018-08-09T14:59:05.999Z",
             "agent": {
+                "ephemeral_id": "abcdef123",
                 "name": "elastic-node",
                 "version": "3.14.0"
             },
@@ -368,6 +370,7 @@
         {
             "@timestamp": "2018-08-01T10:00:00Z",
             "agent": {
+                "ephemeral_id": "justanid",
                 "name": "elastic-ruby",
                 "version": "2.1.3"
             },
@@ -449,6 +452,7 @@
         {
             "@timestamp": "2018-08-09T15:04:05.999Z",
             "agent": {
+                "ephemeral_id": "abcdef123",
                 "name": "elastic-node",
                 "version": "3.14.0"
             },

docs/data/elasticsearch/generated/spans.json

@@ -157,6 +157,7 @@
         {
             "@timestamp": "2018-08-01T10:00:00.00283092Z",
             "agent": {
+                "ephemeral_id": "justanid",
                 "name": "elastic-ruby",
                 "version": "2.2"
             },

docs/data/elasticsearch/generated/transactions.json

@@ -262,6 +262,7 @@
         {
             "@timestamp": "2018-07-30T18:53:42.281Z",
             "agent": {
+                "ephemeral_id": "justanid",
                 "name": "elastic-ruby",
                 "version": "2.2"
             },

docs/fields.asciidoc

@@ -167,7 +167,7 @@ The http method of the request leading to this event.
 --
 type: object
 
-The canonicalized headers of the monitored HTTP request.
+The canonical headers of the monitored HTTP request.
 
 
 Object is not enabled.
@@ -200,7 +200,7 @@ Used by the Node agent to indicate when in the response life cycle an error has
 --
 type: object
 
-The canonicalized headers of the monitored HTTP response.
+The canonical headers of the monitored HTTP response.
 
 
 Object is not enabled.
@@ -389,6 +389,16 @@ type: keyword
 Version of the agent used.
 
 
+--
+
+*`agent.ephemeral_id`*::
++
+--
+type: keyword
+
+The Ephemeral ID identifies a running process.
+
+
 --
 
 [float]

docs/spec/service.json

@@ -16,6 +16,11 @@
                     "description": "Version of the Elastic APM agent, e.g.\"1.0.0\"",
                     "type": ["string", "null"],
                     "maxLength": 1024
+                },
+                "ephemeral_id": {
+                    "description": "Free format ID used for metrics correlation by some agents",
+                    "type": ["string", "null"],
+                    "maxLength": 1024
                 }
             }
         },

docs/spec/spans/span.json

@@ -113,6 +113,11 @@
                                                 "null"
                                             ],
                                             "maxLength": 1024
+                                        },
+                                        "ephemeral_id": {
+                                            "description": "Free format ID used for metrics correlation by some agents",
+                                            "type": ["string", "null"],
+                                            "maxLength": 1024
                                         }
                                     }
                                 },

model/context_test.go

@@ -148,8 +148,9 @@ func TestDecodeContext(t *testing.T) {
 							"version": "1.2.3",
 						},
 						"agent": common.MapStr{
-							"name":    "elastic-node",
-							"version": "1.0.0",
+							"name":         "elastic-node",
+							"version":      "1.0.0",
+							"ephemeral_id": "abcdef123",
 						}},
 					"page": map[string]interface{}{"url": "https://example.com", "referer": "http://refer.example.com"},
 				}},

model/error/generated/schema/error.go

@@ -282,6 +282,11 @@ const ModelSchema = `{
                     "description": "Version of the Elastic APM agent, e.g.\"1.0.0\"",
                     "type": ["string", "null"],
                     "maxLength": 1024
+                },
+                "ephemeral_id": {
+                    "description": "Free format ID used for metrics correlation by some agents",
+                    "type": ["string", "null"],
+                    "maxLength": 1024
                 }
             }
         },

model/metadata/generated/schema/metadata.go

@@ -41,6 +41,11 @@ const ModelSchema = `{
                     "description": "Version of the Elastic APM agent, e.g.\"1.0.0\"",
                     "type": ["string", "null"],
                     "maxLength": 1024
+                },
+                "ephemeral_id": {
+                    "description": "Free format ID used for metrics correlation by some agents",
+                    "type": ["string", "null"],
+                    "maxLength": 1024
                 }
             }
         },

model/metadata/service.go

@@ -47,8 +47,9 @@ type Framework struct {
 	Version *string
 }
 type Agent struct {
-	Name    *string
-	Version *string
+	Name        *string
+	Version     *string
+	EphemeralId *string
 }
 
 func DecodeService(input interface{}, err error) (*Service, error) {
@@ -65,8 +66,9 @@ func DecodeService(input interface{}, err error) (*Service, error) {
 		Version:     decoder.StringPtr(raw, "version"),
 		Environment: decoder.StringPtr(raw, "environment"),
 		Agent: Agent{
-			Name:    decoder.StringPtr(raw, "name", "agent"),
-			Version: decoder.StringPtr(raw, "version", "agent"),
+			Name:        decoder.StringPtr(raw, "name", "agent"),
+			Version:     decoder.StringPtr(raw, "version", "agent"),
+			EphemeralId: decoder.StringPtr(raw, "ephemeral_id", "agent"),
 		},
 		Framework: Framework{
 			Name:    decoder.StringPtr(raw, "name", "framework"),
@@ -133,5 +135,6 @@ func (a *Agent) fields() common.MapStr {
 	agent := common.MapStr{}
 	utility.Set(agent, "name", a.Name)
 	utility.Set(agent, "version", a.Version)
+	utility.Set(agent, "ephemeral_id", a.EphemeralId)
 	return agent
 }

model/span/generated/schema/span.go

@@ -151,6 +151,11 @@ const ModelSchema = `{
                                                 "null"
                                             ],
                                             "maxLength": 1024
+                                        },
+                                        "ephemeral_id": {
+                                            "description": "Free format ID used for metrics correlation by some agents",
+                                            "type": ["string", "null"],
+                                            "maxLength": 1024
                                         }
                                     }
                                 },

model/test_approved_model/context_full_event with experimental=true.approved.json

@@ -56,6 +56,7 @@
     },
     "Service": {
         "Agent": {
+            "EphemeralId": "abcdef123",
             "Name": "elastic-node",
             "Version": "1.0.0"
         },

model/transaction/generated/schema/transaction.go

@@ -278,6 +278,11 @@ const ModelSchema = `{
                     "description": "Version of the Elastic APM agent, e.g.\"1.0.0\"",
                     "type": ["string", "null"],
                     "maxLength": 1024
+                },
+                "ephemeral_id": {
+                    "description": "Free format ID used for metrics correlation by some agents",
+                    "type": ["string", "null"],
+                    "maxLength": 1024
                 }
             }
         },

processor/stream/test_approved_es_documents/testIntakeIntegrationErrors.approved.json

@@ -3,6 +3,7 @@
         {
             "@timestamp": "2017-05-09T15:04:05.999999Z",
             "agent": {
+                "ephemeral_id": "abcdef123",
                 "name": "elastic-node",
                 "version": "3.14.0"
             },
@@ -293,6 +294,7 @@
         {
             "@timestamp": "2018-08-09T14:59:05.999Z",
             "agent": {
+                "ephemeral_id": "abcdef123",
                 "name": "elastic-node",
                 "version": "3.14.0"
             },
@@ -368,6 +370,7 @@
         {
             "@timestamp": "2018-08-01T10:00:00Z",
             "agent": {
+                "ephemeral_id": "justanid",
                 "name": "elastic-ruby",
                 "version": "2.1.3"
             },
@@ -449,6 +452,7 @@
         {
             "@timestamp": "2018-08-09T15:04:05.999Z",
             "agent": {
+                "ephemeral_id": "abcdef123",
                 "name": "elastic-node",
                 "version": "3.14.0"
             },

processor/stream/test_approved_es_documents/testIntakeIntegrationSpans.approved.json

@@ -157,6 +157,7 @@
         {
             "@timestamp": "2018-08-01T10:00:00.00283092Z",
             "agent": {
+                "ephemeral_id": "justanid",
                 "name": "elastic-ruby",
                 "version": "2.2"
             },

processor/stream/test_approved_es_documents/testIntakeIntegrationTransactions.approved.json

@@ -262,6 +262,7 @@
         {
             "@timestamp": "2018-07-30T18:53:42.281Z",
             "agent": {
+                "ephemeral_id": "justanid",
                 "name": "elastic-ruby",
                 "version": "2.2"
             },

testdata/intake-v2/errors.ndjson

@@ -1,5 +1,5 @@
-{"metadata": {"process": {"ppid": 6789, "pid": 1234, "argv": ["node", "server.js"], "title": "node"},  "user": { "id": 123, "username": "bar", "email": "[email protected]"}, "system": {"platform": "darwin", "hostname": "prod1.example.com", "architecture": "x64", "container": {"id": "container-id"}, "kubernetes": {"namespace": "namespace1", "pod": {"uid": "pod-uid", "name": "pod-name"}, "node": {"name": "node-name"}}}, "service": {"name": "1234_service-12a3", "language": {"version": "8", "name": "ecmascript"}, "agent": {"version": "3.14.0", "name": "elastic-node"}, "environment": "staging", "framework": {"version": "1.2.3", "name": "Express"}, "version": "5.1.3", "runtime": {"version": "8.0.0", "name": "node"}}}}
+{"metadata": {"process": {"ppid": 6789, "pid": 1234, "argv": ["node", "server.js"], "title": "node"},  "user": { "id": 123, "username": "bar", "email": "[email protected]"}, "system": {"platform": "darwin", "hostname": "prod1.example.com", "architecture": "x64", "container": {"id": "container-id"}, "kubernetes": {"namespace": "namespace1", "pod": {"uid": "pod-uid", "name": "pod-name"}, "node": {"name": "node-name"}}}, "service": {"name": "1234_service-12a3", "language": {"version": "8", "name": "ecmascript"}, "agent": {"version": "3.14.0", "name": "elastic-node", "ephemeral_id":"abcdef123"}, "environment": "staging", "framework": {"version": "1.2.3", "name": "Express"}, "version": "5.1.3", "runtime": {"version": "8.0.0", "name": "node"}}}}
 {"error": {"id": "0123456789012345", "timestamp": 1494342245999999, "culprit": "my.module.function_name","log": { "message": "My service could not talk to the database named foobar", "param_message": "My service could not talk to the database named %s", "logger_name": "my.logger.name", "level": "warning", "stacktrace": [ { "abs_path": "/real/file/name.py", "filename": "/webpack/file/name.py", "function": "foo", "vars": { "key": "value" }, "pre_context": ["line1", "line2"], "context_line": "line3","library_frame": false,"lineno": 3,"module": "App::MyModule","colno": 4,"post_context": ["line4","line5" ]},{"filename": "lib/instrumentation/index.js","lineno": 102,"function": "instrumented","abs_path": "/Users/watson/code/node_modules/elastic/lib/instrumentation/index.js","vars": {"key": "value"},"pre_context": ["  var trans = this.currentTransaction","","  return instrumented","","  function instrumented () {","    var prev = ins.currentTransaction", "    ins.currentTransaction = trans"],"context_line": "    var result = original.apply(this, arguments)","post_context": ["    ins.currentTransaction = prev","    return result","}","}","","Instrumentation.prototype._recoverTransaction = function (trans) {","  if (this.currentTransaction === trans) return"]}]},"exception": {"message": "The username root is unknown","type": "DbError","module": "__builtins__","code": 42,"handled": false,"attributes": {"foo": "bar" },"stacktrace": [{ "abs_path": "/real/file/name.py","filename": "file/name.py","function": "foo","vars": {"key": "value"},"pre_context": ["line1","line2"],"context_line": "line3", "library_frame": true,"lineno": 3,"module": "App::MyModule","colno": 4,"post_context": ["line4","line5"]},{"filename": "lib/instrumentation/index.js","lineno": 102,"function": "instrumented","abs_path": "/Users/watson/code/node_modules/elastic/lib/instrumentation/index.js","vars": {"key": "value"},"pre_context": ["  var trans = this.currentTransaction","","  return instrumented","","  function instrumented () {", "    var prev = ins.currentTransaction","    ins.currentTransaction = trans"],"context_line": "    var result = original.apply(this, arguments)","post_context": ["    ins.currentTransaction = prev","    return result","}","}","","Instrumentation.prototype._recoverTransaction = function (trans) {","  if (this.currentTransaction === trans) return"]}]},"context": {"page":{"referer":"http://localhost:8000/test/e2e/","url":"http://localhost:8000/test/e2e/general-usecase/"}, "request": {"socket": {"remote_address": "12.53.12.1","encrypted": true},"http_version": "1.1","method": "POST","url": {"protocol": "https:","full": "https://www.example.com/p/a/t/h?query=string#hash","hostname": "www.example.com","port": 8080,"pathname": "/p/a/t/h","search": "?query=string", "hash": "#hash","raw": "/p/a/t/h?query=string#hash"},"headers": {"user-agent": "Mozilla Chrome Edge","content-type": "text/html","cookie": "c1=v1,c2=v2","some-other-header": "foo","array": ["foo","bar","baz"]}, "cookies": {"c1": "v1", "c2": "v2" },"env": {"SERVER_SOFTWARE": "nginx", "GATEWAY_INTERFACE": "CGI/1.1"},"body": "Hello World"},"response": { "status_code": 200, "headers": { "content-type": "application/json" },"headers_sent": true, "finished": true }, "user": { "id": 99, "username": "foo"},"tags": {"organization_uuid": "9f0e9d64-c185-4d21-a6f4-4673ed561ec8"}, "custom": {"my_key": 1,"some_other_value": "foo bar","and_objects": {"foo": ["bar","baz" ] }},"service": {"name": "service1", "language": {"version": "1.2"}, "framework": {"version": "1", "name": "Node"}}}}}
 { "error": {"id": "cdefab0123456789", "trace_id": null, "timestamp": 1533826745999000,"exception": {"message": "Cannot read property 'baz' no defined"}}}
-{ "error": {"id": "cdefab0123456780", "trace_id": "0123456789abcdeffedcba0123456789", "parent_id": "9632587410abcdef", "exception": {"type": "DbError"}, "context":{"service": {"name": "service1", "environment":"testing","language": {"version": "2.5", "name": "ruby"}, "agent": {"version": "2.1.3", "name": "elastic-ruby"}, "framework": {"version": "5.0", "name": "Rails"}, "version": "2", "runtime": {"version": "2.5", "name": "cruby"}}}}}
+{ "error": {"id": "cdefab0123456780", "trace_id": "0123456789abcdeffedcba0123456789", "parent_id": "9632587410abcdef", "exception": {"type": "DbError"}, "context":{"service": {"name": "service1", "environment":"testing","language": {"version": "2.5", "name": "ruby"}, "agent": {"version": "2.1.3", "name": "elastic-ruby", "ephemeral_id":"justanid"}, "framework": {"version": "5.0", "name": "Rails"}, "version": "2", "runtime": {"version": "2.5", "name": "cruby"}}}}}
 { "error": {"id": "abcdef0123456789", "trace_id": "0123456789abcdeffedcba0123456789", "parent_id": "9632587410abcdef", "transaction_id": "1234567890987654", "transaction": { "sampled": true, "type": "request"}, "timestamp": 1533827045999000,"log": {"level": "custom log level","message": "Cannot read property 'baz' of undefined"}}}

testdata/intake-v2/only-metadata.ndjson

@@ -1 +1 @@
-{"metadata": {"process": {"ppid": 6789, "pid": 1234, "argv": ["node", "server.js"], "title": "node"}, "system": {"platform": "darwin", "hostname": "prod1.example.com", "architecture": "x64", "container": {"id": "container-id"}, "kubernetes": {"namespace": "namespace1", "pod": {"uid": "pod-uid", "name": "pod-name"}, "node": {"name": "node-name"}}}, "service": {"name": "1234_service-12a3", "language": {"version": "8", "name": "ecmascript"}, "agent": {"version": "3.14.0", "name": "elastic-node"}, "environment": "staging", "framework": {"version": "1.2.3", "name": "Express"}, "version": "5.1.3", "runtime": {"version": "8.0.0", "name": "node"}}, "labels": {"tag0": null, "tag1": "one", "tag2": 2}, "user": {"id": "99","username": "foo","email": "[email protected]"}}}
+{"metadata": {"process": {"ppid": 6789, "pid": 1234, "argv": ["node", "server.js"], "title": "node"}, "system": {"platform": "darwin", "hostname": "prod1.example.com", "architecture": "x64", "container": {"id": "container-id"}, "kubernetes": {"namespace": "namespace1", "pod": {"uid": "pod-uid", "name": "pod-name"}, "node": {"name": "node-name"}}}, "service": {"name": "1234_service-12a3", "language": {"version": "8", "name": "ecmascript"}, "agent": {"version": "3.14.0", "name": "elastic-node", "ephemeral_id": "123abcdef"}, "environment": "staging", "framework": {"version": "1.2.3", "name": "Express"}, "version": "5.1.3", "runtime": {"version": "8.0.0", "name": "node"}}, "labels": {"tag0": null, "tag1": "one", "tag2": 2}, "user": {"id": "99","username": "foo","email": "[email protected]"}}}