diff --git a/NOTICE.txt b/NOTICE.txt index f0339ff4a39..b015329e4e9 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -509,11 +509,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/elastic/beats/v7 -Version: v7.0.0-alpha2.0.20210208210832-2363df416081 +Version: v7.0.0-alpha2.0.20210210141635-5475407dcdb0 Licence type (autodetected): Elastic -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/v7@v7.0.0-alpha2.0.20210208210832-2363df416081/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/v7@v7.0.0-alpha2.0.20210210141635-5475407dcdb0/LICENSE.txt: Source code in this repository is variously licensed under the Apache License Version 2.0, an Apache compatible license, or the Elastic License. Outside of @@ -15926,11 +15926,11 @@ Contents of probable licence file $GOMODCACHE/github.com/pquerna/cachecontrol@v0 -------------------------------------------------------------------------------- Dependency : github.com/prometheus/procfs -Version: v0.4.1 +Version: v0.5.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/prometheus/procfs@v0.4.1/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/prometheus/procfs@v0.5.0/LICENSE: Apache License Version 2.0, January 2004 diff --git a/apmpackage/apm/0.1.0/_dev/docs/README.template.md b/apmpackage/apm/0.1.0/_dev/docs/README.template.md index 4f93c4988ae..5c5c4c17008 100644 --- a/apmpackage/apm/0.1.0/_dev/docs/README.template.md +++ b/apmpackage/apm/0.1.0/_dev/docs/README.template.md @@ -66,7 +66,8 @@ IMPORTANT: If you run APM Server with Elastic Agent manually in standalone mode, - `Host`: APM Server host and port to listen on. - `Secret token`: Authorization token for sending data to APM Server. See the [documentation](https://www.elastic.co/guide/en/apm/server/current/configuration-rum.html) for details. - `Enable RUM`: Enables support for RUM monitoring. See the [documentation](https://www.elastic.co/guide/en/apm/server/current/configuration-rum.html) for details. - +- `API Key for Central Configuration`: Gives privileges for APM Agent central configuration. See the [documentation](https://www.elastic.co/guide/en/kibana/master/agent-configuration.html) +- `API Key for Sourcemaps`: Gives priveleges to read sourcemaps. See the [documentation](https://www.elastic.co/guide/en/apm/agent/rum-js/current/sourcemap.html). ## Traces diff --git a/apmpackage/apm/0.1.0/agent/input/template.yml.hbs b/apmpackage/apm/0.1.0/agent/input/template.yml.hbs index 082497b05cf..77c50823730 100644 --- a/apmpackage/apm/0.1.0/agent/input/template.yml.hbs +++ b/apmpackage/apm/0.1.0/agent/input/template.yml.hbs @@ -1,4 +1,8 @@ apm-server: host: {{host}} secret_token: {{secret_token}} - rum.enabled: {{enable_rum}} + rum: + enabled: {{enable_rum}} + source_mapping.elasticsearch.api_key: {{sourcemap_api_key}} + kibana: + api_key: {{kibana_api_key}} diff --git a/apmpackage/apm/0.1.0/data_stream/sampled_traces/elasticsearch/ilm/default_policy.json b/apmpackage/apm/0.1.0/data_stream/sampled_traces/elasticsearch/ilm/default_policy.json new file mode 100644 index 00000000000..fc6ad7a9992 --- /dev/null +++ b/apmpackage/apm/0.1.0/data_stream/sampled_traces/elasticsearch/ilm/default_policy.json @@ -0,0 +1,19 @@ +{ + "policy": { + "phases": { + "hot": { + "actions": { + "rollover": { + "max_age": "1h" + } + } + }, + "delete": { + "min_age": "1h", + "actions": { + "delete": {} + } + } + } + } +} diff --git a/apmpackage/apm/0.1.0/data_stream/sampled_traces/elasticsearch/ingest_pipeline/default.json b/apmpackage/apm/0.1.0/data_stream/sampled_traces/elasticsearch/ingest_pipeline/default.json new file mode 100644 index 00000000000..ac07d8ee66e --- /dev/null +++ b/apmpackage/apm/0.1.0/data_stream/sampled_traces/elasticsearch/ingest_pipeline/default.json @@ -0,0 +1,11 @@ +{ + "description": "Ingest pipeline for sampled trace documents", + "processors": [ + { + "set": { + "field": "event.ingested", + "value": "{{_ingest.timestamp}}" + } + } + ] +} diff --git a/apmpackage/apm/0.1.0/data_stream/sampled_traces/fields/base-fields.yml b/apmpackage/apm/0.1.0/data_stream/sampled_traces/fields/base-fields.yml new file mode 100644 index 00000000000..bef973826be --- /dev/null +++ b/apmpackage/apm/0.1.0/data_stream/sampled_traces/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: '@timestamp' + type: date + description: Event timestamp. +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. diff --git a/apmpackage/apm/0.1.0/data_stream/sampled_traces/fields/ecs.yml b/apmpackage/apm/0.1.0/data_stream/sampled_traces/fields/ecs.yml new file mode 100644 index 00000000000..690ea7daf4a --- /dev/null +++ b/apmpackage/apm/0.1.0/data_stream/sampled_traces/fields/ecs.yml @@ -0,0 +1,8 @@ +- name: event.ingested + type: date + description: | + Timestamp when an event arrived in the central data store. +- name: trace.id + type: keyword + description: | + The ID of the sampled trace. diff --git a/apmpackage/apm/0.1.0/data_stream/sampled_traces/fields/fields.yml b/apmpackage/apm/0.1.0/data_stream/sampled_traces/fields/fields.yml new file mode 100644 index 00000000000..370b8752662 --- /dev/null +++ b/apmpackage/apm/0.1.0/data_stream/sampled_traces/fields/fields.yml @@ -0,0 +1,6 @@ +# When changing fields or ILM policy, make sure to update +# x-pack/apm-server/sampling/pubsub/datastream.go. +- name: observer.id + type: keyword + description: | + The ID of the APM Server that indexed the sampled trace ID. diff --git a/apmpackage/apm/0.1.0/data_stream/sampled_traces/manifest.yml b/apmpackage/apm/0.1.0/data_stream/sampled_traces/manifest.yml new file mode 100644 index 00000000000..be3deb14708 --- /dev/null +++ b/apmpackage/apm/0.1.0/data_stream/sampled_traces/manifest.yml @@ -0,0 +1,4 @@ +title: APM tail-sampled traces +type: traces +dataset: sampled +ilm_policy: traces-apm.sampled-default_policy diff --git a/apmpackage/apm/0.1.0/docs/README.md b/apmpackage/apm/0.1.0/docs/README.md index 99fb3c2e7ae..8d26ec5cfd0 100644 --- a/apmpackage/apm/0.1.0/docs/README.md +++ b/apmpackage/apm/0.1.0/docs/README.md @@ -66,7 +66,8 @@ IMPORTANT: If you run APM Server with Elastic Agent manually in standalone mode, - `Host`: APM Server host and port to listen on. - `Secret token`: Authorization token for sending data to APM Server. See the [documentation](https://www.elastic.co/guide/en/apm/server/current/configuration-rum.html) for details. - `Enable RUM`: Enables support for RUM monitoring. See the [documentation](https://www.elastic.co/guide/en/apm/server/current/configuration-rum.html) for details. - +- `API Key for Central Configuration`: Gives privileges for APM Agent central configuration. See the [documentation](https://www.elastic.co/guide/en/kibana/master/agent-configuration.html) +- `API Key for Sourcemaps`: Gives priveleges to read sourcemaps. See the [documentation](https://www.elastic.co/guide/en/apm/agent/rum-js/current/sourcemap.html). ## Traces diff --git a/apmpackage/apm/0.1.0/manifest.yml b/apmpackage/apm/0.1.0/manifest.yml index 8aed4b2ea6e..e6401dd294e 100644 --- a/apmpackage/apm/0.1.0/manifest.yml +++ b/apmpackage/apm/0.1.0/manifest.yml @@ -42,6 +42,18 @@ policy_templates: required: true show_user: true default: false + - name: kibana_api_key + type: string + title: API Key for Central Configuration + required: false + description: Enter as : + show_user: true + - name: sourcemap_api_key + type: string + title: API Key for Sourcemaps + required: false + description: Enter as : + show_user: true template_path: template.yml.hbs owner: github: elastic/apm-server diff --git a/apmpackage/cmd/gen-package/genfields.go b/apmpackage/cmd/gen-package/genfields.go index c093441093a..d5828923c87 100644 --- a/apmpackage/cmd/gen-package/genfields.go +++ b/apmpackage/cmd/gen-package/genfields.go @@ -19,6 +19,7 @@ package main import ( "io/ioutil" + "log" "net/http" "path/filepath" "sort" @@ -42,6 +43,7 @@ func generateFields(version string) map[string][]field { inputFieldsFiles["app_metrics"] = filterInternalMetrics(inputFieldsFiles["internal_metrics"]) for streamType, inputFields := range inputFieldsFiles { + log.Printf("%s", streamType) var ecsFields []field var nonECSFields []field for _, fields := range populateECSInfo(ecsFlatFields, inputFields) { diff --git a/apmpackage/cmd/gen-package/main.go b/apmpackage/cmd/gen-package/main.go index 6e536739ea1..46ecf2ab0a9 100644 --- a/apmpackage/cmd/gen-package/main.go +++ b/apmpackage/cmd/gen-package/main.go @@ -33,6 +33,13 @@ var versionMapping = map[string]string{ "8.0": "0.1.0", } +// Some data streams may not have a counterpart template +// in standalone apm-server, and so it does not make sense +// to maintain a separate fields.yml. +var handwritten = map[string]bool{ + "sampled_traces": true, +} + func main() { stackVersion := common.MustNewVersion(cmd.DefaultSettings().Version) shortVersion := fmt.Sprintf("%d.%d", stackVersion.Major, stackVersion.Minor) @@ -57,11 +64,28 @@ func clear(version string) { panic(err) } for _, f := range fileInfo { - if f.IsDir() { - os.Remove(ecsFilePath(version, f.Name())) - os.Remove(fieldsFilePath(version, f.Name())) - os.RemoveAll(pipelinesPath(version, f.Name())) + if !f.IsDir() { + continue + } + name := f.Name() + if handwritten[name] { + continue } + removeFile(ecsFilePath(version, name)) + removeFile(fieldsFilePath(version, name)) + removeDir(pipelinesPath(version, name)) } ioutil.WriteFile(docsFilePath(version), nil, 0644) } + +func removeFile(path string) { + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + log.Fatal(err) + } +} + +func removeDir(path string) { + if err := os.RemoveAll(path); err != nil && !os.IsNotExist(err) { + log.Fatal(err) + } +} diff --git a/beater/api/config/agent/handler_test.go b/beater/api/config/agent/handler_test.go index cbfe03613f1..6f6883994ec 100644 --- a/beater/api/config/agent/handler_test.go +++ b/beater/api/config/agent/handler_test.go @@ -355,7 +355,7 @@ func TestIfNoneMatch(t *testing.T) { } func TestAgentConfigTraceContext(t *testing.T) { - kibanaCfg := libkibana.DefaultClientConfig() + kibanaCfg := config.KibanaConfig{Enabled: true, ClientConfig: libkibana.DefaultClientConfig()} kibanaCfg.Host = "testKibana:12345" client := kibana.NewConnectingClient(&kibanaCfg) handler := Handler(client, &config.AgentConfig{Cache: &config.Cache{Expiration: 5 * time.Minute}}) diff --git a/beater/api/mux.go b/beater/api/mux.go index 04dbfa0e0ff..cf236ee85bf 100644 --- a/beater/api/mux.go +++ b/beater/api/mux.go @@ -20,9 +20,9 @@ package api import ( "net/http" - "github.com/elastic/beats/v7/libbeat/monitoring" - + "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/libbeat/monitoring" "github.com/elastic/apm-server/beater/api/asset/sourcemap" "github.com/elastic/apm-server/beater/api/config/agent" @@ -64,13 +64,8 @@ const ( IntakeRUMV3Path = "/intake/v3/rum/events" ) -type route struct { - path string - handlerFn func(*config.Config, *authorization.Builder, publish.Reporter) (request.Handler, error) -} - // NewMux registers apm handlers to paths building up the APM Server API. -func NewMux(beaterConfig *config.Config, report publish.Reporter) (*http.ServeMux, error) { +func NewMux(beatInfo beat.Info, beaterConfig *config.Config, report publish.Reporter) (*http.ServeMux, error) { pool := request.NewContextPool() mux := http.NewServeMux() logger := logp.NewLogger(logs.Handler) @@ -80,26 +75,36 @@ func NewMux(beaterConfig *config.Config, report publish.Reporter) (*http.ServeMu return nil, err } + builder := routeBuilder{ + info: beatInfo, + cfg: beaterConfig, + authBuilder: auth, + reporter: report, + } + + type route struct { + path string + handlerFn func() (request.Handler, error) + } routeMap := []route{ - {RootPath, rootHandler}, - {AssetSourcemapPath, sourcemapHandler}, - {AgentConfigPath, backendAgentConfigHandler}, - {AgentConfigRUMPath, rumAgentConfigHandler}, - {IntakeRUMPath, rumIntakeHandler}, - {IntakeRUMV3Path, rumV3IntakeHandler}, - {IntakePath, backendIntakeHandler}, + {RootPath, builder.rootHandler}, + {AssetSourcemapPath, builder.sourcemapHandler}, + {AgentConfigPath, builder.backendAgentConfigHandler}, + {AgentConfigRUMPath, builder.rumAgentConfigHandler}, + {IntakeRUMPath, builder.rumIntakeHandler}, + {IntakeRUMV3Path, builder.rumV3IntakeHandler}, + {IntakePath, builder.backendIntakeHandler}, // The profile endpoint is in Beta - {ProfilePath, profileHandler}, + {ProfilePath, builder.profileHandler}, } for _, route := range routeMap { - h, err := route.handlerFn(beaterConfig, auth, report) + h, err := route.handlerFn() if err != nil { return nil, err } logger.Infof("Path %s added to request handler", route.path) mux.Handle(route.path, pool.HTTPHandler(h)) - } if beaterConfig.Expvar.IsEnabled() { path := beaterConfig.Expvar.URL @@ -109,41 +114,53 @@ func NewMux(beaterConfig *config.Config, report publish.Reporter) (*http.ServeMu return mux, nil } -func profileHandler(cfg *config.Config, builder *authorization.Builder, reporter publish.Reporter) (request.Handler, error) { - h := profile.Handler(reporter) - authHandler := builder.ForPrivilege(authorization.PrivilegeEventWrite.Action) - return middleware.Wrap(h, backendMiddleware(cfg, authHandler, profile.MonitoringMap)...) +type routeBuilder struct { + info beat.Info + cfg *config.Config + authBuilder *authorization.Builder + reporter publish.Reporter +} + +func (r *routeBuilder) profileHandler() (request.Handler, error) { + h := profile.Handler(r.reporter) + authHandler := r.authBuilder.ForPrivilege(authorization.PrivilegeEventWrite.Action) + return middleware.Wrap(h, backendMiddleware(r.cfg, authHandler, profile.MonitoringMap)...) } -func backendIntakeHandler(cfg *config.Config, builder *authorization.Builder, reporter publish.Reporter) (request.Handler, error) { - h := intake.Handler(stream.BackendProcessor(cfg), reporter) - authHandler := builder.ForPrivilege(authorization.PrivilegeEventWrite.Action) - return middleware.Wrap(h, backendMiddleware(cfg, authHandler, intake.MonitoringMap)...) +func (r *routeBuilder) backendIntakeHandler() (request.Handler, error) { + h := intake.Handler(stream.BackendProcessor(r.cfg), r.reporter) + authHandler := r.authBuilder.ForPrivilege(authorization.PrivilegeEventWrite.Action) + return middleware.Wrap(h, backendMiddleware(r.cfg, authHandler, intake.MonitoringMap)...) } -func rumIntakeHandler(cfg *config.Config, _ *authorization.Builder, reporter publish.Reporter) (request.Handler, error) { - h := intake.Handler(stream.RUMV2Processor(cfg), reporter) - return middleware.Wrap(h, rumMiddleware(cfg, nil, intake.MonitoringMap)...) +func (r *routeBuilder) rumIntakeHandler() (request.Handler, error) { + h := intake.Handler(stream.RUMV2Processor(r.cfg), r.reporter) + return middleware.Wrap(h, rumMiddleware(r.cfg, nil, intake.MonitoringMap)...) } -func rumV3IntakeHandler(cfg *config.Config, _ *authorization.Builder, reporter publish.Reporter) (request.Handler, error) { - h := intake.Handler(stream.RUMV3Processor(cfg), reporter) - return middleware.Wrap(h, rumMiddleware(cfg, nil, intake.MonitoringMap)...) +func (r *routeBuilder) rumV3IntakeHandler() (request.Handler, error) { + h := intake.Handler(stream.RUMV3Processor(r.cfg), r.reporter) + return middleware.Wrap(h, rumMiddleware(r.cfg, nil, intake.MonitoringMap)...) } -func sourcemapHandler(cfg *config.Config, builder *authorization.Builder, reporter publish.Reporter) (request.Handler, error) { - h := sourcemap.Handler(reporter) - authHandler := builder.ForPrivilege(authorization.PrivilegeSourcemapWrite.Action) - return middleware.Wrap(h, sourcemapMiddleware(cfg, authHandler)...) +func (r *routeBuilder) sourcemapHandler() (request.Handler, error) { + h := sourcemap.Handler(r.reporter) + authHandler := r.authBuilder.ForPrivilege(authorization.PrivilegeSourcemapWrite.Action) + return middleware.Wrap(h, sourcemapMiddleware(r.cfg, authHandler)...) } -func backendAgentConfigHandler(cfg *config.Config, builder *authorization.Builder, _ publish.Reporter) (request.Handler, error) { - authHandler := builder.ForPrivilege(authorization.PrivilegeAgentConfigRead.Action) - return agentConfigHandler(cfg, authHandler, backendMiddleware) +func (r *routeBuilder) rootHandler() (request.Handler, error) { + h := root.Handler(root.HandlerConfig{Version: r.info.Version}) + return middleware.Wrap(h, rootMiddleware(r.cfg, r.authBuilder.ForAnyOfPrivileges(authorization.ActionAny))...) } -func rumAgentConfigHandler(cfg *config.Config, _ *authorization.Builder, _ publish.Reporter) (request.Handler, error) { - return agentConfigHandler(cfg, nil, rumMiddleware) +func (r *routeBuilder) backendAgentConfigHandler() (request.Handler, error) { + authHandler := r.authBuilder.ForPrivilege(authorization.PrivilegeAgentConfigRead.Action) + return agentConfigHandler(r.cfg, authHandler, backendMiddleware) +} + +func (r *routeBuilder) rumAgentConfigHandler() (request.Handler, error) { + return agentConfigHandler(r.cfg, nil, rumMiddleware) } type middlewareFunc func(*config.Config, *authorization.Handler, map[request.ResultID]*monitoring.Int) []middleware.Middleware @@ -151,7 +168,7 @@ type middlewareFunc func(*config.Config, *authorization.Handler, map[request.Res func agentConfigHandler(cfg *config.Config, authHandler *authorization.Handler, middlewareFunc middlewareFunc) (request.Handler, error) { var client kibana.Client if cfg.Kibana.Enabled { - client = kibana.NewConnectingClient(&cfg.Kibana.ClientConfig) + client = kibana.NewConnectingClient(&cfg.Kibana) } h := agent.Handler(client, cfg.AgentConfig) msg := "Agent remote configuration is disabled. " + @@ -162,11 +179,6 @@ func agentConfigHandler(cfg *config.Config, authHandler *authorization.Handler, return middleware.Wrap(h, append(middlewareFunc(cfg, authHandler, agent.MonitoringMap), ks)...) } -func rootHandler(cfg *config.Config, builder *authorization.Builder, _ publish.Reporter) (request.Handler, error) { - return middleware.Wrap(root.Handler(), - rootMiddleware(cfg, builder.ForAnyOfPrivileges(authorization.ActionAny))...) -} - func apmMiddleware(m map[request.ResultID]*monitoring.Int) []middleware.Middleware { return []middleware.Middleware{ middleware.LogMiddleware(), diff --git a/beater/api/mux_config_agent_test.go b/beater/api/mux_config_agent_test.go index 5ce408722d1..8187d75ec06 100644 --- a/beater/api/mux_config_agent_test.go +++ b/beater/api/mux_config_agent_test.go @@ -19,15 +19,12 @@ package api import ( "net/http" - "net/http/httptest" "testing" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/elastic/apm-server/approvaltest" "github.com/elastic/apm-server/beater/api/config/agent" - "github.com/elastic/apm-server/beater/beatertest" "github.com/elastic/apm-server/beater/config" "github.com/elastic/apm-server/beater/headers" "github.com/elastic/apm-server/beater/request" @@ -72,27 +69,16 @@ func TestConfigAgentHandler_KillSwitchMiddleware(t *testing.T) { } func TestConfigAgentHandler_PanicMiddleware(t *testing.T) { - h := testHandler(t, backendAgentConfigHandler) - rec := &beatertest.WriterPanicOnce{} - c := request.NewContext() - c.Reset(rec, httptest.NewRequest(http.MethodGet, "/", nil)) - h(c) - require.Equal(t, http.StatusInternalServerError, rec.StatusCode) - approvaltest.ApproveJSON(t, approvalPathConfigAgent(t.Name()), rec.Body.Bytes()) + testPanicMiddleware(t, "/config/v1/agents", approvalPathConfigAgent(t.Name())) } func TestConfigAgentHandler_MonitoringMiddleware(t *testing.T) { - h := testHandler(t, backendAgentConfigHandler) - c, _ := beatertest.ContextWithResponseRecorder(http.MethodPost, "/") - - expected := map[request.ResultID]int{ + testMonitoringMiddleware(t, "/config/v1/agents", agent.MonitoringMap, map[request.ResultID]int{ request.IDRequestCount: 1, request.IDResponseCount: 1, request.IDResponseErrorsCount: 1, - request.IDResponseErrorsForbidden: 1} - equal, result := beatertest.CompareMonitoringInt(h, c, expected, agent.MonitoringMap) - assert.True(t, equal, result) - + request.IDResponseErrorsForbidden: 1, + }) } func configEnabledConfigAgent() *config.Config { diff --git a/beater/api/mux_intake_backend_test.go b/beater/api/mux_intake_backend_test.go index 2e6db8e014c..b2bdbc27e4d 100644 --- a/beater/api/mux_intake_backend_test.go +++ b/beater/api/mux_intake_backend_test.go @@ -19,7 +19,6 @@ package api import ( "net/http" - "net/http/httptest" "testing" "github.com/stretchr/testify/assert" @@ -27,7 +26,6 @@ import ( "github.com/elastic/apm-server/approvaltest" "github.com/elastic/apm-server/beater/api/intake" - "github.com/elastic/apm-server/beater/beatertest" "github.com/elastic/apm-server/beater/config" "github.com/elastic/apm-server/beater/headers" "github.com/elastic/apm-server/beater/request" @@ -57,27 +55,17 @@ func TestIntakeBackendHandler_AuthorizationMiddleware(t *testing.T) { } func TestIntakeBackendHandler_PanicMiddleware(t *testing.T) { - h := testHandler(t, backendIntakeHandler) - rec := &beatertest.WriterPanicOnce{} - c := request.NewContext() - c.Reset(rec, httptest.NewRequest(http.MethodGet, "/", nil)) - h(c) - assert.Equal(t, http.StatusInternalServerError, rec.StatusCode) - approvaltest.ApproveJSON(t, approvalPathIntakeBackend(t.Name()), rec.Body.Bytes()) + testPanicMiddleware(t, "/intake/v2/events", approvalPathIntakeBackend(t.Name())) } func TestIntakeBackendHandler_MonitoringMiddleware(t *testing.T) { - h := testHandler(t, backendIntakeHandler) - c, _ := beatertest.ContextWithResponseRecorder(http.MethodGet, "/") // send GET request resulting in 405 MethodNotAllowed error - expected := map[request.ResultID]int{ + testMonitoringMiddleware(t, "/intake/v2/events", intake.MonitoringMap, map[request.ResultID]int{ request.IDRequestCount: 1, request.IDResponseCount: 1, request.IDResponseErrorsCount: 1, - request.IDResponseErrorsMethodNotAllowed: 1} - - equal, result := beatertest.CompareMonitoringInt(h, c, expected, intake.MonitoringMap) - assert.True(t, equal, result) + request.IDResponseErrorsMethodNotAllowed: 1, + }) } func approvalPathIntakeBackend(f string) string { diff --git a/beater/api/mux_intake_rum_test.go b/beater/api/mux_intake_rum_test.go index 87e747c36f0..ad85f9d4004 100644 --- a/beater/api/mux_intake_rum_test.go +++ b/beater/api/mux_intake_rum_test.go @@ -27,7 +27,6 @@ import ( "github.com/elastic/apm-server/approvaltest" "github.com/elastic/apm-server/beater/api/intake" - "github.com/elastic/apm-server/beater/beatertest" "github.com/elastic/apm-server/beater/config" "github.com/elastic/apm-server/beater/headers" "github.com/elastic/apm-server/beater/middleware" @@ -94,39 +93,32 @@ func TestRUMHandler_KillSwitchMiddleware(t *testing.T) { func TestRUMHandler_CORSMiddleware(t *testing.T) { cfg := cfgEnabledRUM() cfg.RumConfig.AllowOrigins = []string{"foo"} - h, err := rumIntakeHandler(cfg, nil, beatertest.NilReporter) - require.NoError(t, err) - c, w := beatertest.ContextWithResponseRecorder(http.MethodPost, "/") - c.Request.Header.Set(headers.Origin, "bar") - h(c) - - assert.Equal(t, http.StatusForbidden, w.Code) + h := newTestMux(t, cfg) + + for _, path := range []string{"/intake/v2/rum/events", "/intake/v3/rum/events"} { + req := httptest.NewRequest(http.MethodPost, path, nil) + req.Header.Set(headers.Origin, "bar") + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + assert.Equal(t, http.StatusForbidden, w.Code) + } } func TestIntakeRUMHandler_PanicMiddleware(t *testing.T) { - h, err := rumIntakeHandler(config.DefaultConfig(), nil, beatertest.NilReporter) - require.NoError(t, err) - rec := &beatertest.WriterPanicOnce{} - c := request.NewContext() - c.Reset(rec, httptest.NewRequest(http.MethodGet, "/", nil)) - h(c) - assert.Equal(t, http.StatusInternalServerError, rec.StatusCode) - approvaltest.ApproveJSON(t, approvalPathIntakeRUM(t.Name()), rec.Body.Bytes()) + testPanicMiddleware(t, "/intake/v2/rum/events", approvalPathIntakeRUM(t.Name())) + testPanicMiddleware(t, "/intake/v3/rum/events", approvalPathIntakeRUM(t.Name())) } func TestRumHandler_MonitoringMiddleware(t *testing.T) { - h, err := rumIntakeHandler(config.DefaultConfig(), nil, beatertest.NilReporter) - require.NoError(t, err) - c, _ := beatertest.ContextWithResponseRecorder(http.MethodPost, "/") // send GET request resulting in 403 Forbidden error - expected := map[request.ResultID]int{ - request.IDRequestCount: 1, - request.IDResponseCount: 1, - request.IDResponseErrorsCount: 1, - request.IDResponseErrorsForbidden: 1} - - equal, result := beatertest.CompareMonitoringInt(h, c, expected, intake.MonitoringMap) - assert.True(t, equal, result) + for _, path := range []string{"/intake/v2/rum/events", "/intake/v3/rum/events"} { + testMonitoringMiddleware(t, path, intake.MonitoringMap, map[request.ResultID]int{ + request.IDRequestCount: 1, + request.IDResponseCount: 1, + request.IDResponseErrorsCount: 1, + request.IDResponseErrorsForbidden: 1, + }) + } } func cfgEnabledRUM() *config.Config { diff --git a/beater/api/mux_root_test.go b/beater/api/mux_root_test.go index 4fdbb8b302f..3dd784856a6 100644 --- a/beater/api/mux_root_test.go +++ b/beater/api/mux_root_test.go @@ -19,7 +19,6 @@ package api import ( "net/http" - "net/http/httptest" "testing" "github.com/stretchr/testify/assert" @@ -27,7 +26,6 @@ import ( "github.com/elastic/apm-server/approvaltest" "github.com/elastic/apm-server/beater/api/root" - "github.com/elastic/apm-server/beater/beatertest" "github.com/elastic/apm-server/beater/config" "github.com/elastic/apm-server/beater/headers" "github.com/elastic/apm-server/beater/request" @@ -54,29 +52,16 @@ func TestRootHandler_AuthorizationMiddleware(t *testing.T) { } func TestRootHandler_PanicMiddleware(t *testing.T) { - h := testHandler(t, rootHandler) - rec := &beatertest.WriterPanicOnce{} - c := request.NewContext() - c.Reset(rec, httptest.NewRequest(http.MethodGet, "/", nil)) - h(c) - - assert.Equal(t, http.StatusInternalServerError, rec.StatusCode) - approvaltest.ApproveJSON(t, approvalPathRoot(t.Name()), rec.Body.Bytes()) + testPanicMiddleware(t, "/", approvalPathRoot(t.Name())) } func TestRootHandler_MonitoringMiddleware(t *testing.T) { - h := testHandler(t, rootHandler) - c, _ := beatertest.ContextWithResponseRecorder(http.MethodGet, "/") - - // send GET request resulting in 403 Forbidden error as RUM is disabled by default - expected := map[request.ResultID]int{ + testMonitoringMiddleware(t, "/", root.MonitoringMap, map[request.ResultID]int{ request.IDRequestCount: 1, request.IDResponseCount: 1, request.IDResponseValidCount: 1, - request.IDResponseValidOK: 1} - - equal, result := beatertest.CompareMonitoringInt(h, c, expected, root.MonitoringMap) - assert.True(t, equal, result) + request.IDResponseValidOK: 1, + }) } func approvalPathRoot(f string) string { return "root/test_approved/integration/" + f } diff --git a/beater/api/mux_sourcemap_handler_test.go b/beater/api/mux_sourcemap_handler_test.go index f17d3eb76de..95b9ef0f60e 100644 --- a/beater/api/mux_sourcemap_handler_test.go +++ b/beater/api/mux_sourcemap_handler_test.go @@ -19,15 +19,12 @@ package api import ( "net/http" - "net/http/httptest" "testing" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/elastic/apm-server/approvaltest" "github.com/elastic/apm-server/beater/api/asset/sourcemap" - "github.com/elastic/apm-server/beater/beatertest" "github.com/elastic/apm-server/beater/config" "github.com/elastic/apm-server/beater/headers" "github.com/elastic/apm-server/beater/request" @@ -82,28 +79,17 @@ func TestSourcemapHandler_KillSwitchMiddleware(t *testing.T) { } func TestSourcemapHandler_PanicMiddleware(t *testing.T) { - h := testHandler(t, sourcemapHandler) - rec := &beatertest.WriterPanicOnce{} - c := request.NewContext() - c.Reset(rec, httptest.NewRequest(http.MethodGet, "/", nil)) - h(c) - require.Equal(t, http.StatusInternalServerError, rec.StatusCode) - approvaltest.ApproveJSON(t, approvalPathAsset(t.Name()), rec.Body.Bytes()) + testPanicMiddleware(t, "/assets/v1/sourcemaps", approvalPathAsset(t.Name())) } func TestSourcemapHandler_MonitoringMiddleware(t *testing.T) { - h := testHandler(t, sourcemapHandler) - c, _ := beatertest.ContextWithResponseRecorder(http.MethodPost, "/") - // send GET request resulting in 403 Forbidden error as RUM is disabled by default - expected := map[request.ResultID]int{ + testMonitoringMiddleware(t, "/assets/v1/sourcemaps", sourcemap.MonitoringMap, map[request.ResultID]int{ request.IDRequestCount: 1, request.IDResponseCount: 1, request.IDResponseErrorsCount: 1, - request.IDResponseErrorsForbidden: 1} - - equal, result := beatertest.CompareMonitoringInt(h, c, expected, sourcemap.MonitoringMap) - assert.True(t, equal, result) + request.IDResponseErrorsForbidden: 1, + }) } func approvalPathAsset(f string) string { return "asset/sourcemap/test_approved/integration/" + f } diff --git a/beater/api/mux_test.go b/beater/api/mux_test.go index b64d6d3c52c..9ce8ba7a5c7 100644 --- a/beater/api/mux_test.go +++ b/beater/api/mux_test.go @@ -22,13 +22,15 @@ import ( "net/http/httptest" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/elastic/apm-server/beater/authorization" + "github.com/elastic/apm-server/approvaltest" "github.com/elastic/apm-server/beater/beatertest" "github.com/elastic/apm-server/beater/config" "github.com/elastic/apm-server/beater/request" - "github.com/elastic/apm-server/publish" + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/monitoring" ) func requestToMuxerWithPattern(cfg *config.Config, pattern string) (*httptest.ResponseRecorder, error) { @@ -44,7 +46,7 @@ func requestToMuxerWithHeader(cfg *config.Config, pattern string, method string, } func requestToMuxer(cfg *config.Config, r *http.Request) (*httptest.ResponseRecorder, error) { - mux, err := NewMux(cfg, beatertest.NilReporter) + mux, err := NewMux(beat.Info{Version: "1.2.3"}, cfg, beatertest.NilReporter) if err != nil { return nil, err } @@ -54,11 +56,30 @@ func requestToMuxer(cfg *config.Config, r *http.Request) (*httptest.ResponseReco return w, nil } -func testHandler(t *testing.T, fn func(*config.Config, *authorization.Builder, publish.Reporter) (request.Handler, error)) request.Handler { - cfg := config.DefaultConfig() - builder, err := authorization.NewBuilder(cfg) - require.NoError(t, err) - h, err := fn(cfg, builder, beatertest.NilReporter) +func testPanicMiddleware(t *testing.T, urlPath string, approvalPath string) { + h := newTestMux(t, config.DefaultConfig()) + req := httptest.NewRequest(http.MethodGet, urlPath, nil) + + var rec beatertest.WriterPanicOnce + h.ServeHTTP(&rec, req) + + assert.Equal(t, http.StatusInternalServerError, rec.StatusCode) + approvaltest.ApproveJSON(t, approvalPath, rec.Body.Bytes()) +} + +func testMonitoringMiddleware(t *testing.T, urlPath string, monitoringMap map[request.ResultID]*monitoring.Int, expected map[request.ResultID]int) { + beatertest.ClearRegistry(monitoringMap) + + h := newTestMux(t, config.DefaultConfig()) + req := httptest.NewRequest(http.MethodGet, urlPath, nil) + h.ServeHTTP(httptest.NewRecorder(), req) + + equal, result := beatertest.CompareMonitoringInt(expected, monitoringMap) + assert.True(t, equal, result) +} + +func newTestMux(t *testing.T, cfg *config.Config) http.Handler { + mux, err := NewMux(beat.Info{Version: "1.2.3"}, cfg, beatertest.NilReporter) require.NoError(t, err) - return h + return mux } diff --git a/beater/api/root/handler.go b/beater/api/root/handler.go index 73620de4a0f..dffde6af59a 100644 --- a/beater/api/root/handler.go +++ b/beater/api/root/handler.go @@ -35,14 +35,20 @@ var ( registry = monitoring.Default.NewRegistry("apm-server.root") ) +// HandlerConfig holds configuration for Handler. +type HandlerConfig struct { + // Version holds the APM Server version. + Version string +} + // Handler returns error if route does not exist, // otherwise returns information about the server. The detail level differs for authorized and non-authorized requests. //TODO: only allow GET, HEAD requests (breaking change) -func Handler() request.Handler { +func Handler(cfg HandlerConfig) request.Handler { serverInfo := common.MapStr{ "build_date": version.BuildTime().Format(time.RFC3339), "build_sha": version.Commit(), - "version": version.GetDefaultVersion(), + "version": cfg.Version, } return func(c *request.Context) { diff --git a/beater/api/root/handler_test.go b/beater/api/root/handler_test.go index 2eb2b28c00b..5844d60d0b3 100644 --- a/beater/api/root/handler_test.go +++ b/beater/api/root/handler_test.go @@ -36,7 +36,7 @@ import ( func TestRootHandler(t *testing.T) { t.Run("404", func(t *testing.T) { c, w := beatertest.ContextWithResponseRecorder(http.MethodGet, "/abc/xyz") - Handler()(c) + Handler(HandlerConfig{Version: "1.2.3"})(c) assert.Equal(t, http.StatusNotFound, w.Code) assert.Equal(t, `{"error":"404 page not found"}`+"\n", w.Body.String()) @@ -45,7 +45,7 @@ func TestRootHandler(t *testing.T) { t.Run("ok", func(t *testing.T) { c, w := beatertest.ContextWithResponseRecorder(http.MethodGet, "/") c.Authorization = &authorization.DenyAuth{} - Handler()(c) + Handler(HandlerConfig{Version: "1.2.3"})(c) assert.Equal(t, http.StatusOK, w.Code) assert.Equal(t, "", w.Body.String()) @@ -54,7 +54,7 @@ func TestRootHandler(t *testing.T) { t.Run("unauthorized", func(t *testing.T) { c, w := beatertest.ContextWithResponseRecorder(http.MethodGet, "/") c.Authorization = authorization.DenyAuth{} - Handler()(c) + Handler(HandlerConfig{Version: "1.2.3"})(c) assert.Equal(t, http.StatusOK, w.Code) assert.Equal(t, "", w.Body.String()) @@ -65,11 +65,11 @@ func TestRootHandler(t *testing.T) { builder, err := authorization.NewBuilder(&config.Config{SecretToken: "abc"}) require.NoError(t, err) c.Authorization = builder.ForPrivilege("").AuthorizationFor("Bearer", "abc") - Handler()(c) + Handler(HandlerConfig{Version: "1.2.3"})(c) assert.Equal(t, http.StatusOK, w.Code) - body := fmt.Sprintf("{\"build_date\":\"0001-01-01T00:00:00Z\",\"build_sha\":\"%s\",\"version\":\"%s\"}\n", - version.Commit(), version.GetDefaultVersion()) + body := fmt.Sprintf("{\"build_date\":\"0001-01-01T00:00:00Z\",\"build_sha\":\"%s\",\"version\":\"1.2.3\"}\n", + version.Commit()) assert.Equal(t, body, w.Body.String()) }) } diff --git a/beater/api/root/test_approved/integration/TestRootHandler_AuthorizationMiddleware/Authorized.approved.json b/beater/api/root/test_approved/integration/TestRootHandler_AuthorizationMiddleware/Authorized.approved.json index 05ae72701a0..a624c8b269d 100644 --- a/beater/api/root/test_approved/integration/TestRootHandler_AuthorizationMiddleware/Authorized.approved.json +++ b/beater/api/root/test_approved/integration/TestRootHandler_AuthorizationMiddleware/Authorized.approved.json @@ -1,5 +1,5 @@ { "build_date": "0001-01-01T00:00:00Z", "build_sha": "unknown", - "version": "8.0.0" + "version": "1.2.3" } diff --git a/beater/beater.go b/beater/beater.go index d4f7e19b8d2..35de9b57036 100644 --- a/beater/beater.go +++ b/beater/beater.go @@ -25,6 +25,8 @@ import ( "sync" "time" + "github.com/elastic/beats/v7/libbeat/kibana" + "github.com/pkg/errors" "go.elastic.co/apm" "golang.org/x/sync/errgroup" @@ -184,6 +186,7 @@ func (bt *beater) start(ctx context.Context, cancelContext context.CancelFunc, b inputs.Stop() } reload.Register.MustRegisterList("inputs", inputs) + } else { // Management disabled, use statically defined config. s, err := newServerRunner(ctx, serverRunnerParams{ @@ -233,6 +236,7 @@ func (s *serverCreator) Create(p beat.PipelineConnector, rawConfig *common.Confi sharedServerRunnerParams: s.args, Namespace: namespace, Pipeline: p, + KibanaConfig: &integrationConfig.Fleet.Kibana, RawConfig: apmServerCommonConfig, }) } @@ -265,9 +269,10 @@ type serverRunner struct { type serverRunnerParams struct { sharedServerRunnerParams - Namespace string - Pipeline beat.PipelineConnector - RawConfig *common.Config + Namespace string + Pipeline beat.PipelineConnector + KibanaConfig *kibana.ClientConfig + RawConfig *common.Config } type sharedServerRunnerParams struct { @@ -284,6 +289,11 @@ func newServerRunner(ctx context.Context, args serverRunnerParams) (*serverRunne if err != nil { return nil, err } + + if cfg.DataStreams.Enabled && args.KibanaConfig != nil { + cfg.Kibana.ClientConfig = *args.KibanaConfig + } + runServerContext, cancel := context.WithCancel(ctx) return &serverRunner{ backgroundContext: ctx, @@ -366,11 +376,13 @@ func (s *serverRunner) run() error { } if err := runServer(s.runServerContext, ServerParams{ - Info: s.beat.Info, - Config: s.config, - Logger: s.logger, - Tracer: s.tracer, - Reporter: reporter, + Info: s.beat.Info, + Config: s.config, + Managed: s.beat.Manager != nil && s.beat.Manager.Enabled(), + Namespace: s.namespace, + Logger: s.logger, + Tracer: s.tracer, + Reporter: reporter, }); err != nil { return err } diff --git a/beater/beater_test.go b/beater/beater_test.go index a0d93d08888..c5b8662d8b9 100644 --- a/beater/beater_test.go +++ b/beater/beater_test.go @@ -42,7 +42,6 @@ import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/instrumentation" "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/beats/v7/libbeat/version" ) type testBeater struct { @@ -68,7 +67,7 @@ func newBeat(t *testing.T, cfg *common.Config, beatConfig *beat.BeatConfig, even info := beat.Info{ Beat: "test-apm-server", IndexPrefix: "test-apm-server", - Version: version.GetDefaultVersion(), + Version: "1.2.3", // hard-coded to avoid changing approvals ID: uuid.Must(uuid.FromString("fbba762a-14dd-412c-b7e9-b79f903eb492")), } diff --git a/beater/beatertest/monitoring.go b/beater/beatertest/monitoring.go index 36ed5bfb08e..8d91b79bffb 100644 --- a/beater/beatertest/monitoring.go +++ b/beater/beatertest/monitoring.go @@ -26,17 +26,14 @@ import ( ) // CompareMonitoringInt matches expected with real monitoring counters and -// returns false and an a string showind diffs if not matching +// returns false and an a string showind diffs if not matching. +// +// The caller is expected to call ClearRegistry before invoking some code +// path that should update monitoring counters. func CompareMonitoringInt( - handler func(c *request.Context), - c *request.Context, expected map[request.ResultID]int, m map[request.ResultID]*monitoring.Int, ) (bool, string) { - - ClearRegistry(m) - handler(c) - var result string for _, id := range AllRequestResultIDs() { monitoringIntVal := int64(0) @@ -52,7 +49,6 @@ func CompareMonitoringInt( result += fmt.Sprintf("[%s] Expected: %d, Received: %d", id, expectedVal, monitoringIntVal) } } - return len(result) == 0, result } diff --git a/beater/config/config.go b/beater/config/config.go index 8ff68759c68..69b0c61f114 100644 --- a/beater/config/config.go +++ b/beater/config/config.go @@ -40,17 +40,18 @@ const ( ) type KibanaConfig struct { - Enabled bool `config:"enabled"` + Enabled bool `config:"enabled"` + APIKey string `config:"api_key"` kibana.ClientConfig `config:",inline"` } func (k *KibanaConfig) Unpack(cfg *common.Config) error { - if err := cfg.Unpack(&k.ClientConfig); err != nil { + type kibanaConfig KibanaConfig + if err := cfg.Unpack((*kibanaConfig)(k)); err != nil { return err } k.Enabled = cfg.Enabled() k.Host = strings.TrimRight(k.Host, "/") - return nil } @@ -119,7 +120,7 @@ func NewConfig(ucfg *common.Config, outputESCfg *common.Config) (*Config, error) return nil, errors.New(msgInvalidConfigAgentCfg) } - if err := c.RumConfig.setup(logger, outputESCfg); err != nil { + if err := c.RumConfig.setup(logger, c.DataStreams.Enabled, outputESCfg); err != nil { return nil, err } diff --git a/beater/config/integration.go b/beater/config/integration.go index 78d80dc7a3f..99c5c321e15 100644 --- a/beater/config/integration.go +++ b/beater/config/integration.go @@ -21,6 +21,7 @@ import ( "errors" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/kibana" ) func NewIntegrationConfig(rootConfig *common.Config) (*IntegrationConfig, error) { @@ -48,6 +49,7 @@ type IntegrationConfig struct { Meta *Meta `config:"meta"` DataStream *DataStream `config:"data_stream"` APMServer *common.Config `config:"apm-server"` + Fleet Fleet `config:"fleet"` } type DataStream struct { @@ -62,3 +64,7 @@ type Package struct { Name string `config:"name"` Version string `config:"version"` } + +type Fleet struct { + Kibana kibana.ClientConfig `config:"kibana"` +} diff --git a/beater/config/rum.go b/beater/config/rum.go index 4b74f3fb5ee..3a2a4458301 100644 --- a/beater/config/rum.go +++ b/beater/config/rum.go @@ -76,7 +76,7 @@ func (s *SourceMapping) IsEnabled() bool { return s == nil || s.Enabled == nil || *s.Enabled } -func (c *RumConfig) setup(log *logp.Logger, outputESCfg *common.Config) error { +func (c *RumConfig) setup(log *logp.Logger, dataStreamsEnabled bool, outputESCfg *common.Config) error { if !c.IsEnabled() { return nil } @@ -88,8 +88,14 @@ func (c *RumConfig) setup(log *logp.Logger, outputESCfg *common.Config) error { return errors.Wrapf(err, "Invalid regex for `exclude_from_grouping`: ") } + var apiKey string if c.SourceMapping == nil || c.SourceMapping.esConfigured { - return nil + if dataStreamsEnabled { + // when running under Fleet, the only setting configured is the api key + apiKey = c.SourceMapping.ESConfig.APIKey + } else { + return nil + } } // fall back to elasticsearch output configuration for sourcemap storage if possible @@ -101,6 +107,9 @@ func (c *RumConfig) setup(log *logp.Logger, outputESCfg *common.Config) error { if err := outputESCfg.Unpack(c.SourceMapping.ESConfig); err != nil { return errors.Wrap(err, "unpacking Elasticsearch config into Sourcemap config") } + if c.SourceMapping.ESConfig.APIKey == "" { + c.SourceMapping.ESConfig.APIKey = apiKey + } return nil } diff --git a/beater/config/rum_test.go b/beater/config/rum_test.go index bb098108679..09db7dae6b8 100644 --- a/beater/config/rum_test.go +++ b/beater/config/rum_test.go @@ -20,6 +20,12 @@ package config import ( "testing" + "github.com/stretchr/testify/require" + + "github.com/elastic/apm-server/elasticsearch" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/stretchr/testify/assert" ) @@ -37,6 +43,22 @@ func TestIsRumEnabled(t *testing.T) { } } +func TestRumSetup(t *testing.T) { + rum := defaultRum() + rum.SourceMapping.esConfigured = true + rum.Enabled = &rum.SourceMapping.esConfigured + rum.SourceMapping.ESConfig = &elasticsearch.Config{APIKey: "id:apikey"} + esCfg := common.MustNewConfigFrom(map[string]interface{}{ + "hosts": []interface{}{"cloud:9200"}, + }) + + err := rum.setup(logp.NewLogger("test"), true, esCfg) + + require.NoError(t, err) + assert.Equal(t, elasticsearch.Hosts{"cloud:9200"}, rum.SourceMapping.ESConfig.Hosts) + assert.Equal(t, "id:apikey", rum.SourceMapping.ESConfig.APIKey) +} + func TestDefaultRum(t *testing.T) { c := DefaultConfig() assert.Equal(t, defaultRum(), c.RumConfig) diff --git a/beater/http.go b/beater/http.go index 2e215f8cde8..c72bdf6f250 100644 --- a/beater/http.go +++ b/beater/http.go @@ -30,6 +30,7 @@ import ( "github.com/elastic/apm-server/beater/api" "github.com/elastic/apm-server/beater/config" "github.com/elastic/apm-server/publish" + "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/gmux" @@ -43,8 +44,8 @@ type httpServer struct { grpcListener net.Listener } -func newHTTPServer(logger *logp.Logger, cfg *config.Config, tracer *apm.Tracer, reporter publish.Reporter) (*httpServer, error) { - mux, err := api.NewMux(cfg, reporter) +func newHTTPServer(logger *logp.Logger, info beat.Info, cfg *config.Config, tracer *apm.Tracer, reporter publish.Reporter) (*httpServer, error) { + mux, err := api.NewMux(info, cfg, reporter) if err != nil { return nil, err } diff --git a/beater/jaeger/server.go b/beater/jaeger/server.go index eb3856f74ff..d22c34692e2 100644 --- a/beater/jaeger/server.go +++ b/beater/jaeger/server.go @@ -102,7 +102,7 @@ func NewServer(logger *logp.Logger, cfg *config.Config, tracer *apm.Tracer, repo var client kibana.Client var fetcher *agentcfg.Fetcher if cfg.Kibana.Enabled { - client = kibana.NewConnectingClient(&cfg.Kibana.ClientConfig) + client = kibana.NewConnectingClient(&cfg.Kibana) fetcher = agentcfg.NewFetcher(client, cfg.AgentConfig.Cache.Expiration) } RegisterGRPCServices( diff --git a/beater/middleware/monitoring_middleware_test.go b/beater/middleware/monitoring_middleware_test.go index f77c58e70fa..4a988611f81 100644 --- a/beater/middleware/monitoring_middleware_test.go +++ b/beater/middleware/monitoring_middleware_test.go @@ -40,8 +40,10 @@ func TestMonitoringHandler(t *testing.T) { expected map[request.ResultID]int, m map[request.ResultID]*monitoring.Int, ) { + beatertest.ClearRegistry(m) c, _ := beatertest.DefaultContextWithResponseRecorder() - equal, result := beatertest.CompareMonitoringInt(Apply(MonitoringMiddleware(m), h), c, expected, m) + Apply(MonitoringMiddleware(m), h)(c) + equal, result := beatertest.CompareMonitoringInt(expected, m) assert.True(t, equal, result) } diff --git a/beater/server.go b/beater/server.go index fe841ea28bd..344ae4d7fba 100644 --- a/beater/server.go +++ b/beater/server.go @@ -52,6 +52,12 @@ type ServerParams struct { // Config is the configuration used for running the APM Server. Config *config.Config + // Managed indicates that the server is managed by Fleet. + Managed bool + + // Namespace holds the data stream namespace for the server. + Namespace string + // Logger is the logger for the beater component. Logger *logp.Logger @@ -66,7 +72,7 @@ type ServerParams struct { // runServer runs the APM Server until a fatal error occurs, or ctx is cancelled. func runServer(ctx context.Context, args ServerParams) error { - srv, err := newServer(args.Logger, args.Config, args.Tracer, args.Reporter) + srv, err := newServer(args.Logger, args.Info, args.Config, args.Tracer, args.Reporter) if err != nil { return err } @@ -92,8 +98,8 @@ type server struct { reporter publish.Reporter } -func newServer(logger *logp.Logger, cfg *config.Config, tracer *apm.Tracer, reporter publish.Reporter) (server, error) { - httpServer, err := newHTTPServer(logger, cfg, tracer, reporter) +func newServer(logger *logp.Logger, info beat.Info, cfg *config.Config, tracer *apm.Tracer, reporter publish.Reporter) (server, error) { + httpServer, err := newHTTPServer(logger, info, cfg, tracer, reporter) if err != nil { return server{}, err } @@ -132,7 +138,7 @@ func newGRPCServer( var kibanaClient kibana.Client var agentcfgFetcher *agentcfg.Fetcher if cfg.Kibana.Enabled { - kibanaClient = kibana.NewConnectingClient(&cfg.Kibana.ClientConfig) + kibanaClient = kibana.NewConnectingClient(&cfg.Kibana) agentcfgFetcher = agentcfg.NewFetcher(kibanaClient, cfg.AgentConfig.Cache.Expiration) } jaeger.RegisterGRPCServices(srv, authBuilder, jaeger.ElasticAuthTag, logger, reporter, kibanaClient, agentcfgFetcher) diff --git a/beater/test_approved_es_documents/TestPublishIntegrationErrors.approved.json b/beater/test_approved_es_documents/TestPublishIntegrationErrors.approved.json index 50e7886e0be..c9de42cfb49 100644 --- a/beater/test_approved_es_documents/TestPublishIntegrationErrors.approved.json +++ b/beater/test_approved_es_documents/TestPublishIntegrationErrors.approved.json @@ -299,8 +299,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "process": { "args": [ @@ -432,8 +432,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "process": { "args": [ @@ -546,8 +546,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "process": { "args": [ @@ -660,8 +660,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "parent": { "id": "9632587410abcdef" @@ -779,8 +779,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "parent": { "id": "9632587410abcdef" diff --git a/beater/test_approved_es_documents/TestPublishIntegrationEvents.approved.json b/beater/test_approved_es_documents/TestPublishIntegrationEvents.approved.json index 2e4175201c6..fa0c605122a 100644 --- a/beater/test_approved_es_documents/TestPublishIntegrationEvents.approved.json +++ b/beater/test_approved_es_documents/TestPublishIntegrationEvents.approved.json @@ -105,8 +105,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "parent": { "id": "abcdefabcdef01234567" @@ -241,8 +241,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "parent": { "id": "abcdef0123456789" @@ -418,8 +418,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "process": { "args": [ @@ -705,8 +705,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "parent": { "id": "9632587410abcdef" diff --git a/beater/test_approved_es_documents/TestPublishIntegrationMetricsets.approved.json b/beater/test_approved_es_documents/TestPublishIntegrationMetricsets.approved.json index 66c17839aa1..7f0799349ff 100644 --- a/beater/test_approved_es_documents/TestPublishIntegrationMetricsets.approved.json +++ b/beater/test_approved_es_documents/TestPublishIntegrationMetricsets.approved.json @@ -47,8 +47,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "process": { "pid": 1234 @@ -132,8 +132,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "process": { "pid": 1234 @@ -178,8 +178,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "process": { "pid": 1234 diff --git a/beater/test_approved_es_documents/TestPublishIntegrationMinimalEvents.approved.json b/beater/test_approved_es_documents/TestPublishIntegrationMinimalEvents.approved.json index 479c1412a7d..6851fa033df 100644 --- a/beater/test_approved_es_documents/TestPublishIntegrationMinimalEvents.approved.json +++ b/beater/test_approved_es_documents/TestPublishIntegrationMinimalEvents.approved.json @@ -23,8 +23,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "transaction", @@ -71,8 +71,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "parent": { "id": "ab23456a89012345" @@ -122,8 +122,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "parent": { "id": "ab23456a89012345" @@ -168,8 +168,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "metric", @@ -203,8 +203,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "error", @@ -243,8 +243,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "error", @@ -283,8 +283,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "error", diff --git a/beater/test_approved_es_documents/TestPublishIntegrationProfileCPUProfile.approved.json b/beater/test_approved_es_documents/TestPublishIntegrationProfileCPUProfile.approved.json index 1f36d4a3e7a..d82de03dc4d 100644 --- a/beater/test_approved_es_documents/TestPublishIntegrationProfileCPUProfile.approved.json +++ b/beater/test_approved_es_documents/TestPublishIntegrationProfileCPUProfile.approved.json @@ -10,8 +10,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -144,8 +144,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -224,8 +224,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -376,8 +376,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -456,8 +456,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -518,8 +518,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -616,8 +616,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -708,8 +708,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -794,8 +794,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -898,8 +898,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1020,8 +1020,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1076,8 +1076,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1144,8 +1144,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1248,8 +1248,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1382,8 +1382,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1444,8 +1444,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1560,8 +1560,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1664,8 +1664,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1756,8 +1756,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1848,8 +1848,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1970,8 +1970,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2026,8 +2026,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2112,8 +2112,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2174,8 +2174,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2242,8 +2242,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2370,8 +2370,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2450,8 +2450,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2506,8 +2506,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2550,8 +2550,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2630,8 +2630,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2752,8 +2752,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2862,8 +2862,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2972,8 +2972,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -3046,8 +3046,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -3162,8 +3162,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -3224,8 +3224,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -3304,8 +3304,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -3372,8 +3372,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -3494,8 +3494,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -3664,8 +3664,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -3744,8 +3744,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -3842,8 +3842,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -4006,8 +4006,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -4062,8 +4062,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -4190,8 +4190,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -4306,8 +4306,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -4368,8 +4368,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -4454,8 +4454,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -4504,8 +4504,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -4578,8 +4578,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -4634,8 +4634,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -4762,8 +4762,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -4818,8 +4818,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -4862,8 +4862,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -4942,8 +4942,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -5052,8 +5052,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -5144,8 +5144,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -5218,8 +5218,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -5322,8 +5322,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -5426,8 +5426,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -5506,8 +5506,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -5628,8 +5628,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -5714,8 +5714,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -5806,8 +5806,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -5892,8 +5892,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -5966,8 +5966,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -6022,8 +6022,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -6132,8 +6132,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -6266,8 +6266,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -6412,8 +6412,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -6498,8 +6498,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -6554,8 +6554,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -6646,8 +6646,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -6798,8 +6798,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -6932,8 +6932,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -7054,8 +7054,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -7116,8 +7116,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -7184,8 +7184,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -7288,8 +7288,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -7326,8 +7326,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -7376,8 +7376,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -7444,8 +7444,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -7518,8 +7518,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -7592,8 +7592,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -7678,8 +7678,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -7788,8 +7788,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -7856,8 +7856,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -7930,8 +7930,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -8046,8 +8046,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -8114,8 +8114,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -8224,8 +8224,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -8316,8 +8316,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -8408,8 +8408,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -8458,8 +8458,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -8562,8 +8562,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -8636,8 +8636,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -8686,8 +8686,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -8820,8 +8820,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", diff --git a/beater/test_approved_es_documents/TestPublishIntegrationProfileCPUProfileMetadata.approved.json b/beater/test_approved_es_documents/TestPublishIntegrationProfileCPUProfileMetadata.approved.json index ba4bfd57e57..f736f27907e 100644 --- a/beater/test_approved_es_documents/TestPublishIntegrationProfileCPUProfileMetadata.approved.json +++ b/beater/test_approved_es_documents/TestPublishIntegrationProfileCPUProfileMetadata.approved.json @@ -17,8 +17,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -161,8 +161,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -251,8 +251,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -413,8 +413,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -503,8 +503,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -575,8 +575,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -683,8 +683,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -785,8 +785,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -881,8 +881,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -995,8 +995,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1127,8 +1127,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1193,8 +1193,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1271,8 +1271,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1385,8 +1385,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1529,8 +1529,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1601,8 +1601,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1727,8 +1727,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1841,8 +1841,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1943,8 +1943,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2045,8 +2045,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2177,8 +2177,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2243,8 +2243,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2339,8 +2339,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2411,8 +2411,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2489,8 +2489,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2627,8 +2627,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2717,8 +2717,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2783,8 +2783,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2837,8 +2837,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2927,8 +2927,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -3059,8 +3059,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -3179,8 +3179,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -3299,8 +3299,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -3383,8 +3383,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -3509,8 +3509,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -3581,8 +3581,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -3671,8 +3671,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -3749,8 +3749,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -3881,8 +3881,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -4061,8 +4061,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -4151,8 +4151,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -4259,8 +4259,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -4433,8 +4433,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -4499,8 +4499,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -4637,8 +4637,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -4763,8 +4763,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -4835,8 +4835,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -4931,8 +4931,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -4991,8 +4991,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -5075,8 +5075,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -5141,8 +5141,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -5279,8 +5279,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -5345,8 +5345,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -5399,8 +5399,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -5489,8 +5489,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -5609,8 +5609,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -5711,8 +5711,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -5795,8 +5795,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -5909,8 +5909,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -6023,8 +6023,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -6113,8 +6113,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -6245,8 +6245,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -6341,8 +6341,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -6443,8 +6443,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -6539,8 +6539,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -6623,8 +6623,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -6689,8 +6689,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -6809,8 +6809,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -6953,8 +6953,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -7109,8 +7109,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -7205,8 +7205,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -7271,8 +7271,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -7373,8 +7373,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -7535,8 +7535,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -7679,8 +7679,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -7811,8 +7811,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -7883,8 +7883,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -7961,8 +7961,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -8075,8 +8075,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -8123,8 +8123,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -8183,8 +8183,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -8261,8 +8261,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -8345,8 +8345,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -8429,8 +8429,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -8525,8 +8525,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -8645,8 +8645,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -8723,8 +8723,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -8807,8 +8807,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -8933,8 +8933,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -9011,8 +9011,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -9131,8 +9131,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -9233,8 +9233,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -9335,8 +9335,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -9395,8 +9395,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -9509,8 +9509,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -9593,8 +9593,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -9653,8 +9653,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -9797,8 +9797,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", diff --git a/beater/test_approved_es_documents/TestPublishIntegrationProfileHeapProfile.approved.json b/beater/test_approved_es_documents/TestPublishIntegrationProfileHeapProfile.approved.json index 1edbeb9a10c..1c2a43e4c49 100644 --- a/beater/test_approved_es_documents/TestPublishIntegrationProfileHeapProfile.approved.json +++ b/beater/test_approved_es_documents/TestPublishIntegrationProfileHeapProfile.approved.json @@ -10,8 +10,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -79,8 +79,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -190,8 +190,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -289,8 +289,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -388,8 +388,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -433,8 +433,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -526,8 +526,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -571,8 +571,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -640,8 +640,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -703,8 +703,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -820,8 +820,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -901,8 +901,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1012,8 +1012,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1093,8 +1093,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1162,8 +1162,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1207,8 +1207,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1252,8 +1252,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1327,8 +1327,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1372,8 +1372,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1585,8 +1585,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1636,8 +1636,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1699,8 +1699,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1834,8 +1834,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -1885,8 +1885,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2014,8 +2014,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2053,8 +2053,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2116,8 +2116,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2185,8 +2185,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2338,8 +2338,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2497,8 +2497,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2704,8 +2704,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2821,8 +2821,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -2908,8 +2908,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -3043,8 +3043,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -3166,8 +3166,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -3271,8 +3271,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -3466,8 +3466,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -3583,8 +3583,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -3700,8 +3700,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -3781,8 +3781,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -3862,8 +3862,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", @@ -3967,8 +3967,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "processor": { "event": "profile", diff --git a/beater/test_approved_es_documents/TestPublishIntegrationSpans.approved.json b/beater/test_approved_es_documents/TestPublishIntegrationSpans.approved.json index 76908a2300d..9d41bd41e97 100644 --- a/beater/test_approved_es_documents/TestPublishIntegrationSpans.approved.json +++ b/beater/test_approved_es_documents/TestPublishIntegrationSpans.approved.json @@ -70,8 +70,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "parent": { "id": "abcdef0123456789" @@ -199,8 +199,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "parent": { "id": "0000000011111111" @@ -332,8 +332,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "parent": { "id": "abcdefabcdef7890" @@ -463,8 +463,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "parent": { "id": "ababcdcdefefabde" @@ -599,8 +599,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "parent": { "id": "abcdef0123456789" @@ -811,8 +811,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "parent": { "id": "abcdef0123456789" diff --git a/beater/test_approved_es_documents/TestPublishIntegrationTransactions.approved.json b/beater/test_approved_es_documents/TestPublishIntegrationTransactions.approved.json index dd3c87ebf5f..5b444a0df7a 100644 --- a/beater/test_approved_es_documents/TestPublishIntegrationTransactions.approved.json +++ b/beater/test_approved_es_documents/TestPublishIntegrationTransactions.approved.json @@ -67,8 +67,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "parent": { "id": "abcdefabcdef01234567" @@ -263,8 +263,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "process": { "args": [ @@ -430,8 +430,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "process": { "args": [ @@ -577,8 +577,8 @@ "hostname": "", "id": "fbba762a-14dd-412c-b7e9-b79f903eb492", "type": "test-apm-server", - "version": "8.0.0", - "version_major": 8 + "version": "1.2.3", + "version_major": 1 }, "parent": { "id": "abcdefabcdef01234567" diff --git a/beater/tracing.go b/beater/tracing.go index 8653ca6c664..dd3de2a3182 100644 --- a/beater/tracing.go +++ b/beater/tracing.go @@ -22,6 +22,7 @@ import ( "net" "net/http" + "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/apm-server/beater/api" @@ -53,7 +54,7 @@ func newTracerServer(listener net.Listener, logger *logp.Logger) (*tracerServer, } } cfg := config.DefaultConfig() - mux, err := api.NewMux(cfg, report) + mux, err := api.NewMux(beat.Info{}, cfg, report) if err != nil { return nil, err } diff --git a/changelogs/head.asciidoc b/changelogs/head.asciidoc index 0163da75866..c3363a94aa2 100644 --- a/changelogs/head.asciidoc +++ b/changelogs/head.asciidoc @@ -6,6 +6,8 @@ https://github.com/elastic/apm-server/compare/7.11\...master[View commits] [float] ==== Breaking Changes * Leading 0s are no longer removed from trace/span ids if they are created by Jaeger {pull}4671[4671] +* Jaeger spans will now have a type of "app" where they previously were "custom" {pull}4711[4711] +* Jaeger spans may now have a (more accurate) outcome of "unknown" where they previously were "success" {pull}4711[4711] [float] ==== Bug fixes @@ -23,6 +25,8 @@ https://github.com/elastic/apm-server/compare/7.11\...master[View commits] * Add a `_doc_count` field to transaction histogram docs {pull}4647[4647] * Upgrade Go to 1.15.7 {pull}4663[4663] * OpenTelemetry Protocol (OTLP) over gRPC is now supported on the standard endpoint (8200) {pull}4677[4677] +* Add initial support for APM central config and sourcemaps when running under Fleet {pull}4670[4670] +* Data stream and ILM policy for tail-based sampling {pull}4707[4707] [float] ==== Deprecated diff --git a/cmd/apikey.go b/cmd/apikey.go index 58a5e50f93e..18b984b3738 100644 --- a/cmd/apikey.go +++ b/cmd/apikey.go @@ -49,9 +49,9 @@ func genApikeyCmd(settings instance.Settings) *cobra.Command { apikeyCmd := cobra.Command{ Use: "apikey", Short: short, - Long: short + `. -Most operations require the "manage_api_key" cluster privilege. Ensure to configure "apm-server.api_key.*" or -"output.elasticsearch.*" appropriately. APM Server will create security privileges for the "apm" application; + Long: short + `. +Most operations require the "manage_api_key" cluster privilege. Ensure to configure "apm-server.api_key.*" or +"output.elasticsearch.*" appropriately. APM Server will create security privileges for the "apm" application; you can freely query them. If you modify or delete apm privileges, APM Server might reject all requests. Check the Elastic Security API documentation for details.`, } diff --git a/go.mod b/go.mod index 7b1d37575f5..7172b7821e0 100644 --- a/go.mod +++ b/go.mod @@ -9,11 +9,11 @@ require ( github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e github.com/dgraph-io/badger/v2 v2.2007.3-0.20201012072640-f5a7e0a1c83b github.com/dlclark/regexp2 v1.4.0 // indirect - github.com/dop251/goja v0.0.0-20210126164150-f5884268f0c0 // indirect + github.com/dop251/goja v0.0.0-20210209213638-ed7b3dc62eb9 // indirect github.com/dop251/goja_nodejs v0.0.0-20201222133159-1629e8d0b836 // indirect github.com/dustin/go-humanize v1.0.0 github.com/elastic/apm-server/approvaltest v0.0.0-00010101000000-000000000000 - github.com/elastic/beats/v7 v7.0.0-alpha2.0.20210208210832-2363df416081 + github.com/elastic/beats/v7 v7.0.0-alpha2.0.20210210141635-5475407dcdb0 github.com/elastic/ecs v1.6.0 github.com/elastic/gmux v0.1.0 github.com/elastic/go-elasticsearch/v7 v7.5.1-0.20201007132508-ff965d99ba02 @@ -41,7 +41,7 @@ require ( github.com/modern-go/reflect2 v1.0.1 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 - github.com/prometheus/procfs v0.4.1 // indirect + github.com/prometheus/procfs v0.5.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/reviewdog/reviewdog v0.9.17 github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735 diff --git a/go.sum b/go.sum index bf37622e2f2..c24c07f8f23 100644 --- a/go.sum +++ b/go.sum @@ -191,6 +191,7 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2 h1:oMCHnXa6CCCafdPDbMh/lWRhRByN0VFLvv+g+ayx1SI= github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI= github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= @@ -201,7 +202,9 @@ github.com/bsm/sarama-cluster v2.1.13+incompatible/go.mod h1:r7ao+4tTNXvWm+VRpRJ github.com/bsm/sarama-cluster v2.1.14-0.20180625083203-7e67d87a6b3f+incompatible/go.mod h1:r7ao+4tTNXvWm+VRpRJchr2kQhqxgmAp2iEX5W96gMM= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= +github.com/cavaliercoder/badio v0.0.0-20160213150051-ce5280129e9e h1:YYUjy5BRwO5zPtfk+aa2gw255FIIoi93zMmuy19o0bc= github.com/cavaliercoder/badio v0.0.0-20160213150051-ce5280129e9e/go.mod h1:V284PjgVwSk4ETmz84rpu9ehpGg7swlIH8npP9k2bGw= +github.com/cavaliercoder/go-rpm v0.0.0-20190131055624-7a9c54e3d83e h1:Gbx+iVCXG/1m5WSnidDGuHgN+vbIwl+6fR092ANU+Y8= github.com/cavaliercoder/go-rpm v0.0.0-20190131055624-7a9c54e3d83e/go.mod h1:AZIh1CCnMrcVm6afFf96PBvE2MRpWFco91z8ObJtgDY= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= @@ -333,8 +336,8 @@ github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFP github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/elastic/beats/v7 v7.0.0-alpha2.0.20210208210832-2363df416081 h1:pyTga57Ud8xBorRChZErH8kLhhlNLVGUkjrQoYN2SDQ= -github.com/elastic/beats/v7 v7.0.0-alpha2.0.20210208210832-2363df416081/go.mod h1:iXNYCPBl2aQsYVJ/pS1/BTsATh7u7i1UucEeBIbkPhA= +github.com/elastic/beats/v7 v7.0.0-alpha2.0.20210210141635-5475407dcdb0 h1:ctgIwmV/rUIwxBNWz+1boGuDXl/ohEEIxyPqtIpveDs= +github.com/elastic/beats/v7 v7.0.0-alpha2.0.20210210141635-5475407dcdb0/go.mod h1:mgtOD69pqbv2ltdrJvmZWOatpTnNBys8/yVTEPnlUXc= github.com/elastic/ecs v1.6.0 h1:8NmgfnsjmKXh9hVsK3H2tZtfUptepNc3msJOAynhtmc= github.com/elastic/ecs v1.6.0/go.mod h1:pgiLbQsijLOJvFR8OTILLu0Ni/R/foUNg0L+T6mU9b4= github.com/elastic/elastic-agent-client/v7 v7.0.0-20200709172729-d43b7ad5833a h1:2NHgf1RUw+f240lpTnLrCp1aBNvq2wDi0E1A423/S1k= @@ -1116,8 +1119,8 @@ github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.4.1 h1:a4oCTNJdGpE6eD4j1mypyS2ZXLFXo8wEVgUabL47Xr0= -github.com/prometheus/procfs v0.4.1/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.5.0 h1:ICtgn8CchRgPjUV2P2qwqAAPVDd5CFZsFOpkBRc1vS0= +github.com/prometheus/procfs v0.5.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/prometheus v1.8.2-0.20201105135750-00f16d1ac3a4/go.mod h1:XYjkJiog7fyQu3puQNivZPI2pNq1C/775EIoHfDvuvY= github.com/prometheus/prometheus v2.5.0+incompatible h1:7QPitgO2kOFG8ecuRn9O/4L9+10He72rVRJvMXrE9Hg= github.com/prometheus/prometheus v2.5.0+incompatible/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s= diff --git a/kibana/connecting_client.go b/kibana/connecting_client.go index 3558f58b6b3..8c827d31680 100644 --- a/kibana/connecting_client.go +++ b/kibana/connecting_client.go @@ -19,6 +19,7 @@ package kibana import ( "context" + "encoding/base64" "errors" "io" "net/http" @@ -26,6 +27,8 @@ import ( "sync" "time" + "github.com/elastic/apm-server/beater/config" + "go.elastic.co/apm" "go.elastic.co/apm/module/apmhttp" @@ -58,12 +61,12 @@ type Client interface { type ConnectingClient struct { m sync.RWMutex client *kibana.Client - cfg *kibana.ClientConfig + cfg *config.KibanaConfig } // NewConnectingClient returns instance of ConnectingClient and starts a background routine trying to connect // to configured Kibana instance, using JitterBackoff for establishing connection. -func NewConnectingClient(cfg *kibana.ClientConfig) Client { +func NewConnectingClient(cfg *config.KibanaConfig) Client { c := &ConnectingClient{cfg: cfg} go func() { log := logp.NewLogger(logs.Kibana) @@ -124,7 +127,7 @@ func (c *ConnectingClient) SupportsVersion(ctx context.Context, v *common.Versio if !retry || upToDate { return upToDate, nil } - client, err := kibana.NewClientWithConfig(c.cfg) + client, err := kibana.NewClientWithConfig(c.clientConfig()) if err != nil { log.Errorf("failed to obtain connection to Kibana: %s", err.Error()) return upToDate, err @@ -145,11 +148,23 @@ func (c *ConnectingClient) connect() error { if c.client != nil { return nil } - client, err := kibana.NewClientWithConfig(c.cfg) + client, err := kibana.NewClientWithConfig(c.clientConfig()) if err != nil { return err } + if c.cfg.APIKey != "" { + client.Headers["Authorization"] = []string{"ApiKey " + base64.StdEncoding.EncodeToString([]byte(c.cfg.APIKey))} + client.Username = "" + client.Password = "" + } client.HTTP = apmhttp.WrapClient(client.HTTP) c.client = client return nil } + +func (c *ConnectingClient) clientConfig() *kibana.ClientConfig { + if c != nil && c.cfg != nil { + return &c.cfg.ClientConfig + } + return nil +} diff --git a/kibana/connecting_client_test.go b/kibana/connecting_client_test.go index fdd3de86d91..894c845f830 100644 --- a/kibana/connecting_client_test.go +++ b/kibana/connecting_client_test.go @@ -26,6 +26,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/elastic/apm-server/beater/config" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/kibana" @@ -39,6 +41,28 @@ func TestNewConnectingClientFrom(t *testing.T) { assert.Equal(t, mockCfg, c.(*ConnectingClient).cfg) } +func TestNewConnectingClientWithAPIKey(t *testing.T) { + cfg := &config.KibanaConfig{ + Enabled: true, + APIKey: "foo-id:bar-apikey", + ClientConfig: kibana.ClientConfig{ + Host: "localhost:5601", + Username: "elastic", + Password: "secret", + IgnoreVersion: true, + }, + } + conn := &ConnectingClient{cfg: cfg} + require.NotNil(t, conn) + err := conn.connect() + require.NoError(t, err) + client := conn.client + require.NotNil(t, client) + assert.Equal(t, "", client.Username) + assert.Equal(t, "", client.Password) + assert.Equal(t, "ApiKey Zm9vLWlkOmJhci1hcGlrZXk=", client.Headers.Get("Authorization")) +} + func TestConnectingClient_Send(t *testing.T) { t.Run("Send", func(t *testing.T) { c := mockClient() @@ -102,8 +126,11 @@ type rt struct { } var ( - mockCfg = &kibana.ClientConfig{ - Host: "non-existing", + mockCfg = &config.KibanaConfig{ + Enabled: true, + ClientConfig: kibana.ClientConfig{ + Host: "non-existing", + }, } mockBody = ioutil.NopCloser(convert.ToReader(`{"response": "ok"}`)) mockStatus = http.StatusOK diff --git a/model/context.go b/model/context.go index ed4dd1ebb94..d8ad67ef900 100644 --- a/model/context.go +++ b/model/context.go @@ -57,17 +57,20 @@ type URL struct { Fragment *string } -func ParseURL(original, hostname string) *URL { +func ParseURL(original, defaultHostname, defaultScheme string) *URL { original = truncate(original) url, err := url.Parse(original) if err != nil { return &URL{Original: &original} } if url.Scheme == "" { - url.Scheme = "http" + url.Scheme = defaultScheme + if url.Scheme == "" { + url.Scheme = "http" + } } if url.Host == "" { - url.Host = hostname + url.Host = defaultHostname } full := truncate(url.String()) out := &URL{ diff --git a/model/error_test.go b/model/error_test.go index 3b9fa17cb8b..50d33b264fe 100644 --- a/model/error_test.go +++ b/model/error_test.go @@ -555,7 +555,7 @@ func TestErrorTransformPage(t *testing.T) { Error: Error{ ID: &id, Page: &Page{ - URL: ParseURL(urlExample, ""), + URL: ParseURL(urlExample, "", ""), Referer: nil, }, }, @@ -572,9 +572,9 @@ func TestErrorTransformPage(t *testing.T) { Error: Error{ ID: &id, Timestamp: time.Now(), - URL: ParseURL("https://localhost:8200/", ""), + URL: ParseURL("https://localhost:8200/", "", ""), Page: &Page{ - URL: ParseURL(urlExample, ""), + URL: ParseURL(urlExample, "", ""), Referer: nil, }, }, diff --git a/model/modeldecoder/rumv3/decoder.go b/model/modeldecoder/rumv3/decoder.go index 13e9ca45d2e..855f0c2c0a8 100644 --- a/model/modeldecoder/rumv3/decoder.go +++ b/model/modeldecoder/rumv3/decoder.go @@ -433,7 +433,7 @@ func mapToMetricsetModel(from *metricset, metadata *model.Metadata, reqTime time func mapToPageModel(from contextPage, out *model.Page) { if from.URL.IsSet() { - out.URL = model.ParseURL(from.URL.Val, "") + out.URL = model.ParseURL(from.URL.Val, "", "") } if from.Referer.IsSet() { referer := from.Referer.Val diff --git a/model/modeldecoder/v2/decoder.go b/model/modeldecoder/v2/decoder.go index acc90735556..35a60cec50e 100644 --- a/model/modeldecoder/v2/decoder.go +++ b/model/modeldecoder/v2/decoder.go @@ -578,7 +578,7 @@ func mapToMetricsetModel(from *metricset, metadata *model.Metadata, reqTime time func mapToPageModel(from contextPage, out *model.Page) { if from.URL.IsSet() { - out.URL = model.ParseURL(from.URL.Val, "") + out.URL = model.ParseURL(from.URL.Val, "", "") } if from.Referer.IsSet() { referer := from.Referer.Val diff --git a/model/transaction_test.go b/model/transaction_test.go index 563ca2b15f1..5fa32b34468 100644 --- a/model/transaction_test.go +++ b/model/transaction_test.go @@ -240,7 +240,7 @@ func TestTransactionTransformPage(t *testing.T) { Type: "tx", Duration: 65.98, Page: &Page{ - URL: ParseURL(urlExample, ""), + URL: ParseURL(urlExample, "", ""), Referer: nil, }, }, @@ -259,9 +259,9 @@ func TestTransactionTransformPage(t *testing.T) { Type: "tx", Timestamp: time.Now(), Duration: 65.98, - URL: ParseURL("https://localhost:8200/", ""), + URL: ParseURL("https://localhost:8200/", "", ""), Page: &Page{ - URL: ParseURL(urlExample, ""), + URL: ParseURL(urlExample, "", ""), Referer: nil, }, }, diff --git a/processor/otel/builder.go b/processor/otel/builder.go new file mode 100644 index 00000000000..cafc7571d91 --- /dev/null +++ b/processor/otel/builder.go @@ -0,0 +1,121 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package otel + +import ( + "fmt" + + "github.com/elastic/apm-server/model" +) + +type transactionBuilder struct { + *model.Transaction + httpURL string + httpHost string + httpScheme string +} + +func (tx *transactionBuilder) setFramework(name, version string) { + if name == "" { + return + } + tx.Metadata.Service.Framework.Name = name + tx.Metadata.Service.Framework.Version = version +} + +func (tx *transactionBuilder) setHTTPMethod(method string) { + tx.ensureHTTPRequest() + tx.HTTP.Request.Method = method +} + +func (tx *transactionBuilder) setHTTPScheme(scheme string) { + tx.ensureHTTP() + tx.httpScheme = scheme +} + +func (tx *transactionBuilder) setHTTPURL(httpURL string) { + tx.ensureHTTP() + tx.httpURL = httpURL +} + +func (tx *transactionBuilder) setHTTPHost(hostport string) { + tx.ensureHTTP() + tx.httpHost = hostport +} + +func (tx transactionBuilder) setHTTPVersion(version string) { + tx.ensureHTTP() + tx.HTTP.Version = &version +} + +func (tx transactionBuilder) setHTTPRemoteAddr(remoteAddr string) { + tx.ensureHTTPRequest() + if tx.HTTP.Request.Socket == nil { + tx.HTTP.Request.Socket = &model.Socket{} + } + tx.HTTP.Request.Socket.RemoteAddress = &remoteAddr +} + +func (tx *transactionBuilder) setHTTPStatusCode(statusCode int) { + tx.ensureHTTP() + if tx.HTTP.Response == nil { + tx.HTTP.Response = &model.Resp{} + } + tx.HTTP.Response.MinimalResp.StatusCode = &statusCode + if tx.Outcome == outcomeUnknown { + if statusCode >= 500 { + tx.Outcome = outcomeFailure + } else { + tx.Outcome = outcomeSuccess + } + } + if tx.Result == "" { + tx.Result = httpStatusCodeResult(statusCode) + } +} + +func (tx *transactionBuilder) ensureHTTPRequest() { + tx.ensureHTTP() + if tx.HTTP.Request == nil { + tx.HTTP.Request = &model.Req{} + } +} + +func (tx *transactionBuilder) ensureHTTP() { + if tx.HTTP == nil { + tx.HTTP = &model.Http{} + } +} + +var standardStatusCodeResults = [...]string{ + "HTTP 1xx", + "HTTP 2xx", + "HTTP 3xx", + "HTTP 4xx", + "HTTP 5xx", +} + +// httpStatusCodeResult returns the transaction result value to use for the +// given HTTP status code. +func httpStatusCodeResult(statusCode int) string { + switch i := statusCode / 100; i { + case 1, 2, 3, 4, 5: + return standardStatusCodeResults[i-1] + } + return fmt.Sprintf("HTTP %d", statusCode) +} diff --git a/processor/otel/consumer.go b/processor/otel/consumer.go index 46fa9a607cd..a32a8c73c30 100644 --- a/processor/otel/consumer.go +++ b/processor/otel/consumer.go @@ -15,6 +15,23 @@ // specific language governing permissions and limitations // under the License. +// Portions copied from OpenTelemetry Collector (contrib), from the +// elastic exporter. +// +// Copyright 2020, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package otel import ( @@ -26,6 +43,7 @@ import ( "strings" "go.opentelemetry.io/collector/consumer/pdata" + "go.opentelemetry.io/collector/translator/conventions" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" @@ -36,8 +54,6 @@ import ( ) const ( - AgentNameJaeger = "Jaeger" - keywordLength = 1024 dot = "." underscore = "_" @@ -83,11 +99,16 @@ func (c *Consumer) convertResourceSpans(resourceSpans pdata.ResourceSpans, out * func (c *Consumer) convertInstrumentationLibrarySpans(in pdata.InstrumentationLibrarySpans, metadata model.Metadata, out *model.Batch) { otelSpans := in.Spans() for i := 0; i < otelSpans.Len(); i++ { - c.convertSpan(otelSpans.At(i), metadata, out) + c.convertSpan(otelSpans.At(i), in.InstrumentationLibrary(), metadata, out) } } -func (c *Consumer) convertSpan(otelSpan pdata.Span, metadata model.Metadata, out *model.Batch) { +func (c *Consumer) convertSpan( + otelSpan pdata.Span, + otelLibrary pdata.InstrumentationLibrary, + metadata model.Metadata, + out *model.Batch, +) { logger := logp.NewLogger(logs.Otel) root := !otelSpan.ParentSpanID().IsValid() @@ -120,8 +141,9 @@ func (c *Consumer) convertSpan(otelSpan pdata.Span, metadata model.Metadata, out Timestamp: startTime, Duration: durationMillis, Name: name, + Outcome: spanStatusOutcome(otelSpan.Status()), } - translateTransaction(otelSpan, metadata, transaction) + translateTransaction(otelSpan, otelLibrary, metadata, &transactionBuilder{Transaction: transaction}) out.Transactions = append(out.Transactions, transaction) } else { span = &model.Span{ @@ -132,7 +154,7 @@ func (c *Consumer) convertSpan(otelSpan pdata.Span, metadata model.Metadata, out Timestamp: startTime, Duration: durationMillis, Name: name, - Outcome: outcomeUnknown, + Outcome: spanStatusOutcome(otelSpan.Status()), } translateSpan(otelSpan, metadata, span) out.Spans = append(out.Spans, span) @@ -144,16 +166,27 @@ func (c *Consumer) convertSpan(otelSpan pdata.Span, metadata model.Metadata, out } } -func translateTransaction(span pdata.Span, metadata model.Metadata, event *model.Transaction) { +func translateTransaction( + span pdata.Span, + library pdata.InstrumentationLibrary, + metadata model.Metadata, + tx *transactionBuilder, +) { isJaeger := strings.HasPrefix(metadata.Service.Agent.Name, "Jaeger") labels := make(common.MapStr) - var http model.Http - var httpStatusCode int + + var ( + netHostName string + netHostPort int + netPeerIP string + netPeerPort int + ) + var message model.Message var component string - var outcome, result string - var isHTTP, isMessaging bool + var isMessaging bool var samplerType, samplerParam pdata.AttributeValue + var httpHostName string span.Attributes().ForEach(func(kDots string, v pdata.AttributeValue) { if isJaeger { switch kDots { @@ -174,43 +207,84 @@ func translateTransaction(span pdata.Span, metadata model.Metadata, event *model labels[k] = v.DoubleVal() case pdata.AttributeValueINT: switch kDots { - case "http.status_code": - httpStatusCode = int(v.IntVal()) - isHTTP = true + case conventions.AttributeHTTPStatusCode: + tx.setHTTPStatusCode(int(v.IntVal())) + case conventions.AttributeNetPeerPort: + netPeerPort = int(v.IntVal()) + case conventions.AttributeNetHostPort: + netHostPort = int(v.IntVal()) default: labels[k] = v.IntVal() } case pdata.AttributeValueSTRING: stringval := truncate(v.StringVal()) switch kDots { - case "span.kind": // filter out - case "http.method": - http.Request = &model.Req{Method: stringval} - isHTTP = true - case "http.url", "http.path": - event.URL = model.ParseURL(stringval, metadata.System.DetectedHostname) - isHTTP = true - case "http.status_code": + // http.* + case conventions.AttributeHTTPMethod: + tx.setHTTPMethod(stringval) + case conventions.AttributeHTTPURL, conventions.AttributeHTTPTarget, "http.path": + tx.setHTTPURL(stringval) + case conventions.AttributeHTTPHost: + tx.setHTTPHost(stringval) + case conventions.AttributeHTTPScheme: + tx.setHTTPScheme(stringval) + case conventions.AttributeHTTPStatusCode: if intv, err := strconv.Atoi(stringval); err == nil { - httpStatusCode = intv + tx.setHTTPStatusCode(intv) } - isHTTP = true case "http.protocol": - if strings.HasPrefix(stringval, "HTTP/") { - version := strings.TrimPrefix(stringval, "HTTP/") - http.Version = &version - } else { + if !strings.HasPrefix(stringval, "HTTP/") { + // Unexpected, store in labels for debugging. labels[k] = stringval + break + } + stringval = strings.TrimPrefix(stringval, "HTTP/") + fallthrough + case conventions.AttributeHTTPFlavor: + tx.setHTTPVersion(stringval) + case conventions.AttributeHTTPServerName: + httpHostName = stringval + case conventions.AttributeHTTPClientIP: + tx.Metadata.Client.IP = net.ParseIP(stringval) + case conventions.AttributeHTTPUserAgent: + tx.Metadata.UserAgent.Original = stringval + case "http.remote_addr": + // NOTE(axw) this is non-standard, sent by opentelemetry-go's othttp. + // It's semanticall equivalent to net.peer.ip+port. Standard attributes + // take precedence. + ip, port, err := net.SplitHostPort(stringval) + if err != nil { + ip = stringval } - isHTTP = true + if net.ParseIP(ip) != nil { + if netPeerIP == "" { + netPeerIP = ip + } + if netPeerPort == 0 { + netPeerPort, _ = strconv.Atoi(port) + } + } + + // net.* + case conventions.AttributeNetPeerIP: + netPeerIP = stringval + case conventions.AttributeNetHostName: + netHostName = stringval + + // messaging + // + // TODO(axw) translate OpenTelemtry messaging conventions. case "message_bus.destination": message.QueueName = &stringval isMessaging = true + + // miscellaneous + case "span.kind": // filter out case "type": - event.Type = stringval - case "service.version": - event.Metadata.Service.Version = stringval - case "component": + tx.Type = stringval + case conventions.AttributeServiceVersion: + tx.Metadata.Service.Version = stringval + case conventions.AttributeComponent: component = stringval fallthrough default: @@ -219,57 +293,81 @@ func translateTransaction(span pdata.Span, metadata model.Metadata, event *model } }) - if event.Type == "" { - if isHTTP { - event.Type = "request" + if tx.Type == "" { + if tx.HTTP != nil { + tx.Type = "request" } else if isMessaging { - event.Type = "messaging" + tx.Type = "messaging" } else if component != "" { - event.Type = component + tx.Type = component } else { - event.Type = "custom" + tx.Type = "custom" } } - if isHTTP { - if httpStatusCode > 0 { - http.Response = &model.Resp{MinimalResp: model.MinimalResp{StatusCode: &httpStatusCode}} - result = statusCodeResult(httpStatusCode) - outcome = serverStatusCodeOutcome(httpStatusCode) + if tx.HTTP != nil { + // Build the model.URL from tx.http{URL,Host,Scheme}. + httpHost := tx.httpHost + if httpHost == "" { + httpHost = httpHostName + if httpHost == "" { + httpHost = netHostName + if httpHost == "" { + httpHost = metadata.System.DetectedHostname + } + } + if httpHost != "" && netHostPort > 0 { + httpHost = net.JoinHostPort(httpHost, strconv.Itoa(netHostPort)) + } + } + tx.URL = model.ParseURL(tx.httpURL, httpHost, tx.httpScheme) + + // Set the remote address from net.peer.* + if tx.HTTP.Request != nil && netPeerIP != "" { + remoteAddr := netPeerIP + if netPeerPort > 0 { + remoteAddr = net.JoinHostPort(remoteAddr, strconv.Itoa(netPeerPort)) + } + tx.setHTTPRemoteAddr(remoteAddr) } - event.HTTP = &http - } else if isMessaging { - event.Message = &message } - if result == "" { - if span.Status().Code() == pdata.StatusCodeError { - result = "Error" - outcome = outcomeFailure - } else { - result = "Success" - outcome = outcomeSuccess - } + if isMessaging { + tx.Message = &message } - event.Result = result - event.Outcome = outcome if samplerType != (pdata.AttributeValue{}) { // The client has reported its sampling rate, so we can use it to extrapolate span metrics. - parseSamplerAttributes(samplerType, samplerParam, &event.RepresentativeCount, labels) + parseSamplerAttributes(samplerType, samplerParam, &tx.RepresentativeCount, labels) } - event.Labels = labels + if tx.Result == "" { + tx.Result = spanStatusResult(span.Status()) + } + tx.setFramework(library.Name(), library.Version()) + tx.Labels = labels } func translateSpan(span pdata.Span, metadata model.Metadata, event *model.Span) { isJaeger := strings.HasPrefix(metadata.Service.Agent.Name, "Jaeger") labels := make(common.MapStr) + var ( + netPeerName string + netPeerIP string + netPeerPort int + ) + + var ( + httpURL string + httpHost string + httpTarget string + httpScheme string = "http" + ) + var http model.HTTP var message model.Message var db model.DB - var destination model.Destination var destinationService model.DestinationService var isDBSpan, isHTTPSpan, isMessagingSpan bool var component string @@ -298,41 +396,56 @@ func translateSpan(span pdata.Span, metadata model.Metadata, event *model.Span) code := int(v.IntVal()) http.StatusCode = &code isHTTPSpan = true - case "peer.port": - port := int(v.IntVal()) - destination.Port = &port + case conventions.AttributeNetPeerPort, "peer.port": + netPeerPort = int(v.IntVal()) default: labels[k] = v.IntVal() } case pdata.AttributeValueSTRING: stringval := truncate(v.StringVal()) switch kDots { - case "span.kind": // filter out - case "http.url": - http.URL = &stringval + // http.* + case conventions.AttributeHTTPHost: + httpHost = stringval + isHTTPSpan = true + case conventions.AttributeHTTPScheme: + httpScheme = stringval + isHTTPSpan = true + case conventions.AttributeHTTPTarget: + httpTarget = stringval isHTTPSpan = true - case "http.method": + case conventions.AttributeHTTPURL: + httpURL = stringval + isHTTPSpan = true + case conventions.AttributeHTTPMethod: http.Method = &stringval isHTTPSpan = true + + // db.* case "sql.query": - db.Statement = &stringval if db.Type == nil { dbType := "sql" db.Type = &dbType } - isDBSpan = true - case "db.statement": + fallthrough + case conventions.AttributeDBStatement: db.Statement = &stringval isDBSpan = true - case "db.instance": + case conventions.AttributeDBName, "db.instance": db.Instance = &stringval isDBSpan = true - case "db.type": + case conventions.AttributeDBSystem, "db.type": db.Type = &stringval isDBSpan = true - case "db.user": + case conventions.AttributeDBUser: db.UserName = &stringval isDBSpan = true + + // net.* + case conventions.AttributeNetPeerName, "peer.hostname": + netPeerName = stringval + case conventions.AttributeNetPeerIP, "peer.ipv4", "peer.ipv6": + netPeerIP = stringval case "peer.address": destinationService.Resource = &stringval if !strings.ContainsRune(stringval, ':') || net.ParseIP(stringval) != nil { @@ -340,20 +453,25 @@ func translateSpan(span pdata.Span, metadata model.Metadata, event *model.Span) // or IP address; it could be something like // a JDBC connection string or ip:port. Ignore // values containing colons, except for IPv6. - destination.Address = &stringval + netPeerName = stringval } - case "peer.hostname", "peer.ipv4", "peer.ipv6": - destination.Address = &stringval - case "peer.service": + + // messaging + // + // TODO(axw) translate OpenTelemtry messaging conventions. + case "message_bus.destination": + message.QueueName = &stringval + isMessagingSpan = true + + // miscellaneous + case "span.kind": // filter out + case conventions.AttributePeerService: destinationService.Name = &stringval if destinationService.Resource == nil { // Prefer using peer.address for resource. destinationService.Resource = &stringval } - case "message_bus.destination": - message.QueueName = &stringval - isMessagingSpan = true - case "component": + case conventions.AttributeComponent: component = stringval fallthrough default: @@ -362,8 +480,34 @@ func translateSpan(span pdata.Span, metadata model.Metadata, event *model.Span) } }) - if http.URL != nil { - if fullURL, err := url.Parse(*http.URL); err == nil { + destPort := netPeerPort + destAddr := netPeerName + if destAddr == "" { + destAddr = netPeerIP + } + + if isHTTPSpan { + var fullURL *url.URL + if httpURL != "" { + fullURL, _ = url.Parse(httpURL) + } else if httpTarget != "" { + // Build http.url from http.scheme, http.target, etc. + if u, err := url.Parse(httpTarget); err == nil { + fullURL = u + fullURL.Scheme = httpScheme + if httpHost == "" { + // Set host from net.peer.* + httpHost = destAddr + if netPeerPort > 0 { + httpHost = net.JoinHostPort(httpHost, strconv.Itoa(destPort)) + } + } + fullURL.Host = httpHost + httpURL = fullURL.String() + } + } + if fullURL != nil { + http.URL = &httpURL url := url.URL{Scheme: fullURL.Scheme, Host: fullURL.Host} hostname := truncate(url.Hostname()) var port int @@ -376,13 +520,12 @@ func translateSpan(span pdata.Span, metadata model.Metadata, event *model.Span) // Set destination.{address,port} from the HTTP URL, // replacing peer.* based values to ensure consistency. - destination = model.Destination{Address: &hostname} + destAddr = hostname if port > 0 { - destination.Port = &port + destPort = port } - // Set destination.service.* from the HTTP URL, - // unless peer.service was specified. + // Set destination.service.* from the HTTP URL, unless peer.service was specified. if destinationService.Name == nil { resource := url.Host if port > 0 && port == schemeDefaultPort(url.Scheme) { @@ -402,14 +545,12 @@ func translateSpan(span pdata.Span, metadata model.Metadata, event *model.Span) } } - if destination != (model.Destination{}) { - event.Destination = &destination - } - switch { case isHTTPSpan: if http.StatusCode != nil { - event.Outcome = clientStatusCodeOutcome(*http.StatusCode) + if event.Outcome == outcomeUnknown { + event.Outcome = clientHTTPStatusCodeOutcome(*http.StatusCode) + } } event.Type = "external" subtype := "http" @@ -419,18 +560,30 @@ func translateSpan(span pdata.Span, metadata model.Metadata, event *model.Span) event.Type = "db" if db.Type != nil && *db.Type != "" { event.Subtype = db.Type + if destinationService.Name == nil { + // For database requests, we currently just identify + // the destination service by db.system. + destinationService.Name = event.Subtype + destinationService.Resource = event.Subtype + } } event.DB = &db case isMessagingSpan: event.Type = "messaging" event.Message = &message default: - event.Type = "custom" + event.Type = "app" if component != "" { event.Subtype = &component } } + if destAddr != "" { + event.Destination = &model.Destination{Address: &destAddr} + if destPort > 0 { + event.Destination.Port = &destPort + } + } if destinationService != (model.DestinationService{}) { if destinationService.Type == nil { // Copy span type to destination.service.type. @@ -479,6 +632,7 @@ func convertSpanEvent( isJaeger := strings.HasPrefix(metadata.Service.Agent.Name, "Jaeger") event.Attributes().ForEach(func(k string, v pdata.AttributeValue) { if !isJaeger { + // TODO(axw) translate OpenTelemetry exception span events. return } if v.Type() != pdata.AttributeValueSTRING { @@ -565,7 +719,7 @@ func addSpanCtxToErr(span *model.Span, hostname string, err *model.Error) { err.HTTP.Request = &model.Req{Method: *span.HTTP.Method} } if span.HTTP.URL != nil { - err.URL = model.ParseURL(*span.HTTP.URL, hostname) + err.URL = model.ParseURL(*span.HTTP.URL, hostname, "") } } } @@ -574,35 +728,34 @@ func replaceDots(s string) string { return strings.ReplaceAll(s, dot, underscore) } -// copied from elastic go-apm agent - -var standardStatusCodeResults = [...]string{ - "HTTP 1xx", - "HTTP 2xx", - "HTTP 3xx", - "HTTP 4xx", - "HTTP 5xx", -} - -// statusCodeResult returns the transaction result value to use for the given status code. -func statusCodeResult(statusCode int) string { - switch i := statusCode / 100; i { - case 1, 2, 3, 4, 5: - return standardStatusCodeResults[i-1] +// spanStatusOutcome returns the outcome for transactions and spans based on +// the given OTLP span status. +func spanStatusOutcome(status pdata.SpanStatus) string { + switch status.Code() { + case pdata.StatusCodeOk: + return outcomeSuccess + case pdata.StatusCodeError: + return outcomeFailure } - return fmt.Sprintf("HTTP %d", statusCode) + return outcomeUnknown } -// serverStatusCodeOutcome returns the transaction outcome value to use for the given status code. -func serverStatusCodeOutcome(statusCode int) string { - if statusCode >= 500 { - return outcomeFailure +// spanStatusResult returns the result for transactions based on the given +// OTLP span status. If the span status is unknown, an empty result string +// is returned. +func spanStatusResult(status pdata.SpanStatus) string { + switch status.Code() { + case pdata.StatusCodeOk: + return "Success" + case pdata.StatusCodeError: + return "Error" } - return outcomeSuccess + return "" } -// clientStatusCodeOutcome returns the span outcome value to use for the given status code. -func clientStatusCodeOutcome(statusCode int) string { +// clientHTTPStatusCodeOutcome returns the span outcome value to use for the +// given HTTP status code. +func clientHTTPStatusCodeOutcome(statusCode int) string { if statusCode >= 400 { return outcomeFailure } diff --git a/processor/otel/consumer_test.go b/processor/otel/consumer_test.go index d64b613c971..f4c26bc76a1 100644 --- a/processor/otel/consumer_test.go +++ b/processor/otel/consumer_test.go @@ -15,11 +15,29 @@ // specific language governing permissions and limitations // under the License. +// Portions copied from OpenTelemetry Collector (contrib), from the +// elastic exporter. +// +// Copyright 2020, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package otel import ( "context" "fmt" + "net" "path/filepath" "testing" "time" @@ -31,6 +49,7 @@ import ( jaegertranslator "go.opentelemetry.io/collector/translator/trace/jaeger" "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/apm-server/approvaltest" "github.com/elastic/apm-server/beater/beatertest" @@ -50,6 +69,360 @@ func TestConsumer_ConsumeTraces_Empty(t *testing.T) { assert.NoError(t, consumer.ConsumeTraces(context.Background(), traces)) } +func TestHTTPTransactionURL(t *testing.T) { + test := func(t *testing.T, expected *model.URL, attrs map[string]pdata.AttributeValue) { + t.Helper() + tx := transformTransactionWithAttributes(t, attrs) + assert.Equal(t, expected, tx.URL) + } + + t.Run("scheme_host_target", func(t *testing.T) { + test(t, &model.URL{ + Scheme: newString("https"), + Original: newString("/foo?bar"), + Full: newString("https://testing.invalid:80/foo?bar"), + Path: newString("/foo"), + Query: newString("bar"), + Domain: newString("testing.invalid"), + Port: newInt(80), + }, map[string]pdata.AttributeValue{ + "http.scheme": pdata.NewAttributeValueString("https"), + "http.host": pdata.NewAttributeValueString("testing.invalid:80"), + "http.target": pdata.NewAttributeValueString("/foo?bar"), + }) + }) + t.Run("scheme_servername_nethostport_target", func(t *testing.T) { + test(t, &model.URL{ + Scheme: newString("https"), + Original: newString("/foo?bar"), + Full: newString("https://testing.invalid:80/foo?bar"), + Path: newString("/foo"), + Query: newString("bar"), + Domain: newString("testing.invalid"), + Port: newInt(80), + }, map[string]pdata.AttributeValue{ + "http.scheme": pdata.NewAttributeValueString("https"), + "http.server_name": pdata.NewAttributeValueString("testing.invalid"), + "net.host.port": pdata.NewAttributeValueInt(80), + "http.target": pdata.NewAttributeValueString("/foo?bar"), + }) + }) + t.Run("scheme_nethostname_nethostport_target", func(t *testing.T) { + test(t, &model.URL{ + Scheme: newString("https"), + Original: newString("/foo?bar"), + Full: newString("https://testing.invalid:80/foo?bar"), + Path: newString("/foo"), + Query: newString("bar"), + Domain: newString("testing.invalid"), + Port: newInt(80), + }, map[string]pdata.AttributeValue{ + "http.scheme": pdata.NewAttributeValueString("https"), + "net.host.name": pdata.NewAttributeValueString("testing.invalid"), + "net.host.port": pdata.NewAttributeValueInt(80), + "http.target": pdata.NewAttributeValueString("/foo?bar"), + }) + }) + t.Run("http.url", func(t *testing.T) { + test(t, &model.URL{ + Scheme: newString("https"), + Original: newString("https://testing.invalid:80/foo?bar"), + Full: newString("https://testing.invalid:80/foo?bar"), + Path: newString("/foo"), + Query: newString("bar"), + Domain: newString("testing.invalid"), + Port: newInt(80), + }, map[string]pdata.AttributeValue{ + "http.url": pdata.NewAttributeValueString("https://testing.invalid:80/foo?bar"), + }) + }) + t.Run("host_no_port", func(t *testing.T) { + test(t, &model.URL{ + Scheme: newString("https"), + Original: newString("/foo"), + Full: newString("https://testing.invalid/foo"), + Path: newString("/foo"), + Domain: newString("testing.invalid"), + }, map[string]pdata.AttributeValue{ + "http.scheme": pdata.NewAttributeValueString("https"), + "http.host": pdata.NewAttributeValueString("testing.invalid"), + "http.target": pdata.NewAttributeValueString("/foo"), + }) + }) + t.Run("ipv6_host_no_port", func(t *testing.T) { + test(t, &model.URL{ + Scheme: newString("https"), + Original: newString("/foo"), + Full: newString("https://[::1]/foo"), + Path: newString("/foo"), + Domain: newString("::1"), + }, map[string]pdata.AttributeValue{ + "http.scheme": pdata.NewAttributeValueString("https"), + "http.host": pdata.NewAttributeValueString("[::1]"), + "http.target": pdata.NewAttributeValueString("/foo"), + }) + }) + t.Run("default_scheme", func(t *testing.T) { + // scheme is set to "http" if it can't be deduced from attributes. + test(t, &model.URL{ + Scheme: newString("http"), + Original: newString("/foo"), + Full: newString("http://testing.invalid/foo"), + Path: newString("/foo"), + Domain: newString("testing.invalid"), + }, map[string]pdata.AttributeValue{ + "http.host": pdata.NewAttributeValueString("testing.invalid"), + "http.target": pdata.NewAttributeValueString("/foo"), + }) + }) +} + +func TestHTTPSpanURL(t *testing.T) { + test := func(t *testing.T, expected string, attrs map[string]pdata.AttributeValue) { + t.Helper() + span := transformSpanWithAttributes(t, attrs) + require.NotNil(t, span.HTTP) + require.NotNil(t, span.HTTP.URL) + assert.Equal(t, expected, *span.HTTP.URL) + } + + t.Run("host.url", func(t *testing.T) { + test(t, "https://testing.invalid:80/foo?bar", map[string]pdata.AttributeValue{ + "http.url": pdata.NewAttributeValueString("https://testing.invalid:80/foo?bar"), + }) + }) + t.Run("scheme_host_target", func(t *testing.T) { + test(t, "https://testing.invalid:80/foo?bar", map[string]pdata.AttributeValue{ + "http.scheme": pdata.NewAttributeValueString("https"), + "http.host": pdata.NewAttributeValueString("testing.invalid:80"), + "http.target": pdata.NewAttributeValueString("/foo?bar"), + }) + }) + t.Run("scheme_netpeername_netpeerport_target", func(t *testing.T) { + test(t, "https://testing.invalid:80/foo?bar", map[string]pdata.AttributeValue{ + "http.scheme": pdata.NewAttributeValueString("https"), + "net.peer.name": pdata.NewAttributeValueString("testing.invalid"), + "net.peer.ip": pdata.NewAttributeValueString("::1"), // net.peer.name preferred + "net.peer.port": pdata.NewAttributeValueInt(80), + "http.target": pdata.NewAttributeValueString("/foo?bar"), + }) + }) + t.Run("scheme_netpeerip_netpeerport_target", func(t *testing.T) { + test(t, "https://[::1]:80/foo?bar", map[string]pdata.AttributeValue{ + "http.scheme": pdata.NewAttributeValueString("https"), + "net.peer.ip": pdata.NewAttributeValueString("::1"), + "net.peer.port": pdata.NewAttributeValueInt(80), + "http.target": pdata.NewAttributeValueString("/foo?bar"), + }) + }) + t.Run("default_scheme", func(t *testing.T) { + // scheme is set to "http" if it can't be deduced from attributes. + test(t, "http://testing.invalid/foo", map[string]pdata.AttributeValue{ + "http.host": pdata.NewAttributeValueString("testing.invalid"), + "http.target": pdata.NewAttributeValueString("/foo"), + }) + }) +} + +func TestHTTPSpanDestination(t *testing.T) { + test := func(t *testing.T, expectedDestination *model.Destination, expectedDestinationService *model.DestinationService, attrs map[string]pdata.AttributeValue) { + t.Helper() + span := transformSpanWithAttributes(t, attrs) + assert.Equal(t, expectedDestination, span.Destination) + assert.Equal(t, expectedDestinationService, span.DestinationService) + } + + t.Run("url_default_port_specified", func(t *testing.T) { + test(t, &model.Destination{ + Address: newString("testing.invalid"), + Port: newInt(443), + }, &model.DestinationService{ + Type: newString("external"), + Name: newString("https://testing.invalid"), + Resource: newString("testing.invalid:443"), + }, map[string]pdata.AttributeValue{ + "http.url": pdata.NewAttributeValueString("https://testing.invalid:443/foo?bar"), + }) + }) + t.Run("url_port_scheme", func(t *testing.T) { + test(t, &model.Destination{ + Address: newString("testing.invalid"), + Port: newInt(443), + }, &model.DestinationService{ + Type: newString("external"), + Name: newString("https://testing.invalid"), + Resource: newString("testing.invalid:443"), + }, map[string]pdata.AttributeValue{ + "http.url": pdata.NewAttributeValueString("https://testing.invalid/foo?bar"), + }) + }) + t.Run("url_non_default_port", func(t *testing.T) { + test(t, &model.Destination{ + Address: newString("testing.invalid"), + Port: newInt(444), + }, &model.DestinationService{ + Type: newString("external"), + Name: newString("https://testing.invalid:444"), + Resource: newString("testing.invalid:444"), + }, map[string]pdata.AttributeValue{ + "http.url": pdata.NewAttributeValueString("https://testing.invalid:444/foo?bar"), + }) + }) + t.Run("scheme_host_target", func(t *testing.T) { + test(t, &model.Destination{ + Address: newString("testing.invalid"), + Port: newInt(444), + }, &model.DestinationService{ + Type: newString("external"), + Name: newString("https://testing.invalid:444"), + Resource: newString("testing.invalid:444"), + }, map[string]pdata.AttributeValue{ + "http.scheme": pdata.NewAttributeValueString("https"), + "http.host": pdata.NewAttributeValueString("testing.invalid:444"), + "http.target": pdata.NewAttributeValueString("/foo?bar"), + }) + }) + t.Run("scheme_netpeername_nethostport_target", func(t *testing.T) { + test(t, &model.Destination{ + Address: newString("::1"), + Port: newInt(444), + }, &model.DestinationService{ + Type: newString("external"), + Name: newString("https://[::1]:444"), + Resource: newString("[::1]:444"), + }, map[string]pdata.AttributeValue{ + "http.scheme": pdata.NewAttributeValueString("https"), + "net.peer.ip": pdata.NewAttributeValueString("::1"), + "net.peer.port": pdata.NewAttributeValueInt(444), + "http.target": pdata.NewAttributeValueString("/foo?bar"), + }) + }) +} + +func TestHTTPTransactionRequestSocketRemoteAddr(t *testing.T) { + test := func(t *testing.T, expected string, attrs map[string]pdata.AttributeValue) { + // "http.method" is a required attribute for HTTP spans, + // and its presence causes the transaction's HTTP request + // context to be built. + attrs["http.method"] = pdata.NewAttributeValueString("POST") + + tx := transformTransactionWithAttributes(t, attrs) + require.NotNil(t, tx.HTTP) + require.NotNil(t, tx.HTTP.Request) + require.NotNil(t, tx.HTTP.Request.Socket) + assert.Equal(t, &expected, tx.HTTP.Request.Socket.RemoteAddress) + } + + t.Run("net.peer.ip_port", func(t *testing.T) { + test(t, "192.168.0.1:1234", map[string]pdata.AttributeValue{ + "net.peer.ip": pdata.NewAttributeValueString("192.168.0.1"), + "net.peer.port": pdata.NewAttributeValueInt(1234), + }) + }) + t.Run("net.peer.ip", func(t *testing.T) { + test(t, "192.168.0.1", map[string]pdata.AttributeValue{ + "net.peer.ip": pdata.NewAttributeValueString("192.168.0.1"), + }) + }) + t.Run("http.remote_addr", func(t *testing.T) { + test(t, "192.168.0.1:1234", map[string]pdata.AttributeValue{ + "http.remote_addr": pdata.NewAttributeValueString("192.168.0.1:1234"), + }) + }) + t.Run("http.remote_addr_no_port", func(t *testing.T) { + test(t, "192.168.0.1", map[string]pdata.AttributeValue{ + "http.remote_addr": pdata.NewAttributeValueString("192.168.0.1"), + }) + }) +} + +func TestHTTPTransactionFlavor(t *testing.T) { + tx := transformTransactionWithAttributes(t, map[string]pdata.AttributeValue{ + "http.flavor": pdata.NewAttributeValueString("1.1"), + }) + assert.Equal(t, newString("1.1"), tx.HTTP.Version) +} + +func TestHTTPTransactionUserAgent(t *testing.T) { + tx := transformTransactionWithAttributes(t, map[string]pdata.AttributeValue{ + "http.user_agent": pdata.NewAttributeValueString("Foo/bar (baz)"), + }) + assert.Equal(t, model.UserAgent{Original: "Foo/bar (baz)"}, tx.Metadata.UserAgent) +} + +func TestHTTPTransactionClientIP(t *testing.T) { + tx := transformTransactionWithAttributes(t, map[string]pdata.AttributeValue{ + "http.client_ip": pdata.NewAttributeValueString("256.257.258.259"), + }) + assert.Equal(t, net.ParseIP("256.257.258.259"), tx.Metadata.Client.IP) +} + +func TestHTTPTransactionStatusCode(t *testing.T) { + tx := transformTransactionWithAttributes(t, map[string]pdata.AttributeValue{ + "http.status_code": pdata.NewAttributeValueInt(200), + }) + assert.Equal(t, newInt(200), tx.HTTP.Response.StatusCode) +} + +func TestDatabaseSpan(t *testing.T) { + // https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/trace/semantic_conventions/database.md#mysql + connectionString := "Server=shopdb.example.com;Database=ShopDb;Uid=billing_user;TableCache=true;UseCompression=True;MinimumPoolSize=10;MaximumPoolSize=50;" + span := transformSpanWithAttributes(t, map[string]pdata.AttributeValue{ + "db.system": pdata.NewAttributeValueString("mysql"), + "db.connection_string": pdata.NewAttributeValueString(connectionString), + "db.user": pdata.NewAttributeValueString("billing_user"), + "db.name": pdata.NewAttributeValueString("ShopDb"), + "db.statement": pdata.NewAttributeValueString("SELECT * FROM orders WHERE order_id = 'o4711'"), + "net.peer.name": pdata.NewAttributeValueString("shopdb.example.com"), + "net.peer.ip": pdata.NewAttributeValueString("192.0.2.12"), + "net.peer.port": pdata.NewAttributeValueInt(3306), + "net.transport": pdata.NewAttributeValueString("IP.TCP"), + }) + + assert.Equal(t, "db", span.Type) + assert.Equal(t, newString("mysql"), span.Subtype) + assert.Nil(t, span.Action) + + assert.Equal(t, &model.DB{ + Instance: newString("ShopDb"), + Statement: newString("SELECT * FROM orders WHERE order_id = 'o4711'"), + Type: newString("mysql"), + UserName: newString("billing_user"), + }, span.DB) + + assert.Equal(t, common.MapStr{ + "db_connection_string": connectionString, + "net_transport": "IP.TCP", + }, span.Labels) + + assert.Equal(t, &model.Destination{ + Address: newString("shopdb.example.com"), + Port: newInt(3306), + }, span.Destination) + + assert.Equal(t, &model.DestinationService{ + Type: newString("db"), + Name: newString("mysql"), + Resource: newString("mysql"), + }, span.DestinationService) +} + +func TestInstrumentationLibrary(t *testing.T) { + traces, spans := newTracesSpans() + spans.InstrumentationLibrary().SetName("library-name") + spans.InstrumentationLibrary().SetVersion("1.2.3") + otelSpan := pdata.NewSpan() + otelSpan.SetTraceID(pdata.NewTraceID([16]byte{1})) + otelSpan.SetSpanID(pdata.NewSpanID([8]byte{2})) + spans.Spans().Append(otelSpan) + events := transformTraces(t, traces) + require.Len(t, events, 1) + tx := events[0].(*model.Transaction) + + assert.Equal(t, "library-name", tx.Metadata.Service.Framework.Name) + assert.Equal(t, "1.2.3", tx.Metadata.Service.Framework.Version) +} + func TestConsumer_JaegerMetadata(t *testing.T) { jaegerBatch := jaegermodel.Batch{ Spans: []*jaegermodel.Span{{ @@ -538,3 +911,55 @@ func jaegerKeyValue(k string, v interface{}) jaegermodel.KeyValue { } return kv } + +func transformTransactionWithAttributes(t *testing.T, attrs map[string]pdata.AttributeValue) *model.Transaction { + traces, spans := newTracesSpans() + otelSpan := pdata.NewSpan() + otelSpan.SetTraceID(pdata.NewTraceID([16]byte{1})) + otelSpan.SetSpanID(pdata.NewSpanID([8]byte{2})) + otelSpan.Attributes().InitFromMap(attrs) + spans.Spans().Append(otelSpan) + events := transformTraces(t, traces) + require.Len(t, events, 1) + return events[0].(*model.Transaction) +} + +func transformSpanWithAttributes(t *testing.T, attrs map[string]pdata.AttributeValue) *model.Span { + traces, spans := newTracesSpans() + otelSpan := pdata.NewSpan() + otelSpan.SetTraceID(pdata.NewTraceID([16]byte{1})) + otelSpan.SetSpanID(pdata.NewSpanID([8]byte{2})) + otelSpan.SetParentSpanID(pdata.NewSpanID([8]byte{3})) + otelSpan.Attributes().InitFromMap(attrs) + spans.Spans().Append(otelSpan) + events := transformTraces(t, traces) + require.Len(t, events, 1) + return events[0].(*model.Span) +} + +func transformTraces(t *testing.T, traces pdata.Traces) []transform.Transformable { + var events []transform.Transformable + reporter := func(ctx context.Context, req publish.PendingReq) error { + events = append(events, req.Transformables...) + return nil + } + require.NoError(t, (&Consumer{Reporter: reporter}).ConsumeTraces(context.Background(), traces)) + return events +} + +func newTracesSpans() (pdata.Traces, pdata.InstrumentationLibrarySpans) { + traces := pdata.NewTraces() + resourceSpans := pdata.NewResourceSpans() + librarySpans := pdata.NewInstrumentationLibrarySpans() + resourceSpans.InstrumentationLibrarySpans().Append(librarySpans) + traces.ResourceSpans().Append(resourceSpans) + return traces, librarySpans +} + +func newString(s string) *string { + return &s +} + +func newInt(v int) *int { + return &v +} diff --git a/processor/otel/metadata.go b/processor/otel/metadata.go index 2949e178bf1..8eb506b6588 100644 --- a/processor/otel/metadata.go +++ b/processor/otel/metadata.go @@ -20,6 +20,7 @@ package otel import ( "fmt" "net" + "regexp" "strings" "go.opentelemetry.io/collector/consumer/pdata" @@ -29,12 +30,20 @@ import ( "github.com/elastic/beats/v7/libbeat/common" ) +const ( + AgentNameJaeger = "Jaeger" +) + +var ( + serviceNameInvalidRegexp = regexp.MustCompile("[^a-zA-Z0-9 _-]") +) + func translateResourceMetadata(resource pdata.Resource, out *model.Metadata) { var exporterVersion string resource.Attributes().ForEach(func(k string, v pdata.AttributeValue) { switch k { case conventions.AttributeServiceName: - out.Service.Name = truncate(v.StringVal()) + out.Service.Name = cleanServiceName(v.StringVal()) case conventions.AttributeServiceVersion: out.Service.Version = truncate(v.StringVal()) case conventions.AttributeServiceInstance: @@ -83,7 +92,7 @@ func translateResourceMetadata(resource pdata.Resource, out *model.Metadata) { if v := versionParts[len(versionParts)-1]; v != "" { out.Service.Agent.Version = v } - out.Service.Agent.Name = "Jaeger" + out.Service.Agent.Name = AgentNameJaeger // Translate known Jaeger labels. if clientUUID, ok := out.Labels["client-uuid"].(string); ok { @@ -115,6 +124,10 @@ func translateResourceMetadata(resource pdata.Resource, out *model.Metadata) { } } +func cleanServiceName(name string) string { + return serviceNameInvalidRegexp.ReplaceAllString(truncate(name), "_") +} + func ifaceAttributeValue(v pdata.AttributeValue) interface{} { switch v.Type() { case pdata.AttributeValueSTRING: diff --git a/processor/otel/test_approved/jaeger_sampling_rate.approved.json b/processor/otel/test_approved/jaeger_sampling_rate.approved.json index ba3c06f4fa0..28bfb67a1b4 100644 --- a/processor/otel/test_approved/jaeger_sampling_rate.approved.json +++ b/processor/otel/test_approved/jaeger_sampling_rate.approved.json @@ -9,7 +9,7 @@ "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { - "outcome": "success" + "outcome": "unknown" }, "host": { "hostname": "host-abc", @@ -36,7 +36,6 @@ "us": 79000000 }, "id": "", - "result": "Success", "sampled": true, "type": "custom" } @@ -50,7 +49,7 @@ "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { - "outcome": "success" + "outcome": "unknown" }, "host": { "hostname": "host-abc", @@ -81,7 +80,6 @@ "us": 79000000 }, "id": "", - "result": "Success", "sampled": true, "type": "custom" } @@ -122,7 +120,7 @@ "us": 79000000 }, "name": "", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576500418000768 diff --git a/processor/otel/test_approved/metadata_jaeger-no-language.approved.json b/processor/otel/test_approved/metadata_jaeger-no-language.approved.json index 966a2ffbbc3..0dc34aa4121 100644 --- a/processor/otel/test_approved/metadata_jaeger-no-language.approved.json +++ b/processor/otel/test_approved/metadata_jaeger-no-language.approved.json @@ -9,7 +9,7 @@ "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { - "outcome": "success" + "outcome": "unknown" }, "processor": { "event": "transaction", @@ -32,7 +32,6 @@ "us": 0 }, "id": "0000000041414646", - "result": "Success", "sampled": true, "type": "custom" } diff --git a/processor/otel/test_approved/metadata_jaeger-version.approved.json b/processor/otel/test_approved/metadata_jaeger-version.approved.json index f04dd014c9e..a648c2fad35 100644 --- a/processor/otel/test_approved/metadata_jaeger-version.approved.json +++ b/processor/otel/test_approved/metadata_jaeger-version.approved.json @@ -9,7 +9,7 @@ "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { - "outcome": "success" + "outcome": "unknown" }, "processor": { "event": "transaction", @@ -32,7 +32,6 @@ "us": 0 }, "id": "0000000041414646", - "result": "Success", "sampled": true, "type": "custom" } diff --git a/processor/otel/test_approved/metadata_jaeger.approved.json b/processor/otel/test_approved/metadata_jaeger.approved.json index 2e288ea1ab2..1a9a8aa2057 100644 --- a/processor/otel/test_approved/metadata_jaeger.approved.json +++ b/processor/otel/test_approved/metadata_jaeger.approved.json @@ -10,7 +10,7 @@ "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { - "outcome": "success" + "outcome": "unknown" }, "host": { "hostname": "host-foo", @@ -45,7 +45,6 @@ "us": 0 }, "id": "0000000041414646", - "result": "Success", "sampled": true, "type": "custom" } diff --git a/processor/otel/test_approved/span_jaeger_custom.approved.json b/processor/otel/test_approved/span_jaeger_custom.approved.json index 5288a2b88ca..5ceab2067b1 100644 --- a/processor/otel/test_approved/span_jaeger_custom.approved.json +++ b/processor/otel/test_approved/span_jaeger_custom.approved.json @@ -37,7 +37,7 @@ }, "id": "0000000041414646", "name": "", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576500418000768 diff --git a/processor/otel/test_approved/span_jaeger_subtype_component.approved.json b/processor/otel/test_approved/span_jaeger_subtype_component.approved.json index 48bd6ecef37..bab6c03e440 100644 --- a/processor/otel/test_approved/span_jaeger_subtype_component.approved.json +++ b/processor/otel/test_approved/span_jaeger_subtype_component.approved.json @@ -41,7 +41,7 @@ "id": "0000000041414646", "name": "", "subtype": "whatever", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576500418000768 diff --git a/processor/otel/test_approved/transaction_jaeger_custom.approved.json b/processor/otel/test_approved/transaction_jaeger_custom.approved.json index 3b531443a3a..5e82aabde88 100644 --- a/processor/otel/test_approved/transaction_jaeger_custom.approved.json +++ b/processor/otel/test_approved/transaction_jaeger_custom.approved.json @@ -9,7 +9,7 @@ "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { - "outcome": "success" + "outcome": "unknown" }, "host": { "hostname": "host-abc", @@ -36,7 +36,6 @@ "us": 0 }, "id": "", - "result": "Success", "sampled": true, "type": "custom" } diff --git a/processor/otel/test_approved/transaction_jaeger_full.approved.json b/processor/otel/test_approved/transaction_jaeger_full.approved.json index 8fb1dfdac67..a7299ac5dea 100644 --- a/processor/otel/test_approved/transaction_jaeger_full.approved.json +++ b/processor/otel/test_approved/transaction_jaeger_full.approved.json @@ -9,7 +9,7 @@ "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { - "outcome": "success" + "outcome": "failure" }, "host": { "hostname": "host-abc", diff --git a/processor/otel/test_approved/transaction_jaeger_type_component.approved.json b/processor/otel/test_approved/transaction_jaeger_type_component.approved.json index 9ae85c5d296..def3ffc0f73 100644 --- a/processor/otel/test_approved/transaction_jaeger_type_component.approved.json +++ b/processor/otel/test_approved/transaction_jaeger_type_component.approved.json @@ -9,7 +9,7 @@ "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { - "outcome": "success" + "outcome": "unknown" }, "host": { "hostname": "host-abc", @@ -36,7 +36,6 @@ "us": 0 }, "id": "", - "result": "Success", "sampled": true, "type": "amqp" } diff --git a/processor/otel/test_approved/transaction_jaeger_type_messaging.approved.json b/processor/otel/test_approved/transaction_jaeger_type_messaging.approved.json index 1fe49b9bcd4..5867df8200e 100644 --- a/processor/otel/test_approved/transaction_jaeger_type_messaging.approved.json +++ b/processor/otel/test_approved/transaction_jaeger_type_messaging.approved.json @@ -9,7 +9,7 @@ "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { - "outcome": "success" + "outcome": "unknown" }, "host": { "hostname": "host-abc", @@ -44,7 +44,6 @@ "name": "queue-abc" } }, - "result": "Success", "sampled": true, "type": "messaging" } diff --git a/publish/pub.go b/publish/pub.go index 3d63ff69e4a..4251141405d 100644 --- a/publish/pub.go +++ b/publish/pub.go @@ -89,17 +89,19 @@ func NewPublisher(pipeline beat.Pipeline, tracer *apm.Tracer, cfg *PublisherConf return nil, errors.Wrap(err, "invalid config") } + observerFields := common.MapStr{ + "type": cfg.Info.Beat, + "hostname": cfg.Info.Hostname, + "version": cfg.Info.Version, + "id": cfg.Info.ID.String(), + "ephemeral_id": cfg.Info.EphemeralID.String(), + } + if version, err := common.NewVersion(cfg.Info.Version); err == nil { + observerFields["version_major"] = version.Major + } + processingCfg := beat.ProcessingConfig{ - Fields: common.MapStr{ - "observer": common.MapStr{ - "type": cfg.Info.Beat, - "hostname": cfg.Info.Hostname, - "version": cfg.Info.Version, - "version_major": 8, - "id": cfg.Info.ID.String(), - "ephemeral_id": cfg.Info.EphemeralID.String(), - }, - }, + Fields: common.MapStr{"observer": observerFields}, Processor: cfg.Processor, } if cfg.TransformConfig.DataStreams { diff --git a/systemtest/approvals/TestOTLPGRPC.approved.json b/systemtest/approvals/TestOTLPGRPC.approved.json index b6a6490b5a0..ffc13186fc1 100644 --- a/systemtest/approvals/TestOTLPGRPC.approved.json +++ b/systemtest/approvals/TestOTLPGRPC.approved.json @@ -26,6 +26,9 @@ "name": "transaction" }, "service": { + "framework": { + "name": "systemtest" + }, "language": { "name": "unknown" }, diff --git a/systemtest/elasticsearch.go b/systemtest/elasticsearch.go index 161765e517d..a7122545a24 100644 --- a/systemtest/elasticsearch.go +++ b/systemtest/elasticsearch.go @@ -91,7 +91,7 @@ func newElasticsearchConfig() elasticsearch.Config { // and deletes the default ILM policy "apm-rollover-30-days". func CleanupElasticsearch(t testing.TB) { const ( - legacyPrefix = "apm*" + legacyPrefix = "apm-*" apmTracesPrefix = "traces-apm*" apmMetricsPrefix = "metrics-apm*" apmLogsPrefix = "logs-apm*" @@ -106,6 +106,7 @@ func CleanupElasticsearch(t testing.TB) { } doParallel := func(requests ...estest.Request) { + t.Helper() var g errgroup.Group for _, req := range requests { req := req // copy for closure @@ -115,18 +116,20 @@ func CleanupElasticsearch(t testing.TB) { t.Fatal(err) } } + + // Delete indices, data streams, and ingest pipelines. + doReq(esapi.IndicesDeleteRequest{Index: []string{legacyPrefix}}) doParallel( - esapi.IndicesDeleteRequest{Index: []string{legacyPrefix}}, + esapi.IndicesDeleteDataStreamRequest{Name: legacyPrefix}, esapi.IndicesDeleteDataStreamRequest{Name: apmTracesPrefix}, esapi.IndicesDeleteDataStreamRequest{Name: apmMetricsPrefix}, esapi.IndicesDeleteDataStreamRequest{Name: apmLogsPrefix}, esapi.IngestDeletePipelineRequest{PipelineID: legacyPrefix}, - esapi.IndicesDeleteTemplateRequest{Name: legacyPrefix}, ) // Delete index templates after deleting data streams. doParallel( - esapi.IndicesDeleteIndexTemplateRequest{Name: legacyPrefix}, // for index template created by tests + esapi.IndicesDeleteTemplateRequest{Name: legacyPrefix}, esapi.IndicesDeleteIndexTemplateRequest{Name: apmTracesPrefix}, esapi.IndicesDeleteIndexTemplateRequest{Name: apmMetricsPrefix}, esapi.IndicesDeleteIndexTemplateRequest{Name: apmLogsPrefix}, diff --git a/systemtest/estest/search.go b/systemtest/estest/search.go index 169aba00b96..5f40ec44ef3 100644 --- a/systemtest/estest/search.go +++ b/systemtest/estest/search.go @@ -71,6 +71,11 @@ func (r *SearchRequest) WithQuery(q interface{}) *SearchRequest { return r } +func (r *SearchRequest) WithSort(fieldDirection ...string) *SearchRequest { + r.Sort = fieldDirection + return r +} + func (r *SearchRequest) WithSize(size int) *SearchRequest { r.Size = &size return r diff --git a/systemtest/monitoring_test.go b/systemtest/monitoring_test.go index db1eecdb369..9b6dcc490d0 100644 --- a/systemtest/monitoring_test.go +++ b/systemtest/monitoring_test.go @@ -18,6 +18,7 @@ package systemtest_test import ( + "context" "encoding/json" "testing" "time" @@ -83,9 +84,13 @@ func getBeatsMonitoringStats(t testing.TB, srv *apmservertest.Server, out interf } func getBeatsMonitoring(t testing.TB, srv *apmservertest.Server, type_ string, out interface{}) *beatsMonitoringDoc { - result := systemtest.Elasticsearch.ExpectDocs(t, ".monitoring-beats-*", + var result estest.SearchResult + req := systemtest.Elasticsearch.Search(".monitoring-beats-*").WithQuery( estest.TermQuery{Field: type_ + ".beat.uuid", Value: srv.BeatUUID}, - ) + ).WithSort("timestamp:desc") + if _, err := req.Do(context.Background(), &result, estest.WithCondition(result.Hits.MinHitsCondition(1))); err != nil { + t.Error(err) + } var doc beatsMonitoringDoc doc.RawSource = []byte(result.Hits.Hits[0].RawSource) diff --git a/systemtest/sampling_test.go b/systemtest/sampling_test.go index 401c2f9047e..288423857c4 100644 --- a/systemtest/sampling_test.go +++ b/systemtest/sampling_test.go @@ -28,6 +28,7 @@ import ( "github.com/stretchr/testify/require" "github.com/tidwall/gjson" "go.elastic.co/apm" + "golang.org/x/sync/errgroup" "github.com/elastic/apm-server/systemtest" "github.com/elastic/apm-server/systemtest/apmservertest" @@ -88,29 +89,6 @@ func TestKeepUnsampledWarning(t *testing.T) { func TestTailSampling(t *testing.T) { systemtest.CleanupElasticsearch(t) - // Create the apm-sampled-traces index for the two servers to coordinate. - _, err := systemtest.Elasticsearch.Do(context.Background(), &esapi.IndicesCreateRequest{ - Index: "apm-sampled-traces", - Body: strings.NewReader(`{ - "mappings": { - "properties": { - "event.ingested": {"type": "date"}, - "observer": { - "properties": { - "id": {"type": "keyword"} - } - }, - "trace": { - "properties": { - "id": {"type": "keyword"} - } - } - } - } -}`), - }, nil) - require.NoError(t, err) - srv1 := apmservertest.NewUnstartedServer(t) srv1.Config.Sampling = &apmservertest.SamplingConfig{ Tail: &apmservertest.TailSamplingConfig{ @@ -148,6 +126,10 @@ func TestTailSampling(t *testing.T) { tracer1.Flush(nil) tracer2.Flush(nil) + // Flush the data stream while the test is running, as we have no + // control over the settings for the sampled traces index template. + refreshPeriodically(t, 250*time.Millisecond, "apm-sampled-traces") + for _, transactionType := range []string{"parent", "child"} { var result estest.SearchResult t.Logf("waiting for %d %q transactions", expected, transactionType) @@ -220,3 +202,33 @@ func TestTailSamplingUnlicensed(t *testing.T) { assert.NoError(t, err) assert.Empty(t, result.Hits.Hits) } + +func refreshPeriodically(t *testing.T, interval time.Duration, index ...string) { + g, ctx := errgroup.WithContext(context.Background()) + ctx, cancel := context.WithCancel(ctx) + t.Cleanup(func() { + cancel() + assert.NoError(t, g.Wait()) + }) + g.Go(func() error { + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + allowNoIndices := true + ignoreUnavailable := true + request := esapi.IndicesRefreshRequest{ + Index: index, + AllowNoIndices: &allowNoIndices, + IgnoreUnavailable: &ignoreUnavailable, + } + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + } + if _, err := systemtest.Elasticsearch.Do(ctx, &request, nil); err != nil { + return err + } + } + }) +} diff --git a/testdata/jaeger/batch_0.approved.json b/testdata/jaeger/batch_0.approved.json index 45221240edf..cf4421d6d24 100644 --- a/testdata/jaeger/batch_0.approved.json +++ b/testdata/jaeger/batch_0.approved.json @@ -8,7 +8,7 @@ "version": "2.20.1" }, "event": { - "outcome": "success" + "outcome": "unknown" }, "host": { "hostname": "host01", @@ -48,7 +48,6 @@ }, "id": "7be2fd98d0973be3", "name": "Driver::findNearest", - "result": "Success", "sampled": true, "type": "custom" } diff --git a/testdata/jaeger/batch_1.approved.json b/testdata/jaeger/batch_1.approved.json index b9926744680..9feec9e765d 100644 --- a/testdata/jaeger/batch_1.approved.json +++ b/testdata/jaeger/batch_1.approved.json @@ -40,7 +40,7 @@ }, "id": "6e09e8bcefd6b828", "name": "FindDriverIDs", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576827704954062 @@ -57,7 +57,7 @@ "version": "2.20.1" }, "event": { - "outcome": "unknown" + "outcome": "failure" }, "host": { "hostname": "host01", @@ -89,7 +89,7 @@ }, "id": "333295bfb438ea03", "name": "GetDriver", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576827704973809 @@ -138,7 +138,7 @@ }, "id": "627c37a97e475c2f", "name": "GetDriver", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576827705007578 @@ -187,7 +187,7 @@ }, "id": "7bd7663d39c5a847", "name": "GetDriver", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576827705016845 @@ -236,7 +236,7 @@ }, "id": "6b4051dd2a5e2366", "name": "GetDriver", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576827705029415 @@ -285,7 +285,7 @@ }, "id": "6df97a86b9b3451b", "name": "GetDriver", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576827705040082 @@ -302,7 +302,7 @@ "version": "2.20.1" }, "event": { - "outcome": "unknown" + "outcome": "failure" }, "host": { "hostname": "host01", @@ -334,7 +334,7 @@ }, "id": "614811d6c498bfb0", "name": "GetDriver", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576827705054046 @@ -383,7 +383,7 @@ }, "id": "231604559da84d61", "name": "GetDriver", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576827705089459 @@ -432,7 +432,7 @@ }, "id": "61f7ecf24d13c36a", "name": "GetDriver", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576827705101278 @@ -481,7 +481,7 @@ }, "id": "2ef335bad24accc2", "name": "GetDriver", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576827705113531 @@ -530,7 +530,7 @@ }, "id": "38ec645e7201224d", "name": "GetDriver", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576827705125567 @@ -547,7 +547,7 @@ "version": "2.20.1" }, "event": { - "outcome": "unknown" + "outcome": "failure" }, "host": { "hostname": "host01", @@ -579,7 +579,7 @@ }, "id": "0242ee3774d9eab1", "name": "GetDriver", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576827705132896 @@ -628,7 +628,7 @@ }, "id": "6a63d1e81cfc7d95", "name": "GetDriver", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576827705172618 @@ -677,7 +677,7 @@ }, "id": "2b4c28f02b272f17", "name": "GetDriver", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576827705186670 diff --git a/tests/system/jaeger_batch_0_auth_tag_removed.approved.json b/tests/system/jaeger_batch_0_auth_tag_removed.approved.json index 44c08a2204e..7ea1b33323d 100644 --- a/tests/system/jaeger_batch_0_auth_tag_removed.approved.json +++ b/tests/system/jaeger_batch_0_auth_tag_removed.approved.json @@ -59,7 +59,6 @@ }, "id": "7be2fd98d0973be3", "name": "Driver::findNearest", - "result": "Success", "sampled": true, "type": "custom" } diff --git a/tests/system/jaeger_batch_0_authorization.approved.json b/tests/system/jaeger_batch_0_authorization.approved.json index 781bc75aeb6..d7bf815c2b1 100644 --- a/tests/system/jaeger_batch_0_authorization.approved.json +++ b/tests/system/jaeger_batch_0_authorization.approved.json @@ -59,7 +59,6 @@ }, "id": "7be2fd98d0973be3", "name": "Driver::findNearest", - "result": "Success", "sampled": true, "type": "custom" } diff --git a/tests/system/jaeger_span.approved.json b/tests/system/jaeger_span.approved.json index d1bbb9aaa8e..ac21b5ee496 100644 --- a/tests/system/jaeger_span.approved.json +++ b/tests/system/jaeger_span.approved.json @@ -55,7 +55,6 @@ }, "id": "5025e08c7fef6542", "name": "test_span", - "result": "Success", "sampled": true, "type": "custom" } diff --git a/x-pack/apm-server/main.go b/x-pack/apm-server/main.go index 9eabf9eed6a..91e02c81b6b 100644 --- a/x-pack/apm-server/main.go +++ b/x-pack/apm-server/main.go @@ -24,6 +24,7 @@ import ( "github.com/elastic/apm-server/x-pack/apm-server/aggregation/txmetrics" "github.com/elastic/apm-server/x-pack/apm-server/cmd" "github.com/elastic/apm-server/x-pack/apm-server/sampling" + "github.com/elastic/apm-server/x-pack/apm-server/sampling/pubsub" ) var ( @@ -108,6 +109,31 @@ func newTailSamplingProcessor(args beater.ServerParams) (*sampling.Processor, er if err != nil { return nil, errors.Wrap(err, "failed to create Elasticsearch client for tail-sampling") } + + var sampledTracesDataStream sampling.DataStreamConfig + if args.Managed { + // Data stream and ILM policy are managed by Fleet. + sampledTracesDataStream = sampling.DataStreamConfig{ + Type: "traces", + Dataset: "sampled", + Namespace: args.Namespace, + } + } else { + sampledTracesDataStream = sampling.DataStreamConfig{ + Type: "apm", + Dataset: "sampled", + Namespace: "traces", + } + if err := pubsub.SetupDataStream(context.Background(), es, + "apm-sampled-traces", // Index template + "apm-sampled-traces", // ILM policy + "apm-sampled-traces", // Index pattern + ); err != nil { + return nil, errors.Wrap(err, "failed to create data stream for tail-sampling") + } + args.Logger.Infof("Created tail-sampling data stream index template") + } + policies := make([]sampling.Policy, len(tailSamplingConfig.Policies)) for i, in := range tailSamplingConfig.Policies { policies[i] = sampling.Policy{ @@ -124,16 +150,14 @@ func newTailSamplingProcessor(args beater.ServerParams) (*sampling.Processor, er BeatID: args.Info.ID.String(), Reporter: args.Reporter, LocalSamplingConfig: sampling.LocalSamplingConfig{ - FlushInterval: tailSamplingConfig.Interval, - // TODO(axw) make MaxDynamicServices configurable? + FlushInterval: tailSamplingConfig.Interval, MaxDynamicServices: 1000, Policies: policies, IngestRateDecayFactor: tailSamplingConfig.IngestRateDecayFactor, }, RemoteSamplingConfig: sampling.RemoteSamplingConfig{ - Elasticsearch: es, - // TODO(axw) make index name configurable? - SampledTracesIndex: "apm-sampled-traces", + Elasticsearch: es, + SampledTracesDataStream: sampledTracesDataStream, }, StorageConfig: sampling.StorageConfig{ StorageDir: paths.Resolve(paths.Data, tailSamplingConfig.StorageDir), diff --git a/x-pack/apm-server/sampling/config.go b/x-pack/apm-server/sampling/config.go index f6d44b7568a..ce713d93ee0 100644 --- a/x-pack/apm-server/sampling/config.go +++ b/x-pack/apm-server/sampling/config.go @@ -11,6 +11,7 @@ import ( "github.com/elastic/apm-server/elasticsearch" "github.com/elastic/apm-server/publish" + "github.com/elastic/apm-server/x-pack/apm-server/sampling/pubsub" ) // Config holds configuration for Processor. @@ -60,9 +61,21 @@ type RemoteSamplingConfig struct { // and subscribing to remote sampling decisions. Elasticsearch elasticsearch.Client - // SampledTracesIndex holds the name of the Elasticsearch index for - // storing and searching sampled trace IDs. - SampledTracesIndex string + // SampledTracesDataStream holds the identifiers for the Elasticsearch + // data stream for storing and searching sampled trace IDs. + SampledTracesDataStream DataStreamConfig +} + +// DataStreamConfig holds configuration to identify a data stream. +type DataStreamConfig struct { + // Type holds the data stream's type. + Type string + + // Dataset holds the data stream's dataset. + Dataset string + + // Namespace holds the data stream's namespace. + Namespace string } // StorageConfig holds Processor configuration related to event storage. @@ -175,12 +188,16 @@ func (config RemoteSamplingConfig) validate() error { if config.Elasticsearch == nil { return errors.New("Elasticsearch unspecified") } - if config.SampledTracesIndex == "" { - return errors.New("SampledTracesIndex unspecified") + if err := config.SampledTracesDataStream.validate(); err != nil { + return errors.New("SampledTracesDataStream unspecified or invalid") } return nil } +func (config DataStreamConfig) validate() error { + return pubsub.DataStreamConfig(config).Validate() +} + func (config StorageConfig) validate() error { if config.StorageDir == "" { return errors.New("StorageDir unspecified") diff --git a/x-pack/apm-server/sampling/config_test.go b/x-pack/apm-server/sampling/config_test.go index 528267ddfc8..7519dbd06b2 100644 --- a/x-pack/apm-server/sampling/config_test.go +++ b/x-pack/apm-server/sampling/config_test.go @@ -57,8 +57,12 @@ func TestNewProcessorConfigInvalid(t *testing.T) { } config.Elasticsearch = elasticsearchClient - assertInvalidConfigError("invalid remote sampling config: SampledTracesIndex unspecified") - config.SampledTracesIndex = "sampled-traces" + assertInvalidConfigError("invalid remote sampling config: SampledTracesDataStream unspecified or invalid") + config.SampledTracesDataStream = sampling.DataStreamConfig{ + Type: "traces", + Dataset: "sampled", + Namespace: "testing", + } assertInvalidConfigError("invalid storage config: StorageDir unspecified") config.StorageDir = "tbs" diff --git a/x-pack/apm-server/sampling/processor.go b/x-pack/apm-server/sampling/processor.go index 4f9fa1cd3e2..712ec58fdeb 100644 --- a/x-pack/apm-server/sampling/processor.go +++ b/x-pack/apm-server/sampling/processor.go @@ -329,10 +329,10 @@ func (p *Processor) Run() error { } pubsub, err := pubsub.New(pubsub.Config{ - BeatID: p.config.BeatID, - Client: p.config.Elasticsearch, - Index: p.config.SampledTracesIndex, - Logger: p.logger, + BeatID: p.config.BeatID, + Client: p.config.Elasticsearch, + DataStream: pubsub.DataStreamConfig(p.config.SampledTracesDataStream), + Logger: p.logger, // Issue pubsub subscriber search requests at twice the frequency // of publishing, so each server observes each other's sampled diff --git a/x-pack/apm-server/sampling/processor_test.go b/x-pack/apm-server/sampling/processor_test.go index b6483da0a7b..ba412ba8ad3 100644 --- a/x-pack/apm-server/sampling/processor_test.go +++ b/x-pack/apm-server/sampling/processor_test.go @@ -596,8 +596,12 @@ func newTempdirConfig(tb testing.TB) sampling.Config { }, }, RemoteSamplingConfig: sampling.RemoteSamplingConfig{ - Elasticsearch: pubsubtest.Client(nil, nil), - SampledTracesIndex: "apm-sampled-traces", + Elasticsearch: pubsubtest.Client(nil, nil), + SampledTracesDataStream: sampling.DataStreamConfig{ + Type: "traces", + Dataset: "sampled", + Namespace: "testing", + }, }, StorageConfig: sampling.StorageConfig{ StorageDir: tempdir, diff --git a/x-pack/apm-server/sampling/pubsub/config.go b/x-pack/apm-server/sampling/pubsub/config.go index b4cd34b61a4..b97e5abc8dd 100644 --- a/x-pack/apm-server/sampling/pubsub/config.go +++ b/x-pack/apm-server/sampling/pubsub/config.go @@ -5,6 +5,7 @@ package pubsub import ( + "fmt" "time" "github.com/pkg/errors" @@ -20,8 +21,8 @@ type Config struct { // trace ID observations. Client elasticsearch.Client - // Index holds the index name. - Index string + // DataStream holds the data stream. + DataStream DataStreamConfig // BeatID holds the APM Server's unique ID, used for filtering out // local observations in the subscriber. @@ -48,13 +49,25 @@ type Config struct { Logger *logp.Logger } +// DataStreamConfig holds data stream configuration for Pubsub. +type DataStreamConfig struct { + // Type holds the data stream's type. + Type string + + // Dataset holds the data stream's dataset. + Dataset string + + // Namespace holds the data stream's namespace. + Namespace string +} + // Validate validates the configuration. func (config Config) Validate() error { if config.Client == nil { return errors.New("Client unspecified") } - if config.Index == "" { - return errors.New("Index unspecified") + if err := config.DataStream.Validate(); err != nil { + return errors.Wrap(err, "DataStream unspecified or invalid") } if config.BeatID == "" { return errors.New("BeatID unspecified") @@ -67,3 +80,22 @@ func (config Config) Validate() error { } return nil } + +// Validate validates the configuration. +func (config DataStreamConfig) Validate() error { + if config.Type == "" { + return errors.New("Type unspecified") + } + if config.Dataset == "" { + return errors.New("Dataset unspecified") + } + if config.Namespace == "" { + return errors.New("Namespace unspecified") + } + return nil +} + +// String returns the data stream as a combined string. +func (config DataStreamConfig) String() string { + return fmt.Sprintf("%s-%s-%s", config.Type, config.Dataset, config.Namespace) +} diff --git a/x-pack/apm-server/sampling/pubsub/config_test.go b/x-pack/apm-server/sampling/pubsub/config_test.go index bbbb8cd8d77..36cca74911d 100644 --- a/x-pack/apm-server/sampling/pubsub/config_test.go +++ b/x-pack/apm-server/sampling/pubsub/config_test.go @@ -32,24 +32,53 @@ func TestConfigInvalid(t *testing.T) { config: pubsub.Config{ Client: elasticsearchClient, }, - err: "Index unspecified", + err: "DataStream unspecified or invalid: Type unspecified", }, { config: pubsub.Config{ Client: elasticsearchClient, - Index: "index", + DataStream: pubsub.DataStreamConfig{ + Type: "type", + }, + }, + err: "DataStream unspecified or invalid: Dataset unspecified", + }, { + config: pubsub.Config{ + Client: elasticsearchClient, + DataStream: pubsub.DataStreamConfig{ + Type: "type", + Dataset: "dataset", + }, + }, + err: "DataStream unspecified or invalid: Namespace unspecified", + }, { + config: pubsub.Config{ + Client: elasticsearchClient, + DataStream: pubsub.DataStreamConfig{ + Type: "type", + Dataset: "dataset", + Namespace: "namespace", + }, }, err: "BeatID unspecified", }, { config: pubsub.Config{ Client: elasticsearchClient, - Index: "index", + DataStream: pubsub.DataStreamConfig{ + Type: "type", + Dataset: "dataset", + Namespace: "namespace", + }, BeatID: "beat_id", }, err: "SearchInterval unspecified or negative", }, { config: pubsub.Config{ - Client: elasticsearchClient, - Index: "index", + Client: elasticsearchClient, + DataStream: pubsub.DataStreamConfig{ + Type: "type", + Dataset: "dataset", + Namespace: "namespace", + }, BeatID: "beat_id", SearchInterval: time.Second, }, diff --git a/x-pack/apm-server/sampling/pubsub/datastream.go b/x-pack/apm-server/sampling/pubsub/datastream.go new file mode 100644 index 00000000000..016848cb0a6 --- /dev/null +++ b/x-pack/apm-server/sampling/pubsub/datastream.go @@ -0,0 +1,113 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package pubsub + +import ( + "context" + "fmt" + "io/ioutil" + "strings" + + "github.com/elastic/apm-server/elasticsearch" + + "github.com/elastic/go-elasticsearch/v7/esapi" +) + +// SetupDataStream ensures that the sampled traces data stream index template +// exists with the given name, creating it and its associated ILM policy if it +// does not. +// +// This should only be called when not running under Fleet. +func SetupDataStream( + ctx context.Context, + client elasticsearch.Client, + indexTemplateName string, + ilmPolicyName string, + indexPattern string, +) error { + // Create/replace ILM policy. + resp, err := esapi.ILMPutLifecycleRequest{ + Policy: ilmPolicyName, + Body: strings.NewReader(ilmPolicy), + }.Do(ctx, client) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.IsError() { + body, _ := ioutil.ReadAll(resp.Body) + return fmt.Errorf("failed to create ILM policy (%d): %s", resp.StatusCode, body) + } + + // Create/replace index template. + dataStreamIndexTemplate := fmt.Sprintf(dataStreamIndexTemplate, indexPattern, ilmPolicyName) + resp, err = esapi.IndicesPutIndexTemplateRequest{ + Name: indexTemplateName, + Body: strings.NewReader(dataStreamIndexTemplate), + }.Do(ctx, client) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.IsError() { + body, _ := ioutil.ReadAll(resp.Body) + return fmt.Errorf("failed to create index template (%d): %s", resp.StatusCode, body) + } + return nil +} + +// NOTE(axw) these replicate the index template and ILM policy created by Fleet, +// and should be kept in sync with apmpackage/apm/.../data_stream/sampled_traces. + +const ilmPolicy = `{ + "policy": { + "phases": { + "hot": { + "actions": { + "rollover": { + "max_age": "1h" + } + } + }, + "delete": { + "min_age": "1h", + "actions": { + "delete": {} + } + } + } + } +}` + +const dataStreamIndexTemplate = `{ + "index_patterns": [%q], + "priority": 1, + "data_stream": {}, + "template": { + "settings": { + "index.lifecycle.name": %q + }, + "mappings": { + "properties": { + "@timestamp": {"type": "date"}, + "event": { + "properties": { + "ingested": {"type": "date"} + } + }, + "observer": { + "properties": { + "id": {"type": "keyword"} + } + }, + "trace": { + "properties": { + "id": {"type": "keyword"} + } + } + } + } + } +}` diff --git a/x-pack/apm-server/sampling/pubsub/pubsub.go b/x-pack/apm-server/sampling/pubsub/pubsub.go index dd5dedec306..5ba4352d1e6 100644 --- a/x-pack/apm-server/sampling/pubsub/pubsub.go +++ b/x-pack/apm-server/sampling/pubsub/pubsub.go @@ -47,7 +47,7 @@ func New(config Config) (*Pubsub, error) { config.Logger = logp.NewLogger(logs.Sampling) } indexer, err := config.Client.NewBulkIndexer(elasticsearch.BulkIndexerConfig{ - Index: config.Index, + Index: config.DataStream.String(), FlushInterval: config.FlushInterval, OnError: func(ctx context.Context, err error) { config.Logger.With(logp.Error(err)).Debug("publishing sampled trace IDs failed") @@ -61,17 +61,12 @@ func New(config Config) (*Pubsub, error) { // PublishSampledTraceIDs bulk indexes traceIDs into Elasticsearch. func (p *Pubsub) PublishSampledTraceIDs(ctx context.Context, traceID ...string) error { + now := time.Now() for _, id := range traceID { - var doc traceIDDocument - doc.Observer.ID = p.config.BeatID - doc.Trace.ID = id - var json fastjson.Writer - if err := doc.MarshalFastJSON(&json); err != nil { - return err - } + p.marshalTraceIDDocument(&json, id, now, p.config.DataStream) if err := p.indexer.Add(ctx, elasticsearch.BulkIndexerItem{ - Action: "index", + Action: "create", Body: bytes.NewReader(json.Bytes()), OnFailure: p.onBulkIndexerItemFailure, }); err != nil { @@ -82,7 +77,7 @@ func (p *Pubsub) PublishSampledTraceIDs(ctx context.Context, traceID ...string) } func (p *Pubsub) onBulkIndexerItemFailure(ctx context.Context, item elasticsearch.BulkIndexerItem, resp elasticsearch.BulkIndexerResponseItem, err error) { - p.config.Logger.With(logp.Error(err)).Debug("publishing sampled trace ID failed") + p.config.Logger.With(logp.Error(err)).Debug("publishing sampled trace ID failed", resp.Error) } // SubscribeSampledTraceIDs subscribes to new sampled trace IDs, sending them to the @@ -146,7 +141,7 @@ func (p *Pubsub) searchTraceIDs(ctx context.Context, out chan<- string, lastSeqN } req := esapi.SearchRequest{ - Index: []string{p.config.Index}, + Index: []string{p.config.DataStream.String()}, Body: esutil.NewJSONReader(searchBody), } resp, err := req.Do(ctx, p.config.Client) @@ -201,6 +196,23 @@ func (p *Pubsub) searchTraceIDs(ctx context.Context, out chan<- string, lastSeqN return n, nil } +func (p *Pubsub) marshalTraceIDDocument(w *fastjson.Writer, traceID string, timestamp time.Time, dataStream DataStreamConfig) { + w.RawString(`{"@timestamp":"`) + w.Time(timestamp.UTC(), time.RFC3339Nano) + w.RawString(`","data_stream.type":`) + w.String(dataStream.Type) + w.RawString(`,"data_stream.dataset":`) + w.String(dataStream.Dataset) + w.RawString(`,"data_stream.namespace":`) + w.String(dataStream.Namespace) + w.RawString(`,"observer":{"id":`) + w.String(p.config.BeatID) + w.RawString(`},`) + w.RawString(`"trace":{"id":`) + w.String(traceID) + w.RawString(`}}`) +} + type traceIDDocument struct { // Observer identifies the entity (typically an APM Server) that observed // and indexed the/ trace ID document. This can be used to filter out local @@ -217,8 +229,11 @@ type traceIDDocument struct { } `json:"trace"` } +/* func (d *traceIDDocument) MarshalFastJSON(w *fastjson.Writer) error { - w.RawString(`{"observer":{"id":`) + w.RawString(`{"@timestamp":"`) + w.Time(d.Timestamp, time.RFC3339Nano) + w.RawString(`","observer":{"id":`) w.String(d.Observer.ID) w.RawString(`},`) w.RawString(`"trace":{"id":`) @@ -226,3 +241,4 @@ func (d *traceIDDocument) MarshalFastJSON(w *fastjson.Writer) error { w.RawString(`}}`) return nil } +*/ diff --git a/x-pack/apm-server/sampling/pubsub/pubsub_integration_test.go b/x-pack/apm-server/sampling/pubsub/pubsub_integration_test.go index 6a7c0447af3..1bdcc44d533 100644 --- a/x-pack/apm-server/sampling/pubsub/pubsub_integration_test.go +++ b/x-pack/apm-server/sampling/pubsub/pubsub_integration_test.go @@ -35,11 +35,16 @@ const ( func TestElasticsearchIntegration_PublishSampledTraceIDs(t *testing.T) { const ( localBeatID = "local_beat_id" - indexName = "apm-testing-sampled-traces" ) + dataStream := pubsub.DataStreamConfig{ + Type: "apm", + Dataset: "sampled_traces", + Namespace: "testing", + } + client := newElasticsearchClient(t) - recreateIndex(t, client, indexName) + recreateDataStream(t, client, dataStream) var input []string for i := 0; i < 50; i++ { @@ -48,7 +53,7 @@ func TestElasticsearchIntegration_PublishSampledTraceIDs(t *testing.T) { es, err := pubsub.New(pubsub.Config{ Client: client, - Index: indexName, + DataStream: dataStream, BeatID: localBeatID, FlushInterval: 100 * time.Millisecond, SearchInterval: time.Minute, @@ -76,7 +81,7 @@ func TestElasticsearchIntegration_PublishSampledTraceIDs(t *testing.T) { for { size := len(input) + 1 resp, err := esapi.SearchRequest{ - Index: []string{indexName}, + Index: []string{dataStream.String()}, Size: &size, }.Do(context.Background(), client) require.NoError(t, err) @@ -105,15 +110,20 @@ func TestElasticsearchIntegration_SubscribeSampledTraceIDs(t *testing.T) { const ( localBeatID = "local_observer_id" remoteBeatID = "remote_observer_id" - indexName = "apm-testing-sampled-traces" ) + dataStream := pubsub.DataStreamConfig{ + Type: "apm", + Dataset: "sampled_traces", + Namespace: "testing", + } + client := newElasticsearchClient(t) - recreateIndex(t, client, indexName) + recreateDataStream(t, client, dataStream) es, err := pubsub.New(pubsub.Config{ Client: client, - Index: indexName, + DataStream: dataStream, BeatID: localBeatID, FlushInterval: time.Minute, SearchInterval: 100 * time.Millisecond, @@ -154,7 +164,7 @@ func TestElasticsearchIntegration_SubscribeSampledTraceIDs(t *testing.T) { assert.NoError(t, enc.Encode(&doc)) } resp, err := esapi.BulkRequest{ - Index: indexName, + Index: dataStream.String(), Body: &body, }.Do(context.Background(), client) require.NoError(t, err) @@ -188,7 +198,7 @@ func TestElasticsearchIntegration_SubscribeSampledTraceIDs(t *testing.T) { } } -func recreateIndex(tb testing.TB, client elasticsearch.Client, indexName string) { +func recreateDataStream(tb testing.TB, client elasticsearch.Client, dataStream pubsub.DataStreamConfig) { body := strings.NewReader(`{ "mappings": { "properties": { @@ -207,14 +217,19 @@ func recreateIndex(tb testing.TB, client elasticsearch.Client, indexName string) } }`) + // NOTE(aww) we cheat and create an index, rather than a + // data stream. System tests will test with data streams, + // and will pick up any resulting discrepancies. + + name := dataStream.String() resp, err := esapi.IndicesDeleteRequest{ - Index: []string{indexName}, + Index: []string{dataStream.String()}, }.Do(context.Background(), client) require.NoError(tb, err) resp.Body.Close() resp, err = esapi.IndicesCreateRequest{ - Index: indexName, + Index: name, Body: body, }.Do(context.Background(), client) require.NoError(tb, err) diff --git a/x-pack/apm-server/sampling/pubsub/pubsub_test.go b/x-pack/apm-server/sampling/pubsub/pubsub_test.go index 78225a4eb7a..5f89787aa69 100644 --- a/x-pack/apm-server/sampling/pubsub/pubsub_test.go +++ b/x-pack/apm-server/sampling/pubsub/pubsub_test.go @@ -28,10 +28,15 @@ import ( func TestPublishSampledTraceIDs(t *testing.T) { const ( - indexName = "trace-ids" - beatID = "beat_id" + beatID = "beat_id" ) + dataStream := pubsub.DataStreamConfig{ + Type: "traces", + Dataset: "sampled", + Namespace: "testing", + } + requests := make(chan *http.Request, 1) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var buf bytes.Buffer @@ -54,7 +59,7 @@ func TestPublishSampledTraceIDs(t *testing.T) { pub, err := pubsub.New(pubsub.Config{ Client: client, - Index: indexName, + DataStream: dataStream, BeatID: beatID, FlushInterval: time.Millisecond, SearchInterval: time.Minute, @@ -83,7 +88,7 @@ func TestPublishSampledTraceIDs(t *testing.T) { case <-deadlineTimer.C: t.Fatal("timed out waiting for events to be received by server") case req := <-requests: - require.Equal(t, fmt.Sprintf("/%s/_bulk", indexName), req.URL.Path) + require.Equal(t, fmt.Sprintf("/%s/_bulk", dataStream.String()), req.URL.Path) d := json.NewDecoder(req.Body) for { @@ -93,11 +98,15 @@ func TestPublishSampledTraceIDs(t *testing.T) { break } assert.NoError(t, err) - assert.Equal(t, map[string]interface{}{"index": map[string]interface{}{}}, action) + assert.Equal(t, map[string]interface{}{"create": map[string]interface{}{}}, action) doc := make(map[string]interface{}) assert.NoError(t, d.Decode(&doc)) + assert.Contains(t, doc, "@timestamp") assert.Equal(t, map[string]interface{}{"id": beatID}, doc["observer"]) + assert.Equal(t, dataStream.Type, doc["data_stream.type"]) + assert.Equal(t, dataStream.Dataset, doc["data_stream.dataset"]) + assert.Equal(t, dataStream.Namespace, doc["data_stream.namespace"]) trace := doc["trace"].(map[string]interface{}) traceID := trace["id"].(string) @@ -105,8 +114,13 @@ func TestPublishSampledTraceIDs(t *testing.T) { delete(trace, "id") assert.Empty(t, trace) // no other fields in "trace" + delete(doc, "@timestamp") + delete(doc, "data_stream.type") + delete(doc, "data_stream.dataset") + delete(doc, "data_stream.namespace") delete(doc, "observer") delete(doc, "trace") + assert.Empty(t, doc) // no other fields in doc } } @@ -119,10 +133,15 @@ func TestPublishSampledTraceIDs(t *testing.T) { func TestSubscribeSampledTraceIDs(t *testing.T) { const ( - indexName = "trace-ids" - beatID = "beat_id" + beatID = "beat_id" ) + dataStream := pubsub.DataStreamConfig{ + Type: "traces", + Dataset: "sampled", + Namespace: "default", + } + var requests []*http.Request responses := make(chan string) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -148,7 +167,7 @@ func TestSubscribeSampledTraceIDs(t *testing.T) { sub, err := pubsub.New(pubsub.Config{ Client: client, - Index: indexName, + DataStream: dataStream, BeatID: beatID, FlushInterval: time.Minute, SearchInterval: time.Millisecond, @@ -221,7 +240,7 @@ func TestSubscribeSampledTraceIDs(t *testing.T) { var bodies []string for _, r := range requests { - assert.Equal(t, fmt.Sprintf("/%s/_search", indexName), r.URL.Path) + assert.Equal(t, fmt.Sprintf("/%s/_search", dataStream.String()), r.URL.Path) var buf bytes.Buffer io.Copy(&buf, r.Body)