diff --git a/beater/beater_test.go b/beater/beater_test.go index f87fc4eebf0..ed1dd08d8ac 100644 --- a/beater/beater_test.go +++ b/beater/beater_test.go @@ -28,9 +28,11 @@ import ( "testing" "time" - "github.com/gofrs/uuid" + "github.com/elastic/beats/libbeat/logp" + + "github.com/elastic/beats/libbeat/publisher/processing" + "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" @@ -39,7 +41,6 @@ import ( "github.com/elastic/beats/libbeat/publisher/pipeline" "github.com/elastic/beats/libbeat/publisher/queue" "github.com/elastic/beats/libbeat/publisher/queue/memqueue" - "github.com/elastic/beats/libbeat/version" ) func TestBeatConfig(t *testing.T) { @@ -293,12 +294,19 @@ func (d *DummyOutputClient) Publish(batch pubs.Batch) error { func (d *DummyOutputClient) Close() error { return nil } func (d *DummyOutputClient) String() string { return "" } -func DummyPipeline(clients ...outputs.Client) *pipeline.Pipeline { +func DummyPipeline(cfg *common.Config, info beat.Info, clients ...outputs.Client) *pipeline.Pipeline { if len(clients) == 0 { clients = []outputs.Client{&DummyOutputClient{}} } + if cfg == nil { + cfg = common.NewConfig() + } + processors, err := processing.MakeDefaultObserverSupport(false)(info, logp.NewLogger("testbeat"), cfg) + if err != nil { + panic(err) + } p, err := pipeline.New( - beat.Info{Name: "test-apm-server"}, + info, pipeline.Monitors{}, func(e queue.Eventer) (queue.Queue, error) { return memqueue.NewBroker(nil, memqueue.Settings{ @@ -314,6 +322,7 @@ func DummyPipeline(clients ...outputs.Client) *pipeline.Pipeline { pipeline.Settings{ WaitClose: 0, WaitCloseMode: pipeline.NoWaitOnClose, + Processors: processors, }, ) if err != nil { @@ -383,21 +392,7 @@ func (bt *beater) smapElasticsearchHosts() []string { return hosts } -func setupBeater(t *testing.T, publisher beat.Pipeline, ucfg *common.Config, beatConfig *beat.BeatConfig) (*beater, func(), error) { - beatId, err := uuid.FromString("fbba762a-14dd-412c-b7e9-b79f903eb492") - require.NoError(t, err) - // create a beat - apmBeat := &beat.Beat{ - Publisher: publisher, - Info: beat.Info{ - Beat: "test-apm-server", - IndexPrefix: "test-apm-server", - Version: version.GetDefaultVersion(), - ID: beatId, - }, - Config: beatConfig, - } - +func setupBeater(t *testing.T, apmBeat *beat.Beat, ucfg *common.Config, beatConfig *beat.BeatConfig) (*beater, func(), error) { // create our beater beatBeater, err := New(apmBeat, ucfg) assert.NoError(t, err) diff --git a/beater/server_test.go b/beater/server_test.go index 5a7f3ec7624..fa53a30af18 100644 --- a/beater/server_test.go +++ b/beater/server_test.go @@ -32,6 +32,10 @@ import ( "testing" "time" + "github.com/gofrs/uuid" + + "github.com/elastic/beats/libbeat/version" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -505,17 +509,33 @@ func setupServer(t *testing.T, cfg *common.Config, beatConfig *beat.BeatConfig, require.NoError(t, err) } + beatId, err := uuid.FromString("fbba762a-14dd-412c-b7e9-b79f903eb492") + require.NoError(t, err) + info := beat.Info{ + Beat: "test-apm-server", + IndexPrefix: "test-apm-server", + Version: version.GetDefaultVersion(), + ID: beatId, + } + var pub beat.Pipeline if events != nil { // capture events pubClient := NewChanClientWith(events) - pub = DummyPipeline(pubClient) + pub = DummyPipeline(cfg, info, pubClient) } else { // don't capture events - pub = DummyPipeline() + pub = DummyPipeline(cfg, info) + } + + // create a beat + apmBeat := &beat.Beat{ + Publisher: pub, + Info: info, + Config: beatConfig, } - btr, stop, err := setupBeater(t, pub, baseConfig, beatConfig) + btr, stop, err := setupBeater(t, apmBeat, baseConfig, beatConfig) if err == nil { assert.NotEqual(t, btr.config.Host, "localhost:0", "config.Host unmodified") } diff --git a/beater/test_approved_es_documents/TestPublishIntegrationErrors.approved.json b/beater/test_approved_es_documents/TestPublishIntegrationErrors.approved.json index c3cb3d4a8dd..158157b77bd 100644 --- a/beater/test_approved_es_documents/TestPublishIntegrationErrors.approved.json +++ b/beater/test_approved_es_documents/TestPublishIntegrationErrors.approved.json @@ -14,6 +14,9 @@ "container": { "id": "container-id" }, + "ecs": { + "version": "1.1.0-dev" + }, "error": { "culprit": "my.module.function_name", "custom": { @@ -317,6 +320,9 @@ "container": { "id": "container-id" }, + "ecs": { + "version": "1.1.0-dev" + }, "error": { "exception": [ { @@ -405,6 +411,9 @@ "container": { "id": "container-id" }, + "ecs": { + "version": "1.1.0-dev" + }, "error": { "exception": [ { @@ -499,6 +508,9 @@ "container": { "id": "container-id" }, + "ecs": { + "version": "1.1.0-dev" + }, "error": { "grouping_key": "d6b3f958dfea98dc9ed2b57d5f0c48bb", "id": "abcdef0123456789", diff --git a/beater/test_approved_es_documents/TestPublishIntegrationEvents.approved.json b/beater/test_approved_es_documents/TestPublishIntegrationEvents.approved.json index 1dd209ae3c3..14e1492cf5f 100644 --- a/beater/test_approved_es_documents/TestPublishIntegrationEvents.approved.json +++ b/beater/test_approved_es_documents/TestPublishIntegrationEvents.approved.json @@ -14,6 +14,9 @@ "container": { "id": "container-id" }, + "ecs": { + "version": "1.1.0-dev" + }, "error": { "grouping_key": "d6b3f958dfea98dc9ed2b57d5f0c48bb", "id": "abcdef0123456789", @@ -74,6 +77,9 @@ "name": "elastic-node", "version": "3.14.0" }, + "ecs": { + "version": "1.1.0-dev" + }, "observer": { "ephemeral_id": "00000000-0000-0000-0000-000000000000", "hostname": "", @@ -128,6 +134,9 @@ "container": { "id": "container-id" }, + "ecs": { + "version": "1.1.0-dev" + }, "host": { "ip": "127.0.0.1" }, @@ -198,6 +207,9 @@ "container": { "id": "container-id" }, + "ecs": { + "version": "1.1.0-dev" + }, "go": { "memstats": { "heap": { diff --git a/beater/test_approved_es_documents/TestPublishIntegrationMetricsets.approved.json b/beater/test_approved_es_documents/TestPublishIntegrationMetricsets.approved.json index 001beba0b6f..5ec1478c35f 100644 --- a/beater/test_approved_es_documents/TestPublishIntegrationMetricsets.approved.json +++ b/beater/test_approved_es_documents/TestPublishIntegrationMetricsets.approved.json @@ -18,6 +18,9 @@ } }, "double_gauge": 3.141592653589793, + "ecs": { + "version": "1.1.0-dev" + }, "float_gauge": 9.16, "integer_gauge": 42767, "labels": { @@ -78,6 +81,9 @@ "name": "elastic-node", "version": "3.14.0" }, + "ecs": { + "version": "1.1.0-dev" + }, "go": { "memstats": { "heap": { diff --git a/beater/test_approved_es_documents/TestPublishIntegrationMinimalEvents.approved.json b/beater/test_approved_es_documents/TestPublishIntegrationMinimalEvents.approved.json index e611b83909c..55e18bf9ff2 100644 --- a/beater/test_approved_es_documents/TestPublishIntegrationMinimalEvents.approved.json +++ b/beater/test_approved_es_documents/TestPublishIntegrationMinimalEvents.approved.json @@ -11,6 +11,9 @@ "name": "elastic-node", "version": "3.14.0" }, + "ecs": { + "version": "1.1.0-dev" + }, "error": { "grouping_key": "0b9cba09845a097a271c6beb4c6207f3", "id": "abcdef0123456789", @@ -51,6 +54,9 @@ "name": "elastic-node", "version": "3.14.0" }, + "ecs": { + "version": "1.1.0-dev" + }, "error": { "exception": [ { @@ -93,6 +99,9 @@ "name": "elastic-node", "version": "3.14.0" }, + "ecs": { + "version": "1.1.0-dev" + }, "error": { "exception": [ { @@ -135,6 +144,9 @@ "name": "elastic-node", "version": "3.14.0" }, + "ecs": { + "version": "1.1.0-dev" + }, "observer": { "ephemeral_id": "00000000-0000-0000-0000-000000000000", "hostname": "", @@ -182,6 +194,9 @@ "name": "elastic-node", "version": "3.14.0" }, + "ecs": { + "version": "1.1.0-dev" + }, "observer": { "ephemeral_id": "00000000-0000-0000-0000-000000000000", "hostname": "", @@ -226,6 +241,9 @@ "name": "elastic-node", "version": "3.14.0" }, + "ecs": { + "version": "1.1.0-dev" + }, "host": { "ip": "127.0.0.1" }, @@ -273,6 +291,9 @@ "name": "elastic-node", "version": "3.14.0" }, + "ecs": { + "version": "1.1.0-dev" + }, "host": { "ip": "127.0.0.1" }, diff --git a/beater/test_approved_es_documents/TestPublishIntegrationSpans.approved.json b/beater/test_approved_es_documents/TestPublishIntegrationSpans.approved.json index e2e5c8a7460..282a2eed7fd 100644 --- a/beater/test_approved_es_documents/TestPublishIntegrationSpans.approved.json +++ b/beater/test_approved_es_documents/TestPublishIntegrationSpans.approved.json @@ -11,6 +11,9 @@ "name": "elastic-node", "version": "3.14.0" }, + "ecs": { + "version": "1.1.0-dev" + }, "observer": { "ephemeral_id": "00000000-0000-0000-0000-000000000000", "hostname": "", @@ -60,6 +63,9 @@ "name": "elastic-node", "version": "3.14.0" }, + "ecs": { + "version": "1.1.0-dev" + }, "observer": { "ephemeral_id": "00000000-0000-0000-0000-000000000000", "hostname": "", @@ -110,6 +116,9 @@ "name": "elastic-node", "version": "3.14.0" }, + "ecs": { + "version": "1.1.0-dev" + }, "labels": { "tag1": "value1", "tag2": 123, @@ -167,6 +176,9 @@ "name": "elastic-node", "version": "3.14.0" }, + "ecs": { + "version": "1.1.0-dev" + }, "observer": { "ephemeral_id": "00000000-0000-0000-0000-000000000000", "hostname": "", @@ -217,6 +229,9 @@ "name": "elastic-ruby", "version": "2.2" }, + "ecs": { + "version": "1.1.0-dev" + }, "observer": { "ephemeral_id": "00000000-0000-0000-0000-000000000000", "hostname": "", diff --git a/beater/test_approved_es_documents/TestPublishIntegrationTransactions.approved.json b/beater/test_approved_es_documents/TestPublishIntegrationTransactions.approved.json index 316648f5ea5..fc28878edec 100644 --- a/beater/test_approved_es_documents/TestPublishIntegrationTransactions.approved.json +++ b/beater/test_approved_es_documents/TestPublishIntegrationTransactions.approved.json @@ -14,6 +14,9 @@ "container": { "id": "container-id" }, + "ecs": { + "version": "1.1.0-dev" + }, "host": { "architecture": "x64", "hostname": "prod1.example.com", @@ -110,6 +113,9 @@ "container": { "id": "container-id" }, + "ecs": { + "version": "1.1.0-dev" + }, "host": { "architecture": "x64", "hostname": "prod1.example.com", @@ -294,6 +300,9 @@ "container": { "id": "container-id" }, + "ecs": { + "version": "1.1.0-dev" + }, "host": { "architecture": "x64", "hostname": "prod1.example.com", diff --git a/changelogs/7.0.asciidoc b/changelogs/7.0.asciidoc index 0142594f017..35ca58e06d0 100644 --- a/changelogs/7.0.asciidoc +++ b/changelogs/7.0.asciidoc @@ -21,10 +21,11 @@ ==== Bugfix - Ensure setup cmd uses expected configuration {pull}1934[1934]. -- Ensure host.name is not added {pull}1934[1934]. +- Ensure host.name is not added {pull}1934[1934] {pull}1982[1982]. - Update Go version to 1.11.5 {pull}1840[1840] {pull}1950[1950]. - Ensure enabling user-agent pipeline indexes data at the same key by default {pull}1966[1966]. - Request and Response headers are stored in a canonicalized way {pull}1966[1966]. +- Use changed processor handling from libbeat {pull}1982[1982]. [[release-notes-7.7.0-rc1]] === APM Server version 7.0.0-rc1 diff --git a/cmd/root.go b/cmd/root.go index 498579abbc0..8ab77e8f3f1 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -20,15 +20,15 @@ package cmd import ( "github.com/spf13/pflag" + "github.com/elastic/apm-server/beater" "github.com/elastic/apm-server/idxmgmt" "github.com/elastic/apm-server/idxmgmt/ilm" - - "github.com/elastic/apm-server/beater" _ "github.com/elastic/apm-server/include" "github.com/elastic/beats/libbeat/cmd" "github.com/elastic/beats/libbeat/cmd/instance" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/monitoring/report" + "github.com/elastic/beats/libbeat/publisher/processing" ) // Name of the beat (apm-server). @@ -76,6 +76,7 @@ func init() { }, ILM: ilm.MakeDefaultSupporter, IndexManagement: idxmgmt.MakeDefaultSupporter, + Processing: processing.MakeDefaultObserverSupport(false), }) // remove dashboard from export commands for _, cmd := range RootCmd.ExportCmd.Commands() { diff --git a/publish/pub.go b/publish/pub.go index b663f2ec24b..e955d4e1504 100644 --- a/publish/pub.go +++ b/publish/pub.go @@ -68,19 +68,18 @@ func NewPublisher(info beat.Info, pipeline beat.Pipeline, shutdownTimeout time.D // If set >0 `Close` will block for the duration or until pipeline is empty WaitClose: shutdownTimeout, - Fields: common.MapStr{ - "observer": common.MapStr{ - "type": info.Beat, - "hostname": info.Hostname, - "version": info.Version, - "version_major": 8, - "id": info.ID.String(), - "ephemeral_id": info.EphemeralID.String(), + Processing: beat.ProcessingConfig{ + Fields: common.MapStr{ + "observer": common.MapStr{ + "type": info.Beat, + "hostname": info.Hostname, + "version": info.Version, + "version_major": 8, + "id": info.ID.String(), + "ephemeral_id": info.EphemeralID.String(), + }, }, }, - SkipAgentMetadata: true, - SkipNormalization: true, - SkipHostName: true, }) if err != nil { return nil, err diff --git a/tests/system/error.approved.json b/tests/system/error.approved.json index 7c18c028198..74851be04d8 100644 --- a/tests/system/error.approved.json +++ b/tests/system/error.approved.json @@ -9,7 +9,7 @@ "us": 1494342245000000 }, "ecs": { - "version": "1.0.0-beta2" + "version": "1.1.0-dev" }, "observer": { "type": "apm-server", @@ -95,7 +95,7 @@ "us": 1494342245999999 }, "ecs":{ - "version":"1.0.0-beta2" + "version":"1.1.0-dev" }, "observer": { "type": "apm-server", @@ -391,7 +391,7 @@ "us": 1494342245000000 }, "ecs":{ - "version":"1.0.0-beta2" + "version":"1.1.0-dev" }, "observer": { "type": "apm-server", @@ -478,7 +478,7 @@ "us": 1494342245999000 }, "ecs":{ - "version":"1.0.0-beta2" + "version":"1.1.0-dev" }, "observer": { "type": "apm-server", diff --git a/tests/system/spans.approved.json b/tests/system/spans.approved.json index 8e04205a936..3368207b781 100644 --- a/tests/system/spans.approved.json +++ b/tests/system/spans.approved.json @@ -9,7 +9,7 @@ "us": 1496170407154000 }, "ecs": { - "version": "1.0.0-beta2" + "version": "1.1.0-dev" }, "observer": { "type": "apm-server", @@ -82,7 +82,7 @@ "us": 1496170422281000 }, "ecs": { - "version": "1.0.0-beta2" + "version": "1.1.0-dev" }, "observer": { "type": "apm-server", @@ -163,7 +163,7 @@ "us": 1496170407154000 }, "ecs": { - "version": "1.0.0-beta2" + "version": "1.1.0-dev" }, "observer": { "type": "apm-server", @@ -234,7 +234,7 @@ "us": 1496170407154000 }, "ecs": { - "version": "1.0.0-beta2" + "version": "1.1.0-dev" }, "observer": { "type": "apm-server", @@ -364,7 +364,7 @@ "us": 1496170407154000 }, "ecs": { - "version": "1.0.0-beta2" + "version": "1.1.0-dev" }, "observer": { "type": "apm-server", diff --git a/tests/system/transaction.approved.json b/tests/system/transaction.approved.json index 2592a281113..2caa115db28 100644 --- a/tests/system/transaction.approved.json +++ b/tests/system/transaction.approved.json @@ -9,7 +9,7 @@ "us": 1496170407154000 }, "ecs": { - "version": "1.0.0-beta2" + "version": "1.1.0-dev" }, "observer": { "type": "apm-server", @@ -189,7 +189,7 @@ "us": 1496170422281999 }, "ecs": { - "version": "1.0.0-beta2" + "version": "1.1.0-dev" }, "observer": { "type": "apm-server", @@ -285,7 +285,7 @@ "us": 1496170422000000 }, "ecs": { - "version": "1.0.0-beta2" + "version": "1.1.0-dev" }, "observer": { "type": "apm-server", @@ -380,7 +380,7 @@ "us": 1496170422281000 }, "ecs": { - "version": "1.0.0-beta2" + "version": "1.1.0-dev" }, "observer": { "type": "apm-server",