From 3a878b7770b5a1bd5e8bfab29460f8a18595610e Mon Sep 17 00:00:00 2001 From: Juan Date: Thu, 3 Dec 2020 13:53:07 +0100 Subject: [PATCH 1/2] fix pipeline names --- .../ingest_pipeline/default.json | 25 ++++++ .../data_stream/internal_metrics/manifest.yml | 1 - .../ingest_pipeline/default.json | 8 +- .../{apm.json => default.json} | 8 +- .../0.1.0/data_stream/metrics/manifest.yml | 1 - .../{apm.json => default.json} | 8 +- .../0.1.0/data_stream/profiles/manifest.yml | 1 - .../elasticsearch/ingest_pipeline/apm.json | 25 ------ .../ingest_pipeline/default.json} | 8 +- .../apm/0.1.0/data_stream/traces/manifest.yml | 1 - apmpackage/cmd/gen-package/genpipelines.go | 86 +++++++++++++++++-- apmpackage/cmd/gen-package/main.go | 7 +- 12 files changed, 119 insertions(+), 60 deletions(-) create mode 100644 apmpackage/apm/0.1.0/data_stream/internal_metrics/elasticsearch/ingest_pipeline/default.json rename apmpackage/apm/0.1.0/data_stream/metrics/elasticsearch/ingest_pipeline/{apm.json => default.json} (52%) rename apmpackage/apm/0.1.0/data_stream/profiles/elasticsearch/ingest_pipeline/{apm.json => default.json} (52%) delete mode 100644 apmpackage/apm/0.1.0/data_stream/traces/elasticsearch/ingest_pipeline/apm.json rename apmpackage/apm/0.1.0/data_stream/{internal_metrics/elasticsearch/ingest_pipeline/apm.json => traces/elasticsearch/ingest_pipeline/default.json} (53%) diff --git a/apmpackage/apm/0.1.0/data_stream/internal_metrics/elasticsearch/ingest_pipeline/default.json b/apmpackage/apm/0.1.0/data_stream/internal_metrics/elasticsearch/ingest_pipeline/default.json new file mode 100644 index 00000000000..1a5bdc999ce --- /dev/null +++ b/apmpackage/apm/0.1.0/data_stream/internal_metrics/elasticsearch/ingest_pipeline/default.json @@ -0,0 +1,25 @@ +{ + "description": "Default enrichment for APM events", + "processors": [ + { + "pipeline": { + "name": "metrics-apm.internal-0.1.0-apm_user_agent" + } + }, + { + "pipeline": { + "name": "metrics-apm.internal-0.1.0-apm_user_geo" + } + }, + { + "pipeline": { + "name": "metrics-apm.internal-0.1.0-apm_ingest_timestamp" + } + }, + { + "pipeline": { + "name": "metrics-apm.internal-0.1.0-apm_remove_span_metadata" + } + } + ] +} \ No newline at end of file diff --git a/apmpackage/apm/0.1.0/data_stream/internal_metrics/manifest.yml b/apmpackage/apm/0.1.0/data_stream/internal_metrics/manifest.yml index e891f9ceaf9..6fb4ec958c5 100644 --- a/apmpackage/apm/0.1.0/data_stream/internal_metrics/manifest.yml +++ b/apmpackage/apm/0.1.0/data_stream/internal_metrics/manifest.yml @@ -1,4 +1,3 @@ title: APM internal metrics type: metrics dataset: apm.internal -ingest_pipeline: apm diff --git a/apmpackage/apm/0.1.0/data_stream/logs/elasticsearch/ingest_pipeline/default.json b/apmpackage/apm/0.1.0/data_stream/logs/elasticsearch/ingest_pipeline/default.json index 05ada716015..9203f00cc2f 100644 --- a/apmpackage/apm/0.1.0/data_stream/logs/elasticsearch/ingest_pipeline/default.json +++ b/apmpackage/apm/0.1.0/data_stream/logs/elasticsearch/ingest_pipeline/default.json @@ -3,22 +3,22 @@ "processors": [ { "pipeline": { - "name": "apm_user_agent" + "name": "logs-apm.error-0.1.0-apm_user_agent" } }, { "pipeline": { - "name": "apm_user_geo" + "name": "logs-apm.error-0.1.0-apm_user_geo" } }, { "pipeline": { - "name": "apm_ingest_timestamp" + "name": "logs-apm.error-0.1.0-apm_ingest_timestamp" } }, { "pipeline": { - "name": "apm_remove_span_metadata" + "name": "logs-apm.error-0.1.0-apm_remove_span_metadata" } } ] diff --git a/apmpackage/apm/0.1.0/data_stream/metrics/elasticsearch/ingest_pipeline/apm.json b/apmpackage/apm/0.1.0/data_stream/metrics/elasticsearch/ingest_pipeline/default.json similarity index 52% rename from apmpackage/apm/0.1.0/data_stream/metrics/elasticsearch/ingest_pipeline/apm.json rename to apmpackage/apm/0.1.0/data_stream/metrics/elasticsearch/ingest_pipeline/default.json index 05ada716015..e848ed68123 100644 --- a/apmpackage/apm/0.1.0/data_stream/metrics/elasticsearch/ingest_pipeline/apm.json +++ b/apmpackage/apm/0.1.0/data_stream/metrics/elasticsearch/ingest_pipeline/default.json @@ -3,22 +3,22 @@ "processors": [ { "pipeline": { - "name": "apm_user_agent" + "name": "metrics-apm-0.1.0-apm_user_agent" } }, { "pipeline": { - "name": "apm_user_geo" + "name": "metrics-apm-0.1.0-apm_user_geo" } }, { "pipeline": { - "name": "apm_ingest_timestamp" + "name": "metrics-apm-0.1.0-apm_ingest_timestamp" } }, { "pipeline": { - "name": "apm_remove_span_metadata" + "name": "metrics-apm-0.1.0-apm_remove_span_metadata" } } ] diff --git a/apmpackage/apm/0.1.0/data_stream/metrics/manifest.yml b/apmpackage/apm/0.1.0/data_stream/metrics/manifest.yml index 07c8e297b8f..ef03a64fd06 100644 --- a/apmpackage/apm/0.1.0/data_stream/metrics/manifest.yml +++ b/apmpackage/apm/0.1.0/data_stream/metrics/manifest.yml @@ -1,4 +1,3 @@ title: APM application metrics type: metrics dataset: apm -ingest_pipeline: apm diff --git a/apmpackage/apm/0.1.0/data_stream/profiles/elasticsearch/ingest_pipeline/apm.json b/apmpackage/apm/0.1.0/data_stream/profiles/elasticsearch/ingest_pipeline/default.json similarity index 52% rename from apmpackage/apm/0.1.0/data_stream/profiles/elasticsearch/ingest_pipeline/apm.json rename to apmpackage/apm/0.1.0/data_stream/profiles/elasticsearch/ingest_pipeline/default.json index 05ada716015..f278bd45b28 100644 --- a/apmpackage/apm/0.1.0/data_stream/profiles/elasticsearch/ingest_pipeline/apm.json +++ b/apmpackage/apm/0.1.0/data_stream/profiles/elasticsearch/ingest_pipeline/default.json @@ -3,22 +3,22 @@ "processors": [ { "pipeline": { - "name": "apm_user_agent" + "name": "profiles-apm-0.1.0-apm_user_agent" } }, { "pipeline": { - "name": "apm_user_geo" + "name": "profiles-apm-0.1.0-apm_user_geo" } }, { "pipeline": { - "name": "apm_ingest_timestamp" + "name": "profiles-apm-0.1.0-apm_ingest_timestamp" } }, { "pipeline": { - "name": "apm_remove_span_metadata" + "name": "profiles-apm-0.1.0-apm_remove_span_metadata" } } ] diff --git a/apmpackage/apm/0.1.0/data_stream/profiles/manifest.yml b/apmpackage/apm/0.1.0/data_stream/profiles/manifest.yml index 2f96abb5b7f..8452356da95 100644 --- a/apmpackage/apm/0.1.0/data_stream/profiles/manifest.yml +++ b/apmpackage/apm/0.1.0/data_stream/profiles/manifest.yml @@ -1,4 +1,3 @@ title: APM profiles type: metrics dataset: apm.profiling -ingest_pipeline: apm diff --git a/apmpackage/apm/0.1.0/data_stream/traces/elasticsearch/ingest_pipeline/apm.json b/apmpackage/apm/0.1.0/data_stream/traces/elasticsearch/ingest_pipeline/apm.json deleted file mode 100644 index 05ada716015..00000000000 --- a/apmpackage/apm/0.1.0/data_stream/traces/elasticsearch/ingest_pipeline/apm.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "description": "Default enrichment for APM events", - "processors": [ - { - "pipeline": { - "name": "apm_user_agent" - } - }, - { - "pipeline": { - "name": "apm_user_geo" - } - }, - { - "pipeline": { - "name": "apm_ingest_timestamp" - } - }, - { - "pipeline": { - "name": "apm_remove_span_metadata" - } - } - ] -} \ No newline at end of file diff --git a/apmpackage/apm/0.1.0/data_stream/internal_metrics/elasticsearch/ingest_pipeline/apm.json b/apmpackage/apm/0.1.0/data_stream/traces/elasticsearch/ingest_pipeline/default.json similarity index 53% rename from apmpackage/apm/0.1.0/data_stream/internal_metrics/elasticsearch/ingest_pipeline/apm.json rename to apmpackage/apm/0.1.0/data_stream/traces/elasticsearch/ingest_pipeline/default.json index 05ada716015..9c816d624e0 100644 --- a/apmpackage/apm/0.1.0/data_stream/internal_metrics/elasticsearch/ingest_pipeline/apm.json +++ b/apmpackage/apm/0.1.0/data_stream/traces/elasticsearch/ingest_pipeline/default.json @@ -3,22 +3,22 @@ "processors": [ { "pipeline": { - "name": "apm_user_agent" + "name": "traces-apm-0.1.0-apm_user_agent" } }, { "pipeline": { - "name": "apm_user_geo" + "name": "traces-apm-0.1.0-apm_user_geo" } }, { "pipeline": { - "name": "apm_ingest_timestamp" + "name": "traces-apm-0.1.0-apm_ingest_timestamp" } }, { "pipeline": { - "name": "apm_remove_span_metadata" + "name": "traces-apm-0.1.0-apm_remove_span_metadata" } } ] diff --git a/apmpackage/apm/0.1.0/data_stream/traces/manifest.yml b/apmpackage/apm/0.1.0/data_stream/traces/manifest.yml index 6fb4b202e7c..d04968d09b4 100644 --- a/apmpackage/apm/0.1.0/data_stream/traces/manifest.yml +++ b/apmpackage/apm/0.1.0/data_stream/traces/manifest.yml @@ -1,4 +1,3 @@ title: APM traces type: traces dataset: apm -ingest_pipeline: apm diff --git a/apmpackage/cmd/gen-package/genpipelines.go b/apmpackage/cmd/gen-package/genpipelines.go index 89e4b4650de..28a3bafc52d 100644 --- a/apmpackage/cmd/gen-package/genpipelines.go +++ b/apmpackage/cmd/gen-package/genpipelines.go @@ -19,11 +19,68 @@ package main import ( "encoding/json" + "errors" "io/ioutil" "os" "path/filepath" ) +var streamMappings = map[string]string{ + "logs": "logs-apm.error", + "traces": "traces-apm", + "metrics": "metrics-apm", + "internal_metrics": "metrics-apm.internal", + "profiles": "profiles-apm", +} + +type PipelineDef struct { + ID string `json:"id"` + Body PipelineBody `json:"body"` +} + +type PipelineBody struct { + Description string `json:"description"` + Processors []Processor `json:"processors"` +} + +type Processor struct { + Pipeline *Pipeline `json:"pipeline,omitempty"` + m map[string]interface{} +} + +type Pipeline struct { + Name string `json:"name"` +} + +type _Processor Processor + +func (p *Processor) UnmarshalJSON(bytes []byte) error { + aux := _Processor{} + err := json.Unmarshal(bytes, &aux) + if err != nil { + return err + } + + *p = Processor(aux) + m := make(map[string]interface{}) + + err = json.Unmarshal(bytes, &m) + if err != nil { + return err + } + delete(m, "pipeline") + p.m = m + return nil +} + +func (p *Processor) MarshalJSON() ([]byte, error) { + aux := _Processor(*p) + if p.Pipeline != nil { + return json.Marshal(aux) + } + return json.Marshal(p.m) +} + func generatePipelines(version, dataStream string) { pipelines, err := os.Open("ingest/pipeline/definition.json") if err != nil { @@ -36,7 +93,7 @@ func generatePipelines(version, dataStream string) { panic(err) } - var definitions = make([]map[string]interface{}, 0) + var definitions = make([]PipelineDef, 0) err = json.Unmarshal(bytes, &definitions) if err != nil { panic(err) @@ -44,21 +101,32 @@ func generatePipelines(version, dataStream string) { os.MkdirAll(pipelinesPath(version, dataStream), 0755) + var apmPipeline PipelineBody for _, definition := range definitions { - pipeline, ok := definition["body"] - if !ok { - continue - } - id, ok := definition["id"] - if !ok { + pipeline := definition.Body + if definition.ID == "apm" { + apmPipeline = pipeline continue } - out, err := json.MarshalIndent(pipeline, "", " ") if err != nil { panic(err) } - fName := filepath.Join(pipelinesPath(version, dataStream), id.(string)+".json") + fName := filepath.Join(pipelinesPath(version, dataStream), definition.ID+".json") ioutil.WriteFile(fName, out, 0644) } + + for _, p := range apmPipeline.Processors { + if p.Pipeline == nil { + // should not happen, lets panic loudly + panic(errors.New("expected pipeline processor")) + } + p.Pipeline.Name = streamMappings[dataStream] + "-" + version + "-" + p.Pipeline.Name + } + out, err := json.MarshalIndent(apmPipeline, "", " ") + if err != nil { + panic(err) + } + fName := filepath.Join(pipelinesPath(version, dataStream), "default.json") + ioutil.WriteFile(fName, out, 0644) } diff --git a/apmpackage/cmd/gen-package/main.go b/apmpackage/cmd/gen-package/main.go index fd438f5396c..6e536739ea1 100644 --- a/apmpackage/cmd/gen-package/main.go +++ b/apmpackage/cmd/gen-package/main.go @@ -23,7 +23,6 @@ import ( "io/ioutil" "log" "os" - "path/filepath" "github.com/elastic/apm-server/cmd" "github.com/elastic/beats/v7/libbeat/common" @@ -46,11 +45,6 @@ func main() { for dataStream := range inputFields { generatePipelines(packageVersion, dataStream) } - // hack, remove when bugfix comes to Kibana - bad := filepath.Join(pipelinesPath(packageVersion, "logs"), "apm.json") - good := filepath.Join(pipelinesPath(packageVersion, "logs"), "default.json") - os.Rename(bad, good) - generateDocs(inputFields, packageVersion) log.Printf("Package fields and docs generated for version %s (stack %s)", packageVersion, stackVersion.String()) } @@ -66,6 +60,7 @@ func clear(version string) { if f.IsDir() { os.Remove(ecsFilePath(version, f.Name())) os.Remove(fieldsFilePath(version, f.Name())) + os.RemoveAll(pipelinesPath(version, f.Name())) } } ioutil.WriteFile(docsFilePath(version), nil, 0644) From fcb613097cb93d7590eb16ca17a6204515727e71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Fri, 4 Dec 2020 09:04:54 +0100 Subject: [PATCH 2/2] Update apmpackage/cmd/gen-package/genpipelines.go --- apmpackage/cmd/gen-package/genpipelines.go | 1 + 1 file changed, 1 insertion(+) diff --git a/apmpackage/cmd/gen-package/genpipelines.go b/apmpackage/cmd/gen-package/genpipelines.go index 28a3bafc52d..329e5cc88c4 100644 --- a/apmpackage/cmd/gen-package/genpipelines.go +++ b/apmpackage/cmd/gen-package/genpipelines.go @@ -121,6 +121,7 @@ func generatePipelines(version, dataStream string) { // should not happen, lets panic loudly panic(errors.New("expected pipeline processor")) } + // name is updated to match the one generated by Fleet when installs the pipelines p.Pipeline.Name = streamMappings[dataStream] + "-" + version + "-" + p.Pipeline.Name } out, err := json.MarshalIndent(apmPipeline, "", " ")