From 2f3eb92a1b030b3cdf0584b9c3e49d24f9dc6a99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Tue, 15 Dec 2020 17:03:39 +0100 Subject: [PATCH] [7.x] Fix pipeline name refs and remove service datasets (#4489, #4491) (#4537) * Fix pipeline name refs (#4489) Update pipeline names referenced in default.json files to match the one generated by Fleet. * Remove service datasets (#4491) Also rename data streams and fix the profiles pipelines referenced names --- .../ingest_pipeline/apm_ingest_timestamp.json | 0 .../apm_remove_span_metadata.json | 0 .../ingest_pipeline/apm_user_agent.json | 0 .../ingest_pipeline/apm_user_geo.json | 0 .../ingest_pipeline/default.json} | 8 +- .../fields/base-fields.yml | 0 .../{metrics => app_metrics}/fields/ecs.yml | 0 .../fields/fields.yml | 0 .../{metrics => app_metrics}/manifest.yml | 1 - .../ingest_pipeline/apm_ingest_timestamp.json | 0 .../apm_remove_span_metadata.json | 0 .../ingest_pipeline/apm_user_agent.json | 0 .../ingest_pipeline/apm_user_geo.json | 0 .../ingest_pipeline/default.json} | 8 +- .../fields/base-fields.yml | 0 .../{logs => error_logs}/fields/ecs.yml | 0 .../{logs => error_logs}/fields/fields.yml | 0 .../{logs => error_logs}/manifest.yml | 0 .../ingest_pipeline/default.json | 25 ++++++ .../data_stream/internal_metrics/manifest.yml | 1 - .../ingest_pipeline/apm_ingest_timestamp.json | 0 .../apm_remove_span_metadata.json | 0 .../ingest_pipeline/apm_user_agent.json | 0 .../ingest_pipeline/apm_user_geo.json | 0 .../ingest_pipeline/default.json | 25 ++++++ .../fields/base-fields.yml | 0 .../fields/ecs.yml | 0 .../fields/fields.yml | 0 .../manifest.yml | 1 - .../elasticsearch/ingest_pipeline/apm.json | 25 ------ .../elasticsearch/ingest_pipeline/apm.json | 25 ------ .../ingest_pipeline/default.json | 8 +- .../apm/0.1.0/data_stream/traces/manifest.yml | 1 - apmpackage/apm/0.1.0/manifest.yml | 4 +- apmpackage/cmd/gen-package/gendocs.go | 10 +-- apmpackage/cmd/gen-package/genfields.go | 6 +- apmpackage/cmd/gen-package/genpipelines.go | 89 +++++++++++++++++-- apmpackage/cmd/gen-package/main.go | 7 +- datastreams/servicename.go | 2 + model/error.go | 4 +- model/error_test.go | 8 +- model/metricset.go | 9 +- model/metricset_test.go | 10 +-- model/profile.go | 7 +- model/profile_test.go | 2 +- model/span.go | 4 +- model/span_test.go | 6 +- model/transaction.go | 5 +- model/transaction_test.go | 2 +- .../test_approved/consume_span.approved.json | 4 +- .../jaeger_sampling_rate.approved.json | 4 +- .../metadata_jaeger-no-language.approved.json | 2 +- .../metadata_jaeger-version.approved.json | 2 +- .../metadata_jaeger.approved.json | 2 +- ...metadata_jaeger_full-traceid.approved.json | 2 +- .../metadata_jaeger_minimal.approved.json | 2 +- .../metadata_minimal.approved.json | 2 +- .../span_jaeger_custom.approved.json | 2 +- .../span_jaeger_db.approved.json | 2 +- .../span_jaeger_http.approved.json | 14 +-- ...span_jaeger_http_status_code.approved.json | 2 +- ...an_jaeger_https_default_port.approved.json | 2 +- .../span_jaeger_messaging.approved.json | 2 +- .../transaction_jaeger_custom.approved.json | 2 +- .../transaction_jaeger_full.approved.json | 14 +-- .../transaction_jaeger_no_attrs.approved.json | 2 +- ...action_jaeger_type_component.approved.json | 2 +- ...action_jaeger_type_messaging.approved.json | 2 +- ...nsaction_jaeger_type_request.approved.json | 2 +- ...n_jaeger_type_request_result.approved.json | 2 +- .../testIntakeIntegrationErrors.approved.json | 10 +-- .../testIntakeIntegrationEvents.approved.json | 8 +- ...ntakeIntegrationInvalidEvent.approved.json | 2 +- ...eIntegrationInvalidJSONEvent.approved.json | 2 +- ...ntegrationMetadataNullValues.approved.json | 2 +- ...tIntakeIntegrationMetricsets.approved.json | 6 +- ...akeIntegrationMinimalService.approved.json | 4 +- ...ntegrationOptionalTimestamps.approved.json | 6 +- ...stIntakeIntegrationRumErrors.approved.json | 2 +- ...keIntegrationRumTransactions.approved.json | 4 +- .../testIntakeIntegrationSpans.approved.json | 12 +-- ...ntakeIntegrationTransactions.approved.json | 8 +- .../testIntakeRUMV3Errors.approved.json | 2 +- .../testIntakeRUMV3Events.approved.json | 26 +++--- .../TestDataStreamsEnabled/true.approved.json | 2 +- systemtest/datastreams_test.go | 2 +- tests/system/test_tls.py | 1 + 87 files changed, 266 insertions(+), 206 deletions(-) rename apmpackage/apm/0.1.0/data_stream/{logs => app_metrics}/elasticsearch/ingest_pipeline/apm_ingest_timestamp.json (100%) rename apmpackage/apm/0.1.0/data_stream/{logs => app_metrics}/elasticsearch/ingest_pipeline/apm_remove_span_metadata.json (100%) rename apmpackage/apm/0.1.0/data_stream/{logs => app_metrics}/elasticsearch/ingest_pipeline/apm_user_agent.json (100%) rename apmpackage/apm/0.1.0/data_stream/{logs => app_metrics}/elasticsearch/ingest_pipeline/apm_user_geo.json (100%) rename apmpackage/apm/0.1.0/data_stream/{internal_metrics/elasticsearch/ingest_pipeline/apm.json => app_metrics/elasticsearch/ingest_pipeline/default.json} (52%) rename apmpackage/apm/0.1.0/data_stream/{logs => app_metrics}/fields/base-fields.yml (100%) rename apmpackage/apm/0.1.0/data_stream/{metrics => app_metrics}/fields/ecs.yml (100%) rename apmpackage/apm/0.1.0/data_stream/{metrics => app_metrics}/fields/fields.yml (100%) rename apmpackage/apm/0.1.0/data_stream/{metrics => app_metrics}/manifest.yml (73%) rename apmpackage/apm/0.1.0/data_stream/{metrics => error_logs}/elasticsearch/ingest_pipeline/apm_ingest_timestamp.json (100%) rename apmpackage/apm/0.1.0/data_stream/{metrics => error_logs}/elasticsearch/ingest_pipeline/apm_remove_span_metadata.json (100%) rename apmpackage/apm/0.1.0/data_stream/{metrics => error_logs}/elasticsearch/ingest_pipeline/apm_user_agent.json (100%) rename apmpackage/apm/0.1.0/data_stream/{metrics => error_logs}/elasticsearch/ingest_pipeline/apm_user_geo.json (100%) rename apmpackage/apm/0.1.0/data_stream/{metrics/elasticsearch/ingest_pipeline/apm.json => error_logs/elasticsearch/ingest_pipeline/default.json} (51%) rename apmpackage/apm/0.1.0/data_stream/{metrics => error_logs}/fields/base-fields.yml (100%) rename apmpackage/apm/0.1.0/data_stream/{logs => error_logs}/fields/ecs.yml (100%) rename apmpackage/apm/0.1.0/data_stream/{logs => error_logs}/fields/fields.yml (100%) rename apmpackage/apm/0.1.0/data_stream/{logs => error_logs}/manifest.yml (100%) create mode 100644 apmpackage/apm/0.1.0/data_stream/internal_metrics/elasticsearch/ingest_pipeline/default.json rename apmpackage/apm/0.1.0/data_stream/{profiles => profile_metrics}/elasticsearch/ingest_pipeline/apm_ingest_timestamp.json (100%) rename apmpackage/apm/0.1.0/data_stream/{profiles => profile_metrics}/elasticsearch/ingest_pipeline/apm_remove_span_metadata.json (100%) rename apmpackage/apm/0.1.0/data_stream/{profiles => profile_metrics}/elasticsearch/ingest_pipeline/apm_user_agent.json (100%) rename apmpackage/apm/0.1.0/data_stream/{profiles => profile_metrics}/elasticsearch/ingest_pipeline/apm_user_geo.json (100%) create mode 100644 apmpackage/apm/0.1.0/data_stream/profile_metrics/elasticsearch/ingest_pipeline/default.json rename apmpackage/apm/0.1.0/data_stream/{profiles => profile_metrics}/fields/base-fields.yml (100%) rename apmpackage/apm/0.1.0/data_stream/{profiles => profile_metrics}/fields/ecs.yml (100%) rename apmpackage/apm/0.1.0/data_stream/{profiles => profile_metrics}/fields/fields.yml (100%) rename apmpackage/apm/0.1.0/data_stream/{profiles => profile_metrics}/manifest.yml (73%) delete mode 100644 apmpackage/apm/0.1.0/data_stream/profiles/elasticsearch/ingest_pipeline/apm.json delete mode 100644 apmpackage/apm/0.1.0/data_stream/traces/elasticsearch/ingest_pipeline/apm.json rename apmpackage/apm/0.1.0/data_stream/{logs => traces}/elasticsearch/ingest_pipeline/default.json (53%) diff --git a/apmpackage/apm/0.1.0/data_stream/logs/elasticsearch/ingest_pipeline/apm_ingest_timestamp.json b/apmpackage/apm/0.1.0/data_stream/app_metrics/elasticsearch/ingest_pipeline/apm_ingest_timestamp.json similarity index 100% rename from apmpackage/apm/0.1.0/data_stream/logs/elasticsearch/ingest_pipeline/apm_ingest_timestamp.json rename to apmpackage/apm/0.1.0/data_stream/app_metrics/elasticsearch/ingest_pipeline/apm_ingest_timestamp.json diff --git a/apmpackage/apm/0.1.0/data_stream/logs/elasticsearch/ingest_pipeline/apm_remove_span_metadata.json b/apmpackage/apm/0.1.0/data_stream/app_metrics/elasticsearch/ingest_pipeline/apm_remove_span_metadata.json similarity index 100% rename from apmpackage/apm/0.1.0/data_stream/logs/elasticsearch/ingest_pipeline/apm_remove_span_metadata.json rename to apmpackage/apm/0.1.0/data_stream/app_metrics/elasticsearch/ingest_pipeline/apm_remove_span_metadata.json diff --git a/apmpackage/apm/0.1.0/data_stream/logs/elasticsearch/ingest_pipeline/apm_user_agent.json b/apmpackage/apm/0.1.0/data_stream/app_metrics/elasticsearch/ingest_pipeline/apm_user_agent.json similarity index 100% rename from apmpackage/apm/0.1.0/data_stream/logs/elasticsearch/ingest_pipeline/apm_user_agent.json rename to apmpackage/apm/0.1.0/data_stream/app_metrics/elasticsearch/ingest_pipeline/apm_user_agent.json diff --git a/apmpackage/apm/0.1.0/data_stream/logs/elasticsearch/ingest_pipeline/apm_user_geo.json b/apmpackage/apm/0.1.0/data_stream/app_metrics/elasticsearch/ingest_pipeline/apm_user_geo.json similarity index 100% rename from apmpackage/apm/0.1.0/data_stream/logs/elasticsearch/ingest_pipeline/apm_user_geo.json rename to apmpackage/apm/0.1.0/data_stream/app_metrics/elasticsearch/ingest_pipeline/apm_user_geo.json diff --git a/apmpackage/apm/0.1.0/data_stream/internal_metrics/elasticsearch/ingest_pipeline/apm.json b/apmpackage/apm/0.1.0/data_stream/app_metrics/elasticsearch/ingest_pipeline/default.json similarity index 52% rename from apmpackage/apm/0.1.0/data_stream/internal_metrics/elasticsearch/ingest_pipeline/apm.json rename to apmpackage/apm/0.1.0/data_stream/app_metrics/elasticsearch/ingest_pipeline/default.json index 05ada716015..e848ed68123 100644 --- a/apmpackage/apm/0.1.0/data_stream/internal_metrics/elasticsearch/ingest_pipeline/apm.json +++ b/apmpackage/apm/0.1.0/data_stream/app_metrics/elasticsearch/ingest_pipeline/default.json @@ -3,22 +3,22 @@ "processors": [ { "pipeline": { - "name": "apm_user_agent" + "name": "metrics-apm-0.1.0-apm_user_agent" } }, { "pipeline": { - "name": "apm_user_geo" + "name": "metrics-apm-0.1.0-apm_user_geo" } }, { "pipeline": { - "name": "apm_ingest_timestamp" + "name": "metrics-apm-0.1.0-apm_ingest_timestamp" } }, { "pipeline": { - "name": "apm_remove_span_metadata" + "name": "metrics-apm-0.1.0-apm_remove_span_metadata" } } ] diff --git a/apmpackage/apm/0.1.0/data_stream/logs/fields/base-fields.yml b/apmpackage/apm/0.1.0/data_stream/app_metrics/fields/base-fields.yml similarity index 100% rename from apmpackage/apm/0.1.0/data_stream/logs/fields/base-fields.yml rename to apmpackage/apm/0.1.0/data_stream/app_metrics/fields/base-fields.yml diff --git a/apmpackage/apm/0.1.0/data_stream/metrics/fields/ecs.yml b/apmpackage/apm/0.1.0/data_stream/app_metrics/fields/ecs.yml similarity index 100% rename from apmpackage/apm/0.1.0/data_stream/metrics/fields/ecs.yml rename to apmpackage/apm/0.1.0/data_stream/app_metrics/fields/ecs.yml diff --git a/apmpackage/apm/0.1.0/data_stream/metrics/fields/fields.yml b/apmpackage/apm/0.1.0/data_stream/app_metrics/fields/fields.yml similarity index 100% rename from apmpackage/apm/0.1.0/data_stream/metrics/fields/fields.yml rename to apmpackage/apm/0.1.0/data_stream/app_metrics/fields/fields.yml diff --git a/apmpackage/apm/0.1.0/data_stream/metrics/manifest.yml b/apmpackage/apm/0.1.0/data_stream/app_metrics/manifest.yml similarity index 73% rename from apmpackage/apm/0.1.0/data_stream/metrics/manifest.yml rename to apmpackage/apm/0.1.0/data_stream/app_metrics/manifest.yml index 07c8e297b8f..ef03a64fd06 100644 --- a/apmpackage/apm/0.1.0/data_stream/metrics/manifest.yml +++ b/apmpackage/apm/0.1.0/data_stream/app_metrics/manifest.yml @@ -1,4 +1,3 @@ title: APM application metrics type: metrics dataset: apm -ingest_pipeline: apm diff --git a/apmpackage/apm/0.1.0/data_stream/metrics/elasticsearch/ingest_pipeline/apm_ingest_timestamp.json b/apmpackage/apm/0.1.0/data_stream/error_logs/elasticsearch/ingest_pipeline/apm_ingest_timestamp.json similarity index 100% rename from apmpackage/apm/0.1.0/data_stream/metrics/elasticsearch/ingest_pipeline/apm_ingest_timestamp.json rename to apmpackage/apm/0.1.0/data_stream/error_logs/elasticsearch/ingest_pipeline/apm_ingest_timestamp.json diff --git a/apmpackage/apm/0.1.0/data_stream/metrics/elasticsearch/ingest_pipeline/apm_remove_span_metadata.json b/apmpackage/apm/0.1.0/data_stream/error_logs/elasticsearch/ingest_pipeline/apm_remove_span_metadata.json similarity index 100% rename from apmpackage/apm/0.1.0/data_stream/metrics/elasticsearch/ingest_pipeline/apm_remove_span_metadata.json rename to apmpackage/apm/0.1.0/data_stream/error_logs/elasticsearch/ingest_pipeline/apm_remove_span_metadata.json diff --git a/apmpackage/apm/0.1.0/data_stream/metrics/elasticsearch/ingest_pipeline/apm_user_agent.json b/apmpackage/apm/0.1.0/data_stream/error_logs/elasticsearch/ingest_pipeline/apm_user_agent.json similarity index 100% rename from apmpackage/apm/0.1.0/data_stream/metrics/elasticsearch/ingest_pipeline/apm_user_agent.json rename to apmpackage/apm/0.1.0/data_stream/error_logs/elasticsearch/ingest_pipeline/apm_user_agent.json diff --git a/apmpackage/apm/0.1.0/data_stream/metrics/elasticsearch/ingest_pipeline/apm_user_geo.json b/apmpackage/apm/0.1.0/data_stream/error_logs/elasticsearch/ingest_pipeline/apm_user_geo.json similarity index 100% rename from apmpackage/apm/0.1.0/data_stream/metrics/elasticsearch/ingest_pipeline/apm_user_geo.json rename to apmpackage/apm/0.1.0/data_stream/error_logs/elasticsearch/ingest_pipeline/apm_user_geo.json diff --git a/apmpackage/apm/0.1.0/data_stream/metrics/elasticsearch/ingest_pipeline/apm.json b/apmpackage/apm/0.1.0/data_stream/error_logs/elasticsearch/ingest_pipeline/default.json similarity index 51% rename from apmpackage/apm/0.1.0/data_stream/metrics/elasticsearch/ingest_pipeline/apm.json rename to apmpackage/apm/0.1.0/data_stream/error_logs/elasticsearch/ingest_pipeline/default.json index 05ada716015..9203f00cc2f 100644 --- a/apmpackage/apm/0.1.0/data_stream/metrics/elasticsearch/ingest_pipeline/apm.json +++ b/apmpackage/apm/0.1.0/data_stream/error_logs/elasticsearch/ingest_pipeline/default.json @@ -3,22 +3,22 @@ "processors": [ { "pipeline": { - "name": "apm_user_agent" + "name": "logs-apm.error-0.1.0-apm_user_agent" } }, { "pipeline": { - "name": "apm_user_geo" + "name": "logs-apm.error-0.1.0-apm_user_geo" } }, { "pipeline": { - "name": "apm_ingest_timestamp" + "name": "logs-apm.error-0.1.0-apm_ingest_timestamp" } }, { "pipeline": { - "name": "apm_remove_span_metadata" + "name": "logs-apm.error-0.1.0-apm_remove_span_metadata" } } ] diff --git a/apmpackage/apm/0.1.0/data_stream/metrics/fields/base-fields.yml b/apmpackage/apm/0.1.0/data_stream/error_logs/fields/base-fields.yml similarity index 100% rename from apmpackage/apm/0.1.0/data_stream/metrics/fields/base-fields.yml rename to apmpackage/apm/0.1.0/data_stream/error_logs/fields/base-fields.yml diff --git a/apmpackage/apm/0.1.0/data_stream/logs/fields/ecs.yml b/apmpackage/apm/0.1.0/data_stream/error_logs/fields/ecs.yml similarity index 100% rename from apmpackage/apm/0.1.0/data_stream/logs/fields/ecs.yml rename to apmpackage/apm/0.1.0/data_stream/error_logs/fields/ecs.yml diff --git a/apmpackage/apm/0.1.0/data_stream/logs/fields/fields.yml b/apmpackage/apm/0.1.0/data_stream/error_logs/fields/fields.yml similarity index 100% rename from apmpackage/apm/0.1.0/data_stream/logs/fields/fields.yml rename to apmpackage/apm/0.1.0/data_stream/error_logs/fields/fields.yml diff --git a/apmpackage/apm/0.1.0/data_stream/logs/manifest.yml b/apmpackage/apm/0.1.0/data_stream/error_logs/manifest.yml similarity index 100% rename from apmpackage/apm/0.1.0/data_stream/logs/manifest.yml rename to apmpackage/apm/0.1.0/data_stream/error_logs/manifest.yml diff --git a/apmpackage/apm/0.1.0/data_stream/internal_metrics/elasticsearch/ingest_pipeline/default.json b/apmpackage/apm/0.1.0/data_stream/internal_metrics/elasticsearch/ingest_pipeline/default.json new file mode 100644 index 00000000000..1a5bdc999ce --- /dev/null +++ b/apmpackage/apm/0.1.0/data_stream/internal_metrics/elasticsearch/ingest_pipeline/default.json @@ -0,0 +1,25 @@ +{ + "description": "Default enrichment for APM events", + "processors": [ + { + "pipeline": { + "name": "metrics-apm.internal-0.1.0-apm_user_agent" + } + }, + { + "pipeline": { + "name": "metrics-apm.internal-0.1.0-apm_user_geo" + } + }, + { + "pipeline": { + "name": "metrics-apm.internal-0.1.0-apm_ingest_timestamp" + } + }, + { + "pipeline": { + "name": "metrics-apm.internal-0.1.0-apm_remove_span_metadata" + } + } + ] +} \ No newline at end of file diff --git a/apmpackage/apm/0.1.0/data_stream/internal_metrics/manifest.yml b/apmpackage/apm/0.1.0/data_stream/internal_metrics/manifest.yml index e891f9ceaf9..6fb4ec958c5 100644 --- a/apmpackage/apm/0.1.0/data_stream/internal_metrics/manifest.yml +++ b/apmpackage/apm/0.1.0/data_stream/internal_metrics/manifest.yml @@ -1,4 +1,3 @@ title: APM internal metrics type: metrics dataset: apm.internal -ingest_pipeline: apm diff --git a/apmpackage/apm/0.1.0/data_stream/profiles/elasticsearch/ingest_pipeline/apm_ingest_timestamp.json b/apmpackage/apm/0.1.0/data_stream/profile_metrics/elasticsearch/ingest_pipeline/apm_ingest_timestamp.json similarity index 100% rename from apmpackage/apm/0.1.0/data_stream/profiles/elasticsearch/ingest_pipeline/apm_ingest_timestamp.json rename to apmpackage/apm/0.1.0/data_stream/profile_metrics/elasticsearch/ingest_pipeline/apm_ingest_timestamp.json diff --git a/apmpackage/apm/0.1.0/data_stream/profiles/elasticsearch/ingest_pipeline/apm_remove_span_metadata.json b/apmpackage/apm/0.1.0/data_stream/profile_metrics/elasticsearch/ingest_pipeline/apm_remove_span_metadata.json similarity index 100% rename from apmpackage/apm/0.1.0/data_stream/profiles/elasticsearch/ingest_pipeline/apm_remove_span_metadata.json rename to apmpackage/apm/0.1.0/data_stream/profile_metrics/elasticsearch/ingest_pipeline/apm_remove_span_metadata.json diff --git a/apmpackage/apm/0.1.0/data_stream/profiles/elasticsearch/ingest_pipeline/apm_user_agent.json b/apmpackage/apm/0.1.0/data_stream/profile_metrics/elasticsearch/ingest_pipeline/apm_user_agent.json similarity index 100% rename from apmpackage/apm/0.1.0/data_stream/profiles/elasticsearch/ingest_pipeline/apm_user_agent.json rename to apmpackage/apm/0.1.0/data_stream/profile_metrics/elasticsearch/ingest_pipeline/apm_user_agent.json diff --git a/apmpackage/apm/0.1.0/data_stream/profiles/elasticsearch/ingest_pipeline/apm_user_geo.json b/apmpackage/apm/0.1.0/data_stream/profile_metrics/elasticsearch/ingest_pipeline/apm_user_geo.json similarity index 100% rename from apmpackage/apm/0.1.0/data_stream/profiles/elasticsearch/ingest_pipeline/apm_user_geo.json rename to apmpackage/apm/0.1.0/data_stream/profile_metrics/elasticsearch/ingest_pipeline/apm_user_geo.json diff --git a/apmpackage/apm/0.1.0/data_stream/profile_metrics/elasticsearch/ingest_pipeline/default.json b/apmpackage/apm/0.1.0/data_stream/profile_metrics/elasticsearch/ingest_pipeline/default.json new file mode 100644 index 00000000000..b0b9f729f40 --- /dev/null +++ b/apmpackage/apm/0.1.0/data_stream/profile_metrics/elasticsearch/ingest_pipeline/default.json @@ -0,0 +1,25 @@ +{ + "description": "Default enrichment for APM events", + "processors": [ + { + "pipeline": { + "name": "metrics-apm.profiling-0.1.0-apm_user_agent" + } + }, + { + "pipeline": { + "name": "metrics-apm.profiling-0.1.0-apm_user_geo" + } + }, + { + "pipeline": { + "name": "metrics-apm.profiling-0.1.0-apm_ingest_timestamp" + } + }, + { + "pipeline": { + "name": "metrics-apm.profiling-0.1.0-apm_remove_span_metadata" + } + } + ] +} \ No newline at end of file diff --git a/apmpackage/apm/0.1.0/data_stream/profiles/fields/base-fields.yml b/apmpackage/apm/0.1.0/data_stream/profile_metrics/fields/base-fields.yml similarity index 100% rename from apmpackage/apm/0.1.0/data_stream/profiles/fields/base-fields.yml rename to apmpackage/apm/0.1.0/data_stream/profile_metrics/fields/base-fields.yml diff --git a/apmpackage/apm/0.1.0/data_stream/profiles/fields/ecs.yml b/apmpackage/apm/0.1.0/data_stream/profile_metrics/fields/ecs.yml similarity index 100% rename from apmpackage/apm/0.1.0/data_stream/profiles/fields/ecs.yml rename to apmpackage/apm/0.1.0/data_stream/profile_metrics/fields/ecs.yml diff --git a/apmpackage/apm/0.1.0/data_stream/profiles/fields/fields.yml b/apmpackage/apm/0.1.0/data_stream/profile_metrics/fields/fields.yml similarity index 100% rename from apmpackage/apm/0.1.0/data_stream/profiles/fields/fields.yml rename to apmpackage/apm/0.1.0/data_stream/profile_metrics/fields/fields.yml diff --git a/apmpackage/apm/0.1.0/data_stream/profiles/manifest.yml b/apmpackage/apm/0.1.0/data_stream/profile_metrics/manifest.yml similarity index 73% rename from apmpackage/apm/0.1.0/data_stream/profiles/manifest.yml rename to apmpackage/apm/0.1.0/data_stream/profile_metrics/manifest.yml index 2f96abb5b7f..8452356da95 100644 --- a/apmpackage/apm/0.1.0/data_stream/profiles/manifest.yml +++ b/apmpackage/apm/0.1.0/data_stream/profile_metrics/manifest.yml @@ -1,4 +1,3 @@ title: APM profiles type: metrics dataset: apm.profiling -ingest_pipeline: apm diff --git a/apmpackage/apm/0.1.0/data_stream/profiles/elasticsearch/ingest_pipeline/apm.json b/apmpackage/apm/0.1.0/data_stream/profiles/elasticsearch/ingest_pipeline/apm.json deleted file mode 100644 index 05ada716015..00000000000 --- a/apmpackage/apm/0.1.0/data_stream/profiles/elasticsearch/ingest_pipeline/apm.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "description": "Default enrichment for APM events", - "processors": [ - { - "pipeline": { - "name": "apm_user_agent" - } - }, - { - "pipeline": { - "name": "apm_user_geo" - } - }, - { - "pipeline": { - "name": "apm_ingest_timestamp" - } - }, - { - "pipeline": { - "name": "apm_remove_span_metadata" - } - } - ] -} \ No newline at end of file diff --git a/apmpackage/apm/0.1.0/data_stream/traces/elasticsearch/ingest_pipeline/apm.json b/apmpackage/apm/0.1.0/data_stream/traces/elasticsearch/ingest_pipeline/apm.json deleted file mode 100644 index 05ada716015..00000000000 --- a/apmpackage/apm/0.1.0/data_stream/traces/elasticsearch/ingest_pipeline/apm.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "description": "Default enrichment for APM events", - "processors": [ - { - "pipeline": { - "name": "apm_user_agent" - } - }, - { - "pipeline": { - "name": "apm_user_geo" - } - }, - { - "pipeline": { - "name": "apm_ingest_timestamp" - } - }, - { - "pipeline": { - "name": "apm_remove_span_metadata" - } - } - ] -} \ No newline at end of file diff --git a/apmpackage/apm/0.1.0/data_stream/logs/elasticsearch/ingest_pipeline/default.json b/apmpackage/apm/0.1.0/data_stream/traces/elasticsearch/ingest_pipeline/default.json similarity index 53% rename from apmpackage/apm/0.1.0/data_stream/logs/elasticsearch/ingest_pipeline/default.json rename to apmpackage/apm/0.1.0/data_stream/traces/elasticsearch/ingest_pipeline/default.json index 05ada716015..9c816d624e0 100644 --- a/apmpackage/apm/0.1.0/data_stream/logs/elasticsearch/ingest_pipeline/default.json +++ b/apmpackage/apm/0.1.0/data_stream/traces/elasticsearch/ingest_pipeline/default.json @@ -3,22 +3,22 @@ "processors": [ { "pipeline": { - "name": "apm_user_agent" + "name": "traces-apm-0.1.0-apm_user_agent" } }, { "pipeline": { - "name": "apm_user_geo" + "name": "traces-apm-0.1.0-apm_user_geo" } }, { "pipeline": { - "name": "apm_ingest_timestamp" + "name": "traces-apm-0.1.0-apm_ingest_timestamp" } }, { "pipeline": { - "name": "apm_remove_span_metadata" + "name": "traces-apm-0.1.0-apm_remove_span_metadata" } } ] diff --git a/apmpackage/apm/0.1.0/data_stream/traces/manifest.yml b/apmpackage/apm/0.1.0/data_stream/traces/manifest.yml index 6fb4b202e7c..d04968d09b4 100644 --- a/apmpackage/apm/0.1.0/data_stream/traces/manifest.yml +++ b/apmpackage/apm/0.1.0/data_stream/traces/manifest.yml @@ -1,4 +1,3 @@ title: APM traces type: traces dataset: apm -ingest_pipeline: apm diff --git a/apmpackage/apm/0.1.0/manifest.yml b/apmpackage/apm/0.1.0/manifest.yml index 774aec5d4b9..803dd6735a7 100644 --- a/apmpackage/apm/0.1.0/manifest.yml +++ b/apmpackage/apm/0.1.0/manifest.yml @@ -4,9 +4,9 @@ title: Elastic APM version: 0.1.0 license: basic description: Ingest APM data -type: solution # integration / solution +type: integration categories: - - monitoring # TODO do we need a new category? + - monitoring release: experimental # experimental / beta / ga conditions: kibana.version: '^7.11.0' diff --git a/apmpackage/cmd/gen-package/gendocs.go b/apmpackage/cmd/gen-package/gendocs.go index 49e5190cafc..9ca93c63c2f 100644 --- a/apmpackage/cmd/gen-package/gendocs.go +++ b/apmpackage/cmd/gen-package/gendocs.go @@ -29,8 +29,8 @@ import ( func generateDocs(inputFields map[string][]field, version string) { data := docsData{ Traces: prepareFields(inputFields, version, "traces"), - Metrics: prepareFields(inputFields, version, "metrics"), - Logs: prepareFields(inputFields, version, "logs"), + Metrics: prepareFields(inputFields, version, "app_metrics"), + Logs: prepareFields(inputFields, version, "error_logs"), TransactionExample: loadExample("transactions.json"), SpanExample: loadExample("spans.json"), MetricsExample: loadExample("metricsets.json"), @@ -65,17 +65,17 @@ type docsData struct { ErrorExample string } -func prepareFields(inputFields map[string][]field, version, streamType string) []field { +func prepareFields(inputFields map[string][]field, version, stream string) []field { extend := func(fs []field) []field { var baseFields []field - for _, f := range loadFieldsFile(baseFieldsFilePath(version, streamType)) { + for _, f := range loadFieldsFile(baseFieldsFilePath(version, stream)) { f.IsECS = true baseFields = append(baseFields, f) } fs = append(baseFields, fs...) return fs } - return extend(inputFields[streamType]) + return extend(inputFields[stream]) } func loadExample(file string) string { diff --git a/apmpackage/cmd/gen-package/genfields.go b/apmpackage/cmd/gen-package/genfields.go index c5094332e87..c093441093a 100644 --- a/apmpackage/cmd/gen-package/genfields.go +++ b/apmpackage/cmd/gen-package/genfields.go @@ -34,12 +34,12 @@ func generateFields(version string) map[string][]field { ecsFlatFields := loadECSFields() inputFieldsFiles := map[string][]field{ - "logs": format("model/error/_meta/fields.yml"), + "error_logs": format("model/error/_meta/fields.yml"), "internal_metrics": format("model/metricset/_meta/fields.yml", "x-pack/apm-server/fields/_meta/fields.yml"), - "profiles": format("model/profile/_meta/fields.yml"), + "profile_metrics": format("model/profile/_meta/fields.yml"), "traces": format("model/transaction/_meta/fields.yml", "model/span/_meta/fields.yml"), } - inputFieldsFiles["metrics"] = filterInternalMetrics(inputFieldsFiles["internal_metrics"]) + inputFieldsFiles["app_metrics"] = filterInternalMetrics(inputFieldsFiles["internal_metrics"]) for streamType, inputFields := range inputFieldsFiles { var ecsFields []field diff --git a/apmpackage/cmd/gen-package/genpipelines.go b/apmpackage/cmd/gen-package/genpipelines.go index 89e4b4650de..ea8ea1cfddf 100644 --- a/apmpackage/cmd/gen-package/genpipelines.go +++ b/apmpackage/cmd/gen-package/genpipelines.go @@ -19,11 +19,70 @@ package main import ( "encoding/json" + "errors" "io/ioutil" "os" "path/filepath" + + "github.com/elastic/apm-server/model" ) +var streamMappings = map[string]string{ + "error_logs": "logs-" + model.ErrorsDataset, + "traces": "traces-" + model.TracesDataset, + "app_metrics": "metrics-" + model.AppMetricsDataset, + "internal_metrics": "metrics-" + model.InternalMetricsDataset, + "profile_metrics": "metrics-" + model.ProfilesDataset, +} + +type PipelineDef struct { + ID string `json:"id"` + Body PipelineBody `json:"body"` +} + +type PipelineBody struct { + Description string `json:"description"` + Processors []Processor `json:"processors"` +} + +type Processor struct { + Pipeline *Pipeline `json:"pipeline,omitempty"` + m map[string]interface{} +} + +type Pipeline struct { + Name string `json:"name"` +} + +type _Processor Processor + +func (p *Processor) UnmarshalJSON(bytes []byte) error { + aux := _Processor{} + err := json.Unmarshal(bytes, &aux) + if err != nil { + return err + } + + *p = Processor(aux) + m := make(map[string]interface{}) + + err = json.Unmarshal(bytes, &m) + if err != nil { + return err + } + delete(m, "pipeline") + p.m = m + return nil +} + +func (p *Processor) MarshalJSON() ([]byte, error) { + aux := _Processor(*p) + if p.Pipeline != nil { + return json.Marshal(aux) + } + return json.Marshal(p.m) +} + func generatePipelines(version, dataStream string) { pipelines, err := os.Open("ingest/pipeline/definition.json") if err != nil { @@ -36,7 +95,7 @@ func generatePipelines(version, dataStream string) { panic(err) } - var definitions = make([]map[string]interface{}, 0) + var definitions = make([]PipelineDef, 0) err = json.Unmarshal(bytes, &definitions) if err != nil { panic(err) @@ -44,21 +103,33 @@ func generatePipelines(version, dataStream string) { os.MkdirAll(pipelinesPath(version, dataStream), 0755) + var apmPipeline PipelineBody for _, definition := range definitions { - pipeline, ok := definition["body"] - if !ok { - continue - } - id, ok := definition["id"] - if !ok { + pipeline := definition.Body + if definition.ID == "apm" { + apmPipeline = pipeline continue } - out, err := json.MarshalIndent(pipeline, "", " ") if err != nil { panic(err) } - fName := filepath.Join(pipelinesPath(version, dataStream), id.(string)+".json") + fName := filepath.Join(pipelinesPath(version, dataStream), definition.ID+".json") ioutil.WriteFile(fName, out, 0644) } + + for _, p := range apmPipeline.Processors { + if p.Pipeline == nil { + // should not happen, lets panic loudly + panic(errors.New("expected pipeline processor")) + } + // name is updated to match the one generated by Fleet when installs the pipelines + p.Pipeline.Name = streamMappings[dataStream] + "-" + version + "-" + p.Pipeline.Name + } + out, err := json.MarshalIndent(apmPipeline, "", " ") + if err != nil { + panic(err) + } + fName := filepath.Join(pipelinesPath(version, dataStream), "default.json") + ioutil.WriteFile(fName, out, 0644) } diff --git a/apmpackage/cmd/gen-package/main.go b/apmpackage/cmd/gen-package/main.go index fd438f5396c..6e536739ea1 100644 --- a/apmpackage/cmd/gen-package/main.go +++ b/apmpackage/cmd/gen-package/main.go @@ -23,7 +23,6 @@ import ( "io/ioutil" "log" "os" - "path/filepath" "github.com/elastic/apm-server/cmd" "github.com/elastic/beats/v7/libbeat/common" @@ -46,11 +45,6 @@ func main() { for dataStream := range inputFields { generatePipelines(packageVersion, dataStream) } - // hack, remove when bugfix comes to Kibana - bad := filepath.Join(pipelinesPath(packageVersion, "logs"), "apm.json") - good := filepath.Join(pipelinesPath(packageVersion, "logs"), "default.json") - os.Rename(bad, good) - generateDocs(inputFields, packageVersion) log.Printf("Package fields and docs generated for version %s (stack %s)", packageVersion, stackVersion.String()) } @@ -66,6 +60,7 @@ func clear(version string) { if f.IsDir() { os.Remove(ecsFilePath(version, f.Name())) os.Remove(fieldsFilePath(version, f.Name())) + os.RemoveAll(pipelinesPath(version, f.Name())) } } ioutil.WriteFile(docsFilePath(version), nil, 0644) diff --git a/datastreams/servicename.go b/datastreams/servicename.go index ac49422727b..d6208237f6e 100644 --- a/datastreams/servicename.go +++ b/datastreams/servicename.go @@ -24,6 +24,8 @@ import "strings" // // Concretely, this function will lowercase the string and replace any // reserved characters with "_". +// +// TODO: use when Fleet supports variables in data streams (see #4492) func NormalizeServiceName(s string) string { s = strings.ToLower(s) s = strings.Map(replaceReservedRune, s) diff --git a/model/error.go b/model/error.go index 11cffa61551..3c10dfda6b8 100644 --- a/model/error.go +++ b/model/error.go @@ -48,6 +48,7 @@ var ( const ( errorProcessorName = "error" errorDocType = "error" + ErrorsDataset = "apm.error" ) type Error struct { @@ -119,9 +120,8 @@ func (e *Error) Transform(ctx context.Context, cfg *transform.Config) []beat.Eve // Errors are stored in an APM errors-specific "logs" data stream, per service. // By storing errors in a "logs" data stream, they can be viewed in the Logs app // in Kibana. - dataset := fmt.Sprintf("apm.error.%s", datastreams.NormalizeServiceName(e.Metadata.Service.Name)) fields[datastreams.TypeField] = datastreams.LogsType - fields[datastreams.DatasetField] = dataset + fields[datastreams.DatasetField] = ErrorsDataset } // first set the generic metadata (order is relevant) diff --git a/model/error_test.go b/model/error_test.go index cf99375dbf0..3b9fa17cb8b 100644 --- a/model/error_test.go +++ b/model/error_test.go @@ -308,7 +308,7 @@ func TestEvents(t *testing.T) { Transformable: &Error{Timestamp: timestamp, Metadata: md}, Output: common.MapStr{ "data_stream.type": "logs", - "data_stream.dataset": "apm.error.myservice", + "data_stream.dataset": "apm.error", "agent": common.MapStr{"name": "go", "version": "1.0"}, "service": common.MapStr{"name": "myservice", "version": "1.0"}, "error": common.MapStr{ @@ -323,7 +323,7 @@ func TestEvents(t *testing.T) { Transformable: &Error{Timestamp: timestamp, Metadata: md, TransactionSampled: &sampledFalse}, Output: common.MapStr{ "data_stream.type": "logs", - "data_stream.dataset": "apm.error.myservice", + "data_stream.dataset": "apm.error", "transaction": common.MapStr{"sampled": false}, "agent": common.MapStr{"name": "go", "version": "1.0"}, "service": common.MapStr{"name": "myservice", "version": "1.0"}, @@ -339,7 +339,7 @@ func TestEvents(t *testing.T) { Transformable: &Error{Timestamp: timestamp, Metadata: md, TransactionType: &transactionType}, Output: common.MapStr{ "data_stream.type": "logs", - "data_stream.dataset": "apm.error.myservice", + "data_stream.dataset": "apm.error", "transaction": common.MapStr{"type": "request"}, "error": common.MapStr{ "grouping_key": "d41d8cd98f00b204e9800998ecf8427e", @@ -370,7 +370,7 @@ func TestEvents(t *testing.T) { Output: common.MapStr{ "data_stream.type": "logs", - "data_stream.dataset": "apm.error.myservice", + "data_stream.dataset": "apm.error", "labels": common.MapStr{"key": true, "label": 101}, "service": common.MapStr{"name": "myservice", "version": "1.0"}, "agent": common.MapStr{"name": "go", "version": "1.0"}, diff --git a/model/metricset.go b/model/metricset.go index 00443bccfdc..2eaa773f33d 100644 --- a/model/metricset.go +++ b/model/metricset.go @@ -38,6 +38,8 @@ const ( metricsetEventKey = "event" metricsetTransactionKey = "transaction" metricsetSpanKey = "span" + AppMetricsDataset = "apm" + InternalMetricsDataset = "apm.internal" ) var ( @@ -186,16 +188,15 @@ func (me *Metricset) Transform(ctx context.Context, cfg *transform.Config) []bea if cfg.DataStreams { // Metrics are stored in "metrics" data streams. - dataset := "apm." if isInternal { // Metrics that include well-defined transaction/span fields // (i.e. breakdown metrics, transaction and span metrics) will // be stored separately from application and runtime metrics. - dataset = "apm.internal." + fields[datastreams.DatasetField] = InternalMetricsDataset + } else { + fields[datastreams.DatasetField] = AppMetricsDataset } - dataset += datastreams.NormalizeServiceName(me.Metadata.Service.Name) fields[datastreams.TypeField] = datastreams.MetricsType - fields[datastreams.DatasetField] = dataset } return []beat.Event{{ diff --git a/model/metricset_test.go b/model/metricset_test.go index 3a95eb46a4a..32e7ce7bba1 100644 --- a/model/metricset_test.go +++ b/model/metricset_test.go @@ -63,7 +63,7 @@ func TestTransform(t *testing.T) { Output: []common.MapStr{ { "data_stream.type": "metrics", - "data_stream.dataset": "apm.myservice", + "data_stream.dataset": "apm", "processor": common.MapStr{"event": "metric", "name": "metric"}, "service": common.MapStr{ "name": "myservice", @@ -91,7 +91,7 @@ func TestTransform(t *testing.T) { Output: []common.MapStr{ { "data_stream.type": "metrics", - "data_stream.dataset": "apm.myservice", + "data_stream.dataset": "apm", "processor": common.MapStr{"event": "metric", "name": "metric"}, "service": common.MapStr{"name": "myservice"}, "labels": common.MapStr{"a_b": "a.b.value"}, @@ -116,7 +116,7 @@ func TestTransform(t *testing.T) { Output: []common.MapStr{ { "data_stream.type": "metrics", - "data_stream.dataset": "apm.internal.myservice", + "data_stream.dataset": "apm.internal", "processor": common.MapStr{"event": "metric", "name": "metric"}, "service": common.MapStr{"name": "myservice"}, "transaction": common.MapStr{"type": trType, "name": trName}, @@ -154,7 +154,7 @@ func TestTransform(t *testing.T) { Output: []common.MapStr{ { "data_stream.type": "metrics", - "data_stream.dataset": "apm.internal.myservice", + "data_stream.dataset": "apm.internal", "processor": common.MapStr{"event": "metric", "name": "metric"}, "service": common.MapStr{"name": "myservice"}, "event": common.MapStr{"outcome": eventOutcome}, @@ -196,7 +196,7 @@ func TestTransform(t *testing.T) { Output: []common.MapStr{ { "data_stream.type": "metrics", - "data_stream.dataset": "apm.internal.myservice", + "data_stream.dataset": "apm.internal", "processor": common.MapStr{"event": "metric", "name": "metric"}, "service": common.MapStr{"name": "myservice"}, "span": common.MapStr{"type": spType, "subtype": spSubtype, diff --git a/model/profile.go b/model/profile.go index 4c6c3a9f494..1c3be53bfa2 100644 --- a/model/profile.go +++ b/model/profile.go @@ -36,6 +36,7 @@ import ( const ( profileProcessorName = "profile" profileDocType = "profile" + ProfilesDataset = "apm.profiling" ) var profileProcessorEntry = common.MapStr{ @@ -70,10 +71,6 @@ func (pp PprofProfile) Transform(ctx context.Context, cfg *transform.Config) []b // Profiles are stored in their own "metrics" data stream, with a data // set per service. This enables managing retention of profiling data // per-service, and indepedently of lower volume metrics. - var dataset string - if cfg.DataStreams { - dataset = fmt.Sprintf("apm.profiling.%s", datastreams.NormalizeServiceName(pp.Metadata.Service.Name)) - } samples := make([]beat.Event, len(pp.Profile.Sample)) for i, sample := range pp.Profile.Sample { @@ -131,7 +128,7 @@ func (pp PprofProfile) Transform(ctx context.Context, cfg *transform.Config) []b } if cfg.DataStreams { event.Fields[datastreams.TypeField] = datastreams.MetricsType - event.Fields[datastreams.DatasetField] = dataset + event.Fields[datastreams.DatasetField] = ProfilesDataset } var profileLabels common.MapStr if len(sample.Label) > 0 { diff --git a/model/profile_test.go b/model/profile_test.go index f5c1d520fae..f4a0b038330 100644 --- a/model/profile_test.go +++ b/model/profile_test.go @@ -98,7 +98,7 @@ func TestPprofProfileTransform(t *testing.T) { Timestamp: timestamp, Fields: common.MapStr{ "data_stream.type": "metrics", - "data_stream.dataset": "apm.profiling.myservice", + "data_stream.dataset": "apm.profiling", "processor": common.MapStr{"event": "profile", "name": "profile"}, "service": common.MapStr{ "name": "myService", diff --git a/model/span.go b/model/span.go index 024e7e2f256..2e3c0377f00 100644 --- a/model/span.go +++ b/model/span.go @@ -19,7 +19,6 @@ package model import ( "context" - "fmt" "net" "time" @@ -198,9 +197,8 @@ func (e *Span) Transform(ctx context.Context, cfg *transform.Config) []beat.Even if cfg.DataStreams { // Spans are stored in a "traces" data stream along with transactions. - dataset := fmt.Sprintf("apm.%s", datastreams.NormalizeServiceName(e.Metadata.Service.Name)) fields[datastreams.TypeField] = datastreams.TracesType - fields[datastreams.DatasetField] = dataset + fields[datastreams.DatasetField] = TracesDataset } // first set the generic metadata diff --git a/model/span_test.go b/model/span_test.go index 64789fac59c..6e6ef285cf1 100644 --- a/model/span_test.go +++ b/model/span_test.go @@ -59,7 +59,7 @@ func TestSpanTransform(t *testing.T) { Span: Span{Timestamp: timestamp, Metadata: metadata}, Output: common.MapStr{ "data_stream.type": "traces", - "data_stream.dataset": "apm.myservice", + "data_stream.dataset": "apm", "processor": common.MapStr{"event": "span", "name": "transaction"}, "service": common.MapStr{"name": serviceName, "environment": env, "version": serviceVersion}, "span": common.MapStr{ @@ -77,7 +77,7 @@ func TestSpanTransform(t *testing.T) { Span: Span{Timestamp: timestamp, Metadata: metadata, Outcome: "success"}, Output: common.MapStr{ "data_stream.type": "traces", - "data_stream.dataset": "apm.myservice", + "data_stream.dataset": "apm", "processor": common.MapStr{"event": "span", "name": "transaction"}, "service": common.MapStr{"name": serviceName, "environment": env, "version": serviceVersion}, "span": common.MapStr{ @@ -126,7 +126,7 @@ func TestSpanTransform(t *testing.T) { }, Output: common.MapStr{ "data_stream.type": "traces", - "data_stream.dataset": "apm.myservice", + "data_stream.dataset": "apm", "span": common.MapStr{ "id": hexID, "duration": common.MapStr{"us": 1200}, diff --git a/model/transaction.go b/model/transaction.go index 3b4640642a6..3a491524eac 100644 --- a/model/transaction.go +++ b/model/transaction.go @@ -19,7 +19,6 @@ package model import ( "context" - "fmt" "time" "github.com/elastic/beats/v7/libbeat/beat" @@ -34,6 +33,7 @@ import ( const ( transactionProcessorName = "transaction" transactionDocType = "transaction" + TracesDataset = "apm" ) var ( @@ -120,9 +120,8 @@ func (e *Transaction) Transform(_ context.Context, cfg *transform.Config) []beat if cfg.DataStreams { // Transactions are stored in a "traces" data stream along with spans. - dataset := fmt.Sprintf("apm.%s", datastreams.NormalizeServiceName(e.Metadata.Service.Name)) fields[datastreams.TypeField] = datastreams.TracesType - fields[datastreams.DatasetField] = dataset + fields[datastreams.DatasetField] = TracesDataset } // first set generic metadata (order is relevant) diff --git a/model/transaction_test.go b/model/transaction_test.go index cef19b0a8ec..563ca2b15f1 100644 --- a/model/transaction_test.go +++ b/model/transaction_test.go @@ -182,7 +182,7 @@ func TestEventsTransformWithMetadata(t *testing.T) { require.Len(t, events, 1) assert.Equal(t, events[0].Fields, common.MapStr{ "data_stream.type": "traces", - "data_stream.dataset": "apm." + serviceName, + "data_stream.dataset": "apm", "user": common.MapStr{"id": "123", "name": "jane"}, "client": common.MapStr{"ip": ip}, "source": common.MapStr{"ip": ip}, diff --git a/processor/otel/test_approved/consume_span.approved.json b/processor/otel/test_approved/consume_span.approved.json index 38d72b66e0b..eea09e4f70d 100644 --- a/processor/otel/test_approved/consume_span.approved.json +++ b/processor/otel/test_approved/consume_span.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.unknown", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "success" @@ -40,7 +40,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.unknown", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "unknown" diff --git a/processor/otel/test_approved/jaeger_sampling_rate.approved.json b/processor/otel/test_approved/jaeger_sampling_rate.approved.json index cb858f95734..95ffa7ac15c 100644 --- a/processor/otel/test_approved/jaeger_sampling_rate.approved.json +++ b/processor/otel/test_approved/jaeger_sampling_rate.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.unknown", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "success" @@ -47,7 +47,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.unknown", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "unknown" diff --git a/processor/otel/test_approved/metadata_jaeger-no-language.approved.json b/processor/otel/test_approved/metadata_jaeger-no-language.approved.json index 7947c0590d6..07902047cdc 100644 --- a/processor/otel/test_approved/metadata_jaeger-no-language.approved.json +++ b/processor/otel/test_approved/metadata_jaeger-no-language.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "3.4.12" }, - "data_stream.dataset": "apm.unknown", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "success" diff --git a/processor/otel/test_approved/metadata_jaeger-version.approved.json b/processor/otel/test_approved/metadata_jaeger-version.approved.json index 1968c9e3e6c..34fcfb1bd2d 100644 --- a/processor/otel/test_approved/metadata_jaeger-version.approved.json +++ b/processor/otel/test_approved/metadata_jaeger-version.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger/PHP", "version": "3.4.12" }, - "data_stream.dataset": "apm.unknown", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "success" diff --git a/processor/otel/test_approved/metadata_jaeger.approved.json b/processor/otel/test_approved/metadata_jaeger.approved.json index 12045a12e69..05692908e79 100644 --- a/processor/otel/test_approved/metadata_jaeger.approved.json +++ b/processor/otel/test_approved/metadata_jaeger.approved.json @@ -7,7 +7,7 @@ "name": "Jaeger/C++", "version": "3.2.1" }, - "data_stream.dataset": "apm.foo", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "success" diff --git a/processor/otel/test_approved/metadata_jaeger_full-traceid.approved.json b/processor/otel/test_approved/metadata_jaeger_full-traceid.approved.json index f08c8b749b5..faf4cd9c2de 100644 --- a/processor/otel/test_approved/metadata_jaeger_full-traceid.approved.json +++ b/processor/otel/test_approved/metadata_jaeger_full-traceid.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.unknown", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "success" diff --git a/processor/otel/test_approved/metadata_jaeger_minimal.approved.json b/processor/otel/test_approved/metadata_jaeger_minimal.approved.json index 2ea014e11d3..423a3674c6a 100644 --- a/processor/otel/test_approved/metadata_jaeger_minimal.approved.json +++ b/processor/otel/test_approved/metadata_jaeger_minimal.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.unknown", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "success" diff --git a/processor/otel/test_approved/metadata_minimal.approved.json b/processor/otel/test_approved/metadata_minimal.approved.json index 34008f72ff6..7588296aa08 100644 --- a/processor/otel/test_approved/metadata_minimal.approved.json +++ b/processor/otel/test_approved/metadata_minimal.approved.json @@ -6,7 +6,7 @@ "name": "Foo", "version": "unknown" }, - "data_stream.dataset": "apm.unknown", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "success" diff --git a/processor/otel/test_approved/span_jaeger_custom.approved.json b/processor/otel/test_approved/span_jaeger_custom.approved.json index 1ac4cb50267..ab0c26d1939 100644 --- a/processor/otel/test_approved/span_jaeger_custom.approved.json +++ b/processor/otel/test_approved/span_jaeger_custom.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.unknown", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "unknown" diff --git a/processor/otel/test_approved/span_jaeger_db.approved.json b/processor/otel/test_approved/span_jaeger_db.approved.json index af989813dc9..2fd7416a916 100644 --- a/processor/otel/test_approved/span_jaeger_db.approved.json +++ b/processor/otel/test_approved/span_jaeger_db.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.unknown", + "data_stream.dataset": "apm", "data_stream.type": "traces", "destination": { "address": "db", diff --git a/processor/otel/test_approved/span_jaeger_http.approved.json b/processor/otel/test_approved/span_jaeger_http.approved.json index 8bc36446961..7628148e5aa 100644 --- a/processor/otel/test_approved/span_jaeger_http.approved.json +++ b/processor/otel/test_approved/span_jaeger_http.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.unknown", + "data_stream.dataset": "apm", "data_stream.type": "traces", "destination": { "address": "foo.bar.com", @@ -81,7 +81,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.error.unknown", + "data_stream.dataset": "apm.error", "data_stream.type": "logs", "error": { "exception": [ @@ -142,7 +142,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.error.unknown", + "data_stream.dataset": "apm.error", "data_stream.type": "logs", "error": { "grouping_key": "23b7ac1bdf1ca957f9f581cfadee467c", @@ -198,7 +198,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.error.unknown", + "data_stream.dataset": "apm.error", "data_stream.type": "logs", "error": { "exception": [ @@ -256,7 +256,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.error.unknown", + "data_stream.dataset": "apm.error", "data_stream.type": "logs", "error": { "exception": [ @@ -314,7 +314,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.error.unknown", + "data_stream.dataset": "apm.error", "data_stream.type": "logs", "error": { "exception": [ @@ -372,7 +372,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.error.unknown", + "data_stream.dataset": "apm.error", "data_stream.type": "logs", "error": { "grouping_key": "c9221918248f05433f6b81c46a666aee", diff --git a/processor/otel/test_approved/span_jaeger_http_status_code.approved.json b/processor/otel/test_approved/span_jaeger_http_status_code.approved.json index 7ee56b57b25..7c13ba7dc46 100644 --- a/processor/otel/test_approved/span_jaeger_http_status_code.approved.json +++ b/processor/otel/test_approved/span_jaeger_http_status_code.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.unknown", + "data_stream.dataset": "apm", "data_stream.type": "traces", "destination": { "address": "foo.bar.com", diff --git a/processor/otel/test_approved/span_jaeger_https_default_port.approved.json b/processor/otel/test_approved/span_jaeger_https_default_port.approved.json index af475472371..0c349033049 100644 --- a/processor/otel/test_approved/span_jaeger_https_default_port.approved.json +++ b/processor/otel/test_approved/span_jaeger_https_default_port.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.unknown", + "data_stream.dataset": "apm", "data_stream.type": "traces", "destination": { "address": "foo.bar.com", diff --git a/processor/otel/test_approved/span_jaeger_messaging.approved.json b/processor/otel/test_approved/span_jaeger_messaging.approved.json index 032f4516de9..799ae7483a1 100644 --- a/processor/otel/test_approved/span_jaeger_messaging.approved.json +++ b/processor/otel/test_approved/span_jaeger_messaging.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.unknown", + "data_stream.dataset": "apm", "data_stream.type": "traces", "destination": { "address": "mq", diff --git a/processor/otel/test_approved/transaction_jaeger_custom.approved.json b/processor/otel/test_approved/transaction_jaeger_custom.approved.json index cf301f83203..31d795e1e78 100644 --- a/processor/otel/test_approved/transaction_jaeger_custom.approved.json +++ b/processor/otel/test_approved/transaction_jaeger_custom.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.unknown", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "success" diff --git a/processor/otel/test_approved/transaction_jaeger_full.approved.json b/processor/otel/test_approved/transaction_jaeger_full.approved.json index 7ffece71652..070a671f2dc 100644 --- a/processor/otel/test_approved/transaction_jaeger_full.approved.json +++ b/processor/otel/test_approved/transaction_jaeger_full.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.unknown", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "success" @@ -76,7 +76,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.error.unknown", + "data_stream.dataset": "apm.error", "data_stream.type": "logs", "error": { "exception": [ @@ -143,7 +143,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.error.unknown", + "data_stream.dataset": "apm.error", "data_stream.type": "logs", "error": { "grouping_key": "23b7ac1bdf1ca957f9f581cfadee467c", @@ -205,7 +205,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.error.unknown", + "data_stream.dataset": "apm.error", "data_stream.type": "logs", "error": { "exception": [ @@ -269,7 +269,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.error.unknown", + "data_stream.dataset": "apm.error", "data_stream.type": "logs", "error": { "exception": [ @@ -333,7 +333,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.error.unknown", + "data_stream.dataset": "apm.error", "data_stream.type": "logs", "error": { "exception": [ @@ -397,7 +397,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.error.unknown", + "data_stream.dataset": "apm.error", "data_stream.type": "logs", "error": { "grouping_key": "c9221918248f05433f6b81c46a666aee", diff --git a/processor/otel/test_approved/transaction_jaeger_no_attrs.approved.json b/processor/otel/test_approved/transaction_jaeger_no_attrs.approved.json index 81eb60c050b..79a8732a45f 100644 --- a/processor/otel/test_approved/transaction_jaeger_no_attrs.approved.json +++ b/processor/otel/test_approved/transaction_jaeger_no_attrs.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.unknown", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "failure" diff --git a/processor/otel/test_approved/transaction_jaeger_type_component.approved.json b/processor/otel/test_approved/transaction_jaeger_type_component.approved.json index b414c1171e4..9ae85c5d296 100644 --- a/processor/otel/test_approved/transaction_jaeger_type_component.approved.json +++ b/processor/otel/test_approved/transaction_jaeger_type_component.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.unknown", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "success" diff --git a/processor/otel/test_approved/transaction_jaeger_type_messaging.approved.json b/processor/otel/test_approved/transaction_jaeger_type_messaging.approved.json index 3ca0d234b8f..d9ca4da167d 100644 --- a/processor/otel/test_approved/transaction_jaeger_type_messaging.approved.json +++ b/processor/otel/test_approved/transaction_jaeger_type_messaging.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.unknown", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "success" diff --git a/processor/otel/test_approved/transaction_jaeger_type_request.approved.json b/processor/otel/test_approved/transaction_jaeger_type_request.approved.json index 9f10e0ec06f..b8ea71c8f77 100644 --- a/processor/otel/test_approved/transaction_jaeger_type_request.approved.json +++ b/processor/otel/test_approved/transaction_jaeger_type_request.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.unknown", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "failure" diff --git a/processor/otel/test_approved/transaction_jaeger_type_request_result.approved.json b/processor/otel/test_approved/transaction_jaeger_type_request_result.approved.json index 679228c4af0..cc9d4b28eed 100644 --- a/processor/otel/test_approved/transaction_jaeger_type_request_result.approved.json +++ b/processor/otel/test_approved/transaction_jaeger_type_request_result.approved.json @@ -6,7 +6,7 @@ "name": "Jaeger", "version": "unknown" }, - "data_stream.dataset": "apm.unknown", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "success" diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationErrors.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationErrors.approved.json index 0389ee8613c..96a9a1e3e91 100644 --- a/processor/stream/test_approved_es_documents/testIntakeIntegrationErrors.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeIntegrationErrors.approved.json @@ -33,7 +33,7 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm.error.service1", + "data_stream.dataset": "apm.error", "data_stream.type": "logs", "error": { "culprit": "my.module.function_name", @@ -377,7 +377,7 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm.error.1234_service_12a3", + "data_stream.dataset": "apm.error", "data_stream.type": "logs", "error": { "grouping_key": "dc8dd667f7036ec5f0bae87bf2188243", @@ -483,7 +483,7 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm.error.1234_service_12a3", + "data_stream.dataset": "apm.error", "data_stream.type": "logs", "error": { "exception": [ @@ -585,7 +585,7 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm.error.service1", + "data_stream.dataset": "apm.error", "data_stream.type": "logs", "error": { "exception": [ @@ -693,7 +693,7 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm.error.1234_service_12a3", + "data_stream.dataset": "apm.error", "data_stream.type": "logs", "error": { "grouping_key": "d6b3f958dfea98dc9ed2b57d5f0c48bb", diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationEvents.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationEvents.approved.json index d44e28bd1f4..b8bdea2433b 100644 --- a/processor/stream/test_approved_es_documents/testIntakeIntegrationEvents.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeIntegrationEvents.approved.json @@ -13,7 +13,7 @@ "container": { "id": "8ec7ceb990749e79b37f6dc6cd3628633618d6ce412553a552a0fa6b69419ad4" }, - "data_stream.dataset": "apm.experimental_java", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "success" @@ -196,7 +196,7 @@ "container": { "id": "8ec7ceb990749e79b37f6dc6cd3628633618d6ce412553a552a0fa6b69419ad4" }, - "data_stream.dataset": "apm.1234_service_12a3", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "success" @@ -343,7 +343,7 @@ "container": { "id": "8ec7ceb990749e79b37f6dc6cd3628633618d6ce412553a552a0fa6b69419ad4" }, - "data_stream.dataset": "apm.internal.1234_service_12a3", + "data_stream.dataset": "apm.internal", "data_stream.type": "metrics", "dotted": { "float": { @@ -469,7 +469,7 @@ "container": { "id": "8ec7ceb990749e79b37f6dc6cd3628633618d6ce412553a552a0fa6b69419ad4" }, - "data_stream.dataset": "apm.error.service1", + "data_stream.dataset": "apm.error", "data_stream.type": "logs", "error": { "culprit": "opbeans.controllers.DTInterceptor.preHandle(DTInterceptor.java:73)", diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationInvalidEvent.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationInvalidEvent.approved.json index f659fa936a9..3c0be4e4581 100644 --- a/processor/stream/test_approved_es_documents/testIntakeIntegrationInvalidEvent.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeIntegrationInvalidEvent.approved.json @@ -6,7 +6,7 @@ "name": "elastic-node", "version": "3.14.0" }, - "data_stream.dataset": "apm.1234_service_12a3", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "unknown" diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationInvalidJSONEvent.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationInvalidJSONEvent.approved.json index f659fa936a9..3c0be4e4581 100644 --- a/processor/stream/test_approved_es_documents/testIntakeIntegrationInvalidJSONEvent.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeIntegrationInvalidJSONEvent.approved.json @@ -6,7 +6,7 @@ "name": "elastic-node", "version": "3.14.0" }, - "data_stream.dataset": "apm.1234_service_12a3", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "unknown" diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationMetadataNullValues.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationMetadataNullValues.approved.json index 28a23968ebf..c15059bb4b6 100644 --- a/processor/stream/test_approved_es_documents/testIntakeIntegrationMetadataNullValues.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeIntegrationMetadataNullValues.approved.json @@ -6,7 +6,7 @@ "name": "elastic-node", "version": "3.14.0" }, - "data_stream.dataset": "apm.error.1234_service_12a3", + "data_stream.dataset": "apm.error", "data_stream.type": "logs", "error": { "grouping_key": "d6b3f958dfea98dc9ed2b57d5f0c48bb", diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationMetricsets.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationMetricsets.approved.json index 7290188947c..3f63c1aea3b 100644 --- a/processor/stream/test_approved_es_documents/testIntakeIntegrationMetricsets.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeIntegrationMetricsets.approved.json @@ -7,7 +7,7 @@ "version": "3.14.0" }, "byte_counter": 1, - "data_stream.dataset": "apm.internal.1234_service_12a3", + "data_stream.dataset": "apm.internal", "data_stream.type": "metrics", "dotted": { "float": { @@ -99,7 +99,7 @@ "name": "elastic-node", "version": "3.14.0" }, - "data_stream.dataset": "apm.1234_service_12a3", + "data_stream.dataset": "apm", "data_stream.type": "metrics", "go": { "memstats": { @@ -145,7 +145,7 @@ "name": "elastic-node", "version": "3.14.0" }, - "data_stream.dataset": "apm.1234_service_12a3", + "data_stream.dataset": "apm", "data_stream.type": "metrics", "host": { "ip": "192.0.0.1" diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationMinimalService.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationMinimalService.approved.json index 640713b5f48..8e288c89924 100644 --- a/processor/stream/test_approved_es_documents/testIntakeIntegrationMinimalService.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeIntegrationMinimalService.approved.json @@ -6,7 +6,7 @@ "name": "elastic-node", "version": "3.14.0" }, - "data_stream.dataset": "apm.1234_service_12a3", + "data_stream.dataset": "apm", "data_stream.type": "metrics", "go": { "memstats": { @@ -37,7 +37,7 @@ "name": "elastic-node", "version": "3.14.0" }, - "data_stream.dataset": "apm.error.1234_service_12a3", + "data_stream.dataset": "apm.error", "data_stream.type": "logs", "error": { "grouping_key": "d6b3f958dfea98dc9ed2b57d5f0c48bb", diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationOptionalTimestamps.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationOptionalTimestamps.approved.json index 00525a34146..1de26b78e7b 100644 --- a/processor/stream/test_approved_es_documents/testIntakeIntegrationOptionalTimestamps.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeIntegrationOptionalTimestamps.approved.json @@ -6,7 +6,7 @@ "name": "elastic-node", "version": "3.14.0" }, - "data_stream.dataset": "apm.backendspans", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "unknown" @@ -83,7 +83,7 @@ "name": "elastic-node", "version": "3.14.0" }, - "data_stream.dataset": "apm.backendspans", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "unknown" @@ -165,7 +165,7 @@ "name": "elastic-node", "version": "3.14.0" }, - "data_stream.dataset": "apm.backendspans", + "data_stream.dataset": "apm", "data_stream.type": "metrics", "host": { "architecture": "x64", diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationRumErrors.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationRumErrors.approved.json index 943b273fc39..502f1b0ba89 100644 --- a/processor/stream/test_approved_es_documents/testIntakeIntegrationRumErrors.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeIntegrationRumErrors.approved.json @@ -9,7 +9,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm.error.apm_agent_js", + "data_stream.dataset": "apm.error", "data_stream.type": "logs", "error": { "culprit": "test/e2e/general-usecase/bundle.js.map", diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationRumTransactions.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationRumTransactions.approved.json index 79051b7e7da..e7c7de01031 100644 --- a/processor/stream/test_approved_es_documents/testIntakeIntegrationRumTransactions.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeIntegrationRumTransactions.approved.json @@ -9,7 +9,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm.apm_agent_js", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "unknown" @@ -77,7 +77,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm.apm_agent_js", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "unknown" diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationSpans.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationSpans.approved.json index f822471e299..f31821ca061 100644 --- a/processor/stream/test_approved_es_documents/testIntakeIntegrationSpans.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeIntegrationSpans.approved.json @@ -34,7 +34,7 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm.backendspans", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "success" @@ -151,7 +151,7 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm.backendspans", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "unknown" @@ -269,7 +269,7 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm.backendspans", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "unknown" @@ -391,7 +391,7 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm.backendspans", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "unknown" @@ -510,7 +510,7 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm.backendspans", + "data_stream.dataset": "apm", "data_stream.type": "traces", "destination": { "address": "0:0::0:1", @@ -711,7 +711,7 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm.backendspans", + "data_stream.dataset": "apm", "data_stream.type": "traces", "destination": { "address": "0:0::0:1", diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationTransactions.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationTransactions.approved.json index 40942881c59..b45ae418722 100644 --- a/processor/stream/test_approved_es_documents/testIntakeIntegrationTransactions.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeIntegrationTransactions.approved.json @@ -29,7 +29,7 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm.1234_service_12a3", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "unknown" @@ -148,7 +148,7 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm.1234_service_12a3", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "success" @@ -358,7 +358,7 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm.service1", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "unknown" @@ -500,7 +500,7 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm.1234_service_12a3", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "unknown" diff --git a/processor/stream/test_approved_es_documents/testIntakeRUMV3Errors.approved.json b/processor/stream/test_approved_es_documents/testIntakeRUMV3Errors.approved.json index 9d4b28ce273..e58bbaba770 100644 --- a/processor/stream/test_approved_es_documents/testIntakeRUMV3Errors.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeRUMV3Errors.approved.json @@ -9,7 +9,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm.error.apm_a_rum_test_e2e_general_usecase", + "data_stream.dataset": "apm.error", "data_stream.type": "logs", "error": { "culprit": "test/e2e/general-usecase/app.e2e-bundle.min.js?token=secret", diff --git a/processor/stream/test_approved_es_documents/testIntakeRUMV3Events.approved.json b/processor/stream/test_approved_es_documents/testIntakeRUMV3Events.approved.json index 6cf3af3b8c7..4711169427a 100644 --- a/processor/stream/test_approved_es_documents/testIntakeRUMV3Events.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeRUMV3Events.approved.json @@ -9,7 +9,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm.apm_a_rum_test_e2e_general_usecase", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "success" @@ -157,7 +157,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm.apm_a_rum_test_e2e_general_usecase", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "unknown" @@ -228,7 +228,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm.apm_a_rum_test_e2e_general_usecase", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "unknown" @@ -299,7 +299,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm.apm_a_rum_test_e2e_general_usecase", + "data_stream.dataset": "apm", "data_stream.type": "traces", "destination": { "address": "localhost", @@ -391,7 +391,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm.apm_a_rum_test_e2e_general_usecase", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "unknown" @@ -461,7 +461,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm.apm_a_rum_test_e2e_general_usecase", + "data_stream.dataset": "apm", "data_stream.type": "traces", "destination": { "address": "localhost", @@ -553,7 +553,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm.apm_a_rum_test_e2e_general_usecase", + "data_stream.dataset": "apm", "data_stream.type": "traces", "destination": { "address": "localhost", @@ -645,7 +645,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm.apm_a_rum_test_e2e_general_usecase", + "data_stream.dataset": "apm", "data_stream.type": "traces", "destination": { "address": "localhost", @@ -738,7 +738,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm.apm_a_rum_test_e2e_general_usecase", + "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { "outcome": "success" @@ -831,7 +831,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm.internal.apm_a_rum_test_e2e_general_usecase", + "data_stream.dataset": "apm.internal", "data_stream.type": "metrics", "labels": { "testTagKey": "testTagValue" @@ -888,7 +888,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm.internal.apm_a_rum_test_e2e_general_usecase", + "data_stream.dataset": "apm.internal", "data_stream.type": "metrics", "labels": { "testTagKey": "testTagValue" @@ -945,7 +945,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm.internal.apm_a_rum_test_e2e_general_usecase", + "data_stream.dataset": "apm.internal", "data_stream.type": "metrics", "labels": { "testTagKey": "testTagValue" @@ -1002,7 +1002,7 @@ "client": { "ip": "192.0.0.1" }, - "data_stream.dataset": "apm.internal.apm_a_rum_test_e2e_general_usecase", + "data_stream.dataset": "apm.internal", "data_stream.type": "metrics", "labels": { "tag1": "value1", diff --git a/systemtest/approvals/TestDataStreamsEnabled/true.approved.json b/systemtest/approvals/TestDataStreamsEnabled/true.approved.json index c648c1bfbc5..00f6eb78089 100644 --- a/systemtest/approvals/TestDataStreamsEnabled/true.approved.json +++ b/systemtest/approvals/TestDataStreamsEnabled/true.approved.json @@ -6,7 +6,7 @@ "name": "go", "version": "0.0.0" }, - "data_stream.dataset": "apm.systemtest", + "data_stream.dataset": "apm", "data_stream.namespace": "", "data_stream.type": "traces", "ecs": { diff --git a/systemtest/datastreams_test.go b/systemtest/datastreams_test.go index 5f7bfa058a2..9ece776264b 100644 --- a/systemtest/datastreams_test.go +++ b/systemtest/datastreams_test.go @@ -98,7 +98,7 @@ func TestDataStreamsEnabled(t *testing.T) { tx.End() tracer.Flush(nil) - result := systemtest.Elasticsearch.ExpectDocs(t, "apm-*,traces-apm.*", estest.TermQuery{ + result := systemtest.Elasticsearch.ExpectDocs(t, "apm-*,traces-apm*", estest.TermQuery{ Field: "processor.event", Value: "transaction", }) systemtest.ApproveEvents( diff --git a/tests/system/test_tls.py b/tests/system/test_tls.py index d7f13e0e0f9..60d0ca4c0ce 100644 --- a/tests/system/test_tls.py +++ b/tests/system/test_tls.py @@ -15,6 +15,7 @@ requests.packages.urllib3.disable_warnings(SubjectAltNameWarning) + @integration_test class TestSecureServerBaseTest(ServerBaseTest): @classmethod