Merge pull request elastic#6 from miloszmarcinkowski/192749-refactor-…

…the-normalize-function-into-per-data-structure-serialization-logic

Otel collector mappings to APM agent

packages/kbn-apm-types/src/es_fields/apm.ts

@@ -8,6 +8,7 @@
  */
 
 export const TIMESTAMP = 'timestamp.us';
+export const AT_TIMESTAMP = '@timestamp';
 export const AGENT = 'agent';
 export const AGENT_NAME = 'agent.name';
 export const AGENT_VERSION = 'agent.version';
@@ -20,12 +21,18 @@ export const CLOUD_AVAILABILITY_ZONE = 'cloud.availability_zone';
 export const CLOUD_PROVIDER = 'cloud.provider';
 export const CLOUD_REGION = 'cloud.region';
 export const CLOUD_MACHINE_TYPE = 'cloud.machine.type';
+export const CLOUD_PROJECT_ID = 'cloud.project.id';
+export const CLOUD_PROJECT_NAME = 'cloud.project.name';
+export const CLOUD_IMAGE_ID = 'cloud.image.id';
 export const CLOUD_ACCOUNT_ID = 'cloud.account.id';
+export const CLOUD_ACCOUNT_NAME = 'cloud.account.name';
 export const CLOUD_INSTANCE_ID = 'cloud.instance.id';
 export const CLOUD_INSTANCE_NAME = 'cloud.instance.name';
 export const CLOUD_SERVICE_NAME = 'cloud.service.name';
 
 export const EVENT_SUCCESS_COUNT = 'event.success_count';
+export const EVENT_SUCCESS_COUNT_SUM = 'event.success_count.sum';
+export const EVENT_SUCCESS_COUNT_VALUE_COUNT = 'event.success_count.value_count';
 
 export const SERVICE = 'service';
 export const SERVICE_NAME = 'service.name';
@@ -39,18 +46,28 @@ export const SERVICE_RUNTIME_VERSION = 'service.runtime.version';
 export const SERVICE_NODE_NAME = 'service.node.name';
 export const SERVICE_VERSION = 'service.version';
 export const SERVICE_TARGET_TYPE = 'service.target.type';
+export const SERVICE_TARGET_NAME = 'service.target.name';
 export const SERVICE_OVERFLOW_COUNT = 'service_transaction.aggregation.overflow_count';
 
 export const URL_FULL = 'url.full';
 export const HTTP_REQUEST_METHOD = 'http.request.method';
 export const HTTP_RESPONSE_STATUS_CODE = 'http.response.status_code';
+
 export const USER_ID = 'user.id';
 export const USER_AGENT_ORIGINAL = 'user_agent.original';
 export const USER_AGENT_NAME = 'user_agent.name';
+export const USER_AGENT_DEVICE_NAME = 'user_agent.device.name';
+export const USER_AGENT_VERSION = 'user_agent.version';
 
 export const OBSERVER_HOSTNAME = 'observer.hostname';
+export const OBSERVER_ID = 'observer.id';
+export const OBSERVER_TYPE = 'observer.type';
+export const OBSERVER_NAME = 'observer.name';
+export const OBSERVER_VERSION = 'observer.version';
 export const OBSERVER_LISTENING = 'observer.listening';
+export const OBSERVER_VERSION_MAJOR = 'observer.version_major';
 export const PROCESSOR_EVENT = 'processor.event';
+export const PROCESSOR_NAME = 'processor.name';
 
 export const TRANSACTION_DURATION = 'transaction.duration.us';
 export const TRANSACTION_DURATION_HISTOGRAM = 'transaction.duration.histogram';
@@ -67,6 +84,9 @@ export const TRANSACTION_OVERFLOW_COUNT = 'transaction.aggregation.overflow_coun
 // for transaction metrics
 export const TRANSACTION_ROOT = 'transaction.root';
 export const TRANSACTION_PROFILER_STACK_TRACE_IDS = 'transaction.profiler_stack_trace_ids';
+export const TRANSACTION_NAME_TEXT = 'transaction.name.text';
+export const TRANSACTION_REPRESENTATIVE_COUNT = 'transaction.representative_count';
+export const TRANSACTION_SPAN_COUNT_STARTED = 'transaction.span_count.started';
 
 export const EVENT_OUTCOME = 'event.outcome';
 
@@ -95,6 +115,7 @@ export const SPAN_COMPOSITE_SUM = 'span.composite.sum.us';
 export const SPAN_COMPOSITE_COMPRESSION_STRATEGY = 'span.composite.compression_strategy';
 
 export const SPAN_SYNC = 'span.sync';
+export const SPAN_REPRESENTATIVE_COUNT = 'span.representative_count';
 
 // Parent ID for a transaction or span
 export const PARENT_ID = 'parent.id';
@@ -138,21 +159,27 @@ export const LABEL_GC = 'labels.gc';
 export const LABEL_TYPE = 'labels.type';
 export const LABEL_TELEMETRY_AUTO_VERSION = 'labels.telemetry_auto_version';
 export const LABEL_LIFECYCLE_STATE = 'labels.lifecycle_state';
+export const LABEL_SOME_RESOURCE_ATTRIBUTE = 'labels.some_resource_attribute';
 
 export const HOST = 'host';
 export const HOST_HOSTNAME = 'host.hostname'; // Do not use. Please use `HOST_NAME` instead.
 export const HOST_NAME = 'host.name';
 export const HOST_OS_PLATFORM = 'host.os.platform';
 export const HOST_ARCHITECTURE = 'host.architecture';
 export const HOST_OS_VERSION = 'host.os.version';
+export const HOST_IP = 'host.ip';
 
 export const CONTAINER_ID = 'container.id';
 export const CONTAINER = 'container';
 export const CONTAINER_IMAGE = 'container.image.name';
+export const CONTAINER_OS = 'container.os';
+export const CONTAINER_TTL_INSTANCES = 'container.totalNumberInstances';
 
 export const KUBERNETES = 'kubernetes';
 export const KUBERNETES_POD_NAME = 'kubernetes.pod.name';
 export const KUBERNETES_POD_UID = 'kubernetes.pod.uid';
+export const KUBERNETES_NAMESPACE = 'kubernetes.namespace';
+export const KUBERNETES_NODE_NAME = 'kubernetes.node.name';
 
 export const FAAS_ID = 'faas.id';
 export const FAAS_NAME = 'faas.name';
@@ -180,6 +207,8 @@ export const VALUE_OTEL_JVM_PROCESS_MEMORY_NON_HEAP = 'non_heap';
 export const TIER = '_tier';
 export const INDEX = '_index';
 export const DATA_STEAM_TYPE = 'data_stream.type';
+export const DATA_STREAM_NAMESPACE = 'data_stream.namespace';
+export const DATA_STREAM_DATASET = 'data_stream.dataset';
 
 // Mobile
 export const NETWORK_CONNECTION_TYPE = 'network.connection.type';
@@ -194,6 +223,7 @@ export const CLIENT_GEO_REGION_ISO_CODE = 'client.geo.region_iso_code';
 export const CLIENT_GEO_COUNTRY_NAME = 'client.geo.country_name';
 export const CLIENT_GEO_CITY_NAME = 'client.geo.city_name';
 export const CLIENT_GEO_REGION_NAME = 'client.geo.region_name';
+export const CLIENT_IP = 'client.ip';
 
 export const CHILD_ID = 'child.id';
 

x-pack/plugins/observability_solution/apm/common/es_fields/infra_metrics.ts

@@ -7,9 +7,12 @@
 
 // Kubernetes
 export const KUBERNETES_CONTAINER_NAME = 'kubernetes.container.name';
+export const KUBERNETES_NODE_NAME = 'kubernetes.node.name';
+export const KUBERNETES_CONTAINER_ID = 'kubernetes.container.id';
 export const KUBERNETES_DEPLOYMENT = 'kubernetes.deployment';
 export const KUBERNETES_DEPLOYMENT_NAME = 'kubernetes.deployment.name';
 export const KUBERNETES_NAMESPACE_NAME = 'kubernetes.namespace.name';
 export const KUBERNETES_NAMESPACE = 'kubernetes.namespace';
 export const KUBERNETES_REPLICASET = 'kubernetes.replicaset';
 export const KUBERNETES_REPLICASET_NAME = 'kubernetes.replicaset.name';
+export const KUBERNETES_POD_NAME = 'kubernetes.pod.name';

x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts

@@ -11,7 +11,6 @@ import { createHash } from 'crypto';
 import { flatten, merge, pickBy, sortBy, sum, uniq } from 'lodash';
 import { SavedObjectsClient } from '@kbn/core/server';
 import type { APMIndices } from '@kbn/apm-data-access-plugin/server';
-import { normalizeFields } from '../../../utils/normalize_fields';
 import { AGENT_NAMES, RUM_AGENT_NAMES } from '../../../../common/agent_name';
 import {
   AGENT_ACTIVATION_METHOD,
@@ -55,10 +54,7 @@ import {
   SavedServiceGroup,
 } from '../../../../common/service_groups';
 import { asMutableArray } from '../../../../common/utils/as_mutable_array';
-import { APMError } from '../../../../typings/es_schemas/ui/apm_error';
 import { AgentName } from '../../../../typings/es_schemas/ui/fields/agent';
-import { Span } from '../../../../typings/es_schemas/ui/span';
-import { Transaction } from '../../../../typings/es_schemas/ui/transaction';
 import {
   APMDataTelemetry,
   APMPerService,
@@ -75,6 +71,7 @@ import {
   APM_CUSTOM_DASHBOARDS_SAVED_OBJECT_TYPE,
   SavedApmCustomDashboard,
 } from '../../../../common/custom_dashboards';
+import { serviceVersionMapping } from '../../../utils/es_fields_mappings';
 
 type ISavedObjectsClient = Pick<SavedObjectsClient, 'find'>;
 const TIME_RANGES = ['1d', 'all'] as const;
@@ -695,10 +692,7 @@ export const tasks: TelemetryTask[] = [
         },
       });
 
-      const hit = normalizeFields(response.hits.hits[0]?.fields) as Pick<
-        Transaction | Span | APMError,
-        'observer'
-      >;
+      const hit = serviceVersionMapping(response.hits.hits[0]?.fields);
 
       if (!hit || !hit.observer?.version) {
         return {};

x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_metadata_for_dependency.ts

@@ -7,11 +7,10 @@
 
 import { rangeQuery } from '@kbn/observability-plugin/server';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
-import type { Span } from '@kbn/apm-types/es_schemas_ui';
 import { maybe } from '../../../common/utils/maybe';
 import { SPAN_DESTINATION_SERVICE_RESOURCE } from '../../../common/es_fields/apm';
 import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
-import { normalizeFields } from '../../utils/normalize_fields';
+import { metadataForDependencyMapping } from '../../utils/es_fields_mappings';
 
 export interface MetadataForDependencyResponse {
   spanType: string | undefined;
@@ -56,7 +55,7 @@ export async function getMetadataForDependency({
   });
 
   const sample = maybe(sampleResponse.hits.hits[0])?.fields;
-  const sampleNorm = sample ? (normalizeFields(sample) as unknown as Span) : null;
+  const sampleNorm = sample ? metadataForDependencyMapping(sample) : null;
 
   return {
     spanType: sampleNorm?.span.type,

x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_top_dependency_spans.ts

@@ -8,8 +8,6 @@
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
 import { kqlQuery, rangeQuery, termQuery, termsQuery } from '@kbn/observability-plugin/server';
 import { keyBy } from 'lodash';
-import { Span } from '@kbn/apm-types/es_schemas_ui';
-import type { TransactionRaw } from '@kbn/apm-types/es_schemas_raw';
 import {
   AGENT_NAME,
   EVENT_OUTCOME,
@@ -30,7 +28,10 @@ import { environmentQuery } from '../../../common/utils/environment_query';
 import { maybe } from '../../../common/utils/maybe';
 import { AgentName } from '../../../typings/es_schemas/ui/fields/agent';
 import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
-import { normalizeFields } from '../../utils/normalize_fields';
+import {
+  topDependencySpansMapping,
+  transactionsForDependencySpansMapping,
+} from '../../utils/es_fields_mappings';
 
 const MAX_NUM_SPANS = 1000;
 
@@ -115,7 +116,7 @@ export async function getTopDependencySpans({
         ],
       },
     })
-  ).hits.hits.map((hit) => normalizeFields(hit?.fields) as unknown as Span);
+  ).hits.hits.map((hit) => topDependencySpansMapping(hit?.fields));
 
   const transactionIds = spans.map((span) => span.transaction!.id);
 
@@ -138,7 +139,7 @@ export async function getTopDependencySpans({
         },
       },
     })
-  ).hits.hits.map((hit) => normalizeFields(hit.fields) as unknown as TransactionRaw);
+  ).hits.hits.map((hit) => transactionsForDependencySpansMapping(hit.fields));
 
   const transactionsById = keyBy(transactions, (transaction) => transaction.transaction.id);
 

x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts

@@ -6,14 +6,75 @@
  */
 
 import { rangeQuery, kqlQuery } from '@kbn/observability-plugin/server';
-import { ERROR_ID, SERVICE_NAME } from '../../../../common/es_fields/apm';
+import { OBSERVER_VERSION_MAJOR } from '@kbn/observability-shared-plugin/common';
+import {
+  ERROR_EXCEPTION_STACKTRACE,
+  ERROR_LOG_STACKTRACE,
+  ERROR_STACK_TRACE,
+} from '@kbn/discover-utils/src/field_constants';
+import {
+  AGENT_NAME,
+  AGENT_VERSION,
+  CONTAINER_ID,
+  CONTAINER_IMAGE,
+  ERROR_ID,
+  HOST_ARCHITECTURE,
+  HOST_HOSTNAME,
+  HOST_NAME,
+  HTTP_RESPONSE_STATUS_CODE,
+  KUBERNETES_POD_UID,
+  OBSERVER_HOSTNAME,
+  OBSERVER_ID,
+  OBSERVER_TYPE,
+  OBSERVER_NAME,
+  OBSERVER_VERSION,
+  PARENT_ID,
+  PROCESSOR_EVENT,
+  PROCESSOR_NAME,
+  SERVICE_NAME,
+  TRACE_ID,
+  SERVICE_ENVIRONMENT,
+  SERVICE_FRAMEWORK_NAME,
+  SERVICE_FRAMEWORK_VERSION,
+  SERVICE_NODE_NAME,
+  SERVICE_RUNTIME_NAME,
+  SERVICE_RUNTIME_VERSION,
+  SERVICE_LANGUAGE_NAME,
+  SERVICE_LANGUAGE_VERSION,
+  SERVICE_VERSION,
+  HOST_OS_PLATFORM,
+  TRANSACTION_ID,
+  TRANSACTION_SAMPLED,
+  TRANSACTION_TYPE,
+  ERROR_CULPRIT,
+  ERROR_PAGE_URL,
+  ERROR_LOG_MESSAGE,
+  URL_FULL,
+  USER_ID,
+  PROCESS_PID,
+  PROCESS_ARGS,
+  PROCESS_TITLE,
+  AT_TIMESTAMP,
+  ERROR_EXC_HANDLED,
+  ERROR_EXC_TYPE,
+  ERROR_EXC_MESSAGE,
+  URL_DOMAIN,
+  URL_ORIGINAL,
+} from '../../../../common/es_fields/apm';
 import { environmentQuery } from '../../../../common/utils/environment_query';
 import { ApmDocumentType } from '../../../../common/document_type';
 import { RollupInterval } from '../../../../common/rollup';
 import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
 import { getTransaction } from '../../transactions/get_transaction';
 import { Transaction } from '../../../../typings/es_schemas/ui/transaction';
 import { APMError } from '../../../../typings/es_schemas/ui/apm_error';
+import {
+  KUBERNETES_CONTAINER_NAME,
+  KUBERNETES_DEPLOYMENT_NAME,
+  KUBERNETES_NAMESPACE,
+  KUBERNETES_REPLICASET_NAME,
+  KUBERNETES_CONTAINER_ID,
+} from '../../../../common/es_fields/infra_metrics';
 
 export interface ErrorSampleDetailsResponse {
   transaction: Transaction | undefined;
@@ -60,7 +121,73 @@ export async function getErrorSampleDetails({
           ],
         },
       },
-      fields: ['*'],
+      fields: [
+        AGENT_VERSION,
+        AGENT_NAME,
+        PARENT_ID,
+        TRACE_ID,
+        // AGENT_EPHEMERAL_ID,
+        // OBSERVER_EPHEMERAL_ID,
+        OBSERVER_HOSTNAME,
+        OBSERVER_ID,
+        OBSERVER_TYPE,
+        OBSERVER_NAME,
+        OBSERVER_VERSION,
+        OBSERVER_VERSION_MAJOR,
+        CONTAINER_ID,
+        CONTAINER_IMAGE,
+        PROCESSOR_NAME,
+        PROCESSOR_EVENT,
+        HOST_ARCHITECTURE,
+        HOST_HOSTNAME,
+        HOST_NAME,
+        'host.ip',
+        'http.request.method',
+        HTTP_RESPONSE_STATUS_CODE,
+        'http.version',
+        KUBERNETES_POD_UID,
+        KUBERNETES_NAMESPACE,
+        KUBERNETES_REPLICASET_NAME,
+        KUBERNETES_DEPLOYMENT_NAME,
+        KUBERNETES_CONTAINER_ID,
+        KUBERNETES_CONTAINER_NAME,
+        SERVICE_NAME,
+        SERVICE_ENVIRONMENT,
+        SERVICE_FRAMEWORK_NAME,
+        SERVICE_FRAMEWORK_VERSION,
+        SERVICE_NODE_NAME,
+        SERVICE_RUNTIME_NAME,
+        SERVICE_RUNTIME_VERSION,
+        SERVICE_LANGUAGE_NAME,
+        SERVICE_LANGUAGE_VERSION,
+        SERVICE_VERSION,
+        PROCESS_ARGS,
+        PROCESS_PID,
+        PROCESS_TITLE,
+        HOST_OS_PLATFORM,
+        AT_TIMESTAMP,
+        TRANSACTION_ID,
+        TRANSACTION_SAMPLED,
+        TRANSACTION_TYPE,
+        ERROR_ID,
+        ERROR_CULPRIT,
+        'error.exception.attributes.response',
+        // ERROR_EXC_ATTRIBUTES_RESPONSE,
+        // ERROR_EXC_CODE,
+        ERROR_EXC_MESSAGE,
+        ERROR_EXC_TYPE,
+        // ERROR_EXCEPTION_MODULE,
+        ERROR_EXC_HANDLED,
+        ERROR_EXCEPTION_STACKTRACE, // todo: fix me
+        ERROR_PAGE_URL,
+        ERROR_LOG_MESSAGE,
+        ERROR_LOG_STACKTRACE, // todo: fixme
+        ERROR_STACK_TRACE,
+        URL_DOMAIN,
+        URL_FULL,
+        URL_ORIGINAL,
+        USER_ID,
+      ],
     },
   };