Merge pull request #2 from bryce-b/192749-refactor-the-normalize-func…

…tion-into-per-data-structure-serialization-logic

added span-link normalization and fixed errors in getTransaction

packages/kbn-apm-types/src/es_fields/apm.ts

@@ -21,7 +21,11 @@ export const CLOUD_AVAILABILITY_ZONE = 'cloud.availability_zone';
 export const CLOUD_PROVIDER = 'cloud.provider';
 export const CLOUD_REGION = 'cloud.region';
 export const CLOUD_MACHINE_TYPE = 'cloud.machine.type';
+export const CLOUD_PROJECT_ID = 'cloud.project.id';
+export const CLOUD_PROJECT_NAME = 'cloud.project.name';
+export const CLOUD_IMAGE_ID = 'cloud.image.id';
 export const CLOUD_ACCOUNT_ID = 'cloud.account.id';
+export const CLOUD_ACCOUNT_NAME = 'cloud.account.name';
 export const CLOUD_INSTANCE_ID = 'cloud.instance.id';
 export const CLOUD_INSTANCE_NAME = 'cloud.instance.name';
 export const CLOUD_SERVICE_NAME = 'cloud.service.name';
@@ -56,9 +60,11 @@ export const USER_AGENT_DEVICE_NAME = 'user_agent.device.name';
 export const USER_AGENT_VERSION = 'user_agent.version';
 
 export const OBSERVER_HOSTNAME = 'observer.hostname';
-export const OBSERVER_LISTENING = 'observer.listening';
+export const OBSERVER_ID = 'observer.id';
 export const OBSERVER_TYPE = 'observer.type';
+export const OBSERVER_NAME = 'observer.name';
 export const OBSERVER_VERSION = 'observer.version';
+export const OBSERVER_LISTENING = 'observer.listening';
 export const OBSERVER_VERSION_MAJOR = 'observer.version_major';
 export const PROCESSOR_EVENT = 'processor.event';
 export const PROCESSOR_NAME = 'processor.name';
@@ -166,6 +172,8 @@ export const HOST_IP = 'host.ip';
 export const CONTAINER_ID = 'container.id';
 export const CONTAINER = 'container';
 export const CONTAINER_IMAGE = 'container.image.name';
+export const CONTAINER_OS = 'container.os';
+export const CONTAINER_TTL_INSTANCES = 'container.totalNumberInstances';
 
 export const KUBERNETES = 'kubernetes';
 export const KUBERNETES_POD_NAME = 'kubernetes.pod.name';

x-pack/plugins/observability_solution/apm/common/correlations/types.ts

@@ -27,7 +27,6 @@ export interface ResponseHitSource {
 }
 
 export interface ResponseHit {
-  _source: ResponseHitSource;
   fields: ResponseHitSource;
 }
 

x-pack/plugins/observability_solution/apm/common/es_fields/infra_metrics.ts

@@ -7,9 +7,12 @@
 
 // Kubernetes
 export const KUBERNETES_CONTAINER_NAME = 'kubernetes.container.name';
+export const KUBERNETES_NODE_NAME = 'kubernetes.node.name';
+export const KUBERNETES_CONTAINER_ID = 'kubernetes.container.id';
 export const KUBERNETES_DEPLOYMENT = 'kubernetes.deployment';
 export const KUBERNETES_DEPLOYMENT_NAME = 'kubernetes.deployment.name';
 export const KUBERNETES_NAMESPACE_NAME = 'kubernetes.namespace.name';
 export const KUBERNETES_NAMESPACE = 'kubernetes.namespace';
 export const KUBERNETES_REPLICASET = 'kubernetes.replicaset';
 export const KUBERNETES_REPLICASET_NAME = 'kubernetes.replicaset.name';
+export const KUBERNETES_POD_NAME = 'kubernetes.pod.name';

x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts

@@ -6,7 +6,61 @@
  */
 
 import { rangeQuery, kqlQuery } from '@kbn/observability-plugin/server';
-import { ERROR_ID, SERVICE_NAME } from '../../../../common/es_fields/apm';
+import { OBSERVER_VERSION_MAJOR } from '@kbn/observability-shared-plugin/common';
+import {
+  ERROR_EXCEPTION_STACKTRACE,
+  ERROR_LOG_STACKTRACE,
+  ERROR_STACK_TRACE,
+} from '@kbn/discover-utils/src/field_constants';
+import {
+  AGENT_NAME,
+  AGENT_VERSION,
+  CONTAINER_ID,
+  CONTAINER_IMAGE,
+  ERROR_ID,
+  HOST_ARCHITECTURE,
+  HOST_HOSTNAME,
+  HOST_NAME,
+  HTTP_RESPONSE_STATUS_CODE,
+  KUBERNETES_POD_UID,
+  OBSERVER_HOSTNAME,
+  OBSERVER_ID,
+  OBSERVER_TYPE,
+  OBSERVER_NAME,
+  OBSERVER_VERSION,
+  PARENT_ID,
+  PROCESSOR_EVENT,
+  PROCESSOR_NAME,
+  SERVICE_NAME,
+  TRACE_ID,
+  SERVICE_ENVIRONMENT,
+  SERVICE_FRAMEWORK_NAME,
+  SERVICE_FRAMEWORK_VERSION,
+  SERVICE_NODE_NAME,
+  SERVICE_RUNTIME_NAME,
+  SERVICE_RUNTIME_VERSION,
+  SERVICE_LANGUAGE_NAME,
+  SERVICE_LANGUAGE_VERSION,
+  SERVICE_VERSION,
+  HOST_OS_PLATFORM,
+  TRANSACTION_ID,
+  TRANSACTION_SAMPLED,
+  TRANSACTION_TYPE,
+  ERROR_CULPRIT,
+  ERROR_PAGE_URL,
+  ERROR_LOG_MESSAGE,
+  URL_FULL,
+  USER_ID,
+  PROCESS_PID,
+  PROCESS_ARGS,
+  PROCESS_TITLE,
+  AT_TIMESTAMP,
+  ERROR_EXC_HANDLED,
+  ERROR_EXC_TYPE,
+  ERROR_EXC_MESSAGE,
+  URL_DOMAIN,
+  URL_ORIGINAL,
+} from '../../../../common/es_fields/apm';
 import { environmentQuery } from '../../../../common/utils/environment_query';
 import { ApmDocumentType } from '../../../../common/document_type';
 import { RollupInterval } from '../../../../common/rollup';
@@ -15,6 +69,13 @@ import { getTransaction } from '../../transactions/get_transaction';
 import { Transaction } from '../../../../typings/es_schemas/ui/transaction';
 import { APMError } from '../../../../typings/es_schemas/ui/apm_error';
 import { errorSampleDetailsMapping } from '../../../utils/es_fields_mappings';
+import {
+  KUBERNETES_CONTAINER_NAME,
+  KUBERNETES_DEPLOYMENT_NAME,
+  KUBERNETES_NAMESPACE,
+  KUBERNETES_REPLICASET_NAME,
+  KUBERNETES_CONTAINER_ID,
+} from '../../../../common/es_fields/infra_metrics';
 
 export interface ErrorSampleDetailsResponse {
   transaction: Transaction | undefined;
@@ -61,7 +122,73 @@ export async function getErrorSampleDetails({
           ],
         },
       },
-      fields: ['*'],
+      fields: [
+        AGENT_VERSION,
+        AGENT_NAME,
+        PARENT_ID,
+        TRACE_ID,
+        // AGENT_EPHEMERAL_ID,
+        // OBSERVER_EPHEMERAL_ID,
+        OBSERVER_HOSTNAME,
+        OBSERVER_ID,
+        OBSERVER_TYPE,
+        OBSERVER_NAME,
+        OBSERVER_VERSION,
+        OBSERVER_VERSION_MAJOR,
+        CONTAINER_ID,
+        CONTAINER_IMAGE,
+        PROCESSOR_NAME,
+        PROCESSOR_EVENT,
+        HOST_ARCHITECTURE,
+        HOST_HOSTNAME,
+        HOST_NAME,
+        'host.ip',
+        'http.request.method',
+        HTTP_RESPONSE_STATUS_CODE,
+        'http.version',
+        KUBERNETES_POD_UID,
+        KUBERNETES_NAMESPACE,
+        KUBERNETES_REPLICASET_NAME,
+        KUBERNETES_DEPLOYMENT_NAME,
+        KUBERNETES_CONTAINER_ID,
+        KUBERNETES_CONTAINER_NAME,
+        SERVICE_NAME,
+        SERVICE_ENVIRONMENT,
+        SERVICE_FRAMEWORK_NAME,
+        SERVICE_FRAMEWORK_VERSION,
+        SERVICE_NODE_NAME,
+        SERVICE_RUNTIME_NAME,
+        SERVICE_RUNTIME_VERSION,
+        SERVICE_LANGUAGE_NAME,
+        SERVICE_LANGUAGE_VERSION,
+        SERVICE_VERSION,
+        PROCESS_ARGS,
+        PROCESS_PID,
+        PROCESS_TITLE,
+        HOST_OS_PLATFORM,
+        AT_TIMESTAMP,
+        TRANSACTION_ID,
+        TRANSACTION_SAMPLED,
+        TRANSACTION_TYPE,
+        ERROR_ID,
+        ERROR_CULPRIT,
+        'error.exception.attributes.response',
+        // ERROR_EXC_ATTRIBUTES_RESPONSE,
+        // ERROR_EXC_CODE,
+        ERROR_EXC_MESSAGE,
+        ERROR_EXC_TYPE,
+        // ERROR_EXCEPTION_MODULE,
+        ERROR_EXC_HANDLED,
+        ERROR_EXCEPTION_STACKTRACE, // todo: fix me
+        ERROR_PAGE_URL,
+        ERROR_LOG_MESSAGE,
+        ERROR_LOG_STACKTRACE, // todo: fixme
+        ERROR_STACK_TRACE,
+        URL_DOMAIN,
+        URL_FULL,
+        URL_ORIGINAL,
+        USER_ID,
+      ],
     },
   };
 

x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts

@@ -7,32 +7,59 @@
 
 import { rangeQuery } from '@kbn/observability-plugin/server';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
+import { serviceMetadataDetailsMapping } from '../../utils/es_fields_mappings';
 import { environmentQuery } from '../../../common/utils/environment_query';
 import {
-  AGENT,
-  CONTAINER,
-  CLOUD,
+  KUBERNETES_CONTAINER_NAME,
+  KUBERNETES_DEPLOYMENT_NAME,
+  KUBERNETES_NAMESPACE,
+  KUBERNETES_REPLICASET_NAME,
+  KUBERNETES_CONTAINER_ID,
+  KUBERNETES_NODE_NAME,
+  KUBERNETES_POD_NAME,
+} from '../../../common/es_fields/infra_metrics';
+import {
+  AGENT_NAME,
+  AGENT_VERSION,
+  CONTAINER_ID,
+  CONTAINER_IMAGE,
   CLOUD_AVAILABILITY_ZONE,
+  CLOUD_INSTANCE_ID,
   CLOUD_REGION,
+  CLOUD_PROJECT_NAME,
   CLOUD_MACHINE_TYPE,
   CLOUD_SERVICE_NAME,
-  CONTAINER_ID,
-  HOST,
-  KUBERNETES,
-  SERVICE,
-  SERVICE_NAME,
+  CLOUD_INSTANCE_NAME,
+  CLOUD_PROVIDER,
+  CLOUD_ACCOUNT_ID,
+  CLOUD_ACCOUNT_NAME,
+  CLOUD_IMAGE_ID,
+  CLOUD_PROJECT_ID,
+  HOST_ARCHITECTURE,
+  HOST_HOSTNAME,
+  HOST_NAME,
+  HOST_IP,
+  SERVICE_ENVIRONMENT,
+  SERVICE_FRAMEWORK_NAME,
+  SERVICE_FRAMEWORK_VERSION,
   SERVICE_NODE_NAME,
+  SERVICE_RUNTIME_NAME,
+  SERVICE_RUNTIME_VERSION,
+  SERVICE_LANGUAGE_NAME,
+  SERVICE_LANGUAGE_VERSION,
   SERVICE_VERSION,
+  SERVICE_NAME,
   FAAS_ID,
   FAAS_TRIGGER_TYPE,
   LABEL_TELEMETRY_AUTO_VERSION,
+  CONTAINER_TTL_INSTANCES,
+  HOST_OS_PLATFORM,
 } from '../../../common/es_fields/apm';
 
 import { ContainerType } from '../../../common/service_metadata';
 import { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw';
 import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
 import { should } from './get_service_metadata_icons';
-import { normalizeFields } from '../../utils/normalize_fields';
 import { isOpenTelemetryAgentName, hasOpenTelemetryPrefix } from '../../../common/agent_name';
 
 type ServiceMetadataDetailsRaw = Pick<
@@ -113,6 +140,7 @@ export async function getServiceMetadataDetails({
     body: {
       track_total_hits: 1,
       size: 1,
+      _source: false,
       query: { bool: { filter, should } },
       aggs: {
         serviceVersions: {
@@ -166,15 +194,56 @@ export async function getServiceMetadataDetails({
         },
         totalNumberInstances: { cardinality: { field: SERVICE_NODE_NAME } },
       },
-      fields: [SERVICE, AGENT, HOST, CONTAINER, KUBERNETES, CLOUD, LABEL_TELEMETRY_AUTO_VERSION],
+      fields: [
+        SERVICE_ENVIRONMENT,
+        SERVICE_FRAMEWORK_NAME,
+        SERVICE_FRAMEWORK_VERSION,
+        SERVICE_NODE_NAME,
+        SERVICE_RUNTIME_NAME,
+        SERVICE_RUNTIME_VERSION,
+        SERVICE_LANGUAGE_NAME,
+        SERVICE_LANGUAGE_VERSION,
+        SERVICE_VERSION,
+        SERVICE_NAME,
+        AGENT_NAME,
+        AGENT_VERSION,
+        CONTAINER_ID,
+        CONTAINER_IMAGE,
+        HOST_ARCHITECTURE,
+        HOST_HOSTNAME,
+        HOST_NAME,
+        HOST_IP,
+        HOST_OS_PLATFORM,
+        CONTAINER_TTL_INSTANCES,
+        KUBERNETES_NAMESPACE,
+        KUBERNETES_NODE_NAME,
+        KUBERNETES_POD_NAME,
+        KUBERNETES_REPLICASET_NAME,
+        KUBERNETES_DEPLOYMENT_NAME,
+        KUBERNETES_CONTAINER_ID,
+        KUBERNETES_CONTAINER_NAME,
+        CLOUD_AVAILABILITY_ZONE,
+        CLOUD_INSTANCE_NAME,
+        CLOUD_INSTANCE_ID,
+        CLOUD_MACHINE_TYPE,
+        CLOUD_PROJECT_ID,
+        CLOUD_PROJECT_NAME,
+        CLOUD_PROVIDER,
+        CLOUD_REGION,
+        CLOUD_ACCOUNT_ID,
+        CLOUD_ACCOUNT_NAME,
+        CLOUD_IMAGE_ID,
+        CLOUD_SERVICE_NAME,
+        LABEL_TELEMETRY_AUTO_VERSION,
+      ],
     },
   };
 
   const response = await apmEventClient.search('get_service_metadata_details', params);
 
   const fields = response.hits.hits[0]?.fields;
   // todo: missing `fields` property?
-  const fieldsNorm = (fields ? normalizeFields(fields) : undefined) as
+  const fieldsNorm = (fields ? serviceMetadataDetailsMapping(fields) : undefined) as
     | ServiceMetadataDetailsRaw
     | undefined;