[APM][Otel] Add service view e2e tests using otel service (#196542)

Closes #193206

## Summary

This PR adds service view e2e tests using otel service. 

To run the tests:
- Start server
`node x-pack/plugins/observability_solution/apm/scripts/test/e2e
--server`
- Open Cypress 
`node x-pack/plugins/observability_solution/apm/scripts/test/e2e
--runner --open`
- Select `otel_service_overview_and_transactions`

packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts

@@ -21,6 +21,7 @@ export interface OtelErrorDocument extends OtelDocument {
     'timestamp.us'?: number;
     'event.name'?: string;
     'error.id'?: string;
+    'error.grouping_key'?: string;
   };
 }
 

packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts

@@ -79,6 +79,7 @@ class Otel extends Serializable<OtelDocument> {
         'timestamp.us': 1726580752010657,
         'event.name': 'exception',
         'error.id': `error-${spanId}`,
+        'error.grouping_key': `errorGroup-${spanId}`,
       },
       data_stream: {
         dataset: 'generic.otel',

packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts

@@ -26,13 +26,25 @@ export class OtelSynthtraceEsClient extends SynthtraceEsClient<OtelDocument> {
     });
     this.dataStreams = ['metrics-generic.otel*', 'traces-generic.otel*', 'logs-generic.otel*'];
   }
+
+  getDefaultPipeline(
+    {
+      includeSerialization,
+    }: {
+      includeSerialization?: boolean;
+    } = { includeSerialization: true }
+  ) {
+    return otelPipeline(includeSerialization);
+  }
 }
 
-function otelPipeline() {
+function otelPipeline(includeSerialization: boolean = true) {
+  const serializationTransform = includeSerialization ? [getSerializeTransform()] : [];
   return (base: Readable) => {
     return pipeline(
+      // @ts-expect-error see apm_pipeline.ts
       base,
-      getSerializeTransform(),
+      ...serializationTransform,
       getRoutingTransform(),
       getDedotTransform(),
       (err: unknown) => {

x-pack/plugins/observability_solution/apm/ftr_e2e/cypress/e2e/service_overview/otel_service_overview_and_transactions.cy.ts

@@ -0,0 +1,143 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import url from 'url';
+import { synthtraceOtel } from '../../../synthtrace';
+import { sendotlp } from '../../fixtures/synthtrace/sendotlp';
+import { checkA11y } from '../../support/commands';
+
+const start = '2021-10-10T00:00:00.000Z';
+const end = '2021-10-10T00:15:00.000Z';
+const serviceInstanceId = '89117ac1-0dbf-4488-9e17-4c2c3b76943a';
+
+const serviceOverviewPath = '/app/apm/services/sendotlp-synth/overview';
+const baseUrl = url.format({
+  pathname: serviceOverviewPath,
+  query: { rangeFrom: start, rangeTo: end },
+});
+
+describe('Service Overview', () => {
+  before(() => {
+    synthtraceOtel.index(
+      sendotlp({
+        from: new Date(start).getTime(),
+        to: new Date(end).getTime(),
+      })
+    );
+  });
+
+  after(() => {
+    synthtraceOtel.clean();
+  });
+
+  describe('renders', () => {
+    beforeEach(() => {
+      cy.loginAsViewerUser();
+      cy.visitKibana(baseUrl);
+    });
+
+    it('renders all components on the page', () => {
+      cy.contains('sendotlp-synth');
+      // set skipFailures to true to not fail the test when there are accessibility failures
+      checkA11y({ skipFailures: true });
+      cy.getByTestSubj('latencyChart');
+      cy.getByTestSubj('throughput');
+      cy.getByTestSubj('transactionsGroupTable');
+      cy.getByTestSubj('serviceOverviewErrorsTable');
+      cy.getByTestSubj('dependenciesTable');
+      cy.getByTestSubj('instancesLatencyDistribution');
+      cy.getByTestSubj('serviceOverviewInstancesTable');
+    });
+  });
+
+  describe('service icons', () => {
+    beforeEach(() => {
+      cy.loginAsViewerUser();
+    });
+
+    it('show information on click', () => {
+      cy.intercept('GET', '/internal/apm/services/sendotlp-synth/metadata/details?*').as(
+        'metadataDetailsRequest'
+      );
+
+      cy.visitKibana(baseUrl);
+
+      cy.getByTestSubj('service').click();
+      cy.wait('@metadataDetailsRequest');
+      cy.contains('dt', 'Framework name');
+      cy.contains('dd', 'sendotlp-synth');
+
+      cy.getByTestSubj('opentelemetry').click();
+      cy.contains('dt', 'Language');
+      cy.contains('dd', 'go');
+    });
+  });
+
+  describe('instances table', () => {
+    beforeEach(() => {
+      cy.loginAsViewerUser();
+    });
+
+    it('has data in the table', () => {
+      cy.visitKibana(baseUrl);
+      cy.contains('sendotlp-synth');
+      cy.getByTestSubj('serviceInstancesTableContainer');
+      cy.contains(serviceInstanceId);
+    });
+  });
+
+  describe('transactions', () => {
+    beforeEach(() => {
+      cy.loginAsViewerUser();
+    });
+
+    it('persists transaction type selected when clicking on Transactions tab', () => {
+      cy.intercept('GET', '/internal/apm/services/sendotlp-synth/transaction_types?*').as(
+        'transactionTypesRequest'
+      );
+
+      cy.visitKibana(baseUrl);
+
+      cy.wait('@transactionTypesRequest');
+
+      cy.getByTestSubj('headerFilterTransactionType').should('have.value', 'unknown');
+      cy.contains('Transactions').click();
+      cy.getByTestSubj('headerFilterTransactionType').should('have.value', 'unknown');
+      cy.contains('parent-synth');
+    });
+
+    it('navigates to transaction detail page', () => {
+      cy.visitKibana(baseUrl);
+      cy.contains('Transactions').click();
+
+      cy.contains('a', 'parent-synth').click();
+      cy.contains('h5', 'parent-synth');
+    });
+  });
+
+  describe('errors', () => {
+    beforeEach(() => {
+      cy.loginAsViewerUser();
+      cy.visitKibana(baseUrl);
+    });
+    it('errors table is populated', () => {
+      cy.contains('sendotlp-synth');
+      cy.contains('*errors.errorString');
+    });
+
+    it('navigates to the errors page', () => {
+      cy.contains('sendotlp-synth');
+      cy.contains('a', 'View errors').click();
+      cy.url().should('include', '/sendotlp-synth/errors');
+    });
+
+    it('navigates to error detail page', () => {
+      cy.contains('a', '*errors.errorString').click();
+      cy.contains('div', 'boom');
+    });
+  });
+});

x-pack/plugins/observability_solution/apm/ftr_e2e/cypress/fixtures/synthtrace/sendotlp.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { generateShortId, otel, timerange } from '@kbn/apm-synthtrace-client';
+import { times } from 'lodash';
+
+export function sendotlp({ from, to }: { from: number; to: number }) {
+  const range = timerange(from, to);
+  const traceId = generateShortId();
+  const spanId = generateShortId();
+
+  const otelSendotlp = times(2).map((index) => otel.create(traceId));
+
+  return range
+    .interval('1s')
+    .rate(1)
+    .generator((timestamp) =>
+      otelSendotlp.flatMap((otelDoc) => {
+        return [
+          otelDoc.metric().timestamp(timestamp),
+          otelDoc.transaction(spanId).timestamp(timestamp),
+          otelDoc.error(spanId).timestamp(timestamp),
+        ];
+      })
+    );
+}

x-pack/plugins/observability_solution/apm/ftr_e2e/setup_cypress_node_events.ts

@@ -4,7 +4,12 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-import { ApmSynthtraceEsClient, createLogger, LogLevel } from '@kbn/apm-synthtrace';
+import {
+  ApmSynthtraceEsClient,
+  OtelSynthtraceEsClient,
+  createLogger,
+  LogLevel,
+} from '@kbn/apm-synthtrace';
 import { createEsClientForTesting } from '@kbn/test';
 // eslint-disable-next-line @kbn/imports/no_unresolvable_imports
 import { initPlugin } from '@frsource/cypress-plugin-visual-regression-diff/plugins';
@@ -28,10 +33,20 @@ export function setupNodeEvents(on: Cypress.PluginEvents, config: Cypress.Plugin
     version: config.env.APM_PACKAGE_VERSION,
   });
 
+  const synthtraceOtelEsClient = new OtelSynthtraceEsClient({
+    client,
+    logger,
+    refreshAfterIndex: true,
+  });
+
   synthtraceEsClient.pipeline(
     synthtraceEsClient.getDefaultPipeline({ includeSerialization: false })
   );
 
+  synthtraceOtelEsClient.pipeline(
+    synthtraceOtelEsClient.getDefaultPipeline({ includeSerialization: false })
+  );
+
   initPlugin(on, config);
 
   on('task', {
@@ -50,6 +65,14 @@ export function setupNodeEvents(on: Cypress.PluginEvents, config: Cypress.Plugin
       await synthtraceEsClient.clean();
       return null;
     },
+    async 'synthtraceOtel:index'(events: Array<Record<string, any>>) {
+      await synthtraceOtelEsClient.index(Readable.from(events));
+      return null;
+    },
+    async 'synthtraceOtel:clean'() {
+      await synthtraceOtelEsClient.clean();
+      return null;
+    },
   });
 
   on('after:spec', (spec, results) => {

x-pack/plugins/observability_solution/apm/ftr_e2e/synthtrace.ts

@@ -14,3 +14,12 @@ export const synthtrace = {
     ),
   clean: () => cy.task('synthtrace:clean'),
 };
+
+export const synthtraceOtel = {
+  index: (events: SynthtraceGenerator<ApmFields> | Array<Serializable<ApmFields>>) =>
+    cy.task(
+      'synthtraceOtel:index',
+      Array.from(events).flatMap((event) => event.serialize())
+    ),
+  clean: () => cy.task('synthtraceOtel:clean'),
+};