[APM] Add API tests for storage explorer (#141391) (#141491)

* Add API tests for storage explorer

(cherry picked from commit 4137fb3)

Co-authored-by: Giorgos Bamparopoulos <[email protected]>

x-pack/test/apm_api_integration/common/authentication.ts

@@ -11,7 +11,6 @@ import { ToolingLog } from '@kbn/tooling-log';
 import { omit } from 'lodash';
 import { KbnClientRequesterError } from '@kbn/test';
 import { AxiosError } from 'axios';
-import { ProcessorEvent } from '@kbn/observability-plugin/common';
 import { SecurityServiceProvider } from '../../../../test/common/services/security';
 
 type SecurityService = Awaited<ReturnType<typeof SecurityServiceProvider>>;
@@ -90,12 +89,7 @@ const customRoles = {
     elasticsearch: {
       indices: [
         {
-          names: [
-            ProcessorEvent.transaction,
-            ProcessorEvent.span,
-            ProcessorEvent.metric,
-            ProcessorEvent.error,
-          ],
+          names: ['traces-apm*', 'logs-apm*', 'metrics-apm*', 'apm-*'],
           privileges: ['monitor'],
         },
       ],

x-pack/test/apm_api_integration/tests/storage_explorer/storage_details.spec.ts

@@ -0,0 +1,162 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import expect from '@kbn/expect';
+import { IndexLifecyclePhaseSelectOption } from '@kbn/apm-plugin/common/storage_explorer_types';
+import { apm, timerange } from '@kbn/apm-synthtrace';
+import { ProcessorEvent } from '@kbn/observability-plugin/common';
+import {
+  APIReturnType,
+  APIClientRequestParamsOf,
+} from '@kbn/apm-plugin/public/services/rest/create_call_apm_api';
+import { RecursivePartial } from '@kbn/apm-plugin/typings/common';
+import { keyBy } from 'lodash';
+import { FtrProviderContext } from '../../common/ftr_provider_context';
+
+type StorageDetails = APIReturnType<'GET /internal/apm/services/{serviceName}/storage_details'>;
+
+export default function ApiTest({ getService }: FtrProviderContext) {
+  const registry = getService('registry');
+  const apmApiClient = getService('apmApiClient');
+  const synthtraceEsClient = getService('synthtraceEsClient');
+
+  const start = new Date('2021-01-01T00:00:00.000Z').getTime();
+  const end = new Date('2021-01-01T00:15:00.000Z').getTime() - 1;
+  const serviceName = 'opbeans-go';
+
+  function areProcessorEventStatsEmpty(storageDetails: StorageDetails) {
+    return storageDetails.processorEventStats.every(({ docs, size }) => docs === 0 && size === 0);
+  }
+
+  async function callApi(
+    overrides?: RecursivePartial<
+      APIClientRequestParamsOf<'GET /internal/apm/services/{serviceName}/storage_details'>['params']
+    >
+  ) {
+    return await apmApiClient.monitorIndicesUser({
+      endpoint: 'GET /internal/apm/services/{serviceName}/storage_details',
+      params: {
+        path: {
+          serviceName,
+          ...overrides?.path,
+        },
+        query: {
+          indexLifecyclePhase: IndexLifecyclePhaseSelectOption.All,
+          probability: 1,
+          environment: 'ENVIRONMENT_ALL',
+          start: new Date(start).toISOString(),
+          end: new Date(end).toISOString(),
+          kuery: '',
+          ...overrides?.query,
+        },
+      },
+    });
+  }
+
+  registry.when(
+    'Storage details when data is not loaded',
+    { config: 'basic', archives: [] },
+    () => {
+      it('handles empty state', async () => {
+        const { status, body } = await callApi();
+
+        expect(status).to.be(200);
+        expect(body.processorEventStats).to.have.length(4);
+        expect(areProcessorEventStatsEmpty(body)).to.be(true);
+      });
+    }
+  );
+
+  registry.when('Storage details', { config: 'basic', archives: [] }, () => {
+    describe('when data is loaded', () => {
+      before(async () => {
+        const serviceGo = apm
+          .service({ name: serviceName, environment: 'production', agentName: 'go' })
+          .instance('instance');
+
+        await synthtraceEsClient.index([
+          timerange(start, end)
+            .interval('5m')
+            .rate(1)
+            .generator((timestamp) =>
+              serviceGo
+                .transaction({ transactionName: 'GET /api/product/list' })
+                .duration(2000)
+                .timestamp(timestamp)
+                .children(
+                  serviceGo
+                    .span({ spanName: '/_search', spanType: 'db', spanSubtype: 'elasticsearch' })
+                    .destination('elasticsearch')
+                    .duration(100)
+                    .success()
+                    .timestamp(timestamp),
+                  serviceGo
+                    .span({ spanName: '/_search', spanType: 'db', spanSubtype: 'elasticsearch' })
+                    .destination('elasticsearch')
+                    .duration(300)
+                    .success()
+                    .timestamp(timestamp)
+                )
+                .errors(
+                  serviceGo.error({ message: 'error 1', type: 'foo' }).timestamp(timestamp),
+                  serviceGo.error({ message: 'error 2', type: 'foo' }).timestamp(timestamp),
+                  serviceGo.error({ message: 'error 3', type: 'bar' }).timestamp(timestamp)
+                )
+            ),
+        ]);
+      });
+
+      after(() => synthtraceEsClient.clean());
+
+      it('returns correct stats for processor events', async () => {
+        const { status, body } = await callApi();
+        expect(status).to.be(200);
+        expect(body.processorEventStats).to.have.length(4);
+
+        const stats = keyBy(body.processorEventStats, 'processorEvent');
+
+        expect(stats[ProcessorEvent.transaction]?.docs).to.be(3);
+        expect(stats[ProcessorEvent.transaction]?.size).to.be.greaterThan(0);
+        expect(stats[ProcessorEvent.span]?.docs).to.be(6);
+        expect(stats[ProcessorEvent.span]?.size).to.be.greaterThan(0);
+        expect(stats[ProcessorEvent.error]?.docs).to.be(9);
+        expect(stats[ProcessorEvent.error]?.size).to.be.greaterThan(0);
+        expect(stats[ProcessorEvent.metric]?.docs).to.be.greaterThan(0);
+        expect(stats[ProcessorEvent.metric]?.size).to.be.greaterThan(0);
+      });
+
+      it('returns empty stats when there is no matching environment', async () => {
+        const { status, body } = await callApi({
+          query: {
+            environment: 'test',
+          },
+        });
+        expect(status).to.be(200);
+        expect(areProcessorEventStatsEmpty(body)).to.be(true);
+      });
+
+      it('returns empty stats when there is no matching lifecycle phase', async () => {
+        const { status, body } = await callApi({
+          query: {
+            indexLifecyclePhase: IndexLifecyclePhaseSelectOption.Warm,
+          },
+        });
+        expect(status).to.be(200);
+        expect(areProcessorEventStatsEmpty(body)).to.be(true);
+      });
+
+      it('returns empty stats when there is no matching kql filter', async () => {
+        const { status, body } = await callApi({
+          query: {
+            kuery: 'service.name : opbeans-node',
+          },
+        });
+        expect(status).to.be(200);
+        expect(areProcessorEventStatsEmpty(body)).to.be(true);
+      });
+    });
+  });
+}

x-pack/test/apm_api_integration/tests/storage_explorer/storage_explorer.spec.ts

@@ -0,0 +1,164 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import expect from '@kbn/expect';
+import { IndexLifecyclePhaseSelectOption } from '@kbn/apm-plugin/common/storage_explorer_types';
+import { apm, timerange } from '@kbn/apm-synthtrace';
+import { APIClientRequestParamsOf } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api';
+import { RecursivePartial } from '@kbn/apm-plugin/typings/common';
+import { keyBy } from 'lodash';
+import { FtrProviderContext } from '../../common/ftr_provider_context';
+
+export default function ApiTest({ getService }: FtrProviderContext) {
+  const registry = getService('registry');
+  const apmApiClient = getService('apmApiClient');
+  const synthtraceEsClient = getService('synthtraceEsClient');
+
+  const start = new Date('2021-01-01T00:00:00.000Z').getTime();
+  const end = new Date('2021-01-01T00:15:00.000Z').getTime() - 1;
+  const goServiceName = 'opbeans-go';
+  const nodeServiceName = 'opbeans-node';
+
+  async function callApi(
+    overrides?: RecursivePartial<
+      APIClientRequestParamsOf<'GET /internal/apm/storage_explorer'>['params']
+    >
+  ) {
+    return await apmApiClient.monitorIndicesUser({
+      endpoint: 'GET /internal/apm/storage_explorer',
+      params: {
+        query: {
+          indexLifecyclePhase: IndexLifecyclePhaseSelectOption.All,
+          probability: 1,
+          environment: 'ENVIRONMENT_ALL',
+          start: new Date(start).toISOString(),
+          end: new Date(end).toISOString(),
+          kuery: '',
+          ...overrides?.query,
+        },
+      },
+    });
+  }
+
+  registry.when(
+    'Storage explorer when data is not loaded',
+    { config: 'basic', archives: [] },
+    () => {
+      it('handles empty state', async () => {
+        const { status, body } = await callApi();
+
+        expect(status).to.be(200);
+        expect(body.serviceStatistics).to.be.empty();
+      });
+    }
+  );
+
+  registry.when('Storage explorer', { config: 'basic', archives: [] }, () => {
+    describe('when data is loaded', () => {
+      before(async () => {
+        const serviceGo = apm
+          .service({ name: goServiceName, environment: 'production', agentName: 'go' })
+          .instance('instance-go');
+
+        const serviceNodeStaging = apm
+          .service({ name: nodeServiceName, environment: 'staging', agentName: 'node' })
+          .instance('instance-node-staging');
+
+        const serviceNodeDev = apm
+          .service({ name: nodeServiceName, environment: 'dev', agentName: 'node' })
+          .instance('instance-node-dev');
+
+        await synthtraceEsClient.index([
+          timerange(start, end)
+            .interval('5m')
+            .rate(1)
+            .generator((timestamp) =>
+              serviceGo
+                .transaction({ transactionName: 'GET /api/product/list' })
+                .duration(2000)
+                .timestamp(timestamp)
+            ),
+          timerange(start, end)
+            .interval('5m')
+            .rate(1)
+            .generator((timestamp) =>
+              serviceNodeStaging
+                .transaction({ transactionName: 'GET /api/users/list' })
+                .duration(2000)
+                .timestamp(timestamp)
+            ),
+          timerange(start, end)
+            .interval('5m')
+            .rate(1)
+            .generator((timestamp) =>
+              serviceNodeDev
+                .transaction({ transactionName: 'GET /api/users/list' })
+                .duration(2000)
+                .timestamp(timestamp)
+            ),
+        ]);
+      });
+
+      after(() => synthtraceEsClient.clean());
+
+      it('returns correct stats', async () => {
+        const { status, body } = await callApi();
+        expect(status).to.be(200);
+
+        expect(body.serviceStatistics).to.have.length(2);
+
+        const stats = keyBy(body.serviceStatistics, 'serviceName');
+
+        const goServiceStats = stats[goServiceName];
+        expect(goServiceStats?.environments).to.have.length(1);
+        expect(goServiceStats?.environments).to.contain('production');
+        expect(goServiceStats?.agentName).to.be('go');
+        expect(goServiceStats?.size).to.be.greaterThan(0);
+        expect(goServiceStats?.sampling).to.be(1);
+
+        const nodeServiceStats = stats[nodeServiceName];
+        expect(nodeServiceStats?.environments).to.have.length(2);
+        expect(nodeServiceStats?.environments).to.contain('staging');
+        expect(nodeServiceStats?.environments).to.contain('dev');
+        expect(nodeServiceStats?.agentName).to.be('node');
+        expect(nodeServiceStats?.size).to.be.greaterThan(0);
+        expect(nodeServiceStats?.sampling).to.be(1);
+      });
+
+      it('returns only node service stats when there is a matching environment', async () => {
+        const { status, body } = await callApi({
+          query: {
+            environment: 'dev',
+          },
+        });
+        expect(status).to.be(200);
+        expect(body.serviceStatistics).to.have.length(1);
+        expect(body.serviceStatistics[0]?.serviceName).to.be(nodeServiceName);
+      });
+
+      it('returns empty stats when there is no matching lifecycle phase', async () => {
+        const { status, body } = await callApi({
+          query: {
+            indexLifecyclePhase: IndexLifecyclePhaseSelectOption.Warm,
+          },
+        });
+        expect(status).to.be(200);
+        expect(body.serviceStatistics).to.be.empty();
+      });
+
+      it('returns only go service stats when there is a matching kql filter', async () => {
+        const { status, body } = await callApi({
+          query: {
+            kuery: `service.name : ${goServiceName}`,
+          },
+        });
+        expect(status).to.be(200);
+        expect(body.serviceStatistics).to.have.length(1);
+        expect(body.serviceStatistics[0]?.serviceName).to.be(goServiceName);
+      });
+    });
+  });
+}

x-pack/test/apm_api_integration/tests/storage_explorer/storage_explorer_privileges.spec.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import expect from '@kbn/expect';
+import { FtrProviderContext } from '../../common/ftr_provider_context';
+import { ApmApiSupertest } from '../../common/apm_api_supertest';
+
+export default function ApiTest({ getService }: FtrProviderContext) {
+  const registry = getService('registry');
+  const apmApiClient = getService('apmApiClient');
+
+  async function callApi(apiClient: ApmApiSupertest) {
+    return await apiClient({
+      endpoint: 'GET /internal/apm/storage_explorer/privileges',
+    });
+  }
+
+  registry.when('Storage explorer privileges', { config: 'basic', archives: [] }, () => {
+    it('returns true when the user has the required indices privileges', async () => {
+      const { status, body } = await callApi(apmApiClient.monitorIndicesUser);
+      expect(status).to.be(200);
+      expect(body.hasPrivileges).to.be(true);
+    });
+
+    it(`returns false when the user doesn't have the required indices privileges`, async () => {
+      const { status, body } = await callApi(apmApiClient.writeUser);
+      expect(status).to.be(200);
+      expect(body.hasPrivileges).to.be(false);
+    });
+  });
+}