[8.5] [APM] Limit the number of source map artifacts (#144963) (#145062)

# Backport This will backport the following commits from `main` to `8.5`: - [[APM] Limit the number of source map artifacts (#144963)](#144963)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: Søren Louv-Jansen <[email protected]>
elastic · Nov 12, 2022 · 3b7df37 · 3b7df37
1 parent c23f85a
commit 3b7df37
Show file tree

Hide file tree

Showing 3 changed files with 168 additions and 29 deletions.
diff --git a/x-pack/plugins/apm/server/routes/fleet/source_maps.ts b/x-pack/plugins/apm/server/routes/fleet/source_maps.ts
@@ -13,7 +13,6 @@ import {
 import { promisify } from 'util';
 import { unzip } from 'zlib';
 import { Artifact } from '@kbn/fleet-plugin/server';
-import { isEmpty } from 'lodash';
 import { SourceMap } from '../source_maps/route';
 import { APMPluginStartDependencies } from '../../types';
 import { getApmPackagePolicies } from './get_apm_package_policies';
@@ -55,32 +54,15 @@ export async function listArtifacts({
   fleetPluginStart: FleetPluginStart;
 }) {
   const apmArtifactClient = getApmArtifactClient(fleetPluginStart);
-
-  const artifacts = [];
-  const perPage = 100;
-  let page = 1;
-
-  let fleetArtifactsResponse = await apmArtifactClient.listArtifacts({
+  const fleetArtifactsResponse = await apmArtifactClient.listArtifacts({
     kuery: 'type: sourcemap',
-    perPage,
-    page,
+    perPage: 20,
+    page: 1,
+    sortOrder: 'desc',
+    sortField: 'created',
   });
-  artifacts.push(...fleetArtifactsResponse.items);
-
-  while (
-    fleetArtifactsResponse.total > artifacts.length &&
-    !isEmpty(fleetArtifactsResponse.items)
-  ) {
-    page += 1;
-    fleetArtifactsResponse = await apmArtifactClient.listArtifacts({
-      kuery: 'type: sourcemap',
-      perPage,
-      page,
-    });
-    artifacts.push(...fleetArtifactsResponse.items);
-  }
 
-  return decodeArtifacts(artifacts);
+  return decodeArtifacts(fleetArtifactsResponse.items);
 }
 
 export async function createApmArtifact({

diff --git a/x-pack/test/apm_api_integration/common/apm_api_supertest.ts b/x-pack/test/apm_api_integration/common/apm_api_supertest.ts
@@ -18,22 +18,35 @@ import type { APIEndpoint } from '@kbn/apm-plugin/server';
 export function createApmApiClient(st: supertest.SuperTest<supertest.Test>) {
   return async <TEndpoint extends APIEndpoint>(
     options: {
+      type?: 'form-data';
       endpoint: TEndpoint;
     } & APIClientRequestParamsOf<TEndpoint> & { params?: { query?: { _inspect?: boolean } } }
   ): Promise<SupertestReturnType<TEndpoint>> => {
-    const { endpoint } = options;
+    const { endpoint, type } = options;
 
     const params = 'params' in options ? (options.params as Record<string, any>) : {};
 
     const { method, pathname } = parseEndpoint(endpoint, params?.path);
     const url = format({ pathname, query: params?.query });
 
-    const res = params.body
-      ? await st[method](url).send(params.body).set('kbn-xsrf', 'foo')
-      : await st[method](url).set('kbn-xsrf', 'foo');
+    let res: request.Response;
+    if (type === 'form-data') {
+      const fields: Array<[string, any]> = Object.entries(params.body);
+      const formDataRequest = st[method](url)
+        .set('kbn-xsrf', 'foo')
+        .set('Content-type', 'multipart/form-data');
+      for (const field of fields) {
+        formDataRequest.field(field[0], field[1]);
+      }
+      res = await formDataRequest;
+    } else if (params.body) {
+      res = await st[method](url).send(params.body).set('kbn-xsrf', 'foo');
+    } else {
+      res = await st[method](url).set('kbn-xsrf', 'foo');
+    }
 
     // supertest doesn't throw on http errors
-    if (res.status !== 200) {
+    if (res?.status !== 200) {
       throw new ApmApiError(res, endpoint);
     }
 

diff --git a/x-pack/test/apm_api_integration/tests/sourcemaps/sourcemaps.ts b/x-pack/test/apm_api_integration/tests/sourcemaps/sourcemaps.ts
@@ -0,0 +1,144 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api';
+import type { SourceMap } from '@kbn/apm-plugin/server/routes/source_maps/route';
+import expect from '@kbn/expect';
+import { times } from 'lodash';
+import { FtrProviderContext } from '../../common/ftr_provider_context';
+
+export default function ApiTest({ getService }: FtrProviderContext) {
+  const registry = getService('registry');
+  const apmApiClient = getService('apmApiClient');
+
+  async function uploadSourcemap({
+    bundleFilePath,
+    serviceName,
+    serviceVersion,
+    sourcemap,
+  }: {
+    bundleFilePath: string;
+    serviceName: string;
+    serviceVersion: string;
+    sourcemap: SourceMap;
+  }) {
+    const response = await apmApiClient.writeUser({
+      endpoint: 'POST /api/apm/sourcemaps',
+      type: 'form-data',
+      params: {
+        body: {
+          bundle_filepath: bundleFilePath,
+          service_name: serviceName,
+          service_version: serviceVersion,
+          sourcemap: JSON.stringify(sourcemap),
+        },
+      },
+    });
+    return response.body;
+  }
+
+  async function deleteSourcemap(id: string) {
+    await apmApiClient.writeUser({
+      endpoint: 'DELETE /api/apm/sourcemaps/{id}',
+      params: { path: { id } },
+    });
+  }
+
+  async function listSourcemaps() {
+    const response = await apmApiClient.readUser({
+      endpoint: 'GET /api/apm/sourcemaps',
+    });
+    return response.body.artifacts;
+  }
+
+  registry.when('source maps', { config: 'basic', archives: [] }, () => {
+    let resp: APIReturnType<'POST /api/apm/sourcemaps'>;
+    describe('upload source map', () => {
+      after(async () => {
+        await apmApiClient.writeUser({
+          endpoint: 'DELETE /api/apm/sourcemaps/{id}',
+          params: { path: { id: resp.id } },
+        });
+      });
+
+      it('can upload a source map', async () => {
+        resp = await uploadSourcemap({
+          serviceName: 'foo',
+          serviceVersion: '1.0.0',
+          bundleFilePath: 'bar',
+          sourcemap: {
+            version: 123,
+            sources: [''],
+            mappings: '',
+          },
+        });
+        expect(resp).to.not.empty();
+      });
+    });
+
+    describe('list source maps', () => {
+      const uploadedSourcemapIds: string[] = [];
+      before(async () => {
+        const sourcemapCount = times(2);
+        for (const i of sourcemapCount) {
+          const sourcemap = await uploadSourcemap({
+            serviceName: 'foo',
+            serviceVersion: `1.0.${i}`,
+            bundleFilePath: 'bar',
+            sourcemap: {
+              version: 123,
+              sources: [''],
+              mappings: '',
+            },
+          });
+          uploadedSourcemapIds.push(sourcemap.id);
+          await sleep(100);
+        }
+      });
+
+      after(async () => {
+        await Promise.all(uploadedSourcemapIds.map((id) => deleteSourcemap(id)));
+      });
+
+      it('can list source maps', async () => {
+        const sourcemaps = await listSourcemaps();
+        expect(sourcemaps).to.not.empty();
+      });
+
+      it('returns newest source maps first', async () => {
+        const response = await apmApiClient.readUser({
+          endpoint: 'GET /api/apm/sourcemaps',
+        });
+
+        const timestamps = response.body.artifacts.map((a) => new Date(a.created).getTime());
+        expect(timestamps[0]).to.be.greaterThan(timestamps[1]);
+      });
+    });
+
+    describe('delete source maps', () => {
+      it('can delete a source map', async () => {
+        const sourcemap = await uploadSourcemap({
+          serviceName: 'foo',
+          serviceVersion: '1.0.0',
+          bundleFilePath: 'bar',
+          sourcemap: {
+            version: 123,
+            sources: [''],
+            mappings: '',
+          },
+        });
+
+        await deleteSourcemap(sourcemap.id);
+        const sourcemaps = await listSourcemaps();
+        expect(sourcemaps).to.be.empty();
+      });
+    });
+  });
+}
+
+function sleep(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}