[APM] Script for creating functional test archive (#76926)

* [APM] Script for creating functional test archive

* Lock down variables; add documentation

* Update tests

x-pack/plugins/apm/dev_docs/updating_functional_tests_archives.md

@@ -0,0 +1,8 @@
+### Updating functional tests archives
+
+Some of our API tests use an archive generated by the [`esarchiver`](https://www.elastic.co/guide/en/kibana/current/development-functional-tests.html) script. Updating the main archive (`apm_8.0.0`) is a scripted process, where a 30m snapshot is downloaded from a cluster running the [APM Integration Testing server](https://github.com/elastic/apm-integration-testing). The script will copy the generated archives into the `fixtures/es_archiver` folders of our test suites (currently `basic` and `trial`). It will also generate a file that contains metadata about the archive, that can be imported to get the time range of the snapshot.
+
+Usage:
+`node x-pack/plugins/apm/scripts/create-functional-tests-archive --es-url=https://admin:changeme@localhost:9200 --kibana-url=https://localhost:5601`
+
+

x-pack/plugins/apm/scripts/aggregate-latency-metrics/index.ts

@@ -4,15 +4,12 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { Client } from '@elastic/elasticsearch';
 import { argv } from 'yargs';
 import pLimit from 'p-limit';
 import pRetry from 'p-retry';
-import { parse, format } from 'url';
 import { set } from '@elastic/safer-lodash-set';
 import { uniq, without, merge, flatten } from 'lodash';
 import * as histogram from 'hdr-histogram-js';
-import { ESSearchResponse } from '../../typings/elasticsearch';
 import {
   HOST_NAME,
   SERVICE_NAME,
@@ -28,6 +25,8 @@ import {
 } from '../../common/elasticsearch_fieldnames';
 import { stampLogger } from '../shared/stamp-logger';
 import { createOrUpdateIndex } from '../shared/create-or-update-index';
+import { parseIndexUrl } from '../shared/parse_index_url';
+import { ESClient, getEsClient } from '../shared/get_es_client';
 
 // This script will try to estimate how many latency metric documents
 // will be created based on the available transaction documents.
@@ -125,41 +124,18 @@ export async function aggregateLatencyMetrics() {
   const source = String(argv.source ?? '');
   const dest = String(argv.dest ?? '');
 
-  function getClientOptionsFromIndexUrl(
-    url: string
-  ): { node: string; index: string } {
-    const parsed = parse(url);
-    const { pathname, ...rest } = parsed;
+  const sourceOptions = parseIndexUrl(source);
 
-    return {
-      node: format(rest),
-      index: pathname!.replace('/', ''),
-    };
-  }
-
-  const sourceOptions = getClientOptionsFromIndexUrl(source);
-
-  const sourceClient = new Client({
-    node: sourceOptions.node,
-    ssl: {
-      rejectUnauthorized: false,
-    },
-    requestTimeout: 120000,
-  });
+  const sourceClient = getEsClient({ node: sourceOptions.node });
 
-  let destClient: Client | undefined;
+  let destClient: ESClient | undefined;
   let destOptions: { node: string; index: string } | undefined;
 
   const uploadMetrics = !!dest;
 
   if (uploadMetrics) {
-    destOptions = getClientOptionsFromIndexUrl(dest);
-    destClient = new Client({
-      node: destOptions.node,
-      ssl: {
-        rejectUnauthorized: false,
-      },
-    });
+    destOptions = parseIndexUrl(dest);
+    destClient = getEsClient({ node: destOptions.node });
 
     const mappings = (
       await sourceClient.indices.getMapping({
@@ -298,10 +274,9 @@ export async function aggregateLatencyMetrics() {
               },
             };
 
-            const response = (await sourceClient.search(params))
-              .body as ESSearchResponse<unknown, typeof params>;
+            const response = await sourceClient.search(params);
 
-            const { aggregations } = response;
+            const { aggregations } = response.body;
 
             if (!aggregations) {
               return buckets;
@@ -333,10 +308,9 @@ export async function aggregateLatencyMetrics() {
               },
             };
 
-            const response = (await sourceClient.search(params))
-              .body as ESSearchResponse<unknown, typeof params>;
+            const response = await sourceClient.search(params);
 
-            return response.hits.total.value;
+            return response.body.hits.total.value;
           }
 
           const [buckets, numberOfTransactionDocuments] = await Promise.all([

x-pack/plugins/apm/scripts/create-functional-tests-archive.js

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+// compile typescript on the fly
+// eslint-disable-next-line import/no-extraneous-dependencies
+require('@babel/register')({
+  extensions: ['.js', '.ts'],
+  plugins: ['@babel/plugin-proposal-optional-chaining'],
+  presets: [
+    '@babel/typescript',
+    ['@babel/preset-env', { targets: { node: 'current' } }],
+  ],
+});
+
+require('./create-functional-tests-archive/index.ts');

x-pack/plugins/apm/scripts/create-functional-tests-archive/index.ts

@@ -0,0 +1,179 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { argv } from 'yargs';
+import { execSync } from 'child_process';
+import moment from 'moment';
+import path from 'path';
+import fs from 'fs';
+import { stampLogger } from '../shared/stamp-logger';
+
+async function run() {
+  stampLogger();
+
+  const archiveName = 'apm_8.0.0';
+
+  // include important APM data and ML data
+  const indices =
+    'apm-*-transaction,apm-*-span,apm-*-error,apm-*-metric,.ml-anomalies*,.ml-config';
+
+  const esUrl = argv['es-url'] as string | undefined;
+
+  if (!esUrl) {
+    throw new Error('--es-url is not set');
+  }
+  const kibanaUrl = argv['kibana-url'] as string | undefined;
+
+  if (!kibanaUrl) {
+    throw new Error('--kibana-url is not set');
+  }
+  const gte = moment().subtract(1, 'hour').toISOString();
+  const lt = moment(gte).add(30, 'minutes').toISOString();
+
+  // eslint-disable-next-line no-console
+  console.log(`Archiving from ${gte} to ${lt}...`);
+
+  // APM data uses '@timestamp' (ECS), ML data uses 'timestamp'
+
+  const rangeQueries = [
+    {
+      range: {
+        '@timestamp': {
+          gte,
+          lt,
+        },
+      },
+    },
+    {
+      range: {
+        timestamp: {
+          gte,
+          lt,
+        },
+      },
+    },
+  ];
+
+  // some of the data is timeless/content
+  const query = {
+    bool: {
+      should: [
+        ...rangeQueries,
+        {
+          bool: {
+            must_not: [
+              {
+                exists: {
+                  field: '@timestamp',
+                },
+              },
+              {
+                exists: {
+                  field: 'timestamp',
+                },
+              },
+            ],
+          },
+        },
+      ],
+      minimum_should_match: 1,
+    },
+  };
+
+  const archivesDir = path.join(__dirname, '.archives');
+  const root = path.join(__dirname, '../../../../..');
+
+  // create the archive
+
+  execSync(
+    `node scripts/es_archiver save ${archiveName} ${indices} --dir=${archivesDir} --kibana-url=${kibanaUrl} --es-url=${esUrl} --query='${JSON.stringify(
+      query
+    )}'`,
+    {
+      cwd: root,
+      stdio: 'inherit',
+    }
+  );
+
+  const targetDirs = ['trial', 'basic'];
+
+  // copy the archives to the test fixtures
+
+  await Promise.all(
+    targetDirs.map(async (target) => {
+      const targetPath = path.resolve(
+        __dirname,
+        '../../../../test/apm_api_integration/',
+        target
+      );
+      const targetArchivesPath = path.resolve(
+        targetPath,
+        'fixtures/es_archiver',
+        archiveName
+      );
+
+      if (!fs.existsSync(targetArchivesPath)) {
+        fs.mkdirSync(targetArchivesPath);
+      }
+
+      fs.copyFileSync(
+        path.join(archivesDir, archiveName, 'data.json.gz'),
+        path.join(targetArchivesPath, 'data.json.gz')
+      );
+      fs.copyFileSync(
+        path.join(archivesDir, archiveName, 'mappings.json'),
+        path.join(targetArchivesPath, 'mappings.json')
+      );
+
+      const currentConfig = {};
+
+      // get the current metadata and extend/override metadata for the new archive
+      const configFilePath = path.join(targetPath, 'archives_metadata.ts');
+
+      try {
+        Object.assign(currentConfig, (await import(configFilePath)).default);
+      } catch (error) {
+        // do nothing
+      }
+
+      const newConfig = {
+        ...currentConfig,
+        [archiveName]: {
+          start: gte,
+          end: lt,
+        },
+      };
+
+      fs.writeFileSync(
+        configFilePath,
+        `export default ${JSON.stringify(newConfig, null, 2)}`,
+        { encoding: 'utf-8' }
+      );
+    })
+  );
+
+  fs.unlinkSync(path.join(archivesDir, archiveName, 'data.json.gz'));
+  fs.unlinkSync(path.join(archivesDir, archiveName, 'mappings.json'));
+  fs.rmdirSync(path.join(archivesDir, archiveName));
+  fs.rmdirSync(archivesDir);
+
+  // run ESLint on the generated metadata files
+
+  execSync('node scripts/eslint **/*/archives_metadata.ts --fix', {
+    cwd: root,
+    stdio: 'inherit',
+  });
+}
+
+run()
+  .then(() => {
+    process.exit(0);
+  })
+  .catch((err) => {
+    // eslint-disable-next-line no-console
+    console.log(err);
+    process.exit(1);
+  });

x-pack/plugins/apm/scripts/shared/create-or-update-index.ts

@@ -4,15 +4,15 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { Client } from '@elastic/elasticsearch';
+import { ESClient } from './get_es_client';
 
 export async function createOrUpdateIndex({
   client,
   clear,
   indexName,
   template,
 }: {
-  client: Client;
+  client: ESClient;
   clear: boolean;
   indexName: string;
   template: any;

x-pack/plugins/apm/scripts/shared/get_es_client.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { Client } from '@elastic/elasticsearch';
+import { ApiKeyAuth, BasicAuth } from '@elastic/elasticsearch/lib/pool';
+import { ESSearchResponse, ESSearchRequest } from '../../typings/elasticsearch';
+
+export type ESClient = ReturnType<typeof getEsClient>;
+
+export function getEsClient({
+  node,
+  auth,
+}: {
+  node: string;
+  auth?: BasicAuth | ApiKeyAuth;
+}) {
+  const client = new Client({
+    node,
+    ssl: {
+      rejectUnauthorized: false,
+    },
+    requestTimeout: 120000,
+    auth,
+  });
+
+  return {
+    ...client,
+    async search<TDocument, TSearchRequest extends ESSearchRequest>(
+      request: TSearchRequest
+    ) {
+      const response = await client.search(request as any);
+
+      return {
+        ...response,
+        body: response.body as ESSearchResponse<TDocument, TSearchRequest>,
+      };
+    },
+  };
+}

x-pack/plugins/apm/scripts/shared/parse_index_url.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { parse, format } from 'url';
+
+export function parseIndexUrl(url: string): { node: string; index: string } {
+  const parsed = parse(url);
+  const { pathname, ...rest } = parsed;
+
+  return {
+    node: format(rest),
+    index: pathname!.replace('/', ''),
+  };
+}

x-pack/test/apm_api_integration/basic/archives_metadata.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export default {
+  'apm_8.0.0': {
+    start: '2020-09-09T06:11:22.998Z',
+    end: '2020-09-09T06:41:22.998Z',
+  },
+};