[ftr] support filtering tests by es version (#123289) (#123419)

Co-authored-by: Kibana Machine <[email protected]>
(cherry picked from commit 12e63dd)

packages/kbn-dev-utils/src/serializers/recursive_serializer.ts

@@ -6,11 +6,26 @@
  * Side Public License, v 1.
  */
 
-export function createRecursiveSerializer(test: (v: any) => boolean, print: (v: any) => string) {
+class RawPrint {
+  static fromString(s: string) {
+    return new RawPrint(s);
+  }
+  constructor(public readonly v: string) {}
+}
+
+export function createRecursiveSerializer(
+  test: (v: any) => boolean,
+  print: (v: any, printRaw: (v: string) => RawPrint) => string | RawPrint
+) {
   return {
     test: (v: any) => test(v),
     serialize: (v: any, ...rest: any[]) => {
-      const replacement = print(v);
+      const replacement = print(v, RawPrint.fromString);
+
+      if (replacement instanceof RawPrint) {
+        return replacement.v;
+      }
+
       const printer = rest.pop()!;
       return printer(replacement, ...rest);
     },

packages/kbn-es-archiver/src/cli.ts

@@ -18,7 +18,7 @@ import readline from 'readline';
 import Fs from 'fs';
 
 import { RunWithCommands, createFlagError, CA_CERT_PATH } from '@kbn/dev-utils';
-import { readConfigFile, KbnClient } from '@kbn/test';
+import { readConfigFile, KbnClient, EsVersion } from '@kbn/test';
 import { Client, HttpConnection } from '@elastic/elasticsearch';
 
 import { EsArchiver } from './es_archiver';
@@ -45,7 +45,7 @@ export function runCli() {
       if (typeof configPath !== 'string') {
         throw createFlagError('--config must be a string');
       }
-      const config = await readConfigFile(log, Path.resolve(configPath));
+      const config = await readConfigFile(log, EsVersion.getDefault(), Path.resolve(configPath));
       statsMeta.set('ftrConfigPath', configPath);
 
       let esUrl = flags['es-url'];

packages/kbn-test/BUILD.bazel

@@ -69,6 +69,7 @@ RUNTIME_DEPS = [
     "@npm//react-router-dom",
     "@npm//redux",
     "@npm//rxjs",
+    "@npm//semver",
     "@npm//strip-ansi",
     "@npm//xmlbuilder",
     "@npm//xml2js",
@@ -108,6 +109,7 @@ TYPES_DEPS = [
   "@npm//@types/react-dom",
   "@npm//@types/react-redux",
   "@npm//@types/react-router-dom",
+  "@npm//@types/semver",
   "@npm//@types/xml2js",
 ]
 

packages/kbn-test/src/functional_test_runner/cli.ts

@@ -35,6 +35,11 @@ export function runFtrCli() {
   const reportTime = getTimeReporter(toolingLog, 'scripts/functional_test_runner');
   run(
     async ({ flags, log }) => {
+      const esVersion = flags['es-version'] || undefined; // convert "" to undefined
+      if (esVersion !== undefined && typeof esVersion !== 'string') {
+        throw createFlagError('expected --es-version to be a string');
+      }
+
       const functionalTestRunner = new FunctionalTestRunner(
         log,
         makeAbsolutePath(flags.config as string),
@@ -57,7 +62,8 @@ export function runFtrCli() {
           },
           updateBaselines: flags.updateBaselines || flags.u,
           updateSnapshots: flags.updateSnapshots || flags.u,
-        }
+        },
+        esVersion
       );
 
       if (flags.throttle) {
@@ -131,6 +137,7 @@ export function runFtrCli() {
           'include-tag',
           'exclude-tag',
           'kibana-install-dir',
+          'es-version',
         ],
         boolean: [
           'bail',
@@ -150,6 +157,7 @@ export function runFtrCli() {
           --bail             stop tests after the first failure
           --grep <pattern>   pattern used to select which tests to run
           --invert           invert grep to exclude tests
+          --es-version       the elasticsearch version, formatted as "x.y.z"
           --include=file     a test file to be included, pass multiple times for multiple files
           --exclude=file     a test file to be excluded, pass multiple times for multiple files
           --include-tag=tag  a tag to be included, pass multiple times for multiple tags. Only

packages/kbn-test/src/functional_test_runner/functional_test_runner.ts

@@ -6,6 +6,7 @@
  * Side Public License, v 1.
  */
 
+import type { Client as EsClient } from '@elastic/elasticsearch';
 import { ToolingLog } from '@kbn/dev-utils';
 
 import { Suite, Test } from './fake_mocha_types';
@@ -21,24 +22,33 @@ import {
   DockerServersService,
   Config,
   SuiteTracker,
+  EsVersion,
 } from './lib';
 
 export class FunctionalTestRunner {
   public readonly lifecycle = new Lifecycle();
   public readonly failureMetadata = new FailureMetadata(this.lifecycle);
   private closed = false;
 
+  private readonly esVersion: EsVersion;
   constructor(
     private readonly log: ToolingLog,
     private readonly configFile: string,
-    private readonly configOverrides: any
+    private readonly configOverrides: any,
+    esVersion?: string | EsVersion
   ) {
     for (const [key, value] of Object.entries(this.lifecycle)) {
       if (value instanceof LifecyclePhase) {
         value.before$.subscribe(() => log.verbose('starting %j lifecycle phase', key));
         value.after$.subscribe(() => log.verbose('starting %j lifecycle phase', key));
       }
     }
+    this.esVersion =
+      esVersion === undefined
+        ? EsVersion.getDefault()
+        : esVersion instanceof EsVersion
+        ? esVersion
+        : new EsVersion(esVersion);
   }
 
   async run() {
@@ -51,6 +61,27 @@ export class FunctionalTestRunner {
         ...readProviderSpec('PageObject', config.get('pageObjects')),
       ]);
 
+      // validate es version
+      if (providers.hasService('es')) {
+        const es = (await providers.getService('es')) as unknown as EsClient;
+        let esInfo;
+        try {
+          esInfo = await es.info();
+        } catch (error) {
+          throw new Error(
+            `attempted to use the "es" service to fetch Elasticsearch version info but the request failed: ${error.stack}`
+          );
+        }
+
+        if (!this.esVersion.eql(esInfo.version.number)) {
+          throw new Error(
+            `ES reports a version number "${
+              esInfo.version.number
+            }" which doesn't match supplied es version "${this.esVersion.toString()}"`
+          );
+        }
+      }
+
       await providers.loadAll();
 
       const customTestRunner = config.get('testRunner');
@@ -61,7 +92,7 @@ export class FunctionalTestRunner {
         return (await providers.invokeProviderFn(customTestRunner)) || 0;
       }
 
-      const mocha = await setupMocha(this.lifecycle, this.log, config, providers);
+      const mocha = await setupMocha(this.lifecycle, this.log, config, providers, this.esVersion);
       await this.lifecycle.beforeTests.trigger(mocha.suite);
       this.log.info('Starting tests');
 
@@ -107,14 +138,14 @@ export class FunctionalTestRunner {
         ...readStubbedProviderSpec('PageObject', config.get('pageObjects'), []),
       ]);
 
-      const mocha = await setupMocha(this.lifecycle, this.log, config, providers);
+      const mocha = await setupMocha(this.lifecycle, this.log, config, providers, this.esVersion);
 
       const countTests = (suite: Suite): number =>
         suite.suites.reduce((sum, s) => sum + countTests(s), suite.tests.length);
 
       return {
         testCount: countTests(mocha.suite),
-        excludedTests: mocha.excludedTests.map((t: Test) => t.fullTitle()),
+        testsExcludedByTag: mocha.testsExcludedByTag.map((t: Test) => t.fullTitle()),
       };
     });
   }
@@ -125,7 +156,12 @@ export class FunctionalTestRunner {
     let runErrorOccurred = false;
 
     try {
-      const config = await readConfigFile(this.log, this.configFile, this.configOverrides);
+      const config = await readConfigFile(
+        this.log,
+        this.esVersion,
+        this.configFile,
+        this.configOverrides
+      );
       this.log.info('Config loaded');
 
       if (
@@ -148,6 +184,7 @@ export class FunctionalTestRunner {
         failureMetadata: () => this.failureMetadata,
         config: () => config,
         dockerServers: () => dockerServers,
+        esVersion: () => this.esVersion,
       });
 
       return await handler(config, coreProviders);

packages/kbn-test/src/functional_test_runner/index.ts

@@ -7,7 +7,7 @@
  */
 
 export { FunctionalTestRunner } from './functional_test_runner';
-export { readConfigFile, Config } from './lib';
+export { readConfigFile, Config, EsVersion } from './lib';
 export { runFtrCli } from './cli';
 export * from './lib/docker_servers';
 export * from './public_types';

packages/kbn-test/src/functional_test_runner/lib/config/read_config_file.test.js

@@ -9,34 +9,41 @@
 import { ToolingLog } from '@kbn/dev-utils';
 import { readConfigFile } from './read_config_file';
 import { Config } from './config';
+import { EsVersion } from '../es_version';
 
 const log = new ToolingLog();
+const esVersion = new EsVersion('8.0.0');
 
 describe('readConfigFile()', () => {
   it('reads config from a file, returns an instance of Config class', async () => {
-    const config = await readConfigFile(log, require.resolve('./__fixtures__/config.1'));
+    const config = await readConfigFile(log, esVersion, require.resolve('./__fixtures__/config.1'));
     expect(config instanceof Config).toBeTruthy();
     expect(config.get('testFiles')).toEqual(['config.1']);
   });
 
   it('merges setting overrides into log', async () => {
-    const config = await readConfigFile(log, require.resolve('./__fixtures__/config.1'), {
-      screenshots: {
-        directory: 'foo.bar',
-      },
-    });
+    const config = await readConfigFile(
+      log,
+      esVersion,
+      require.resolve('./__fixtures__/config.1'),
+      {
+        screenshots: {
+          directory: 'foo.bar',
+        },
+      }
+    );
 
     expect(config.get('screenshots.directory')).toBe('foo.bar');
   });
 
   it('supports loading config files from within config files', async () => {
-    const config = await readConfigFile(log, require.resolve('./__fixtures__/config.2'));
+    const config = await readConfigFile(log, esVersion, require.resolve('./__fixtures__/config.2'));
     expect(config.get('testFiles')).toEqual(['config.1', 'config.2']);
   });
 
   it('throws if settings are invalid', async () => {
     try {
-      await readConfigFile(log, require.resolve('./__fixtures__/config.invalid'));
+      await readConfigFile(log, esVersion, require.resolve('./__fixtures__/config.invalid'));
       throw new Error('expected readConfigFile() to fail');
     } catch (err) {
       expect(err.message).toMatch(/"foo"/);

packages/kbn-test/src/functional_test_runner/lib/config/read_config_file.ts

@@ -10,10 +10,16 @@ import { ToolingLog } from '@kbn/dev-utils';
 import { defaultsDeep } from 'lodash';
 
 import { Config } from './config';
+import { EsVersion } from '../es_version';
 
 const cache = new WeakMap();
 
-async function getSettingsFromFile(log: ToolingLog, path: string, settingOverrides: any) {
+async function getSettingsFromFile(
+  log: ToolingLog,
+  esVersion: EsVersion,
+  path: string,
+  settingOverrides: any
+) {
   const configModule = require(path); // eslint-disable-line @typescript-eslint/no-var-requires
   const configProvider = configModule.__esModule ? configModule.default : configModule;
 
@@ -23,9 +29,10 @@ async function getSettingsFromFile(log: ToolingLog, path: string, settingOverrid
       configProvider,
       configProvider({
         log,
+        esVersion,
         async readConfigFile(p: string, o: any) {
           return new Config({
-            settings: await getSettingsFromFile(log, p, o),
+            settings: await getSettingsFromFile(log, esVersion, p, o),
             primary: false,
             path: p,
           });
@@ -43,9 +50,14 @@ async function getSettingsFromFile(log: ToolingLog, path: string, settingOverrid
   return settingsWithDefaults;
 }
 
-export async function readConfigFile(log: ToolingLog, path: string, settingOverrides: any = {}) {
+export async function readConfigFile(
+  log: ToolingLog,
+  esVersion: EsVersion,
+  path: string,
+  settingOverrides: any = {}
+) {
   return new Config({
-    settings: await getSettingsFromFile(log, path, settingOverrides),
+    settings: await getSettingsFromFile(log, esVersion, path, settingOverrides),
     primary: true,
     path,
   });

packages/kbn-test/src/functional_test_runner/lib/es_version.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import semver from 'semver';
+import { kibanaPackageJson } from '@kbn/utils';
+
+export class EsVersion {
+  static getDefault() {
+    // example: https://storage.googleapis.com/kibana-ci-es-snapshots-daily/8.0.0/manifest-latest-verified.json
+    const manifestUrl = process.env.ES_SNAPSHOT_MANIFEST;
+    if (manifestUrl) {
+      const match = manifestUrl.match(/\d+\.\d+\.\d+/);
+      if (!match) {
+        throw new Error('unable to extract es version from ES_SNAPSHOT_MANIFEST_URL');
+      }
+      return new EsVersion(match[0]);
+    }
+
+    return new EsVersion(process.env.TEST_ES_BRANCH || kibanaPackageJson.version);
+  }
+
+  public readonly parsed: semver.SemVer;
+
+  constructor(version: string) {
+    const parsed = semver.coerce(version);
+    if (!parsed) {
+      throw new Error(`unable to parse es version [${version}]`);
+    }
+    this.parsed = parsed;
+  }
+
+  toString() {
+    return this.parsed.version;
+  }
+
+  /**
+   * Determine if the ES version matches a semver range, like >=7 or ^8.1.0
+   */
+  matchRange(range: string) {
+    return semver.satisfies(this.parsed, range);
+  }
+
+  /**
+   * Determine if the ES version matches a specific version, ignores things like -SNAPSHOT
+   */
+  eql(version: string) {
+    const other = semver.coerce(version);
+    return other && semver.compareLoose(this.parsed, other) === 0;
+  }
+}

packages/kbn-test/src/functional_test_runner/lib/index.ts

@@ -17,3 +17,4 @@ export * from './docker_servers';
 export { SuiteTracker } from './suite_tracker';
 
 export type { Provider } from './providers';
+export * from './es_version';

packages/kbn-test/src/functional_test_runner/lib/mocha/decorate_mocha_ui.js

@@ -84,6 +84,9 @@ export function decorateMochaUi(log, lifecycle, context, { isDockerGroup, rootTa
 
             this._tags = [...this._tags, ...tagsToAdd];
           };
+          this.onlyEsVersion = (semver) => {
+            this._esVersionRequirement = semver;
+          };
 
           provider.call(this);