diff --git a/packages/kbn-dev-utils/src/serializers/recursive_serializer.ts b/packages/kbn-dev-utils/src/serializers/recursive_serializer.ts index 6e6572addbc83..15d3f033a85a1 100644 --- a/packages/kbn-dev-utils/src/serializers/recursive_serializer.ts +++ b/packages/kbn-dev-utils/src/serializers/recursive_serializer.ts @@ -6,11 +6,26 @@ * Side Public License, v 1. */ -export function createRecursiveSerializer(test: (v: any) => boolean, print: (v: any) => string) { +class RawPrint { + static fromString(s: string) { + return new RawPrint(s); + } + constructor(public readonly v: string) {} +} + +export function createRecursiveSerializer( + test: (v: any) => boolean, + print: (v: any, printRaw: (v: string) => RawPrint) => string | RawPrint +) { return { test: (v: any) => test(v), serialize: (v: any, ...rest: any[]) => { - const replacement = print(v); + const replacement = print(v, RawPrint.fromString); + + if (replacement instanceof RawPrint) { + return replacement.v; + } + const printer = rest.pop()!; return printer(replacement, ...rest); }, diff --git a/packages/kbn-es-archiver/src/cli.ts b/packages/kbn-es-archiver/src/cli.ts index e54b4d5fbdb52..fbb5784afe5ac 100644 --- a/packages/kbn-es-archiver/src/cli.ts +++ b/packages/kbn-es-archiver/src/cli.ts @@ -18,7 +18,7 @@ import readline from 'readline'; import Fs from 'fs'; import { RunWithCommands, createFlagError, CA_CERT_PATH } from '@kbn/dev-utils'; -import { readConfigFile, KbnClient } from '@kbn/test'; +import { readConfigFile, KbnClient, EsVersion } from '@kbn/test'; import { Client, HttpConnection } from '@elastic/elasticsearch'; import { EsArchiver } from './es_archiver'; @@ -45,7 +45,7 @@ export function runCli() { if (typeof configPath !== 'string') { throw createFlagError('--config must be a string'); } - const config = await readConfigFile(log, Path.resolve(configPath)); + const config = await readConfigFile(log, EsVersion.getDefault(), Path.resolve(configPath)); statsMeta.set('ftrConfigPath', configPath); let esUrl = flags['es-url']; diff --git a/packages/kbn-test/BUILD.bazel b/packages/kbn-test/BUILD.bazel index eae0fe2cdf5dc..1564270a8e588 100644 --- a/packages/kbn-test/BUILD.bazel +++ b/packages/kbn-test/BUILD.bazel @@ -69,6 +69,7 @@ RUNTIME_DEPS = [ "@npm//react-router-dom", "@npm//redux", "@npm//rxjs", + "@npm//semver", "@npm//strip-ansi", "@npm//xmlbuilder", "@npm//xml2js", @@ -108,6 +109,7 @@ TYPES_DEPS = [ "@npm//@types/react-dom", "@npm//@types/react-redux", "@npm//@types/react-router-dom", + "@npm//@types/semver", "@npm//@types/xml2js", ] diff --git a/packages/kbn-test/src/functional_test_runner/cli.ts b/packages/kbn-test/src/functional_test_runner/cli.ts index d9938bebea5bb..e013085e1b39a 100644 --- a/packages/kbn-test/src/functional_test_runner/cli.ts +++ b/packages/kbn-test/src/functional_test_runner/cli.ts @@ -35,6 +35,11 @@ export function runFtrCli() { const reportTime = getTimeReporter(toolingLog, 'scripts/functional_test_runner'); run( async ({ flags, log }) => { + const esVersion = flags['es-version'] || undefined; // convert "" to undefined + if (esVersion !== undefined && typeof esVersion !== 'string') { + throw createFlagError('expected --es-version to be a string'); + } + const functionalTestRunner = new FunctionalTestRunner( log, makeAbsolutePath(flags.config as string), @@ -57,7 +62,8 @@ export function runFtrCli() { }, updateBaselines: flags.updateBaselines || flags.u, updateSnapshots: flags.updateSnapshots || flags.u, - } + }, + esVersion ); if (flags.throttle) { @@ -131,6 +137,7 @@ export function runFtrCli() { 'include-tag', 'exclude-tag', 'kibana-install-dir', + 'es-version', ], boolean: [ 'bail', @@ -150,6 +157,7 @@ export function runFtrCli() { --bail stop tests after the first failure --grep pattern used to select which tests to run --invert invert grep to exclude tests + --es-version the elasticsearch version, formatted as "x.y.z" --include=file a test file to be included, pass multiple times for multiple files --exclude=file a test file to be excluded, pass multiple times for multiple files --include-tag=tag a tag to be included, pass multiple times for multiple tags. Only diff --git a/packages/kbn-test/src/functional_test_runner/functional_test_runner.ts b/packages/kbn-test/src/functional_test_runner/functional_test_runner.ts index 4130cd8d138b8..ea55a2672d670 100644 --- a/packages/kbn-test/src/functional_test_runner/functional_test_runner.ts +++ b/packages/kbn-test/src/functional_test_runner/functional_test_runner.ts @@ -6,6 +6,7 @@ * Side Public License, v 1. */ +import type { Client as EsClient } from '@elastic/elasticsearch'; import { ToolingLog } from '@kbn/dev-utils'; import { Suite, Test } from './fake_mocha_types'; @@ -21,6 +22,7 @@ import { DockerServersService, Config, SuiteTracker, + EsVersion, } from './lib'; export class FunctionalTestRunner { @@ -28,10 +30,12 @@ export class FunctionalTestRunner { public readonly failureMetadata = new FailureMetadata(this.lifecycle); private closed = false; + private readonly esVersion: EsVersion; constructor( private readonly log: ToolingLog, private readonly configFile: string, - private readonly configOverrides: any + private readonly configOverrides: any, + esVersion?: string | EsVersion ) { for (const [key, value] of Object.entries(this.lifecycle)) { if (value instanceof LifecyclePhase) { @@ -39,6 +43,12 @@ export class FunctionalTestRunner { value.after$.subscribe(() => log.verbose('starting %j lifecycle phase', key)); } } + this.esVersion = + esVersion === undefined + ? EsVersion.getDefault() + : esVersion instanceof EsVersion + ? esVersion + : new EsVersion(esVersion); } async run() { @@ -51,6 +61,27 @@ export class FunctionalTestRunner { ...readProviderSpec('PageObject', config.get('pageObjects')), ]); + // validate es version + if (providers.hasService('es')) { + const es = (await providers.getService('es')) as unknown as EsClient; + let esInfo; + try { + esInfo = await es.info(); + } catch (error) { + throw new Error( + `attempted to use the "es" service to fetch Elasticsearch version info but the request failed: ${error.stack}` + ); + } + + if (!this.esVersion.eql(esInfo.version.number)) { + throw new Error( + `ES reports a version number "${ + esInfo.version.number + }" which doesn't match supplied es version "${this.esVersion.toString()}"` + ); + } + } + await providers.loadAll(); const customTestRunner = config.get('testRunner'); @@ -61,7 +92,7 @@ export class FunctionalTestRunner { return (await providers.invokeProviderFn(customTestRunner)) || 0; } - const mocha = await setupMocha(this.lifecycle, this.log, config, providers); + const mocha = await setupMocha(this.lifecycle, this.log, config, providers, this.esVersion); await this.lifecycle.beforeTests.trigger(mocha.suite); this.log.info('Starting tests'); @@ -107,14 +138,14 @@ export class FunctionalTestRunner { ...readStubbedProviderSpec('PageObject', config.get('pageObjects'), []), ]); - const mocha = await setupMocha(this.lifecycle, this.log, config, providers); + const mocha = await setupMocha(this.lifecycle, this.log, config, providers, this.esVersion); const countTests = (suite: Suite): number => suite.suites.reduce((sum, s) => sum + countTests(s), suite.tests.length); return { testCount: countTests(mocha.suite), - excludedTests: mocha.excludedTests.map((t: Test) => t.fullTitle()), + testsExcludedByTag: mocha.testsExcludedByTag.map((t: Test) => t.fullTitle()), }; }); } @@ -125,7 +156,12 @@ export class FunctionalTestRunner { let runErrorOccurred = false; try { - const config = await readConfigFile(this.log, this.configFile, this.configOverrides); + const config = await readConfigFile( + this.log, + this.esVersion, + this.configFile, + this.configOverrides + ); this.log.info('Config loaded'); if ( @@ -148,6 +184,7 @@ export class FunctionalTestRunner { failureMetadata: () => this.failureMetadata, config: () => config, dockerServers: () => dockerServers, + esVersion: () => this.esVersion, }); return await handler(config, coreProviders); diff --git a/packages/kbn-test/src/functional_test_runner/index.ts b/packages/kbn-test/src/functional_test_runner/index.ts index 268c6b2bd9a67..1718b5f7a4bc5 100644 --- a/packages/kbn-test/src/functional_test_runner/index.ts +++ b/packages/kbn-test/src/functional_test_runner/index.ts @@ -7,7 +7,7 @@ */ export { FunctionalTestRunner } from './functional_test_runner'; -export { readConfigFile, Config } from './lib'; +export { readConfigFile, Config, EsVersion } from './lib'; export { runFtrCli } from './cli'; export * from './lib/docker_servers'; export * from './public_types'; diff --git a/packages/kbn-test/src/functional_test_runner/lib/config/read_config_file.test.js b/packages/kbn-test/src/functional_test_runner/lib/config/read_config_file.test.js index 60c307b58aee6..27434ce5a09ca 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/config/read_config_file.test.js +++ b/packages/kbn-test/src/functional_test_runner/lib/config/read_config_file.test.js @@ -9,34 +9,41 @@ import { ToolingLog } from '@kbn/dev-utils'; import { readConfigFile } from './read_config_file'; import { Config } from './config'; +import { EsVersion } from '../es_version'; const log = new ToolingLog(); +const esVersion = new EsVersion('8.0.0'); describe('readConfigFile()', () => { it('reads config from a file, returns an instance of Config class', async () => { - const config = await readConfigFile(log, require.resolve('./__fixtures__/config.1')); + const config = await readConfigFile(log, esVersion, require.resolve('./__fixtures__/config.1')); expect(config instanceof Config).toBeTruthy(); expect(config.get('testFiles')).toEqual(['config.1']); }); it('merges setting overrides into log', async () => { - const config = await readConfigFile(log, require.resolve('./__fixtures__/config.1'), { - screenshots: { - directory: 'foo.bar', - }, - }); + const config = await readConfigFile( + log, + esVersion, + require.resolve('./__fixtures__/config.1'), + { + screenshots: { + directory: 'foo.bar', + }, + } + ); expect(config.get('screenshots.directory')).toBe('foo.bar'); }); it('supports loading config files from within config files', async () => { - const config = await readConfigFile(log, require.resolve('./__fixtures__/config.2')); + const config = await readConfigFile(log, esVersion, require.resolve('./__fixtures__/config.2')); expect(config.get('testFiles')).toEqual(['config.1', 'config.2']); }); it('throws if settings are invalid', async () => { try { - await readConfigFile(log, require.resolve('./__fixtures__/config.invalid')); + await readConfigFile(log, esVersion, require.resolve('./__fixtures__/config.invalid')); throw new Error('expected readConfigFile() to fail'); } catch (err) { expect(err.message).toMatch(/"foo"/); diff --git a/packages/kbn-test/src/functional_test_runner/lib/config/read_config_file.ts b/packages/kbn-test/src/functional_test_runner/lib/config/read_config_file.ts index 374edea7a8db7..fd836f338edf0 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/config/read_config_file.ts +++ b/packages/kbn-test/src/functional_test_runner/lib/config/read_config_file.ts @@ -10,10 +10,16 @@ import { ToolingLog } from '@kbn/dev-utils'; import { defaultsDeep } from 'lodash'; import { Config } from './config'; +import { EsVersion } from '../es_version'; const cache = new WeakMap(); -async function getSettingsFromFile(log: ToolingLog, path: string, settingOverrides: any) { +async function getSettingsFromFile( + log: ToolingLog, + esVersion: EsVersion, + path: string, + settingOverrides: any +) { const configModule = require(path); // eslint-disable-line @typescript-eslint/no-var-requires const configProvider = configModule.__esModule ? configModule.default : configModule; @@ -23,9 +29,10 @@ async function getSettingsFromFile(log: ToolingLog, path: string, settingOverrid configProvider, configProvider({ log, + esVersion, async readConfigFile(p: string, o: any) { return new Config({ - settings: await getSettingsFromFile(log, p, o), + settings: await getSettingsFromFile(log, esVersion, p, o), primary: false, path: p, }); @@ -43,9 +50,14 @@ async function getSettingsFromFile(log: ToolingLog, path: string, settingOverrid return settingsWithDefaults; } -export async function readConfigFile(log: ToolingLog, path: string, settingOverrides: any = {}) { +export async function readConfigFile( + log: ToolingLog, + esVersion: EsVersion, + path: string, + settingOverrides: any = {} +) { return new Config({ - settings: await getSettingsFromFile(log, path, settingOverrides), + settings: await getSettingsFromFile(log, esVersion, path, settingOverrides), primary: true, path, }); diff --git a/packages/kbn-test/src/functional_test_runner/lib/es_version.ts b/packages/kbn-test/src/functional_test_runner/lib/es_version.ts new file mode 100644 index 0000000000000..8b3acde47a4dc --- /dev/null +++ b/packages/kbn-test/src/functional_test_runner/lib/es_version.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import semver from 'semver'; +import { kibanaPackageJson } from '@kbn/utils'; + +export class EsVersion { + static getDefault() { + // example: https://storage.googleapis.com/kibana-ci-es-snapshots-daily/8.0.0/manifest-latest-verified.json + const manifestUrl = process.env.ES_SNAPSHOT_MANIFEST; + if (manifestUrl) { + const match = manifestUrl.match(/\d+\.\d+\.\d+/); + if (!match) { + throw new Error('unable to extract es version from ES_SNAPSHOT_MANIFEST_URL'); + } + return new EsVersion(match[0]); + } + + return new EsVersion(process.env.TEST_ES_BRANCH || kibanaPackageJson.version); + } + + public readonly parsed: semver.SemVer; + + constructor(version: string) { + const parsed = semver.coerce(version); + if (!parsed) { + throw new Error(`unable to parse es version [${version}]`); + } + this.parsed = parsed; + } + + toString() { + return this.parsed.version; + } + + /** + * Determine if the ES version matches a semver range, like >=7 or ^8.1.0 + */ + matchRange(range: string) { + return semver.satisfies(this.parsed, range); + } + + /** + * Determine if the ES version matches a specific version, ignores things like -SNAPSHOT + */ + eql(version: string) { + const other = semver.coerce(version); + return other && semver.compareLoose(this.parsed, other) === 0; + } +} diff --git a/packages/kbn-test/src/functional_test_runner/lib/index.ts b/packages/kbn-test/src/functional_test_runner/lib/index.ts index 1cb1e58a265d5..98b5fec0597e4 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/index.ts +++ b/packages/kbn-test/src/functional_test_runner/lib/index.ts @@ -17,3 +17,4 @@ export * from './docker_servers'; export { SuiteTracker } from './suite_tracker'; export type { Provider } from './providers'; +export * from './es_version'; diff --git a/packages/kbn-test/src/functional_test_runner/lib/mocha/decorate_mocha_ui.js b/packages/kbn-test/src/functional_test_runner/lib/mocha/decorate_mocha_ui.js index 7610ca9128694..e12ffdc8cd616 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/mocha/decorate_mocha_ui.js +++ b/packages/kbn-test/src/functional_test_runner/lib/mocha/decorate_mocha_ui.js @@ -84,6 +84,9 @@ export function decorateMochaUi(log, lifecycle, context, { isDockerGroup, rootTa this._tags = [...this._tags, ...tagsToAdd]; }; + this.onlyEsVersion = (semver) => { + this._esVersionRequirement = semver; + }; provider.call(this); diff --git a/packages/kbn-test/src/functional_test_runner/lib/mocha/filter_suites_by_tags.test.js b/packages/kbn-test/src/functional_test_runner/lib/mocha/filter_suites.test.js similarity index 86% rename from packages/kbn-test/src/functional_test_runner/lib/mocha/filter_suites_by_tags.test.js rename to packages/kbn-test/src/functional_test_runner/lib/mocha/filter_suites.test.js index 10030a1c05632..191503af123d0 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/mocha/filter_suites_by_tags.test.js +++ b/packages/kbn-test/src/functional_test_runner/lib/mocha/filter_suites.test.js @@ -12,9 +12,10 @@ import Mocha from 'mocha'; import { create as createSuite } from 'mocha/lib/suite'; import Test from 'mocha/lib/test'; -import { filterSuitesByTags } from './filter_suites_by_tags'; +import { filterSuites } from './filter_suites'; +import { EsVersion } from '../es_version'; -function setup({ include, exclude }) { +function setup({ include, exclude, esVersion }) { return new Promise((resolve) => { const history = []; @@ -55,6 +56,7 @@ function setup({ include, exclude }) { const level1b = createSuite(level1, 'level 1b'); level1b._tags = ['level1b']; + level1b._esVersionRequirement = '<=8'; level1b.addTest(new Test('test 1b', () => {})); const level2 = createSuite(mocha.suite, 'level 2'); @@ -62,7 +64,7 @@ function setup({ include, exclude }) { level2a._tags = ['level2a']; level2a.addTest(new Test('test 2a', () => {})); - filterSuitesByTags({ + filterSuites({ log: { info(...args) { history.push(`info: ${format(...args)}`); @@ -71,6 +73,7 @@ function setup({ include, exclude }) { mocha, include, exclude, + esVersion, }); mocha.run(); @@ -208,3 +211,27 @@ it('does nothing if everything excluded', async () => { ] `); }); + +it(`excludes tests which don't meet the esVersionRequirement`, async () => { + const { history } = await setup({ + include: [], + exclude: [], + esVersion: new EsVersion('9.0.0'), + }); + + expect(history).toMatchInlineSnapshot(` + Array [ + "info: Only running suites which are compatible with ES version 9.0.0", + "suite: ", + "suite: level 1", + "suite: level 1 level 1a", + "hook: \\"before each\\" hook: rootBeforeEach for \\"test 1a\\"", + "hook: level 1 \\"before each\\" hook: level1BeforeEach for \\"test 1a\\"", + "test: level 1 level 1a test 1a", + "suite: level 2", + "suite: level 2 level 2a", + "hook: \\"before each\\" hook: rootBeforeEach for \\"test 2a\\"", + "test: level 2 level 2a test 2a", + ] + `); +}); diff --git a/packages/kbn-test/src/functional_test_runner/lib/mocha/filter_suites_by_tags.js b/packages/kbn-test/src/functional_test_runner/lib/mocha/filter_suites.ts similarity index 55% rename from packages/kbn-test/src/functional_test_runner/lib/mocha/filter_suites_by_tags.js rename to packages/kbn-test/src/functional_test_runner/lib/mocha/filter_suites.ts index 9724956e121f3..90bb3a894bc6c 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/mocha/filter_suites_by_tags.js +++ b/packages/kbn-test/src/functional_test_runner/lib/mocha/filter_suites.ts @@ -6,6 +6,24 @@ * Side Public License, v 1. */ +import { ToolingLog } from '@kbn/dev-utils'; +import { Suite, Test } from '../../fake_mocha_types'; +import { EsVersion } from '../es_version'; + +interface SuiteInternal extends Suite { + _tags?: string[]; + _esVersionRequirement?: string; + suites: SuiteInternal[]; +} + +interface Options { + log: ToolingLog; + mocha: any; + include: string[]; + exclude: string[]; + esVersion?: EsVersion; +} + /** * Given a mocha instance that has already loaded all of its suites, filter out * the suites based on the include/exclude tags. If there are include tags then @@ -16,23 +34,50 @@ * @param options.include an array of tags that suites must be tagged with to be run * @param options.exclude an array of tags that will be used to exclude suites from the run */ -export function filterSuitesByTags({ log, mocha, include, exclude }) { - mocha.excludedTests = []; +export function filterSuites({ log, mocha, include, exclude, esVersion }: Options) { + mocha.testsExcludedByTag = []; + mocha.testsExcludedByEsVersion = []; + // collect all the tests from some suite, including it's children - const collectTests = (suite) => + const collectTests = (suite: SuiteInternal): Test[] => suite.suites.reduce((acc, s) => acc.concat(collectTests(s)), suite.tests); + if (esVersion) { + // traverse the test graph and exclude any tests which don't meet their esVersionRequirement + log.info('Only running suites which are compatible with ES version', esVersion.toString()); + (function recurse(parentSuite: SuiteInternal) { + const children = parentSuite.suites; + parentSuite.suites = []; + + const meetsEsVersionRequirement = (suite: SuiteInternal) => + !suite._esVersionRequirement || esVersion.matchRange(suite._esVersionRequirement); + + for (const child of children) { + if (meetsEsVersionRequirement(child)) { + parentSuite.suites.push(child); + recurse(child); + } else { + mocha.testsExcludedByEsVersion = mocha.testsExcludedByEsVersion.concat( + collectTests(child) + ); + } + } + })(mocha.suite); + } + // if include tags were provided, filter the tree once to // only include branches that are included at some point if (include.length) { log.info('Only running suites (and their sub-suites) if they include the tag(s):', include); - const isIncluded = (suite) => + const isIncludedByTags = (suite: SuiteInternal) => !suite._tags ? false : suite._tags.some((t) => include.includes(t)); - const isChildIncluded = (suite) => + + const isIncluded = (suite: SuiteInternal) => isIncludedByTags(suite); + const isChildIncluded = (suite: SuiteInternal): boolean => suite.suites.some((s) => isIncluded(s) || isChildIncluded(s)); - (function recurse(parentSuite) { + (function recurse(parentSuite: SuiteInternal) { const children = parentSuite.suites; parentSuite.suites = []; @@ -47,13 +92,13 @@ export function filterSuitesByTags({ log, mocha, include, exclude }) { // itself, so strip out its tests and recurse to filter // out child suites which are not included if (isChildIncluded(child)) { - mocha.excludedTests = mocha.excludedTests.concat(child.tests); + mocha.testsExcludedByTag = mocha.testsExcludedByTag.concat(child.tests); child.tests = []; parentSuite.suites.push(child); recurse(child); continue; } else { - mocha.excludedTests = mocha.excludedTests.concat(collectTests(child)); + mocha.testsExcludedByTag = mocha.testsExcludedByTag.concat(collectTests(child)); } } })(mocha.suite); @@ -64,9 +109,10 @@ export function filterSuitesByTags({ log, mocha, include, exclude }) { if (exclude.length) { log.info('Filtering out any suites that include the tag(s):', exclude); - const isNotExcluded = (suite) => !suite._tags || !suite._tags.some((t) => exclude.includes(t)); + const isNotExcluded = (suite: SuiteInternal) => + !suite._tags || !suite._tags.some((t) => exclude.includes(t)); - (function recurse(parentSuite) { + (function recurse(parentSuite: SuiteInternal) { const children = parentSuite.suites; parentSuite.suites = []; @@ -77,7 +123,7 @@ export function filterSuitesByTags({ log, mocha, include, exclude }) { parentSuite.suites.push(child); recurse(child); } else { - mocha.excludedTests = mocha.excludedTests.concat(collectTests(child)); + mocha.testsExcludedByTag = mocha.testsExcludedByTag.concat(collectTests(child)); } } })(mocha.suite); diff --git a/packages/kbn-test/src/functional_test_runner/lib/mocha/setup_mocha.js b/packages/kbn-test/src/functional_test_runner/lib/mocha/setup_mocha.js index 65b7c09242fdd..8d88410cb2c1d 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/mocha/setup_mocha.js +++ b/packages/kbn-test/src/functional_test_runner/lib/mocha/setup_mocha.js @@ -11,7 +11,7 @@ import { relative } from 'path'; import { REPO_ROOT } from '@kbn/utils'; import { loadTestFiles } from './load_test_files'; -import { filterSuitesByTags } from './filter_suites_by_tags'; +import { filterSuites } from './filter_suites'; import { MochaReporterProvider } from './reporter'; import { validateCiGroupTags } from './validate_ci_group_tags'; @@ -22,9 +22,10 @@ import { validateCiGroupTags } from './validate_ci_group_tags'; * @param {ToolingLog} log * @param {Config} config * @param {ProviderCollection} providers + * @param {EsVersion} esVersion * @return {Promise} */ -export async function setupMocha(lifecycle, log, config, providers) { +export async function setupMocha(lifecycle, log, config, providers, esVersion) { // configure mocha const mocha = new Mocha({ ...config.get('mochaOpts'), @@ -50,18 +51,26 @@ export async function setupMocha(lifecycle, log, config, providers) { // valiate that there aren't any tests in multiple ciGroups validateCiGroupTags(log, mocha); + filterSuites({ + log, + mocha, + include: [], + exclude: [], + esVersion, + }); + // Each suite has a tag that is the path relative to the root of the repo // So we just need to take input paths, make them relative to the root, and use them as tags // Also, this is a separate filterSuitesByTags() call so that the test suites will be filtered first by // files, then by tags. This way, you can target tags (like smoke) in a specific file. - filterSuitesByTags({ + filterSuites({ log, mocha, include: config.get('suiteFiles.include').map((file) => relative(REPO_ROOT, file)), exclude: config.get('suiteFiles.exclude').map((file) => relative(REPO_ROOT, file)), }); - filterSuitesByTags({ + filterSuites({ log, mocha, include: config.get('suiteTags.include').map((tag) => tag.replace(/-\d+$/, '')), diff --git a/packages/kbn-test/src/functional_test_runner/public_types.ts b/packages/kbn-test/src/functional_test_runner/public_types.ts index d1a0f7998b0a9..6cb6d5adf4b19 100644 --- a/packages/kbn-test/src/functional_test_runner/public_types.ts +++ b/packages/kbn-test/src/functional_test_runner/public_types.ts @@ -6,10 +6,10 @@ * Side Public License, v 1. */ -import { ToolingLog } from '@kbn/dev-utils'; +import type { ToolingLog } from '@kbn/dev-utils'; -import { Config, Lifecycle, FailureMetadata, DockerServersService } from './lib'; -import { Test, Suite } from './fake_mocha_types'; +import type { Config, Lifecycle, FailureMetadata, DockerServersService, EsVersion } from './lib'; +import type { Test, Suite } from './fake_mocha_types'; export { Lifecycle, Config, FailureMetadata }; @@ -57,7 +57,7 @@ export interface GenericFtrProviderContext< * @param serviceName */ hasService( - serviceName: 'config' | 'log' | 'lifecycle' | 'failureMetadata' | 'dockerServers' + serviceName: 'config' | 'log' | 'lifecycle' | 'failureMetadata' | 'dockerServers' | 'esVersion' ): true; hasService(serviceName: K): serviceName is K; hasService(serviceName: string): serviceName is Extract; @@ -72,6 +72,7 @@ export interface GenericFtrProviderContext< getService(serviceName: 'lifecycle'): Lifecycle; getService(serviceName: 'dockerServers'): DockerServersService; getService(serviceName: 'failureMetadata'): FailureMetadata; + getService(serviceName: 'esVersion'): EsVersion; getService(serviceName: T): ServiceMap[T]; /** @@ -100,6 +101,7 @@ export class GenericFtrService; } diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap b/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap index ad2f82de87b82..fb00908e0c754 100644 --- a/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap +++ b/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap @@ -37,6 +37,7 @@ Object { ], "createLogger": [Function], "esFrom": "snapshot", + "esVersion": "999.999.999", "extraKbnOpts": undefined, "suiteFiles": Object { "exclude": Array [], @@ -58,6 +59,7 @@ Object { ], "createLogger": [Function], "esFrom": "snapshot", + "esVersion": "999.999.999", "extraKbnOpts": undefined, "suiteFiles": Object { "exclude": Array [], @@ -80,6 +82,7 @@ Object { "createLogger": [Function], "debug": true, "esFrom": "snapshot", + "esVersion": "999.999.999", "extraKbnOpts": undefined, "suiteFiles": Object { "exclude": Array [], @@ -101,6 +104,7 @@ Object { ], "createLogger": [Function], "esFrom": "snapshot", + "esVersion": "999.999.999", "extraKbnOpts": undefined, "suiteFiles": Object { "exclude": Array [], @@ -124,6 +128,7 @@ Object { ], "createLogger": [Function], "esFrom": "snapshot", + "esVersion": "999.999.999", "extraKbnOpts": Object { "server.foo": "bar", }, @@ -146,6 +151,7 @@ Object { ], "createLogger": [Function], "esFrom": "snapshot", + "esVersion": "999.999.999", "extraKbnOpts": undefined, "quiet": true, "suiteFiles": Object { @@ -167,6 +173,7 @@ Object { ], "createLogger": [Function], "esFrom": "snapshot", + "esVersion": "999.999.999", "extraKbnOpts": undefined, "silent": true, "suiteFiles": Object { @@ -188,6 +195,7 @@ Object { ], "createLogger": [Function], "esFrom": "source", + "esVersion": "999.999.999", "extraKbnOpts": undefined, "suiteFiles": Object { "exclude": Array [], @@ -208,6 +216,7 @@ Object { ], "createLogger": [Function], "esFrom": "source", + "esVersion": "999.999.999", "extraKbnOpts": undefined, "suiteFiles": Object { "exclude": Array [], @@ -228,6 +237,7 @@ Object { ], "createLogger": [Function], "esFrom": "snapshot", + "esVersion": "999.999.999", "extraKbnOpts": undefined, "installDir": "foo", "suiteFiles": Object { @@ -249,6 +259,7 @@ Object { ], "createLogger": [Function], "esFrom": "snapshot", + "esVersion": "999.999.999", "extraKbnOpts": undefined, "grep": "management", "suiteFiles": Object { @@ -270,6 +281,7 @@ Object { ], "createLogger": [Function], "esFrom": "snapshot", + "esVersion": "999.999.999", "extraKbnOpts": undefined, "suiteFiles": Object { "exclude": Array [], @@ -291,6 +303,7 @@ Object { ], "createLogger": [Function], "esFrom": "snapshot", + "esVersion": "999.999.999", "extraKbnOpts": undefined, "suiteFiles": Object { "exclude": Array [], diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/args.js b/packages/kbn-test/src/functional_tests/cli/run_tests/args.js index 901ff6394649d..497a9b9c6c533 100644 --- a/packages/kbn-test/src/functional_tests/cli/run_tests/args.js +++ b/packages/kbn-test/src/functional_tests/cli/run_tests/args.js @@ -10,6 +10,7 @@ import { resolve } from 'path'; import dedent from 'dedent'; import { ToolingLog, pickLevelFromFlags } from '@kbn/dev-utils'; +import { EsVersion } from '../../../functional_test_runner'; const options = { help: { desc: 'Display this menu and exit.' }, @@ -147,6 +148,7 @@ export function processOptions(userOptions, defaultConfigPaths) { configs: configs.map((c) => resolve(c)), createLogger, extraKbnOpts: userOptions._, + esVersion: EsVersion.getDefault(), }; } diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js b/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js index 7786aee5af552..72ba541466960 100644 --- a/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js +++ b/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js @@ -6,9 +6,20 @@ * Side Public License, v 1. */ -import { displayHelp, processOptions } from './args'; import { createAbsolutePathSerializer } from '@kbn/dev-utils'; +import { displayHelp, processOptions } from './args'; + +jest.mock('../../../functional_test_runner/lib/es_version', () => { + return { + EsVersion: class { + static getDefault() { + return '999.999.999'; + } + }, + }; +}); + expect.addSnapshotSerializer(createAbsolutePathSerializer(process.cwd())); const INITIAL_TEST_ES_FROM = process.env.TEST_ES_FROM; diff --git a/packages/kbn-test/src/functional_tests/lib/run_ftr.ts b/packages/kbn-test/src/functional_tests/lib/run_ftr.ts index f9e109928ddc0..77d6cd8e357a5 100644 --- a/packages/kbn-test/src/functional_tests/lib/run_ftr.ts +++ b/packages/kbn-test/src/functional_tests/lib/run_ftr.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ import type { ToolingLog } from '@kbn/dev-utils'; -import { FunctionalTestRunner, readConfigFile } from '../../functional_test_runner'; +import { FunctionalTestRunner, readConfigFile, EsVersion } from '../../functional_test_runner'; import { CliError } from './run_cli'; export interface CreateFtrOptions { @@ -26,6 +26,7 @@ export interface CreateFtrOptions { exclude?: string[]; }; updateSnapshots?: boolean; + esVersion: EsVersion; } export interface CreateFtrParams { @@ -34,31 +35,46 @@ export interface CreateFtrParams { } async function createFtr({ configPath, - options: { installDir, log, bail, grep, updateBaselines, suiteFiles, suiteTags, updateSnapshots }, + options: { + installDir, + log, + bail, + grep, + updateBaselines, + suiteFiles, + suiteTags, + updateSnapshots, + esVersion, + }, }: CreateFtrParams) { - const config = await readConfigFile(log, configPath); + const config = await readConfigFile(log, esVersion, configPath); return { config, - ftr: new FunctionalTestRunner(log, configPath, { - mochaOpts: { - bail: !!bail, - grep, + ftr: new FunctionalTestRunner( + log, + configPath, + { + mochaOpts: { + bail: !!bail, + grep, + }, + kbnTestServer: { + installDir, + }, + updateBaselines, + updateSnapshots, + suiteFiles: { + include: [...(suiteFiles?.include || []), ...config.get('suiteFiles.include')], + exclude: [...(suiteFiles?.exclude || []), ...config.get('suiteFiles.exclude')], + }, + suiteTags: { + include: [...(suiteTags?.include || []), ...config.get('suiteTags.include')], + exclude: [...(suiteTags?.exclude || []), ...config.get('suiteTags.exclude')], + }, }, - kbnTestServer: { - installDir, - }, - updateBaselines, - updateSnapshots, - suiteFiles: { - include: [...(suiteFiles?.include || []), ...config.get('suiteFiles.include')], - exclude: [...(suiteFiles?.exclude || []), ...config.get('suiteFiles.exclude')], - }, - suiteTags: { - include: [...(suiteTags?.include || []), ...config.get('suiteTags.include')], - exclude: [...(suiteTags?.exclude || []), ...config.get('suiteTags.exclude')], - }, - }), + esVersion + ), }; } @@ -71,15 +87,15 @@ export async function assertNoneExcluded({ configPath, options }: CreateFtrParam } const stats = await ftr.getTestStats(); - if (stats.excludedTests.length > 0) { + if (stats.testsExcludedByTag.length > 0) { throw new CliError(` - ${stats.excludedTests.length} tests in the ${configPath} config + ${stats.testsExcludedByTag.length} tests in the ${configPath} config are excluded when filtering by the tags run on CI. Make sure that all suites are tagged with one of the following tags: ${JSON.stringify(options.suiteTags)} - - ${stats.excludedTests.join('\n - ')} + - ${stats.testsExcludedByTag.join('\n - ')} `); } } diff --git a/packages/kbn-test/src/functional_tests/tasks.ts b/packages/kbn-test/src/functional_tests/tasks.ts index 6a6c7edb98c79..a8ec1d4be25bc 100644 --- a/packages/kbn-test/src/functional_tests/tasks.ts +++ b/packages/kbn-test/src/functional_tests/tasks.ts @@ -23,7 +23,7 @@ import { CreateFtrOptions, } from './lib'; -import { readConfigFile } from '../functional_test_runner/lib'; +import { readConfigFile, EsVersion } from '../functional_test_runner/lib'; const makeSuccessMessage = (options: StartServerOptions) => { const installDirFlag = options.installDir ? ` --kibana-install-dir=${options.installDir}` : ''; @@ -55,6 +55,7 @@ interface RunTestsParams extends CreateFtrOptions { configs: string[]; /** run from source instead of snapshot */ esFrom?: string; + esVersion: EsVersion; createLogger: () => ToolingLog; extraKbnOpts: string[]; assertNoneExcluded: boolean; @@ -105,7 +106,7 @@ export async function runTests(options: RunTestsParams) { log.write(`--- [${progress}] Running ${relative(REPO_ROOT, configPath)}`); await withProcRunner(log, async (procs) => { - const config = await readConfigFile(log, configPath); + const config = await readConfigFile(log, options.esVersion, configPath); let es; try { @@ -143,6 +144,7 @@ interface StartServerOptions { createLogger: () => ToolingLog; extraKbnOpts: string[]; useDefaultConfig?: boolean; + esVersion: EsVersion; } export async function startServers({ ...options }: StartServerOptions) { @@ -160,7 +162,7 @@ export async function startServers({ ...options }: StartServerOptions) { }; await withProcRunner(log, async (procs) => { - const config = await readConfigFile(log, options.config); + const config = await readConfigFile(log, options.esVersion, options.config); const es = await runElasticsearch({ config, options: opts }); await runKibanaServer({ diff --git a/packages/kbn-test/src/kbn_archiver_cli.ts b/packages/kbn-test/src/kbn_archiver_cli.ts index 80e35efaec976..f7f17900efcff 100644 --- a/packages/kbn-test/src/kbn_archiver_cli.ts +++ b/packages/kbn-test/src/kbn_archiver_cli.ts @@ -12,7 +12,7 @@ import Url from 'url'; import { RunWithCommands, createFlagError, Flags } from '@kbn/dev-utils'; import { KbnClient } from './kbn_client'; -import { readConfigFile } from './functional_test_runner'; +import { readConfigFile, EsVersion } from './functional_test_runner'; function getSinglePositionalArg(flags: Flags) { const positional = flags._; @@ -57,7 +57,7 @@ export function runKbnArchiverCli() { throw createFlagError('expected --config to be a string'); } - config = await readConfigFile(log, Path.resolve(flags.config)); + config = await readConfigFile(log, EsVersion.getDefault(), Path.resolve(flags.config)); statsMeta.set('ftrConfigPath', flags.config); } diff --git a/packages/kbn-test/types/ftr_globals/mocha.d.ts b/packages/kbn-test/types/ftr_globals/mocha.d.ts index ac9e33d4b9dcc..d5895b40f1245 100644 --- a/packages/kbn-test/types/ftr_globals/mocha.d.ts +++ b/packages/kbn-test/types/ftr_globals/mocha.d.ts @@ -14,5 +14,11 @@ declare module 'mocha' { * Assign tags to the test suite to determine in which CI job it should be run. */ tags(tags: string[] | string): void; + /** + * Define the ES versions for which this test requires, any version which doesn't meet this range will + * cause these tests to be skipped + * @param semver any valid semver range, like ">=8" + */ + onlyEsVersion(semver: string): void; } }