diff --git a/packages/kbn-test/src/functional_test_runner/functional_test_runner.ts b/packages/kbn-test/src/functional_test_runner/functional_test_runner.ts index e566a9a4af262..98b299c4d5279 100644 --- a/packages/kbn-test/src/functional_test_runner/functional_test_runner.ts +++ b/packages/kbn-test/src/functional_test_runner/functional_test_runner.ts @@ -30,6 +30,7 @@ import { setupMocha, runTests, Config, + SuiteTracker, } from './lib'; export class FunctionalTestRunner { @@ -52,6 +53,8 @@ export class FunctionalTestRunner { async run() { return await this._run(async (config, coreProviders) => { + SuiteTracker.startTracking(this.lifecycle, this.configFile); + const providers = new ProviderCollection(this.log, [ ...coreProviders, ...readProviderSpec('Service', config.get('services')), diff --git a/packages/kbn-test/src/functional_test_runner/lib/index.ts b/packages/kbn-test/src/functional_test_runner/lib/index.ts index 8940eccad503a..2e534974e1d76 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/index.ts +++ b/packages/kbn-test/src/functional_test_runner/lib/index.ts @@ -23,3 +23,4 @@ export { readConfigFile, Config } from './config'; export { readProviderSpec, ProviderCollection, Provider } from './providers'; export { runTests, setupMocha } from './mocha'; export { FailureMetadata } from './failure_metadata'; +export { SuiteTracker } from './suite_tracker'; diff --git a/packages/kbn-test/src/functional_test_runner/lib/suite_tracker.test.ts b/packages/kbn-test/src/functional_test_runner/lib/suite_tracker.test.ts new file mode 100644 index 0000000000000..b6c2c0a6d511d --- /dev/null +++ b/packages/kbn-test/src/functional_test_runner/lib/suite_tracker.test.ts @@ -0,0 +1,197 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import fs from 'fs'; +import { join, resolve } from 'path'; + +jest.mock('fs'); +jest.mock('@kbn/dev-utils', () => { + return { REPO_ROOT: '/dev/null/root' }; +}); + +import { REPO_ROOT } from '@kbn/dev-utils'; +import { Lifecycle } from './lifecycle'; +import { SuiteTracker } from './suite_tracker'; + +const DEFAULT_TEST_METADATA_PATH = join(REPO_ROOT, 'target', 'test_metadata.json'); +const MOCK_CONFIG_PATH = join('test', 'config.js'); +const MOCK_TEST_PATH = join('test', 'apps', 'test.js'); +const ENVS_TO_RESET = ['TEST_METADATA_PATH']; + +describe('SuiteTracker', () => { + const originalEnvs: Record = {}; + + beforeEach(() => { + for (const env of ENVS_TO_RESET) { + if (env in process.env) { + originalEnvs[env] = process.env[env] || ''; + delete process.env[env]; + } + } + }); + + afterEach(() => { + for (const env of ENVS_TO_RESET) { + delete process.env[env]; + } + + for (const env of Object.keys(originalEnvs)) { + process.env[env] = originalEnvs[env]; + } + + jest.resetAllMocks(); + }); + + let MOCKS: Record; + + const createMock = (overrides = {}) => { + return { + file: resolve(REPO_ROOT, MOCK_TEST_PATH), + title: 'A Test', + suiteTag: MOCK_TEST_PATH, + ...overrides, + }; + }; + + const runLifecycleWithMocks = async (mocks: object[], fn: (objs: any) => any = () => {}) => { + const lifecycle = new Lifecycle(); + const suiteTracker = SuiteTracker.startTracking( + lifecycle, + resolve(REPO_ROOT, MOCK_CONFIG_PATH) + ); + + const ret = { lifecycle, suiteTracker }; + + for (const mock of mocks) { + await lifecycle.beforeTestSuite.trigger(mock); + } + + if (fn) { + fn(ret); + } + + for (const mock of mocks.reverse()) { + await lifecycle.afterTestSuite.trigger(mock); + } + + return ret; + }; + + beforeEach(() => { + MOCKS = { + WITH_TESTS: createMock({ tests: [{}] }), // i.e. a describe with tests in it + WITHOUT_TESTS: createMock(), // i.e. a describe with only other describes in it + }; + }); + + it('collects metadata for a single suite with multiple describe()s', async () => { + const { suiteTracker } = await runLifecycleWithMocks([MOCKS.WITHOUT_TESTS, MOCKS.WITH_TESTS]); + + const suites = suiteTracker.getAllFinishedSuites(); + expect(suites.length).toBe(1); + const suite = suites[0]; + + expect(suite).toMatchObject({ + config: MOCK_CONFIG_PATH, + file: MOCK_TEST_PATH, + tag: MOCK_TEST_PATH, + hasTests: true, + success: true, + }); + }); + + it('writes metadata to a file when cleanup is triggered', async () => { + const { lifecycle, suiteTracker } = await runLifecycleWithMocks([MOCKS.WITH_TESTS]); + await lifecycle.cleanup.trigger(); + + const suites = suiteTracker.getAllFinishedSuites(); + + const call = (fs.writeFileSync as jest.Mock).mock.calls[0]; + expect(call[0]).toEqual(DEFAULT_TEST_METADATA_PATH); + expect(call[1]).toEqual(JSON.stringify(suites, null, 2)); + }); + + it('respects TEST_METADATA_PATH env var for metadata target override', async () => { + process.env.TEST_METADATA_PATH = resolve(REPO_ROOT, '../fake-test-path'); + const { lifecycle } = await runLifecycleWithMocks([MOCKS.WITH_TESTS]); + await lifecycle.cleanup.trigger(); + + expect((fs.writeFileSync as jest.Mock).mock.calls[0][0]).toEqual( + process.env.TEST_METADATA_PATH + ); + }); + + it('identifies suites with tests as leaf suites', async () => { + const root = createMock({ title: 'root', file: join(REPO_ROOT, 'root.js') }); + const parent = createMock({ parent: root }); + const withTests = createMock({ parent, tests: [{}] }); + + const { suiteTracker } = await runLifecycleWithMocks([root, parent, withTests]); + const suites = suiteTracker.getAllFinishedSuites(); + + const finishedRoot = suites.find(s => s.title === 'root'); + const finishedWithTests = suites.find(s => s.title !== 'root'); + + expect(finishedRoot).toBeTruthy(); + expect(finishedRoot?.hasTests).toBeFalsy(); + expect(finishedWithTests?.hasTests).toBe(true); + }); + + describe('with a failing suite', () => { + let root: any; + let parent: any; + let failed: any; + + beforeEach(() => { + root = createMock({ file: join(REPO_ROOT, 'root.js') }); + parent = createMock({ parent: root }); + failed = createMock({ parent, tests: [{}] }); + }); + + it('marks parent suites as not successful when a test fails', async () => { + const { suiteTracker } = await runLifecycleWithMocks( + [root, parent, failed], + async ({ lifecycle }) => { + await lifecycle.testFailure.trigger(Error('test'), { parent: failed }); + } + ); + + const suites = suiteTracker.getAllFinishedSuites(); + expect(suites.length).toBe(2); + for (const suite of suites) { + expect(suite.success).toBeFalsy(); + } + }); + + it('marks parent suites as not successful when a test hook fails', async () => { + const { suiteTracker } = await runLifecycleWithMocks( + [root, parent, failed], + async ({ lifecycle }) => { + await lifecycle.testHookFailure.trigger(Error('test'), { parent: failed }); + } + ); + + const suites = suiteTracker.getAllFinishedSuites(); + expect(suites.length).toBe(2); + for (const suite of suites) { + expect(suite.success).toBeFalsy(); + } + }); + }); +}); diff --git a/packages/kbn-test/src/functional_test_runner/lib/suite_tracker.ts b/packages/kbn-test/src/functional_test_runner/lib/suite_tracker.ts new file mode 100644 index 0000000000000..8967251ea78de --- /dev/null +++ b/packages/kbn-test/src/functional_test_runner/lib/suite_tracker.ts @@ -0,0 +1,147 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import fs from 'fs'; +import { dirname, relative, resolve } from 'path'; + +import { REPO_ROOT } from '@kbn/dev-utils'; + +import { Lifecycle } from './lifecycle'; + +export interface SuiteInProgress { + startTime?: Date; + endTime?: Date; + success?: boolean; +} + +export interface SuiteWithMetadata { + config: string; + file: string; + tag: string; + title: string; + startTime: Date; + endTime: Date; + duration: number; + success: boolean; + hasTests: boolean; +} + +const getTestMetadataPath = () => { + return process.env.TEST_METADATA_PATH || resolve(REPO_ROOT, 'target', 'test_metadata.json'); +}; + +export class SuiteTracker { + finishedSuitesByConfig: Record> = {}; + inProgressSuites: Map = new Map(); + + static startTracking(lifecycle: Lifecycle, configPath: string): SuiteTracker { + const suiteTracker = new SuiteTracker(lifecycle, configPath); + return suiteTracker; + } + + getTracked(suite: object): SuiteInProgress { + if (!this.inProgressSuites.has(suite)) { + this.inProgressSuites.set(suite, { success: undefined } as SuiteInProgress); + } + return this.inProgressSuites.get(suite)!; + } + + constructor(lifecycle: Lifecycle, configPathAbsolute: string) { + if (fs.existsSync(getTestMetadataPath())) { + fs.unlinkSync(getTestMetadataPath()); + } else { + fs.mkdirSync(dirname(getTestMetadataPath()), { recursive: true }); + } + + const config = relative(REPO_ROOT, configPathAbsolute); + + lifecycle.beforeTestSuite.add(suite => { + const tracked = this.getTracked(suite); + tracked.startTime = new Date(); + }); + + // If a test fails, we want to make sure all of the ancestors, all the way up to the root, get marked as failed + // This information is not available on the mocha objects without traversing all descendants of a given node + const handleFailure = (_: any, test: any) => { + let parent = test.parent; + + // Infinite loop protection, just in case + for (let i = 0; i < 500 && parent; i++) { + if (this.inProgressSuites.has(parent)) { + this.getTracked(parent).success = false; + } + parent = parent.parent; + } + }; + + lifecycle.testFailure.add(handleFailure); + lifecycle.testHookFailure.add(handleFailure); + + lifecycle.afterTestSuite.add(suite => { + const tracked = this.getTracked(suite); + tracked.endTime = new Date(); + + // The suite ended without any children failing, so we can mark it as successful + if (typeof tracked.success === 'undefined') { + tracked.success = true; + } + + let duration = tracked.endTime.getTime() - (tracked.startTime || new Date()).getTime(); + duration = Math.floor(duration / 1000); + + const file = relative(REPO_ROOT, suite.file); + + this.finishedSuitesByConfig[config] = this.finishedSuitesByConfig[config] || {}; + + // This will get called multiple times for a test file that has multiple describes in it or similar + // This is okay, because the last one that fires is always the root of the file, which is is the one we ultimately want + this.finishedSuitesByConfig[config][file] = { + ...tracked, + duration, + config, + file, + tag: suite.suiteTag, + title: suite.title, + hasTests: !!( + (suite.tests && suite.tests.length) || + // The below statement is so that `hasTests` will bubble up nested describes in the same file + (this.finishedSuitesByConfig[config][file] && + this.finishedSuitesByConfig[config][file].hasTests) + ), + } as SuiteWithMetadata; + }); + + lifecycle.cleanup.add(() => { + const suites = this.getAllFinishedSuites(); + + fs.writeFileSync(getTestMetadataPath(), JSON.stringify(suites, null, 2)); + }); + } + + getAllFinishedSuites() { + const flattened: SuiteWithMetadata[] = []; + for (const byFile of Object.values(this.finishedSuitesByConfig)) { + for (const suite of Object.values(byFile)) { + flattened.push(suite); + } + } + + flattened.sort((a, b) => b.duration - a.duration); + return flattened; + } +}