[ftr] remove testMetadata and maintain a unique lifecycle instance pe…

…r run

packages/kbn-test/src/functional_test_runner/cli.ts

@@ -98,11 +98,7 @@ export function runFtrCli() {
           });
         }
 
-        try {
-          await functionalTestRunner.close();
-        } finally {
-          process.exit();
-        }
+        process.exit();
       };
 
       process.on('unhandledRejection', (err) =>

packages/kbn-test/src/functional_test_runner/functional_test_runner.ts

@@ -14,8 +14,6 @@ import { REPO_ROOT } from '@kbn/utils';
 import { Suite, Test } from './fake_mocha_types';
 import {
   Lifecycle,
-  LifecyclePhase,
-  TestMetadata,
   readConfigFile,
   ProviderCollection,
   Providers,
@@ -30,23 +28,13 @@ import {
 import { createEsClientForFtrConfig } from '../es';
 
 export class FunctionalTestRunner {
-  public readonly lifecycle = new Lifecycle();
-  public readonly testMetadata = new TestMetadata(this.lifecycle);
-  private closed = false;
-
   private readonly esVersion: EsVersion;
   constructor(
     private readonly log: ToolingLog,
     private readonly configFile: string,
     private readonly configOverrides: any,
     esVersion?: string | EsVersion
   ) {
-    for (const [key, value] of Object.entries(this.lifecycle)) {
-      if (value instanceof LifecyclePhase) {
-        value.before$.subscribe(() => log.verbose('starting %j lifecycle phase', key));
-        value.after$.subscribe(() => log.verbose('starting %j lifecycle phase', key));
-      }
-    }
     this.esVersion =
       esVersion === undefined
         ? EsVersion.getDefault()
@@ -58,8 +46,8 @@ export class FunctionalTestRunner {
   async run() {
     const testStats = await this.getTestStats();
 
-    return await this._run(async (config, coreProviders) => {
-      SuiteTracker.startTracking(this.lifecycle, this.configFile);
+    return await this.runHarness(async (config, lifecycle, coreProviders) => {
+      SuiteTracker.startTracking(lifecycle, this.configFile);
 
       const realServices =
         !!config.get('testRunner') ||
@@ -101,7 +89,7 @@ export class FunctionalTestRunner {
       }
 
       const mocha = await setupMocha(
-        this.lifecycle,
+        lifecycle,
         this.log,
         config,
         providers,
@@ -119,10 +107,10 @@ export class FunctionalTestRunner {
         return this.simulateMochaDryRun(mocha);
       }
 
-      await this.lifecycle.beforeTests.trigger(mocha.suite);
+      await lifecycle.beforeTests.trigger(mocha.suite);
       this.log.info('Starting tests');
 
-      return await runTests(this.lifecycle, mocha);
+      return await runTests(lifecycle, mocha);
     });
   }
 
@@ -154,13 +142,13 @@ export class FunctionalTestRunner {
   }
 
   async getTestStats() {
-    return await this._run(async (config, coreProviders) => {
+    return await this.runHarness(async (config, lifecycle, coreProviders) => {
       if (config.get('testRunner')) {
         throw new Error('Unable to get test stats for config that uses a custom test runner');
       }
 
       const providers = this.getStubProviderCollection(config, coreProviders);
-      const mocha = await setupMocha(this.lifecycle, this.log, config, providers, this.esVersion);
+      const mocha = await setupMocha(lifecycle, this.log, config, providers, this.esVersion);
 
       const queue = new Set([mocha.suite]);
       const allTests: Test[] = [];
@@ -216,10 +204,11 @@ export class FunctionalTestRunner {
     ]);
   }
 
-  async _run<T = any>(
-    handler: (config: Config, coreProviders: Providers) => Promise<T>
+  private async runHarness<T = any>(
+    handler: (config: Config, lifecycle: Lifecycle, coreProviders: Providers) => Promise<T>
   ): Promise<T> {
     let runErrorOccurred = false;
+    const lifecycle = new Lifecycle(this.log);
 
     try {
       const config = await readConfigFile(
@@ -240,26 +229,25 @@ export class FunctionalTestRunner {
       const dockerServers = new DockerServersService(
         config.get('dockerServers'),
         this.log,
-        this.lifecycle
+        lifecycle
       );
 
       // base level services that functional_test_runner exposes
       const coreProviders = readProviderSpec('Service', {
-        lifecycle: () => this.lifecycle,
+        lifecycle: () => lifecycle,
         log: () => this.log,
-        testMetadata: () => this.testMetadata,
         config: () => config,
         dockerServers: () => dockerServers,
         esVersion: () => this.esVersion,
       });
 
-      return await handler(config, coreProviders);
+      return await handler(config, lifecycle, coreProviders);
     } catch (runError) {
       runErrorOccurred = true;
       throw runError;
     } finally {
       try {
-        await this.close();
+        await lifecycle.cleanup.trigger();
       } catch (closeError) {
         if (runErrorOccurred) {
           this.log.error('failed to close functional_test_runner');
@@ -272,13 +260,6 @@ export class FunctionalTestRunner {
     }
   }
 
-  async close() {
-    if (this.closed) return;
-
-    this.closed = true;
-    await this.lifecycle.cleanup.trigger();
-  }
-
   simulateMochaDryRun(mocha: any) {
     interface TestEntry {
       file: string;

packages/kbn-test/src/functional_test_runner/index.ts

@@ -15,7 +15,6 @@ export {
   Lifecycle,
   LifecyclePhase,
 } from './lib';
-export type { ScreenshotRecord } from './lib';
 export { runFtrCli } from './cli';
 export * from './lib/docker_servers';
 export * from './public_types';

packages/kbn-test/src/functional_test_runner/lib/index.ts

@@ -12,7 +12,6 @@ export { readConfigFile, Config } from './config';
 export * from './providers';
 // @internal
 export { runTests, setupMocha } from './mocha';
-export * from './test_metadata';
 export * from './docker_servers';
 export { SuiteTracker } from './suite_tracker';
 

packages/kbn-test/src/functional_test_runner/lib/lifecycle.ts

@@ -6,29 +6,48 @@
  * Side Public License, v 1.
  */
 
+import * as Rx from 'rxjs';
+import { ToolingLog } from '@kbn/tooling-log';
+
 import { LifecyclePhase } from './lifecycle_phase';
 
 import { Suite, Test } from '../fake_mocha_types';
 
 export class Lifecycle {
+  /** root subscription to cleanup lifecycle phases when lifecycle completes */
+  private readonly sub = new Rx.Subscription();
+
   /** lifecycle phase that will run handlers once before tests execute */
-  public readonly beforeTests = new LifecyclePhase<[Suite]>({
+  public readonly beforeTests = new LifecyclePhase<[Suite]>(this.sub, {
     singular: true,
   });
   /** lifecycle phase that runs handlers before each runnable (test and hooks) */
-  public readonly beforeEachRunnable = new LifecyclePhase<[Test]>();
+  public readonly beforeEachRunnable = new LifecyclePhase<[Test]>(this.sub);
   /** lifecycle phase that runs handlers before each suite */
-  public readonly beforeTestSuite = new LifecyclePhase<[Suite]>();
+  public readonly beforeTestSuite = new LifecyclePhase<[Suite]>(this.sub);
   /** lifecycle phase that runs handlers before each test */
-  public readonly beforeEachTest = new LifecyclePhase<[Test]>();
+  public readonly beforeEachTest = new LifecyclePhase<[Test]>(this.sub);
   /** lifecycle phase that runs handlers after each suite */
-  public readonly afterTestSuite = new LifecyclePhase<[Suite]>();
+  public readonly afterTestSuite = new LifecyclePhase<[Suite]>(this.sub);
   /** lifecycle phase that runs handlers after a test fails */
-  public readonly testFailure = new LifecyclePhase<[Error, Test]>();
+  public readonly testFailure = new LifecyclePhase<[Error, Test]>(this.sub);
   /** lifecycle phase that runs handlers after a hook fails */
-  public readonly testHookFailure = new LifecyclePhase<[Error, Test]>();
+  public readonly testHookFailure = new LifecyclePhase<[Error, Test]>(this.sub);
   /** lifecycle phase that runs handlers at the very end of execution */
-  public readonly cleanup = new LifecyclePhase<[]>({
+  public readonly cleanup = new LifecyclePhase<[]>(this.sub, {
     singular: true,
   });
+
+  constructor(log: ToolingLog) {
+    for (const [name, phase] of Object.entries(this)) {
+      if (phase instanceof LifecyclePhase) {
+        phase.before$.subscribe(() => log.verbose('starting %j lifecycle phase', name));
+        phase.after$.subscribe(() => log.verbose('starting %j lifecycle phase', name));
+      }
+    }
+
+    this.cleanup.after$.subscribe(() => {
+      this.sub.unsubscribe();
+    });
+  }
 }

packages/kbn-test/src/functional_test_runner/lib/lifecycle_phase.test.ts

@@ -26,7 +26,7 @@ describe('with randomness', () => {
   });
 
   it('calls handlers in random order', async () => {
-    const phase = new LifecyclePhase();
+    const phase = new LifecyclePhase(new Rx.Subscription());
     const order: string[] = [];
 
     phase.add(
@@ -69,7 +69,7 @@ describe('without randomness', () => {
   afterEach(() => jest.restoreAllMocks());
 
   it('calls all handlers and throws first error', async () => {
-    const phase = new LifecyclePhase();
+    const phase = new LifecyclePhase(new Rx.Subscription());
     const fn1 = jest.fn();
     phase.add(fn1);
 
@@ -88,7 +88,7 @@ describe('without randomness', () => {
   });
 
   it('triggers before$ just before calling handler and after$ once it resolves', async () => {
-    const phase = new LifecyclePhase();
+    const phase = new LifecyclePhase(new Rx.Subscription());
     const order: string[] = [];
 
     const beforeSub = jest.fn(() => order.push('before'));
@@ -116,7 +116,7 @@ describe('without randomness', () => {
   });
 
   it('completes before$ and after$ if phase is singular', async () => {
-    const phase = new LifecyclePhase({ singular: true });
+    const phase = new LifecyclePhase(new Rx.Subscription(), { singular: true });
 
     const beforeNotifs: Array<Rx.Notification<unknown>> = [];
     phase.before$.pipe(materialize()).subscribe((n) => beforeNotifs.push(n));
@@ -160,7 +160,7 @@ describe('without randomness', () => {
   });
 
   it('completes before$ subscribers after trigger of singular phase', async () => {
-    const phase = new LifecyclePhase({ singular: true });
+    const phase = new LifecyclePhase(new Rx.Subscription(), { singular: true });
     await phase.trigger();
 
     await expect(phase.before$.pipe(materialize(), toArray()).toPromise()).resolves
@@ -177,7 +177,7 @@ describe('without randomness', () => {
   });
 
   it('replays after$ event subscribers after trigger of singular phase', async () => {
-    const phase = new LifecyclePhase({ singular: true });
+    const phase = new LifecyclePhase(new Rx.Subscription(), { singular: true });
     await phase.trigger();
 
     await expect(phase.after$.pipe(materialize(), toArray()).toPromise()).resolves

packages/kbn-test/src/functional_test_runner/lib/lifecycle_phase.ts

@@ -26,6 +26,7 @@ export class LifecyclePhase<Args extends readonly any[]> {
   public readonly after$: Rx.Observable<void>;
 
   constructor(
+    sub: Rx.Subscription,
     private readonly options: {
       singular?: boolean;
     } = {}
@@ -35,6 +36,12 @@ export class LifecyclePhase<Args extends readonly any[]> {
 
     this.afterSubj = this.options.singular ? new Rx.ReplaySubject<void>(1) : new Rx.Subject<void>();
     this.after$ = this.afterSubj.asObservable();
+
+    sub.add(() => {
+      this.beforeSubj.complete();
+      this.afterSubj.complete();
+      this.handlers.length = 0;
+    });
   }
 
   public add(fn: (...args: Args) => Promise<void> | void) {

packages/kbn-test/src/functional_test_runner/lib/mocha/filter_suites.test.js

@@ -69,6 +69,9 @@ function setup({ include, exclude, esVersion }) {
         info(...args) {
           history.push(`info: ${format(...args)}`);
         },
+        debug(...args) {
+          history.push(`debg: ${format(...args)}`);
+        },
       },
       mocha,
       include,
@@ -221,7 +224,7 @@ it(`excludes tests which don't meet the esVersionRequirement`, async () => {
 
   expect(history).toMatchInlineSnapshot(`
     Array [
-      "info: Only running suites which are compatible with ES version 9.0.0",
+      "debg: Only running suites which are compatible with ES version 9.0.0",
       "suite: ",
       "suite: level 1",
       "suite: level 1 level 1a",

packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/ci_stats_ftr_reporter.ts

@@ -17,7 +17,6 @@ import {
 
 import { Config } from '../../config';
 import { Runner } from '../../../fake_mocha_types';
-import { TestMetadata, ScreenshotRecord } from '../../test_metadata';
 import { Lifecycle } from '../../lifecycle';
 import { getSnapshotOfRunnableLogs } from '../../../../mocha';
 
@@ -36,7 +35,6 @@ interface Runnable {
   file: string;
   title: string;
   parent: Suite;
-  _screenshots?: ScreenshotRecord[];
 }
 
 function getHookType(hook: Runnable): CiStatsTestType {
@@ -60,13 +58,11 @@ export function setupCiStatsFtrTestGroupReporter({
   config,
   lifecycle,
   runner,
-  testMetadata,
   reporter,
 }: {
   config: Config;
   lifecycle: Lifecycle;
   runner: Runner;
-  testMetadata: TestMetadata;
   reporter: CiStatsReporter;
 }) {
   const testGroupType = process.env.TEST_GROUP_TYPE_FUNCTIONAL;
@@ -111,10 +107,6 @@ export function setupCiStatsFtrTestGroupReporter({
       type,
       error: error?.stack,
       stdout: getSnapshotOfRunnableLogs(runnable),
-      screenshots: testMetadata.getScreenshots(runnable).map((s) => ({
-        base64Png: s.base64Png,
-        name: s.name,
-      })),
     });
   }
 

packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/reporter.js

@@ -24,7 +24,6 @@ export function MochaReporterProvider({ getService }) {
   const log = getService('log');
   const config = getService('config');
   const lifecycle = getService('lifecycle');
-  const testMetadata = getService('testMetadata');
   let originalLogWriters;
   let reporterCaptureStartTime;
 
@@ -61,7 +60,6 @@ export function MochaReporterProvider({ getService }) {
             config,
             lifecycle,
             runner,
-            testMetadata,
           });
         }
       }