diff --git a/packages/allure-cypress/src/index.ts b/packages/allure-cypress/src/index.ts index 93c682e8d..32b860e57 100644 --- a/packages/allure-cypress/src/index.ts +++ b/packages/allure-cypress/src/index.ts @@ -1,9 +1,10 @@ import { getMessageAndTraceFromError, getStatusFromError } from "allure-js-commons/sdk"; -import type { CypressCommand, CypressHook, CypressTest } from "./model.js"; import { ALLURE_REPORT_SYSTEM_HOOK } from "./model.js"; +import type { CypressCommand, CypressHook, CypressSuite, CypressTest } from "./model.js"; import { + completeHookErrorReporting, + completeSpecIfNoAfterHookLeft, enableScopeLevelAfterHookReporting, - flushFinalRuntimeMessages, flushRuntimeMessages, initTestRuntime, reportCommandEnd, @@ -22,7 +23,7 @@ import { reportUnfinishedSteps, } from "./runtime.js"; import { isAllureInitialized, setAllureInitialized } from "./state.js"; -import { applyTestPlan, isTestReported, shouldCommandBeSkipped } from "./utils.js"; +import { applyTestPlan, isAllureHook, isRootAfterAllHook, isTestReported, shouldCommandBeSkipped } from "./utils.js"; const { EVENT_RUN_BEGIN, @@ -35,7 +36,6 @@ const { EVENT_TEST_FAIL, EVENT_TEST_PENDING, EVENT_TEST_END, - EVENT_RUN_END, } = Mocha.Runner.constants; const initializeAllure = () => { @@ -52,24 +52,24 @@ const initializeAllure = () => { initTestRuntime(); reportRunStart(); }) - .on(EVENT_SUITE_BEGIN, (suite: Mocha.Suite) => { - if (!suite.parent) { + .on(EVENT_SUITE_BEGIN, (suite: CypressSuite) => { + if (suite.root) { applyTestPlan(Cypress.spec, suite); } reportSuiteStart(suite); }) - .on(EVENT_SUITE_END, (suite: Mocha.Suite) => { + .on(EVENT_SUITE_END, (suite: CypressSuite) => { reportSuiteEnd(suite); }) .on(EVENT_HOOK_BEGIN, (hook: CypressHook) => { - if (hook.title.includes(ALLURE_REPORT_SYSTEM_HOOK)) { + if (isAllureHook(hook)) { return; } reportHookStart(hook); }) .on(EVENT_HOOK_END, (hook: CypressHook) => { - if (hook.title.includes(ALLURE_REPORT_SYSTEM_HOOK)) { + if (isAllureHook(hook)) { return; } @@ -85,22 +85,35 @@ const initializeAllure = () => { .on(EVENT_TEST_PASS, () => { reportTestPass(); }) - .on(EVENT_TEST_FAIL, (_: CypressTest, err: Error) => { + .on(EVENT_TEST_FAIL, (testOrHook: CypressTest | CypressHook, err: Error) => { + const isHook = "hookName" in testOrHook; + if (isHook && isRootAfterAllHook(testOrHook)) { + // Errors in spec-level 'after all' hooks are handled by Allure wrappers. + return; + } + + const isAllureHookFailure = isHook && isAllureHook(testOrHook); + + if (isAllureHookFailure) { + // Normally, Allure hooks are skipped from the report. + // In case of errors, it will be helpful to see them. + reportHookStart(testOrHook, Date.now() - (testOrHook.duration ?? 0)); + } + + // This will mark the fixture and the test (if any) as failed/broken. reportTestOrHookFail(err); + + if (isHook) { + // This will end the fixture and test (if any) and will report the remaining + // tests in the hook's suite (the ones that will be skipped by Cypress/Mocha). + completeHookErrorReporting(testOrHook, err); + } }) .on(EVENT_TEST_PENDING, (test: CypressTest) => { reportTestSkip(test); }) .on(EVENT_TEST_END, (test: CypressTest) => { reportTestEnd(test); - }) - .on(EVENT_RUN_END, () => { - // after:spec isn't called in interactive mode by default. - // We're using the 'end' event instead to report the remaining messages - // (the root 'suite end', mainly). - if (Cypress.config("isInteractive")) { - flushFinalRuntimeMessages(); - } }); Cypress.Screenshot.defaults({ @@ -132,11 +145,11 @@ const initializeAllure = () => { reportCommandEnd(); }); - afterEach(ALLURE_REPORT_SYSTEM_HOOK, () => { - flushRuntimeMessages(); - }); - after(ALLURE_REPORT_SYSTEM_HOOK, () => { + afterEach(ALLURE_REPORT_SYSTEM_HOOK, flushRuntimeMessages); + + after(ALLURE_REPORT_SYSTEM_HOOK, function (this: Mocha.Context) { flushRuntimeMessages(); + completeSpecIfNoAfterHookLeft(this); }); enableScopeLevelAfterHookReporting(); diff --git a/packages/allure-cypress/src/model.ts b/packages/allure-cypress/src/model.ts index 67eac033e..d8152ebe6 100644 --- a/packages/allure-cypress/src/model.ts +++ b/packages/allure-cypress/src/model.ts @@ -10,18 +10,22 @@ export type AllureCypressConfig = ReporterConfig & { videoOnFailOnly?: boolean; }; +export type CypressSuite = Mocha.Suite & { + id: string; + parent: CypressSuite | undefined; + tests: CypressTest[]; + suites: CypressSuite[]; +}; + export type CypressTest = Mocha.Test & { wallClockStartedAt?: Date; - hookName?: string; - id: string; + parent: CypressSuite | undefined; }; export type CypressHook = Mocha.Hook & { - id: string; hookId: string; - parent: Mocha.Suite & { - id: string; - }; + hookName: string; + parent: CypressSuite | undefined; }; export type CypressCommand = { @@ -41,6 +45,7 @@ export type CupressRunStart = { export type CypressSuiteStartMessage = { type: "cypress_suite_start"; data: { + id: string; name: string; root: boolean; start: number; @@ -59,6 +64,8 @@ export type CypressHookStartMessage = { type: "cypress_hook_start"; data: { name: string; + scopeType: "each" | "all"; + position: "before" | "after"; start: number; }; }; @@ -90,7 +97,9 @@ export type CypressFailMessage = { export type CypressTestSkipMessage = { type: "cypress_test_skip"; - data: object; + data: { + statusDetails?: StatusDetails; + }; }; export type CypressTestPassMessage = { @@ -98,6 +107,15 @@ export type CypressTestPassMessage = { data: object; }; +export type CypressSkippedTestMessage = { + type: "cypress_skipped_test"; + data: CypressTestStartMessage["data"] & + CypressFailMessage["data"] & + CypressTestEndMessage["data"] & { + suites: string[]; + }; +}; + export type CypressTestEndMessage = { type: "cypress_test_end"; data: { @@ -137,6 +155,7 @@ export type CypressMessage = | CypressTestPassMessage | CypressFailMessage | CypressTestSkipMessage + | CypressSkippedTestMessage | CypressTestEndMessage; export type SpecContext = { @@ -146,6 +165,8 @@ export type SpecContext = { fixture: string | undefined; commandSteps: string[]; videoScope: string; + suiteIdToScope: Map; + suiteScopeToId: Map; suiteScopes: string[]; testScope: string | undefined; suiteNames: string[]; @@ -156,17 +177,13 @@ export type AllureSpecState = { initialized: boolean; testPlan: TestPlanV1 | null | undefined; messages: CypressMessage[]; + currentTest?: CypressTest; }; -export type HookPosition = "before" | "after"; - -export type HookScopeType = "all" | "each"; - -export type HookType = [position: HookPosition, scopeType: HookScopeType]; - export type AllureCypressTaskArgs = { absolutePath: string; messages: readonly CypressMessage[]; + isInteractive: boolean; }; export type CypressSuiteFunction = ( @@ -174,3 +191,6 @@ export type CypressSuiteFunction = ( configOrFn?: Cypress.SuiteConfigOverrides | ((this: Mocha.Suite) => void), fn?: (this: Mocha.Suite) => void, ) => Mocha.Suite; + +export type DirectHookImplementation = Mocha.AsyncFunc | ((this: Mocha.Context) => void); +export type HookImplementation = Mocha.Func | DirectHookImplementation; diff --git a/packages/allure-cypress/src/reporter.ts b/packages/allure-cypress/src/reporter.ts index b1328160f..702114514 100644 --- a/packages/allure-cypress/src/reporter.ts +++ b/packages/allure-cypress/src/reporter.ts @@ -1,6 +1,7 @@ import type Cypress from "cypress"; import path from "node:path"; import { ContentType, Stage, Status } from "allure-js-commons"; +import type { TestResult } from "allure-js-commons"; import type { RuntimeMessage } from "allure-js-commons/sdk"; import { ReporterRuntime, @@ -23,13 +24,15 @@ import type { CypressFailMessage, CypressHookEndMessage, CypressHookStartMessage, + CypressSkippedTestMessage, CypressSuiteEndMessage, CypressSuiteStartMessage, CypressTestEndMessage, + CypressTestSkipMessage, CypressTestStartMessage, SpecContext, } from "./model.js"; -import { getHookType, last } from "./utils.js"; +import { last } from "./utils.js"; export class AllureCypress { allureRuntime: ReporterRuntime; @@ -54,7 +57,11 @@ export class AllureCypress { }, reportFinalAllureCypressSpecMessages: (args: AllureCypressTaskArgs) => { this.#applyAllureCypressMessages(args); - this.endSpec(args.absolutePath); + if (args.isInteractive) { + // In non-interactive mode the spec is ended via the 'after:spec' event instead + // to get the spec's video. + this.endSpec(args.absolutePath); + } return null; }, }); @@ -73,8 +80,9 @@ export class AllureCypress { * you need to define your own handler or combine Allure Cypress with other * plugins. More info [here](https://github.com/allure-framework/allure-js/blob/main/packages/allure-cypress/README.md#setupnodeevents-limitations). * @param spec The first argument of the `after:spec` event. - * @param results The second argument of the `after:spec` event. + * @param results The second argument of the `after:spec` event. It's `undefined` in interactive mode. * @example + * ```javascript * import { defineConfig } from "cypress"; * import { allureCypress } from "allure-cypress/reporter"; * @@ -88,17 +96,19 @@ export class AllureCypress { * } * // ... * }); + * ``` */ - onAfterSpec = (spec: Cypress.Spec, results: CypressCommandLine.RunResult) => { - this.endSpec(spec.absolute, results.video ?? undefined); + onAfterSpec = (spec: Cypress.Spec, results: CypressCommandLine.RunResult | undefined) => { + this.endSpec(spec.absolute, results?.video ?? undefined); }; /** * Forward the `after:run` event into Allure Cypress using this function if * you need to define your own handler or combine Allure Cypress with other * plugins. More info [here](https://github.com/allure-framework/allure-js/blob/main/packages/allure-cypress/README.md#setupnodeevents-limitations). - * @param results The argument of the `after:run` event. + * @param results The argument of the `after:run` event. It's `undefined` in interactive mode. * @example + * ```javascript * import { defineConfig } from "cypress"; * import { allureCypress } from "allure-cypress/reporter"; * @@ -112,9 +122,12 @@ export class AllureCypress { * } * // ... * }); + * ``` */ - // eslint-disable-next-line @typescript-eslint/no-unused-vars - onAfterRun = (results: CypressCommandLine.CypressFailedRunResult | CypressCommandLine.CypressRunResult) => { + onAfterRun = ( + // eslint-disable-next-line @typescript-eslint/no-unused-vars + results: CypressCommandLine.CypressFailedRunResult | CypressCommandLine.CypressRunResult | undefined, + ) => { this.endRun(); }; @@ -171,10 +184,13 @@ export class AllureCypress { this.#passTest(context); break; case "cypress_fail": - this.#failTestOrHook(context, message); + this.#failHookAndTest(context, message); break; case "cypress_test_skip": - this.#skipTest(context); + this.#skipTest(context, message); + break; + case "cypress_skipped_test": + this.#addSkippedTest(context, message); break; case "cypress_test_end": this.#stopTest(context, message); @@ -201,43 +217,56 @@ export class AllureCypress { this.#initializeSpecContext(absolutePath); }; - #startSuite = (context: SpecContext, { data: { name, root } }: CypressSuiteStartMessage) => { - const scope = this.allureRuntime.startScope(); - context.suiteScopes.push(scope); + #startSuite = (context: SpecContext, { data: { id, name, root } }: CypressSuiteStartMessage) => { + this.#pushNewSuiteScope(context, id); if (!root) { this.#emitPreviousTestScope(context); context.suiteNames.push(name); } }; + #pushNewSuiteScope = (context: SpecContext, suiteId: string) => { + const scope = this.allureRuntime.startScope(); + context.suiteScopes.push(scope); + context.suiteIdToScope.set(suiteId, scope); + context.suiteScopeToId.set(scope, suiteId); + return scope; + }; + #stopSuite = (context: SpecContext, { data: { root } }: CypressSuiteEndMessage) => { this.#emitPreviousTestScope(context); if (!root) { context.suiteNames.pop(); } + this.#writeLastSuiteScope(context); + }; + + #writeLastSuiteScope = (context: SpecContext) => { const scope = context.suiteScopes.pop(); if (scope) { + const suiteId = context.suiteScopeToId.get(scope); + if (suiteId) { + context.suiteScopeToId.delete(scope); + context.suiteIdToScope.delete(suiteId); + } this.allureRuntime.writeScope(scope); } }; - #startHook = (context: SpecContext, { data: { name, start } }: CypressHookStartMessage) => { - const [hookPosition, hookScopeType] = getHookType(name); - if (hookPosition) { - const isEach = hookScopeType === "each"; - const isAfterEach = hookPosition === "after" && isEach; - if (!isAfterEach) { - this.#emitPreviousTestScope(context); - } + #startHook = (context: SpecContext, { data: { name, scopeType, position, start } }: CypressHookStartMessage) => { + const isEach = scopeType === "each"; + const isAfterEach = position === "after" && isEach; + if (!isAfterEach) { + this.#emitPreviousTestScope(context); + } - const scope = isEach ? context.testScope : last(context.suiteScopes); - if (scope) { - context.fixture = this.allureRuntime.startFixture(scope, hookPosition, { - name, - start, - status: undefined, - }); - } + const scope = isEach ? context.testScope : last(context.suiteScopes); + if (scope) { + context.fixture = this.allureRuntime.startFixture(scope, position, { + name, + start, + status: undefined, + }); } }; @@ -252,18 +281,20 @@ export class AllureCypress { } }; - #startTest = ( - context: SpecContext, - { data: { name, fullName, start, labels: metadataLabels } }: CypressTestStartMessage, - ) => { + #startTest = (context: SpecContext, { data }: CypressTestStartMessage) => { this.#emitPreviousTestScope(context); const testScope = this.allureRuntime.startScope(); context.testScope = testScope; - context.test = this.allureRuntime.startTest( + context.test = this.#addNewTestResult(context, data, [context.videoScope, ...context.suiteScopes, testScope]); + }; + + #addNewTestResult = ( + context: SpecContext, + { labels: metadataLabels = [], ...otherTestData }: Partial, + scopes: string[], + ) => + this.allureRuntime.startTest( { - name, - start, - fullName, stage: Stage.RUNNING, labels: [ getLanguageLabel(), @@ -276,12 +307,12 @@ export class AllureCypress { getThreadLabel(), getPackageLabel(context.specPath), ], + ...otherTestData, }, - [context.videoScope, ...context.suiteScopes, testScope], + scopes, ); - }; - #failTestOrHook = (context: SpecContext, { data: { status, statusDetails } }: CypressFailMessage) => { + #failHookAndTest = (context: SpecContext, { data: { status, statusDetails } }: CypressFailMessage) => { const setError = (result: object) => Object.assign(result, { status, statusDetails }); const fixtureUuid = context.fixture; @@ -306,35 +337,54 @@ export class AllureCypress { } }; - #skipTest = (context: SpecContext) => { + #skipTest = (context: SpecContext, { data: { statusDetails } }: CypressTestSkipMessage) => { const testUuid = context.test; if (testUuid) { this.allureRuntime.updateTest(testUuid, (testResult) => { testResult.status = Status.SKIPPED; - testResult.statusDetails = { message: "The test was skipped" }; + if (statusDetails) { + testResult.statusDetails = statusDetails; + } }); } }; - #stopTest = (context: SpecContext, { data: { retries, duration } }: CypressTestEndMessage) => { + #addSkippedTest = ( + context: SpecContext, + { data: { suites, duration, retries, ...testResultData } }: CypressSkippedTestMessage, + ) => { + // Tests skipped because of a hook error may share all suites of the current context + // or just a part thereof (if it's from a sibling suite). + const scopes = suites.map((s) => context.suiteIdToScope.get(s)).filter((s): s is string => Boolean(s)); + + const testUuid = this.#addNewTestResult(context, testResultData, [context.videoScope, ...scopes]); + this.#stopExistingTestResult(testUuid, { duration, retries }); + this.allureRuntime.writeTest(testUuid); + }; + + #stopTest = (context: SpecContext, { data }: CypressTestEndMessage) => { const testUuid = context.test; if (testUuid) { - this.allureRuntime.updateTest(testUuid, (testResult) => { - if (retries > 0) { - testResult.parameters.push({ - name: "Retry", - value: retries.toString(), - excluded: true, - }); - } - testResult.stage = Stage.FINISHED; - }); - this.allureRuntime.stopTest(testUuid, { duration }); + this.#stopExistingTestResult(testUuid, data); this.allureRuntime.writeTest(testUuid); context.test = undefined; } }; + #stopExistingTestResult = (testUuid: string, { retries, duration }: CypressTestEndMessage["data"]) => { + this.allureRuntime.updateTest(testUuid, (testResult) => { + if (retries > 0) { + testResult.parameters.push({ + name: "Retry", + value: retries.toString(), + excluded: true, + }); + } + testResult.stage = Stage.FINISHED; + }); + this.allureRuntime.stopTest(testUuid, { duration }); + }; + #startCommand = (context: SpecContext, { data: { name, args } }: CypressCommandStartMessage) => { const rootUuid = this.#resolveRootUuid(context); if (rootUuid) { @@ -378,7 +428,7 @@ export class AllureCypress { * method in the following cases: * - when an `after` hook of the test starts (`after` hooks are called later than `afterEach`) * - when a `before` or `beforeEach` hook of the next test starts (in case the next test has `before`/`beforeEach` hooks) - * - when the next test start (in case the next test doesn't have `before`/`beforeEach` hooks) + * - when the next test starts (in case the next test doesn't have `before`/`beforeEach` hooks) * - when the test's suite ends (in case the test is the last one in its suite, including the root suite of the spec) * - when a nested suite starts * - when the spec ends @@ -421,13 +471,15 @@ export class AllureCypress { #initializeSpecContext = (absolutePath: string) => { const specPathElements = path.relative(process.cwd(), absolutePath).split(path.sep); - const context = { + const context: SpecContext = { specPath: specPathElements.join("/"), package: specPathElements.join("."), test: undefined, fixture: undefined, commandSteps: [], videoScope: this.allureRuntime.startScope(), + suiteIdToScope: new Map(), + suiteScopeToId: new Map(), suiteScopes: [], testScope: undefined, suiteNames: [], @@ -458,6 +510,7 @@ export const enableTestPlan = (config: Cypress.PluginConfigOptions) => { * @param cypressConfig The Cypress configuration (the second argument of `setupNodeEvents`). If provided, the selective run feature will be enabled. * @param allureConfig An Allure configuration object (optional). * @example + * ```javascript * import { defineConfig } from "cypress"; * import { allureCypress } from "allure-cypress/reporter"; * @@ -470,6 +523,7 @@ export const enableTestPlan = (config: Cypress.PluginConfigOptions) => { * // ... * } * }); + * ``` */ export const allureCypress = ( on: Cypress.PluginEvents, diff --git a/packages/allure-cypress/src/runtime.ts b/packages/allure-cypress/src/runtime.ts index a054d5e00..d4a0cae5d 100644 --- a/packages/allure-cypress/src/runtime.ts +++ b/packages/allure-cypress/src/runtime.ts @@ -1,19 +1,47 @@ import { ContentType, Status } from "allure-js-commons"; import type { AttachmentOptions, Label, Link, ParameterMode, ParameterOptions, StatusDetails } from "allure-js-commons"; -import { getMessageAndTraceFromError, getStatusFromError, getUnfinishedStepsMessages } from "allure-js-commons/sdk"; +import { + getMessageAndTraceFromError, + getStatusFromError, + getUnfinishedStepsMessages, + isPromise, +} from "allure-js-commons/sdk"; import type { RuntimeMessage } from "allure-js-commons/sdk"; import { getGlobalTestRuntime, setGlobalTestRuntime } from "allure-js-commons/sdk/runtime"; import type { TestRuntime } from "allure-js-commons/sdk/runtime"; import type { + AllureCypressTaskArgs, CypressCommand, CypressCommandEndMessage, + CypressFailMessage, + CypressHook, CypressMessage, + CypressSuite, CypressSuiteFunction, CypressTest, + DirectHookImplementation, + HookImplementation, } from "./model.js"; -import { ALLURE_REPORT_STEP_COMMAND, ALLURE_REPORT_SYSTEM_HOOK } from "./model.js"; -import { enqueueRuntimeMessage, getRuntimeMessages, setRuntimeMessages } from "./state.js"; -import { getNamesAndLabels, isTestReported, markTestAsReported, uint8ArrayToBase64 } from "./utils.js"; +import { ALLURE_REPORT_STEP_COMMAND } from "./model.js"; +import { + dropCurrentTest, + enqueueRuntimeMessage, + getCurrentTest, + getRuntimeMessages, + setCurrentTest, + setRuntimeMessages, +} from "./state.js"; +import { + getSuites, + getTestSkipData, + getTestStartData, + getTestStopData, + isAllureHook, + isTestReported, + iterateTests, + markTestAsReported, + uint8ArrayToBase64, +} from "./utils.js"; export class AllureCypressTestRuntime implements TestRuntime { constructor() { @@ -198,16 +226,23 @@ export class AllureCypressTestRuntime implements TestRuntime { }); } - flushMessages = (): PromiseLike => this.#moveMessagesToAllureCypressTask("reportAllureCypressSpecMessages"); - - flushFinalMessages = (): PromiseLike => - this.#moveMessagesToAllureCypressTask("reportFinalAllureCypressSpecMessages"); + flushAllureMessagesToTask = (taskName: string) => { + const messages = this.#dequeueAllMessages(); + if (messages.length) { + cy.task(taskName, { absolutePath: Cypress.spec.absolute, messages }, { log: false }); + } + }; - #moveMessagesToAllureCypressTask = (taskName: string) => { + flushAllureMessagesToTaskAsync = (taskName: string): Cypress.Chainable | undefined => { const messages = this.#dequeueAllMessages(); - return messages.length - ? cy.task(taskName, { absolutePath: Cypress.spec.absolute, messages }, { log: false }) - : Cypress.Promise.resolve(); + if (messages.length) { + const args: AllureCypressTaskArgs = { + absolutePath: Cypress.spec.absolute, + messages, + isInteractive: Cypress.config("isInteractive"), + }; + return cy.task(taskName, args, { log: false }); + } }; #resetMessages = () => setRuntimeMessages([]); @@ -228,10 +263,6 @@ export const initTestRuntime = () => setGlobalTestRuntime(new AllureCypressTestR export const getTestRuntime = () => getGlobalTestRuntime() as AllureCypressTestRuntime; -export const flushRuntimeMessages = () => getTestRuntime().flushMessages(); - -export const flushFinalRuntimeMessages = () => getTestRuntime().flushFinalMessages(); - export const reportRunStart = () => { enqueueRuntimeMessage({ type: "cypress_run_start", @@ -239,38 +270,41 @@ export const reportRunStart = () => { }); }; -export const reportSuiteStart = (suite: Mocha.Suite) => { +export const reportSuiteStart = (suite: CypressSuite) => { enqueueRuntimeMessage({ type: "cypress_suite_start", data: { + id: suite.id, name: suite.title, - root: !suite.parent, + root: suite.root, start: Date.now(), }, }); }; -export const reportSuiteEnd = (suite: Mocha.Suite) => { +export const reportSuiteEnd = (suite: CypressSuite) => { enqueueRuntimeMessage({ type: "cypress_suite_end", data: { - root: !suite.parent, + root: suite.root, stop: Date.now(), }, }); }; -export const reportHookStart = (hook: Mocha.Hook) => { +export const reportHookStart = (hook: CypressHook, start?: number) => { enqueueRuntimeMessage({ type: "cypress_hook_start", data: { name: hook.title, - start: Date.now(), + scopeType: hook.hookName.includes("each") ? "each" : "all", + position: hook.hookName.includes("before") ? "before" : "after", + start: start ?? Date.now(), }, }); }; -export const reportHookEnd = (hook: Mocha.Hook) => { +export const reportHookEnd = (hook: CypressHook) => { enqueueRuntimeMessage({ type: "cypress_hook_end", data: { @@ -279,17 +313,11 @@ export const reportHookEnd = (hook: Mocha.Hook) => { }); }; -export const reportTestStart = (test: CypressTest, flag?: string) => { - const x = getNamesAndLabels(Cypress.spec, test); - if (flag) { - x.labels.push({ name: "reported", value: flag }); - } +export const reportTestStart = (test: CypressTest) => { + setCurrentTest(test); enqueueRuntimeMessage({ type: "cypress_test_start", - data: { - ...x, - start: test.wallClockStartedAt?.getTime() || Date.now(), - }, + data: getTestStartData(test), }); markTestAsReported(test); }; @@ -328,7 +356,7 @@ export const reportTestSkip = (test: CypressTest) => { enqueueRuntimeMessage({ type: "cypress_test_skip", - data: {}, + data: getTestSkipData(), }); }; @@ -403,6 +431,118 @@ export const reportTestEnd = (test: CypressTest) => { retries: (test as any)._retries ?? 0, }, }); + dropCurrentTest(); +}; + +export const completeHookErrorReporting = (hook: CypressHook, err: Error) => { + const isEachHook = hook.hookName.includes("each"); + const suite = hook.parent!; + const testFailData = getStatusDataOfTestSkippedByHookError(hook.title, isEachHook, err, suite); + + // Cypress doens't emit 'hook end' if the hook has failed. + reportHookEnd(hook); + + // Cypress doens't emit 'test end' if the hook has failed. + // We must report the test's end manualy in case of a 'before each' hook. + reportCurrentTestIfAny(); + + // Cypress skips the remaining tests in the suite of a failed hook. + // We should include them to the report manually. + reportRemainingTests(suite, testFailData); +}; + +/** + * Patches the `after` function, to inject an extra `after` hook after each spec-level + * `after` hook defined by the user. + */ +export const enableScopeLevelAfterHookReporting = () => { + const [getSuiteDepth, incSuiteDepth, decSuiteDepth] = createSuiteDepthCounterState(); + patchDescribe(incSuiteDepth, decSuiteDepth); + patchAfter(getSuiteDepth); +}; + +export const flushRuntimeMessages = () => getTestRuntime().flushAllureMessagesToTask("reportAllureCypressSpecMessages"); + +export const completeSpec = () => + getTestRuntime().flushAllureMessagesToTaskAsync("reportFinalAllureCypressSpecMessages"); + +export const completeSpecIfNoAfterHookLeft = (context: Mocha.Context) => { + if (isLastRootAfterHook(context)) { + const hook = context.test as CypressHook; + if (!isAllureHook(hook)) { + reportHookEnd(hook); + } + return completeSpec(); + } +}; + +const completeSpecOnAfterHookFailure = ( + context: Mocha.Context, + hookError: Error, +): Cypress.Chainable | undefined => { + try { + reportTestOrHookFail(hookError); + completeHookErrorReporting(context.test as CypressHook, hookError); + + // cy.task's then doesn't have onrejected, that's why we don't log async Allure errors here. + return completeSpec(); + } catch (allureError) { + logAllureRootAfterError(context, allureError); + } +}; + +const reportCurrentTestIfAny = () => { + const currentTest = getCurrentTest(); + if (currentTest) { + reportTestEnd(currentTest); + } +}; + +const reportRemainingTests = (suite: CypressSuite, testFailData: CypressFailMessage["data"]) => { + for (const test of iterateTests(suite)) { + // Some tests in the suite might've been already reported. + if (!isTestReported(test)) { + reportTestsSkippedByHookError( + test, + test.pending ? { ...getTestSkipData(), status: Status.SKIPPED } : testFailData, + ); + } + } +}; + +const reportTestsSkippedByHookError = (test: CypressTest, testFailData: CypressFailMessage["data"]) => { + enqueueRuntimeMessage({ + type: "cypress_skipped_test", + data: { + ...getTestStartData(test), + ...testFailData, + ...getTestStopData(test), + suites: getSuites(test).map((s) => s.id), + }, + }); + markTestAsReported(test); +}; + +const getStatusDataOfTestSkippedByHookError = ( + hookTitle: string, + isEachHook: boolean, + err: Error, + suite: CypressSuite, +) => { + const status = isEachHook ? Status.SKIPPED : getStatusFromError(err); + const { message, trace } = getMessageAndTraceFromError(err); + return { + status, + statusDetails: { + message: isEachHook ? getSkipReason(hookTitle, suite) : message, + trace, + }, + }; +}; + +const getSkipReason = (hookTitle: string, suite: CypressSuite) => { + const suiteName = suite.title ? `'${suite.title}'` : "root"; + return `'${hookTitle}' defined in the ${suiteName} suite has failed`; }; const forwardDescribeCall = (target: CypressSuiteFunction, ...args: Parameters) => { @@ -451,26 +591,96 @@ const createSuiteDepthCounterState = (): [get: () => number, inc: () => void, de const patchAfter = (getSuiteDepth: () => number) => { const originalAfter = globalThis.after; - const patchedAfter = (nameOrFn: string | Mocha.Func | Mocha.AsyncFunc, fn?: Mocha.Func | Mocha.AsyncFunc): void => { - try { - return typeof nameOrFn === "string" ? originalAfter(nameOrFn, fn) : originalAfter(nameOrFn); - } finally { - if (getSuiteDepth() === 0) { - originalAfter(ALLURE_REPORT_SYSTEM_HOOK, () => { - flushRuntimeMessages(); - }); + const patchedAfter = (nameOrFn: string | HookImplementation, fn?: HookImplementation): void => { + return typeof nameOrFn === "string" + ? originalAfter(nameOrFn, wrapRootAfterFn(getSuiteDepth, fn)) + : originalAfter(wrapRootAfterFn(getSuiteDepth, nameOrFn)!); + }; + globalThis.after = patchedAfter; +}; + +const wrapRootAfterFn = (getSuiteDepth: () => number, fn?: HookImplementation): HookImplementation | undefined => { + if (getSuiteDepth() === 0 && fn) { + const wrappedFn = fn.length ? wrapAfterFnWithCallback(fn) : wrapAfterFnWithoutArgs(fn as DirectHookImplementation); + Object.defineProperty(wrappedFn, "name", { value: fn.name }); + return wrappedFn; + } + return fn; +}; + +const wrapAfterFnWithCallback = (fn: Mocha.Func): Mocha.Func => { + return function (this: Mocha.Context, done: Mocha.Done) { + const wrappedDone = (hookError?: Error) => { + if (hookError) { + if (!completeSpecOnAfterHookFailure(this, hookError)?.then(() => done(hookError))) { + done(hookError); + } + return; } + + try { + if (completeSpecIfNoAfterHookLeft(this)?.then(() => done())) { + return; + } + } catch (allureError) { + done(allureError); + return; + } + + done(); + }; + return fn.bind(this)(wrappedDone); + }; +}; + +const wrapAfterFnWithoutArgs = (fn: DirectHookImplementation) => { + return function (this: Mocha.Context) { + let result; + let syncError: any; + + try { + result = fn.bind(this)(); + } catch (e) { + syncError = e; + } + + if (syncError) { + throwAfterSpecCompletion(this, syncError); + } else if (isPromise(result)) { + return result.then( + () => completeSpecIfNoAfterHookLeft(this), + (asyncError) => throwAfterSpecCompletion(this, asyncError), + ); + } else { + completeSpecIfNoAfterHookLeft(this); + return result; } }; - globalThis.after = patchedAfter; }; -/** - * Patches the `after` function, to inject an extra `after` hook after each spec-level - * `after` hook defined by the user. - */ -export const enableScopeLevelAfterHookReporting = () => { - const [getSuiteDepth, incSuiteDepth, decSuiteDepth] = createSuiteDepthCounterState(); - patchDescribe(incSuiteDepth, decSuiteDepth); - patchAfter(getSuiteDepth); +const throwAfterSpecCompletion = (context: Mocha.Context, err: any) => { + const chain = completeSpecOnAfterHookFailure(context, err as Error)?.then(() => { + throw err; + }); + if (!chain) { + throw err; + } +}; + +const logAllureRootAfterError = (context: Mocha.Context, err: unknown) => { + // We play safe and swallow errors here to keep the original 'after all' error. + try { + // eslint-disable-next-line no-console + console.error(`Unexpected error when reporting the failure of ${context.test?.title ?? "'after all'"}`); + // eslint-disable-next-line no-console + console.error(err); + } catch {} +}; + +const isLastRootAfterHook = (context: Mocha.Context) => { + const currentAfterAll = context.test as CypressHook; + const rootSuite = (context.test as CypressHook).parent!; + const hooks = (rootSuite as any).hooks as CypressHook[]; + const lastAfterAll = hooks.findLast((h) => h.hookName === "after all"); + return lastAfterAll?.hookId === currentAfterAll.hookId; }; diff --git a/packages/allure-cypress/src/state.ts b/packages/allure-cypress/src/state.ts index 0b887742a..bc4cd9367 100644 --- a/packages/allure-cypress/src/state.ts +++ b/packages/allure-cypress/src/state.ts @@ -1,4 +1,4 @@ -import type { AllureSpecState, CypressMessage } from "./model.js"; +import type { AllureSpecState, CypressMessage, CypressTest } from "./model.js"; export const getAllureState = () => { let state = Cypress.env("allure") as AllureSpecState; @@ -7,6 +7,7 @@ export const getAllureState = () => { initialized: false, messages: [], testPlan: undefined, + currentTest: undefined, }; Cypress.env("allure", state); } @@ -30,3 +31,13 @@ export const enqueueRuntimeMessage = (message: CypressMessage) => { }; export const getAllureTestPlan = () => getAllureState().testPlan; + +export const getCurrentTest = () => getAllureState().currentTest; + +export const setCurrentTest = (test: CypressTest) => { + getAllureState().currentTest = test; +}; + +export const dropCurrentTest = () => { + getAllureState().currentTest = undefined; +}; diff --git a/packages/allure-cypress/src/utils.ts b/packages/allure-cypress/src/utils.ts index 181f71ef0..84d6e73bc 100644 --- a/packages/allure-cypress/src/utils.ts +++ b/packages/allure-cypress/src/utils.ts @@ -1,8 +1,8 @@ import { LabelName, Status } from "allure-js-commons"; import { extractMetadataFromString, getMessageAndTraceFromError, getStatusFromError } from "allure-js-commons/sdk"; import type { TestPlanV1 } from "allure-js-commons/sdk"; -import { ALLURE_REPORT_STEP_COMMAND } from "./model.js"; -import type { CypressCommand, HookPosition, HookScopeType, HookType } from "./model.js"; +import { ALLURE_REPORT_STEP_COMMAND, ALLURE_REPORT_SYSTEM_HOOK } from "./model.js"; +import type { CypressCommand, CypressHook, CypressSuite, CypressTest } from "./model.js"; import { getAllureTestPlan } from "./state.js"; export const uint8ArrayToBase64 = (data: unknown) => { @@ -17,21 +17,20 @@ export const uint8ArrayToBase64 = (data: unknown) => { return btoa(String.fromCharCode.apply(null, data as number[])); }; -export const getSuitePath = (test: Mocha.Test): string[] => { - const path: string[] = []; - let currentSuite: Mocha.Suite | undefined = test.parent; - - while (currentSuite) { - if (currentSuite.title) { - path.unshift(currentSuite.title); - } - - currentSuite = currentSuite.parent; +export const getSuites = (test: CypressTest) => { + const suites: CypressSuite[] = []; + for (let s: CypressSuite | undefined = test.parent; s; s = s.parent) { + suites.push(s); } - - return path; + suites.reverse(); + return suites; }; +export const getSuitePath = (test: CypressTest): string[] => + getSuites(test) + .filter((s) => s.title) + .map((s) => s.title); + export const shouldCommandBeSkipped = (command: CypressCommand) => { if (last(command.attributes.args)?.log === false) { return true; @@ -64,15 +63,11 @@ export const toReversed = (arr: T[]): T[] => { return result; }; -export const isGlobalHook = (hookName: string) => { - return /(before|after) all/.test(hookName); -}; - export const last = (arr: T[]): T | undefined => { return arr[arr.length - 1]; }; -export const getNamesAndLabels = (spec: Cypress.Spec, test: Mocha.Test) => { +export const getNamesAndLabels = (spec: Cypress.Spec, test: CypressTest) => { const rawName = test.title; const { cleanTitle: name, labels } = extractMetadataFromString(rawName); const suites = test.titlePath().slice(0, -1); @@ -80,14 +75,26 @@ export const getNamesAndLabels = (spec: Cypress.Spec, test: Mocha.Test) => { return { name, labels, fullName }; }; -export const applyTestPlan = (spec: Cypress.Spec, root: Mocha.Suite) => { +export const getTestStartData = (test: CypressTest) => ({ + ...getNamesAndLabels(Cypress.spec, test), + start: test.wallClockStartedAt?.getTime() || Date.now(), +}); + +export const getTestStopData = (test: CypressTest) => ({ + duration: test.duration ?? 0, + retries: (test as any)._retries ?? 0, +}); + +export const getTestSkipData = () => ({ + statusDetails: { message: "This is a pending test" }, +}); + +export const applyTestPlan = (spec: Cypress.Spec, root: CypressSuite) => { const testPlan = getAllureTestPlan(); if (testPlan) { - const suiteQueue = []; - for (let s: Mocha.Suite | undefined = root; s; s = suiteQueue.shift()) { - const indicesToRemove = getIndicesOfDeselectedTests(testPlan, spec, s.tests); - removeSortedIndices(s.tests, indicesToRemove); - suiteQueue.push(...s.suites); + for (const suite of iterateSuites(root)) { + const indicesToRemove = getIndicesOfDeselectedTests(testPlan, spec, suite.tests); + removeSortedIndices(suite.tests, indicesToRemove); } } }; @@ -102,26 +109,39 @@ export const resolveStatusWithDetails = (error: Error | undefined) => const testReportedKey = Symbol("The test was reported to Allure"); -export const markTestAsReported = (test: Mocha.Test) => { +export const markTestAsReported = (test: CypressTest) => { (test as any)[testReportedKey] = true; }; -export const isTestReported = (test: Mocha.Test) => (test as any)[testReportedKey] === true; +export const isTestReported = (test: CypressTest) => (test as any)[testReportedKey] === true; -const hookTypeRegexp = /^"(before|after) (all|each)"/; +export const iterateSuites = function* (parent: CypressSuite) { + const suiteStack: CypressSuite[] = []; + for (let s: CypressSuite | undefined = parent; s; s = suiteStack.pop()) { + yield s; + + // Pushing in reverse allows us to maintain depth-first pre-order traversal - + // the same order as used by Mocha & Cypress. + for (let i = s.suites.length - 1; i >= 0; i--) { + suiteStack.push(s.suites[i]); + } + } +}; -export const getHookType = (name: string): HookType | [] => { - const match = hookTypeRegexp.exec(name); - if (match) { - return [match[1] as HookPosition, match[2] as HookScopeType]; +export const iterateTests = function* (parent: CypressSuite) { + for (const suite of iterateSuites(parent)) { + yield* suite.tests; } - return []; }; +export const isAllureHook = (hook: CypressHook) => hook.title.includes(ALLURE_REPORT_SYSTEM_HOOK); + +export const isRootAfterAllHook = (hook: CypressHook) => hook.parent!.root && hook.hookName === "after all"; + const includedInTestPlan = (testPlan: TestPlanV1, fullName: string, allureId: string | undefined): boolean => testPlan.tests.some((test) => (allureId && test.id?.toString() === allureId) || test.selector === fullName); -const getIndicesOfDeselectedTests = (testPlan: TestPlanV1, spec: Cypress.Spec, tests: readonly Mocha.Test[]) => { +const getIndicesOfDeselectedTests = (testPlan: TestPlanV1, spec: Cypress.Spec, tests: readonly CypressTest[]) => { const indicesToRemove: number[] = []; tests.forEach((test, index) => { const { fullName, labels } = getNamesAndLabels(spec, test); diff --git a/packages/allure-cypress/test/spec/hooks.test.ts b/packages/allure-cypress/test/spec/hooks.test.ts index df1af9ffb..b3ff93017 100644 --- a/packages/allure-cypress/test/spec/hooks.test.ts +++ b/packages/allure-cypress/test/spec/hooks.test.ts @@ -1,3 +1,5 @@ +/* eslint max-lines: off */ +import { describe } from "node:test"; import { expect, it } from "vitest"; import { Stage, Status, issue } from "allure-js-commons"; import { runCypressInlineTest } from "../utils.js"; @@ -349,3 +351,879 @@ it("reports manually skipped tests in hooks", async () => { expect(tests[0].status).toBe(Status.SKIPPED); expect(tests[0].stage).toBe(Stage.FINISHED); }); + +describe("hook errors", () => { + it("should report a failed spec-level beforeEach", async () => { + await issue("1072"); + const { tests, groups } = await runCypressInlineTest({ + "cypress/e2e/sample.cy.js": () => ` + beforeEach(function () { + if (this.currentTest.title === "baz") { + throw new Error("Lorem Ipsum"); + } + }); + after("spec-level", () => {}); + + it("foo", () => {}); // should pass and get beforeEach and after("spec-level") + describe("suite 1", () => { + after("suite 1", () => {}); + it("bar", () => {}); // should pass and get beforeEach, after("suite 1"), and after("spec-level") + it("baz", () => {}); // should break and get beforeEach, after("suite 1"), and after("spec-level") + it("qux", () => {}); // should skip and get after("suite 1") and after("spec-level") + }); + describe("suite 2", () => { + after("suite 2", () => {}); // shouldn't be executed at all + it("quux", () => {}); // should skip and get after("spec-level") + }); + `, + }); + + expect(tests).toHaveLength(5); + expect(tests).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + name: "foo", + status: Status.PASSED, + stage: Stage.FINISHED, + }), + expect.objectContaining({ + name: "bar", + status: Status.PASSED, + stage: Stage.FINISHED, + }), + expect.objectContaining({ + name: "baz", + status: Status.BROKEN, + stage: Stage.FINISHED, + statusDetails: expect.objectContaining({ + message: "Lorem Ipsum", + }), + }), + expect.objectContaining({ + name: "qux", + status: Status.SKIPPED, + stage: Stage.FINISHED, + statusDetails: expect.objectContaining({ + message: String.raw`'"before each" hook for "baz"' defined in the root suite has failed`, + }), + }), + expect.objectContaining({ + name: "quux", + status: Status.SKIPPED, + stage: Stage.FINISHED, + statusDetails: expect.objectContaining({ + message: String.raw`'"before each" hook for "baz"' defined in the root suite has failed`, + }), + }), + ]), + ); + const fooUuid = tests.find((t) => t.name === "foo")!.uuid; + const barUuid = tests.find((t) => t.name === "bar")!.uuid; + const bazUuid = tests.find((t) => t.name === "baz")!.uuid; + const quxUuid = tests.find((t) => t.name === "qux")!.uuid; + const quuxUuid = tests.find((t) => t.name === "quux")!.uuid; + expect(groups).toHaveLength(5); + expect(groups).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + children: [fooUuid], + befores: [ + expect.objectContaining({ + name: String.raw`"before each" hook`, + status: Status.PASSED, + }), + ], + }), + expect.objectContaining({ + children: [barUuid], + befores: [ + expect.objectContaining({ + name: String.raw`"before each" hook`, + status: Status.PASSED, + }), + ], + }), + expect.objectContaining({ + children: [bazUuid], + befores: [ + expect.objectContaining({ + name: String.raw`"before each" hook`, + status: Status.BROKEN, + statusDetails: expect.objectContaining({ + message: "Lorem Ipsum", + }), + }), + ], + }), + expect.objectContaining({ + children: [barUuid, bazUuid, quxUuid], + afters: [ + expect.objectContaining({ + name: String.raw`"after all" hook: suite 1`, + status: Status.PASSED, + }), + ], + }), + expect.objectContaining({ + children: [fooUuid, barUuid, bazUuid, quxUuid, quuxUuid], + afters: [ + expect.objectContaining({ + name: String.raw`"after all" hook: spec-level`, + status: Status.PASSED, + }), + ], + }), + ]), + ); + }); + + it("should report a failed spec-level afterEach", async () => { + await issue("1072"); + const { tests, groups } = await runCypressInlineTest({ + "cypress/e2e/sample.cy.js": () => ` + afterEach(function () { + if (this.currentTest.title === "baz") { + throw new Error("Lorem Ipsum"); + } + }); + after("spec-level", () => {}); + + it("foo", () => {}); // should pass and get afterEach and after("spec-level") + describe("suite 1", () => { + after("suite 1", () => {}); + it("bar", () => {}); // should pass and get afterEach, after("suite 1"), and after("spec-level") + it("baz", () => {}); // should pass and get afterEach, after("suite 1"), and after("spec-level") + it("qux", () => {}); // should skip and get after("suite 1") and after("spec-level") + }); + describe("suite 2", () => { + after("suite 2", () => {}); // shouldn't be executed at all + it("quux", () => {}); // should skip and get after("spec-level") + }); + `, + }); + + expect(tests).toHaveLength(5); + expect(tests).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + name: "foo", + status: Status.PASSED, + stage: Stage.FINISHED, + }), + expect.objectContaining({ + name: "bar", + status: Status.PASSED, + stage: Stage.FINISHED, + }), + expect.objectContaining({ + name: "baz", + status: Status.PASSED, // after/afterEach don't affect the test's status + stage: Stage.FINISHED, + }), + expect.objectContaining({ + name: "qux", + status: Status.SKIPPED, + stage: Stage.FINISHED, + statusDetails: expect.objectContaining({ + message: String.raw`'"after each" hook for "baz"' defined in the root suite has failed`, + }), + }), + expect.objectContaining({ + name: "quux", + status: Status.SKIPPED, + stage: Stage.FINISHED, + statusDetails: expect.objectContaining({ + message: String.raw`'"after each" hook for "baz"' defined in the root suite has failed`, + }), + }), + ]), + ); + const fooUuid = tests.find((t) => t.name === "foo")!.uuid; + const barUuid = tests.find((t) => t.name === "bar")!.uuid; + const bazUuid = tests.find((t) => t.name === "baz")!.uuid; + const quxUuid = tests.find((t) => t.name === "qux")!.uuid; + const quuxUuid = tests.find((t) => t.name === "quux")!.uuid; + expect(groups).toHaveLength(5); + expect(groups).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + children: [fooUuid], + afters: [ + expect.objectContaining({ + name: String.raw`"after each" hook`, + status: Status.PASSED, + }), + ], + }), + expect.objectContaining({ + children: [barUuid], + afters: [ + expect.objectContaining({ + name: String.raw`"after each" hook`, + status: Status.PASSED, + }), + ], + }), + expect.objectContaining({ + children: [bazUuid], + afters: [ + expect.objectContaining({ + name: String.raw`"after each" hook`, + status: Status.BROKEN, + statusDetails: expect.objectContaining({ + message: "Lorem Ipsum", + }), + }), + ], + }), + expect.objectContaining({ + children: [barUuid, bazUuid, quxUuid], + afters: [ + expect.objectContaining({ + name: String.raw`"after all" hook: suite 1`, + status: Status.PASSED, + }), + ], + }), + expect.objectContaining({ + children: [fooUuid, barUuid, bazUuid, quxUuid, quuxUuid], + afters: [ + expect.objectContaining({ + name: String.raw`"after all" hook: spec-level`, + status: Status.PASSED, + }), + ], + }), + ]), + ); + }); + + it("should report a failed spec-level before", async () => { + await issue("1072"); + const { tests, groups } = await runCypressInlineTest({ + "cypress/e2e/sample.cy.js": () => ` + before(function () { + throw new Error("Lorem Ipsum"); + }); + after(() => { + throw new Error("Ipsum Lorem"); // doesn't overwrite the error in 'before all' + }); + + it("foo", () => {}); + describe("suite 1", () => { + it("bar", () => {}); + }); + describe("suite 2", () => { + it("baz", () => {}); + }); + `, + }); + + expect(tests).toHaveLength(3); + expect(tests).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + name: "foo", + status: Status.BROKEN, + stage: Stage.FINISHED, + statusDetails: expect.objectContaining({ + message: "Lorem Ipsum", + }), + }), + expect.objectContaining({ + name: "bar", + status: Status.BROKEN, + stage: Stage.FINISHED, + statusDetails: expect.objectContaining({ + message: "Lorem Ipsum", + }), + }), + expect.objectContaining({ + name: "baz", + status: Status.BROKEN, + stage: Stage.FINISHED, + statusDetails: expect.objectContaining({ + message: "Lorem Ipsum", + }), + }), + ]), + ); + const fooUuid = tests.find((t) => t.name === "foo")!.uuid; + const barUuid = tests.find((t) => t.name === "bar")!.uuid; + const bazUuid = tests.find((t) => t.name === "baz")!.uuid; + expect(groups).toHaveLength(2); + expect(groups).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + children: [fooUuid, barUuid, bazUuid], + befores: [ + expect.objectContaining({ + name: String.raw`"before all" hook`, + status: Status.BROKEN, + statusDetails: expect.objectContaining({ + message: "Lorem Ipsum", + }), + }), + ], + }), + expect.objectContaining({ + children: [fooUuid, barUuid, bazUuid], + afters: [ + expect.objectContaining({ + name: String.raw`"after all" hook`, + status: Status.BROKEN, + statusDetails: expect.objectContaining({ + message: "Ipsum Lorem", + }), + }), + ], + }), + ]), + ); + }); + + it("should report a failed spec-level after", async () => { + await issue("1072"); + const { tests, groups } = await runCypressInlineTest({ + "cypress/e2e/sample.cy.js": () => ` + after(function () { + throw new Error("Lorem Ipsum"); + }); + + it("foo", () => {}); + describe("suite 1", () => { + it("bar", () => {}); + }); + describe("suite 2", () => { + it("baz", () => {}); + }); + `, + }); + + expect(tests).toHaveLength(3); + expect(tests).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + name: "foo", + status: Status.PASSED, // after/afterEach don't affect the test's status + stage: Stage.FINISHED, + }), + expect.objectContaining({ + name: "bar", + status: Status.PASSED, // after/afterEach don't affect the test's status + stage: Stage.FINISHED, + }), + expect.objectContaining({ + name: "baz", + status: Status.PASSED, // after/afterEach don't affect the test's status + stage: Stage.FINISHED, + }), + ]), + ); + const fooUuid = tests.find((t) => t.name === "foo")!.uuid; + const barUuid = tests.find((t) => t.name === "bar")!.uuid; + const bazUuid = tests.find((t) => t.name === "baz")!.uuid; + expect(groups).toHaveLength(1); + expect(groups).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + children: [fooUuid, barUuid, bazUuid], + afters: [ + expect.objectContaining({ + name: String.raw`"after all" hook`, + status: Status.BROKEN, + statusDetails: expect.objectContaining({ + message: "Lorem Ipsum", + }), + }), + ], + }), + ]), + ); + }); + + it("should report failed nested hooks", async () => { + await issue("1072"); + const { tests, groups } = await runCypressInlineTest({ + "cypress/e2e/sample.cy.js": () => ` + afterEach("spec-level", () => {}); + after("spec-level", () => {}); + + describe("before", () => { + before(() => { + throw new Error("Lorem Ipsum before"); + }); + it("foo before", () => {}); + describe("before 1", () => { + it("bar before", () => {}); + }); + describe("before 2", () => { + it("baz before", () => {}); + }); + }); + + describe("after", () => { + after(() => { + throw new Error("Lorem Ipsum after"); + }); + it("foo after", () => {}); + describe("after 1", () => { + it("bar after", () => {}); + }); + describe("after 2", () => { + it("baz after", () => {}); + }); + }); + + describe("beforeEach", () => { + beforeEach(() => { + throw new Error("Lorem Ipsum beforeEach"); + }); + it("foo beforeEach", () => {}); + it("bar beforeEach", () => {}); + describe("beforeEach 1", () => { + it("baz beforeEach", () => {}); + }); + describe("beforeEach 2", () => { + it("qux beforeEach", () => {}); + }); + }); + + describe("afterEach", () => { + afterEach(() => { + throw new Error("Lorem Ipsum afterEach"); + }); + it("foo afterEach", () => {}); + it("bar afterEach", () => {}); + describe("afterEach 1", () => { + it("baz afterEach", () => {}); + }); + describe("afterEach 2", () => { + it("qux afterEach", () => {}); + }); + }); + `, + }); + + expect(tests).toHaveLength(14); + expect(tests).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + name: "foo before", + status: Status.BROKEN, + stage: Stage.FINISHED, + statusDetails: expect.objectContaining({ + message: "Lorem Ipsum before", + }), + }), + expect.objectContaining({ + name: "bar before", + status: Status.BROKEN, + stage: Stage.FINISHED, + statusDetails: expect.objectContaining({ + message: "Lorem Ipsum before", + }), + }), + expect.objectContaining({ + name: "baz before", + status: Status.BROKEN, + stage: Stage.FINISHED, + statusDetails: expect.objectContaining({ + message: "Lorem Ipsum before", + }), + }), + expect.objectContaining({ + name: "foo after", + status: Status.PASSED, // after/afterEach don't affect the test's status + stage: Stage.FINISHED, + }), + expect.objectContaining({ + name: "bar after", + status: Status.PASSED, // after/afterEach don't affect the test's status + stage: Stage.FINISHED, + }), + expect.objectContaining({ + name: "baz after", + status: Status.PASSED, // after/afterEach don't affect the test's status + stage: Stage.FINISHED, + }), + expect.objectContaining({ + name: "foo beforeEach", + status: Status.BROKEN, + stage: Stage.FINISHED, + statusDetails: expect.objectContaining({ + message: "Lorem Ipsum beforeEach", + }), + }), + expect.objectContaining({ + name: "bar beforeEach", + status: Status.SKIPPED, + stage: Stage.FINISHED, + statusDetails: expect.objectContaining({ + message: "'\"before each\" hook for \"foo beforeEach\"' defined in the 'beforeEach' suite has failed", + }), + }), + expect.objectContaining({ + name: "baz beforeEach", + status: Status.SKIPPED, + stage: Stage.FINISHED, + statusDetails: expect.objectContaining({ + message: "'\"before each\" hook for \"foo beforeEach\"' defined in the 'beforeEach' suite has failed", + }), + }), + expect.objectContaining({ + name: "qux beforeEach", + status: Status.SKIPPED, + stage: Stage.FINISHED, + statusDetails: expect.objectContaining({ + message: "'\"before each\" hook for \"foo beforeEach\"' defined in the 'beforeEach' suite has failed", + }), + }), + expect.objectContaining({ + name: "foo afterEach", + status: Status.PASSED, // after/afterEach don't affect the test's status + stage: Stage.FINISHED, + }), + expect.objectContaining({ + name: "bar afterEach", + status: Status.SKIPPED, + stage: Stage.FINISHED, + statusDetails: expect.objectContaining({ + message: "'\"after each\" hook for \"foo afterEach\"' defined in the 'afterEach' suite has failed", + }), + }), + expect.objectContaining({ + name: "baz afterEach", + status: Status.SKIPPED, + stage: Stage.FINISHED, + statusDetails: expect.objectContaining({ + message: "'\"after each\" hook for \"foo afterEach\"' defined in the 'afterEach' suite has failed", + }), + }), + expect.objectContaining({ + name: "qux afterEach", + status: Status.SKIPPED, + stage: Stage.FINISHED, + statusDetails: expect.objectContaining({ + message: "'\"after each\" hook for \"foo afterEach\"' defined in the 'afterEach' suite has failed", + }), + }), + ]), + ); + + const fooBeforeUuid = tests.find((t) => t.name === "foo before")!.uuid; + const barBeforeUuid = tests.find((t) => t.name === "bar before")!.uuid; + const bazBeforeUuid = tests.find((t) => t.name === "baz before")!.uuid; + const fooAfterUuid = tests.find((t) => t.name === "foo after")!.uuid; + const barAfterUuid = tests.find((t) => t.name === "bar after")!.uuid; + const bazAfterUuid = tests.find((t) => t.name === "baz after")!.uuid; + const fooBeforeEachUuid = tests.find((t) => t.name === "foo beforeEach")!.uuid; + const barBeforeEachUuid = tests.find((t) => t.name === "bar beforeEach")!.uuid; + const bazBeforeEachUuid = tests.find((t) => t.name === "baz beforeEach")!.uuid; + const quxBeforeEachUuid = tests.find((t) => t.name === "qux beforeEach")!.uuid; + const fooAfterEachUuid = tests.find((t) => t.name === "foo afterEach")!.uuid; + const barAfterEachUuid = tests.find((t) => t.name === "bar afterEach")!.uuid; + const bazAfterEachUuid = tests.find((t) => t.name === "baz afterEach")!.uuid; + const quxAfterEachUuid = tests.find((t) => t.name === "qux afterEach")!.uuid; + + expect(groups).toHaveLength(10); + expect(groups).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + children: [fooBeforeUuid, barBeforeUuid, bazBeforeUuid], + befores: [ + expect.objectContaining({ + name: String.raw`"before all" hook`, + status: Status.BROKEN, + statusDetails: expect.objectContaining({ + message: "Lorem Ipsum before", + }), + }), + ], + }), + expect.objectContaining({ + children: [fooAfterUuid, barAfterUuid, bazAfterUuid], + afters: [ + expect.objectContaining({ + name: String.raw`"after all" hook`, + status: Status.BROKEN, + statusDetails: expect.objectContaining({ + message: "Lorem Ipsum after", + }), + }), + ], + }), + expect.objectContaining({ + children: [fooBeforeEachUuid], + befores: [ + expect.objectContaining({ + name: String.raw`"before each" hook`, + status: Status.BROKEN, + statusDetails: expect.objectContaining({ + message: "Lorem Ipsum beforeEach", + }), + }), + ], + }), + expect.objectContaining({ + children: [fooAfterEachUuid], + afters: [ + expect.objectContaining({ + name: String.raw`"after each" hook`, + status: Status.BROKEN, + statusDetails: expect.objectContaining({ + message: "Lorem Ipsum afterEach", + }), + }), + ], + }), + expect.objectContaining({ + children: [fooAfterUuid], + afters: [ + expect.objectContaining({ + name: String.raw`"after each" hook: spec-level`, + status: Status.PASSED, + }), + ], + }), + expect.objectContaining({ + children: [barAfterUuid], + afters: [ + expect.objectContaining({ + name: String.raw`"after each" hook: spec-level`, + status: Status.PASSED, + }), + ], + }), + expect.objectContaining({ + children: [bazAfterUuid], + afters: [ + expect.objectContaining({ + name: String.raw`"after each" hook: spec-level`, + status: Status.PASSED, + }), + ], + }), + expect.objectContaining({ + children: [fooBeforeEachUuid], + afters: [ + expect.objectContaining({ + name: String.raw`"after each" hook: spec-level`, + status: Status.PASSED, + }), + ], + }), + expect.objectContaining({ + children: [fooAfterEachUuid], + afters: [ + expect.objectContaining({ + name: String.raw`"after each" hook: spec-level`, + status: Status.PASSED, + }), + ], + }), + expect.objectContaining({ + children: [ + fooBeforeUuid, + barBeforeUuid, + bazBeforeUuid, + fooAfterUuid, + barAfterUuid, + bazAfterUuid, + fooBeforeEachUuid, + barBeforeEachUuid, + bazBeforeEachUuid, + quxBeforeEachUuid, + fooAfterEachUuid, + barAfterEachUuid, + bazAfterEachUuid, + quxAfterEachUuid, + ], + afters: [ + expect.objectContaining({ + name: String.raw`"after all" hook: spec-level`, + status: Status.PASSED, + }), + ], + }), + ]), + ); + }); + + it("should report the spec on spec-level after failures", async () => { + const { tests, groups } = await runCypressInlineTest({ + "cypress/e2e/promise.cy.js": () => ` + after(() => new Cypress.Promise((_, reject) => setTimeout(() => reject(new Error("Lorem Ipsum promise")), 0))); + it("foo", () => {}); + `, + "cypress/e2e/sync.cy.js": () => ` + after(() => { + throw new Error("Lorem Ipsum sync"); + }); + it("bar", () => {}); + `, + "cypress/e2e/callback.cy.js": () => ` + after((done) => setTimeout(() => done(new Error("Lorem Ipsum callback")), 0)); + it("baz", () => {}); + `, + }); + + expect(tests).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + name: "foo", + status: Status.PASSED, + }), + expect.objectContaining({ + name: "bar", + status: Status.PASSED, + }), + expect.objectContaining({ + name: "baz", + status: Status.PASSED, + }), + ]), + ); + const fooUuid = tests.find((t) => t.name === "foo")!.uuid; + const barUuid = tests.find((t) => t.name === "bar")!.uuid; + const bazUuid = tests.find((t) => t.name === "baz")!.uuid; + expect(groups).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + children: [fooUuid], + afters: [ + expect.objectContaining({ + name: String.raw`"after all" hook`, + status: Status.BROKEN, + statusDetails: expect.objectContaining({ + message: "Lorem Ipsum promise", + }), + }), + ], + }), + expect.objectContaining({ + children: [barUuid], + afters: [ + expect.objectContaining({ + name: String.raw`"after all" hook`, + status: Status.BROKEN, + statusDetails: expect.objectContaining({ + message: "Lorem Ipsum sync", + }), + }), + ], + }), + expect.objectContaining({ + children: [bazUuid], + afters: [ + expect.objectContaining({ + name: String.raw`"after all" hook`, + status: Status.BROKEN, + statusDetails: expect.objectContaining({ + message: "Lorem Ipsum callback", + }), + }), + ], + }), + ]), + ); + }); +}); + +it("should complete the spec from the last after hook", async () => { + const { tests, groups } = await runCypressInlineTest({ + "cypress/e2e/promise.cy.js": () => ` + after("foo promise", () => new Cypress.Promise((resolve) => setTimeout(resolve, 0))); + after("bar promise", () => new Cypress.Promise((resolve) => setTimeout(resolve, 0))); + it("baz promise", () => {}); + `, + "cypress/e2e/sync.cy.js": () => ` + after("foo sync", () => {}); + after("bar sync", () => {}); + it("baz sync", () => {}); + `, + "cypress/e2e/callback.cy.js": () => ` + after("foo callback", (done) => setTimeout(done, 0)); + after("bar callback", (done) => setTimeout(done, 0)); + it("baz callback", () => {}); + `, + }); + + expect(tests).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + name: "baz promise", + status: Status.PASSED, + }), + expect.objectContaining({ + name: "baz sync", + status: Status.PASSED, + }), + expect.objectContaining({ + name: "baz callback", + status: Status.PASSED, + }), + ]), + ); + const promiseUuid = tests.find((t) => t.name === "baz promise")!.uuid; + const syncUuid = tests.find((t) => t.name === "baz sync")!.uuid; + const callbackUuid = tests.find((t) => t.name === "baz callback")!.uuid; + expect(groups).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + children: [promiseUuid], + afters: [ + expect.objectContaining({ + name: String.raw`"after all" hook: foo promise`, + status: Status.PASSED, + }), + ], + }), + expect.objectContaining({ + children: [promiseUuid], + afters: [ + expect.objectContaining({ + name: String.raw`"after all" hook: bar promise`, + status: Status.PASSED, + }), + ], + }), + expect.objectContaining({ + children: [syncUuid], + afters: [ + expect.objectContaining({ + name: String.raw`"after all" hook: foo sync`, + status: Status.PASSED, + }), + ], + }), + expect.objectContaining({ + children: [syncUuid], + afters: [ + expect.objectContaining({ + name: String.raw`"after all" hook: bar sync`, + status: Status.PASSED, + }), + ], + }), + expect.objectContaining({ + children: [callbackUuid], + afters: [ + expect.objectContaining({ + name: String.raw`"after all" hook: foo callback`, + status: Status.PASSED, + }), + ], + }), + expect.objectContaining({ + children: [callbackUuid], + afters: [ + expect.objectContaining({ + name: String.raw`"after all" hook: bar callback`, + status: Status.PASSED, + }), + ], + }), + ]), + ); +});