fix(allure-cypress): missing tests and fixtures when an error occurs …

…in a hook (fixes #1072, via #1123)
allure-framework · Aug 29, 2024 · 0c4d92a · 0c4d92a
1 parent 3e7b6f7
commit 0c4d92a
Show file tree

Hide file tree

Showing 7 changed files with 1,382 additions and 176 deletions.
diff --git a/packages/allure-cypress/src/index.ts b/packages/allure-cypress/src/index.ts
@@ -1,9 +1,10 @@
 import { getMessageAndTraceFromError, getStatusFromError } from "allure-js-commons/sdk";
-import type { CypressCommand, CypressHook, CypressTest } from "./model.js";
 import { ALLURE_REPORT_SYSTEM_HOOK } from "./model.js";
+import type { CypressCommand, CypressHook, CypressSuite, CypressTest } from "./model.js";
 import {
+  completeHookErrorReporting,
+  completeSpecIfNoAfterHookLeft,
   enableScopeLevelAfterHookReporting,
-  flushFinalRuntimeMessages,
   flushRuntimeMessages,
   initTestRuntime,
   reportCommandEnd,
@@ -22,7 +23,7 @@ import {
   reportUnfinishedSteps,
 } from "./runtime.js";
 import { isAllureInitialized, setAllureInitialized } from "./state.js";
-import { applyTestPlan, isTestReported, shouldCommandBeSkipped } from "./utils.js";
+import { applyTestPlan, isAllureHook, isRootAfterAllHook, isTestReported, shouldCommandBeSkipped } from "./utils.js";
 
 const {
   EVENT_RUN_BEGIN,
@@ -35,7 +36,6 @@ const {
   EVENT_TEST_FAIL,
   EVENT_TEST_PENDING,
   EVENT_TEST_END,
-  EVENT_RUN_END,
 } = Mocha.Runner.constants;
 
 const initializeAllure = () => {
@@ -52,24 +52,24 @@ const initializeAllure = () => {
       initTestRuntime();
       reportRunStart();
     })
-    .on(EVENT_SUITE_BEGIN, (suite: Mocha.Suite) => {
-      if (!suite.parent) {
+    .on(EVENT_SUITE_BEGIN, (suite: CypressSuite) => {
+      if (suite.root) {
         applyTestPlan(Cypress.spec, suite);
       }
       reportSuiteStart(suite);
     })
-    .on(EVENT_SUITE_END, (suite: Mocha.Suite) => {
+    .on(EVENT_SUITE_END, (suite: CypressSuite) => {
       reportSuiteEnd(suite);
     })
     .on(EVENT_HOOK_BEGIN, (hook: CypressHook) => {
-      if (hook.title.includes(ALLURE_REPORT_SYSTEM_HOOK)) {
+      if (isAllureHook(hook)) {
         return;
       }
 
       reportHookStart(hook);
     })
     .on(EVENT_HOOK_END, (hook: CypressHook) => {
-      if (hook.title.includes(ALLURE_REPORT_SYSTEM_HOOK)) {
+      if (isAllureHook(hook)) {
         return;
       }
 
@@ -85,22 +85,35 @@ const initializeAllure = () => {
     .on(EVENT_TEST_PASS, () => {
       reportTestPass();
     })
-    .on(EVENT_TEST_FAIL, (_: CypressTest, err: Error) => {
+    .on(EVENT_TEST_FAIL, (testOrHook: CypressTest | CypressHook, err: Error) => {
+      const isHook = "hookName" in testOrHook;
+      if (isHook && isRootAfterAllHook(testOrHook)) {
+        // Errors in spec-level 'after all' hooks are handled by Allure wrappers.
+        return;
+      }
+
+      const isAllureHookFailure = isHook && isAllureHook(testOrHook);
+
+      if (isAllureHookFailure) {
+        // Normally, Allure hooks are skipped from the report.
+        // In case of errors, it will be helpful to see them.
+        reportHookStart(testOrHook, Date.now() - (testOrHook.duration ?? 0));
+      }
+
+      // This will mark the fixture and the test (if any) as failed/broken.
       reportTestOrHookFail(err);
+
+      if (isHook) {
+        // This will end the fixture and test (if any) and will report the remaining
+        // tests in the hook's suite (the ones that will be skipped by Cypress/Mocha).
+        completeHookErrorReporting(testOrHook, err);
+      }
     })
     .on(EVENT_TEST_PENDING, (test: CypressTest) => {
       reportTestSkip(test);
     })
     .on(EVENT_TEST_END, (test: CypressTest) => {
       reportTestEnd(test);
-    })
-    .on(EVENT_RUN_END, () => {
-      // after:spec isn't called in interactive mode by default.
-      // We're using the 'end' event instead to report the remaining messages
-      // (the root 'suite end', mainly).
-      if (Cypress.config("isInteractive")) {
-        flushFinalRuntimeMessages();
-      }
     });
 
   Cypress.Screenshot.defaults({
@@ -132,11 +145,11 @@ const initializeAllure = () => {
     reportCommandEnd();
   });
 
-  afterEach(ALLURE_REPORT_SYSTEM_HOOK, () => {
-    flushRuntimeMessages();
-  });
-  after(ALLURE_REPORT_SYSTEM_HOOK, () => {
+  afterEach(ALLURE_REPORT_SYSTEM_HOOK, flushRuntimeMessages);
+
+  after(ALLURE_REPORT_SYSTEM_HOOK, function (this: Mocha.Context) {
     flushRuntimeMessages();
+    completeSpecIfNoAfterHookLeft(this);
   });
 
   enableScopeLevelAfterHookReporting();

diff --git a/packages/allure-cypress/src/model.ts b/packages/allure-cypress/src/model.ts
@@ -10,18 +10,22 @@ export type AllureCypressConfig = ReporterConfig & {
   videoOnFailOnly?: boolean;
 };
 
+export type CypressSuite = Mocha.Suite & {
+  id: string;
+  parent: CypressSuite | undefined;
+  tests: CypressTest[];
+  suites: CypressSuite[];
+};
+
 export type CypressTest = Mocha.Test & {
   wallClockStartedAt?: Date;
-  hookName?: string;
-  id: string;
+  parent: CypressSuite | undefined;
 };
 
 export type CypressHook = Mocha.Hook & {
-  id: string;
   hookId: string;
-  parent: Mocha.Suite & {
-    id: string;
-  };
+  hookName: string;
+  parent: CypressSuite | undefined;
 };
 
 export type CypressCommand = {
@@ -41,6 +45,7 @@ export type CupressRunStart = {
 export type CypressSuiteStartMessage = {
   type: "cypress_suite_start";
   data: {
+    id: string;
     name: string;
     root: boolean;
     start: number;
@@ -59,6 +64,8 @@ export type CypressHookStartMessage = {
   type: "cypress_hook_start";
   data: {
     name: string;
+    scopeType: "each" | "all";
+    position: "before" | "after";
     start: number;
   };
 };
@@ -90,14 +97,25 @@ export type CypressFailMessage = {
 
 export type CypressTestSkipMessage = {
   type: "cypress_test_skip";
-  data: object;
+  data: {
+    statusDetails?: StatusDetails;
+  };
 };
 
 export type CypressTestPassMessage = {
   type: "cypress_test_pass";
   data: object;
 };
 
+export type CypressSkippedTestMessage = {
+  type: "cypress_skipped_test";
+  data: CypressTestStartMessage["data"] &
+    CypressFailMessage["data"] &
+    CypressTestEndMessage["data"] & {
+      suites: string[];
+    };
+};
+
 export type CypressTestEndMessage = {
   type: "cypress_test_end";
   data: {
@@ -137,6 +155,7 @@ export type CypressMessage =
   | CypressTestPassMessage
   | CypressFailMessage
   | CypressTestSkipMessage
+  | CypressSkippedTestMessage
   | CypressTestEndMessage;
 
 export type SpecContext = {
@@ -146,6 +165,8 @@ export type SpecContext = {
   fixture: string | undefined;
   commandSteps: string[];
   videoScope: string;
+  suiteIdToScope: Map<string, string>;
+  suiteScopeToId: Map<string, string>;
   suiteScopes: string[];
   testScope: string | undefined;
   suiteNames: string[];
@@ -156,21 +177,20 @@ export type AllureSpecState = {
   initialized: boolean;
   testPlan: TestPlanV1 | null | undefined;
   messages: CypressMessage[];
+  currentTest?: CypressTest;
 };
 
-export type HookPosition = "before" | "after";
-
-export type HookScopeType = "all" | "each";
-
-export type HookType = [position: HookPosition, scopeType: HookScopeType];
-
 export type AllureCypressTaskArgs = {
   absolutePath: string;
   messages: readonly CypressMessage[];
+  isInteractive: boolean;
 };
 
 export type CypressSuiteFunction = (
   title: string,
   configOrFn?: Cypress.SuiteConfigOverrides | ((this: Mocha.Suite) => void),
   fn?: (this: Mocha.Suite) => void,
 ) => Mocha.Suite;
+
+export type DirectHookImplementation = Mocha.AsyncFunc | ((this: Mocha.Context) => void);
+export type HookImplementation = Mocha.Func | DirectHookImplementation;