[kbn/test/failed_test_reporter] handle cypress junit better (#72968)

Co-authored-by: spalger <[email protected]>

packages/kbn-test/src/failed_tests_reporter/__fixtures__/cypress_report.xml

@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites name="Mocha Tests" time="16.198" tests="2" failures="1">
+  <testsuite name="Root Suite" timestamp="2020-07-22T15:06:26" tests="0" file="cypress/integration/timeline_flyout_button.spec.ts" failures="0" time="0">
+  </testsuite>
+  <testsuite name="timeline flyout button" timestamp="2020-07-22T15:06:26" tests="2" failures="1" time="16.198">
+    <testcase name="timeline flyout button toggles open the timeline" time="8.099" classname="toggles open the timeline">
+    </testcase>
+    <testcase name="timeline flyout button &quot;after each&quot; hook for &quot;toggles open the timeline&quot;" time="8.099" classname="&quot;after each&quot; hook for &quot;toggles open the timeline&quot;">
+      <failure message="Timed out retrying: `cy.click()` could not be issued because this element is currently animating:
+
+`&lt;button class=&quot;euiButtonEmpty euiButtonEmpty--text&quot; type=&quot;button&quot; data-test-subj=&quot;timeline-new&quot;&gt;...&lt;/button&gt;`
+
+You can fix this problem by:
+  - Passing `{force: true}` which disables all error checking
+  - Passing `{waitForAnimations: false}` which disables waiting on animations
+  - Passing `{animationDistanceThreshold: 20}` which decreases the sensitivity
+
+https://on.cypress.io/element-is-animating
+
+Because this error occurred during a `after each` hook we are skipping the remaining tests in the current suite: `timeline flyout button`" type="CypressError"><![CDATA[CypressError: Timed out retrying: `cy.click()` could not be issued because this element is currently animating:
+
+`<button class="euiButtonEmpty euiButtonEmpty--text" type="button" data-test-subj="timeline-new">...</button>`
+
+You can fix this problem by:
+  - Passing `{force: true}` which disables all error checking
+  - Passing `{waitForAnimations: false}` which disables waiting on animations
+  - Passing `{animationDistanceThreshold: 20}` which decreases the sensitivity
+
+https://on.cypress.io/element-is-animating
+
+Because this error occurred during a `after each` hook we are skipping the remaining tests in the current suite: `timeline flyout button`
+    at cypressErr (http://elastic:changeme@localhost:61141/__cypress/runner/cypress_runner.js:146621:16)
+    at cypressErrByPath (http://elastic:changeme@localhost:61141/__cypress/runner/cypress_runner.js:146630:10)
+    at Object.throwErrByPath (http://elastic:changeme@localhost:61141/__cypress/runner/cypress_runner.js:146593:11)
+    at Object.ensureElementIsNotAnimating (http://elastic:changeme@localhost:61141/__cypress/runner/cypress_runner.js:137560:24)
+    at ensureNotAnimating (http://elastic:changeme@localhost:61141/__cypress/runner/cypress_runner.js:127434:13)
+    at runAllChecks (http://elastic:changeme@localhost:61141/__cypress/runner/cypress_runner.js:127522:9)
+    at retryActionability (http://elastic:changeme@localhost:61141/__cypress/runner/cypress_runner.js:127542:16)
+    at tryCatcher (http://elastic:changeme@localhost:61141/__cypress/runner/cypress_runner.js:9065:23)
+    at Function.Promise.attempt.Promise.try (http://elastic:changeme@localhost:61141/__cypress/runner/cypress_runner.js:6339:29)
+    at tryFn (http://elastic:changeme@localhost:61141/__cypress/runner/cypress_runner.js:140680:21)
+    at whenStable (http://elastic:changeme@localhost:61141/__cypress/runner/cypress_runner.js:140715:12)
+    at http://elastic:changeme@localhost:61141/__cypress/runner/cypress_runner.js:140259:16
+    at tryCatcher (http://elastic:changeme@localhost:61141/__cypress/runner/cypress_runner.js:9065:23)
+    at Promise._settlePromiseFromHandler (http://elastic:changeme@localhost:61141/__cypress/runner/cypress_runner.js:7000:31)
+    at Promise._settlePromise (http://elastic:changeme@localhost:61141/__cypress/runner/cypress_runner.js:7057:18)
+    at Promise._settlePromise0 (http://elastic:changeme@localhost:61141/__cypress/runner/cypress_runner.js:7102:10)]]></failure>
+    </testcase>
+  </testsuite>
+</testsuites>

packages/kbn-test/src/failed_tests_reporter/__fixtures__/index.ts

@@ -23,3 +23,4 @@ export const FTR_REPORT = Fs.readFileSync(require.resolve('./ftr_report.xml'), '
 export const JEST_REPORT = Fs.readFileSync(require.resolve('./jest_report.xml'), 'utf8');
 export const KARMA_REPORT = Fs.readFileSync(require.resolve('./karma_report.xml'), 'utf8');
 export const MOCHA_REPORT = Fs.readFileSync(require.resolve('./mocha_report.xml'), 'utf8');
+export const CYPRESS_REPORT = Fs.readFileSync(require.resolve('./cypress_report.xml'), 'utf8');

packages/kbn-test/src/failed_tests_reporter/add_messages_to_report.test.ts

@@ -39,7 +39,13 @@ jest.mock('fs', () => {
   };
 });
 
-import { FTR_REPORT, JEST_REPORT, MOCHA_REPORT, KARMA_REPORT } from './__fixtures__';
+import {
+  FTR_REPORT,
+  JEST_REPORT,
+  MOCHA_REPORT,
+  KARMA_REPORT,
+  CYPRESS_REPORT,
+} from './__fixtures__';
 import { parseTestReport } from './test_report';
 import { addMessagesToReport } from './add_messages_to_report';
 
@@ -270,6 +276,69 @@ it('rewrites mocha reports with minimal changes', async () => {
   `);
 });
 
+it('rewrites cypress reports with minimal changes', async () => {
+  const xml = await addMessagesToReport({
+    messages: [
+      {
+        classname: '"after each" hook for "toggles open the timeline"',
+        name: 'timeline flyout button "after each" hook for "toggles open the timeline"',
+        message: 'Some extra content\n',
+      },
+    ],
+    report: await parseTestReport(CYPRESS_REPORT),
+    log,
+    reportPath: Path.resolve(__dirname, './__fixtures__/cypress_report.xml'),
+  });
+
+  expect(createPatch('cypress.xml', CYPRESS_REPORT, xml, { context: 0 })).toMatchInlineSnapshot(`
+    Index: cypress.xml
+    ===================================================================
+    --- cypress.xml	[object Object]
+    +++ cypress.xml
+    @@ -1,25 +1,16 @@
+    -‹?xml version="1.0" encoding="UTF-8"?›
+    +‹?xml version="1.0" encoding="utf-8"?›
+     ‹testsuites name="Mocha Tests" time="16.198" tests="2" failures="1"›
+    -  ‹testsuite name="Root Suite" timestamp="2020-07-22T15:06:26" tests="0" file="cypress/integration/timeline_flyout_button.spec.ts" failures="0" time="0"›
+    -  ‹/testsuite›
+    +  ‹testsuite name="Root Suite" timestamp="2020-07-22T15:06:26" tests="0" file="cypress/integration/timeline_flyout_button.spec.ts" failures="0" time="0"/›
+       ‹testsuite name="timeline flyout button" timestamp="2020-07-22T15:06:26" tests="2" failures="1" time="16.198"›
+    -    ‹testcase name="timeline flyout button toggles open the timeline" time="8.099" classname="toggles open the timeline"›
+    -    ‹/testcase›
+    +    ‹testcase name="timeline flyout button toggles open the timeline" time="8.099" classname="toggles open the timeline"/›
+         ‹testcase name="timeline flyout button "after each" hook for "toggles open the timeline"" time="8.099" classname=""after each" hook for "toggles open the timeline""›
+    -      ‹failure message="Timed out retrying: \`cy.click()\` could not be issued because this element is currently animating:
+    +      ‹failure message="Timed out retrying: \`cy.click()\` could not be issued because this element is currently animating:

\`<button class="euiButtonEmpty euiButtonEmpty--text" type="button" data-test-subj="timeline-new"›...</button›\`

You can fix this problem by:
  - Passing \`{force: true}\` which disables all error checking
  - Passing \`{waitForAnimations: false}\` which disables waiting on animations
  - Passing \`{animationDistanceThreshold: 20}\` which decreases the sensitivity

https://on.cypress.io/element-is-animating

Because this error occurred during a \`after each\` hook we are skipping the remaining tests in the current suite: \`timeline flyout button\`" type="CypressError"›‹![CDATA[Failed Tests Reporter:
+    +  - Some extra content
+
+    -\`<button class="euiButtonEmpty euiButtonEmpty--text" type="button" data-test-subj="timeline-new">...</button>\`
+
+    -You can fix this problem by:
+    -  - Passing \`{force: true}\` which disables all error checking
+    -  - Passing \`{waitForAnimations: false}\` which disables waiting on animations
+    -  - Passing \`{animationDistanceThreshold: 20}\` which decreases the sensitivity
+    +CypressError: Timed out retrying: \`cy.click()\` could not be issued because this element is currently animating:
+
+    -https://on.cypress.io/element-is-animating
+    -
+    -Because this error occurred during a \`after each\` hook we are skipping the remaining tests in the current suite: \`timeline flyout button\`" type="CypressError"›‹![CDATA[CypressError: Timed out retrying: \`cy.click()\` could not be issued because this element is currently animating:
+    -
+     \`‹button class="euiButtonEmpty euiButtonEmpty--text" type="button" data-test-subj="timeline-new"›...‹/button›\`
+
+     You can fix this problem by:
+       - Passing \`{force: true}\` which disables all error checking
+    @@ -46,5 +37,5 @@
+         at Promise._settlePromise (http://elastic:changeme@localhost:61141/__cypress/runner/cypress_runner.js:7057:18)
+         at Promise._settlePromise0 (http://elastic:changeme@localhost:61141/__cypress/runner/cypress_runner.js:7102:10)]]›‹/failure›
+         ‹/testcase›
+       ‹/testsuite›
+    -‹/testsuites›
+    +‹/testsuites›
+    \\ No newline at end of file
+
+  `);
+});
+
 it('rewrites karma reports with minimal changes', async () => {
   const xml = await addMessagesToReport({
     report: await parseTestReport(KARMA_REPORT),

packages/kbn-test/src/failed_tests_reporter/add_messages_to_report.ts

@@ -59,10 +59,14 @@ export async function addMessagesToReport(options: {
     log.info(`${classname} - ${name}:${messageList}`);
     const output = `Failed Tests Reporter:${messageList}\n\n`;
 
-    if (!testCase['system-out']) {
-      testCase['system-out'] = [output];
+    if (typeof testCase.failure[0] === 'object' && testCase.failure[0].$.message) {
+      // failure with "messages" ignore the system-out on jenkins
+      // so we instead extend the failure message
+      testCase.failure[0]._ = output + testCase.failure[0]._;
+    } else if (!testCase['system-out']) {
+      testCase['system-out'] = [{ _: output }];
     } else if (typeof testCase['system-out'][0] === 'string') {
-      testCase['system-out'][0] = output + String(testCase['system-out'][0]);
+      testCase['system-out'][0] = { _: output + testCase['system-out'][0] };
     } else {
       testCase['system-out'][0]._ = output + testCase['system-out'][0]._;
     }

packages/kbn-test/src/failed_tests_reporter/run_failed_tests_reporter_cli.ts

@@ -72,6 +72,7 @@ export function runFailedTestsReporterCli() {
       }
 
       const patterns = flags._.length ? flags._ : DEFAULT_PATTERNS;
+      log.info('Searching for reports at', patterns);
       const reportPaths = await globby(patterns, {
         absolute: true,
       });
@@ -80,6 +81,7 @@ export function runFailedTestsReporterCli() {
         throw createFailError(`Unable to find any junit reports with patterns [${patterns}]`);
       }
 
+      log.info('found', reportPaths.length, 'junit reports', reportPaths);
       const newlyCreatedIssues: Array<{
         failure: TestFailure;
         newIssue: GithubIssueMini;

packages/kbn-test/src/failed_tests_reporter/test_report.ts

@@ -70,7 +70,7 @@ export interface TestCase {
 }
 
 export interface FailedTestCase extends TestCase {
-  failure: Array<string | { _: string }>;
+  failure: Array<string | { $: { message?: string }; _: string }>;
 }
 
 /**