From c47097b99844db4e5c53dca924cb18c7cb5de879 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Tue, 2 Jul 2024 18:43:16 +0200 Subject: [PATCH] [8.14] [CI] Display command on failure page (#186999) (#187351) # Backport This will backport the following commits from `main` to `8.14`: - [[CI] Display command on failure page (#186999)](https://github.com/elastic/kibana/pull/186999) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) --- .../__fixtures__/ftr_report.xml | 4 +-- .../__fixtures__/jest_report.xml | 4 +-- .../__fixtures__/mocha_report.xml | 4 +-- .../add_messages_to_report.test.ts | 10 ++++---- .../get_failures.test.ts | 6 +++++ .../failed_tests_reporter/get_failures.ts | 21 ++++++++++++++-- .../report_failures_to_file.ts | 21 +++++++++++----- .../failed_tests_reporter/test_report.ts | 4 +++ .../src/jest/junit_reporter/junit_reporter.ts | 3 +++ .../src/mocha/junit_report_generation.js | 19 +++++++++++--- .../src/mocha/junit_report_generation.test.js | 25 +++++++++---------- .../kbn-test/src/prettify_command_line.ts | 22 ++++++++++++++++ 12 files changed, 108 insertions(+), 35 deletions(-) create mode 100644 packages/kbn-test/src/prettify_command_line.ts diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/__fixtures__/ftr_report.xml b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/__fixtures__/ftr_report.xml index 3bfc686f9e845..07f1e79b0f5df 100644 --- a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/__fixtures__/ftr_report.xml +++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/__fixtures__/ftr_report.xml @@ -1,6 +1,6 @@ - - + + - - + + diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/__fixtures__/mocha_report.xml b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/__fixtures__/mocha_report.xml index 64cb7ce551ee5..699ad47c58240 100644 --- a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/__fixtures__/mocha_report.xml +++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/__fixtures__/mocha_report.xml @@ -1,6 +1,6 @@ - - + + diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/add_messages_to_report.test.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/add_messages_to_report.test.ts index 02197cf54fab0..42f3dcd120e9c 100644 --- a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/add_messages_to_report.test.ts +++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/add_messages_to_report.test.ts @@ -60,8 +60,8 @@ it('rewrites ftr reports with minimal changes', async () => { +++ ftr.xml @@ -1,53 +1,56 @@ ‹?xml version="1.0" encoding="utf-8"?› - ‹testsuites› - ‹testsuite timestamp="2019-06-05T23:37:10" time="903.670" tests="129" failures="5" skipped="71"› + ‹testsuites name="ftr" timestamp="2019-06-05T23:37:10" time="903.670" tests="129" failures="5" skipped="71" command-line="node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts"› + ‹testsuite timestamp="2019-06-05T23:37:10" time="903.670" tests="129" failures="5" skipped="71" command-line="node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts"› ‹testcase name="maps app maps loaded from sample data ecommerce "before all" hook" classname="Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps/sample_data·js" time="154.378"› - ‹system-out› - ‹![CDATA[[00:00:00] │ @@ -155,7 +155,7 @@ it('rewrites jest reports with minimal changes', async () => { --- jest.xml +++ jest.xml @@ -3,13 +3,17 @@ - ‹testsuite name="x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts" timestamp="2019-06-07T03:42:21" time="14.504" tests="5" failures="1" skipped="0" file="/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts"› + ‹testsuite name="x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts" timestamp="2019-06-07T03:42:21" time="14.504" tests="5" failures="1" skipped="0" file="/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts" command-line="node scripts/jest --config some/jest/config.ts"› ‹testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can start and end a process" time="1.316"/› ‹testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can force kill the process if langServer can not exit" time="3.182"/› ‹testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can reconnect if process died" time="7.060"› @@ -203,8 +203,8 @@ it('rewrites mocha reports with minimal changes', async () => { +++ mocha.xml @@ -1,13 +1,16 @@ ‹?xml version="1.0" encoding="utf-8"?› - ‹testsuites› - ‹testsuite timestamp="2019-06-13T23:29:36" time="30.739" tests="1444" failures="2" skipped="3"› + ‹testsuites command-line="node scripts/functional_tests --config super-mocha-test.config.js"› + ‹testsuite timestamp="2019-06-13T23:29:36" time="30.739" tests="1444" failures="2" skipped="3" command-line="node scripts/functional_tests --config super-mocha-test.config.js"› ‹testcase name="code in multiple nodes "before all" hook" classname="X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts" time="0.121"› - ‹system-out› - ‹![CDATA[]]› diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/get_failures.test.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/get_failures.test.ts index 77d7cd93ce4b0..d76662c600724 100644 --- a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/get_failures.test.ts +++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/get_failures.test.ts @@ -16,6 +16,7 @@ it('discovers failures in ftr report', async () => { Array [ Object { "classname": "Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps/sample_data·js", + "commandLine": "node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts", "failure": " Error: retry.try timeout: TimeoutError: Waiting for element to be located By(css selector, [data-test-subj~=\\"layerTocActionsPanelToggleButtonRoad_Map_-_Bright\\"]) Wait timed out after 10055ms @@ -37,6 +38,7 @@ it('discovers failures in ftr report', async () => { }, Object { "classname": "Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps", + "commandLine": "node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts", "failure": " { NoSuchSessionError: This driver instance does not have a valid session ID (did you call WebDriver.quit()?) and may no longer be used. at promise.finally (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/node_modules/selenium-webdriver/lib/webdriver.js:726:38) @@ -56,6 +58,7 @@ it('discovers failures in ftr report', async () => { }, Object { "classname": "Firefox XPack UI Functional Tests.x-pack/test/functional/apps/machine_learning/anomaly_detection/saved_search_job·ts", + "commandLine": "node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts", "failure": "{ NoSuchSessionError: Tried to run command without establishing a connection at Object.throwDecodedError (/dev/shm/workspace/kibana/node_modules/selenium-webdriver/lib/error.js:550:15) at parseHttpResponse (/dev/shm/workspace/kibana/node_modules/selenium-webdriver/lib/http.js:563:13) @@ -76,6 +79,7 @@ it('discovers failures in jest report', async () => { Array [ Object { "classname": "X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp", + "commandLine": "node scripts/jest --config some/jest/config.ts", "failure": " TypeError: Cannot read property '0' of undefined at Object..test (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts:166:10) @@ -95,6 +99,7 @@ it('discovers failures in mocha report', async () => { Array [ Object { "classname": "X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts", + "commandLine": "node scripts/functional_tests --config super-mocha-test.config.js", "failure": " Error: Unable to read artifact info from https://artifacts-api.elastic.co/v1/versions/8.0.0-SNAPSHOT/builds/latest/projects/elasticsearch: Service Temporarily Unavailable @@ -117,6 +122,7 @@ it('discovers failures in mocha report', async () => { }, Object { "classname": "X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts", + "commandLine": "node scripts/functional_tests --config super-mocha-test.config.js", "failure": " TypeError: Cannot read property 'shutdown' of undefined at Context.shutdown (plugins/code/server/__tests__/multi_node.ts:125:23) diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/get_failures.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/get_failures.ts index e3230e3cdddce..f0955b17f9953 100644 --- a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/get_failures.ts +++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/get_failures.ts @@ -16,6 +16,7 @@ export type TestFailure = FailedTestCase['$'] & { 'system-out'?: string; githubIssue?: string; failureCount?: number; + commandLine?: string; }; const getText = (node?: Array) => { @@ -71,19 +72,35 @@ const isLikelyIrrelevant = (name: string, failure: string) => { export function getFailures(report: TestReport) { const failures: TestFailure[] = []; + const commandLine = getCommandLineFromReport(report); + for (const testCase of makeFailedTestCaseIter(report)) { const failure = getText(testCase.failure); const likelyIrrelevant = isLikelyIrrelevant(testCase.$.name, failure); - failures.push({ + const failureObj = { // unwrap xml weirdness ...testCase.$, // Strip ANSI color characters failure, likelyIrrelevant, 'system-out': getText(testCase['system-out']), - }); + commandLine, + }; + + // cleaning up duplicates + delete failureObj['command-line']; + + failures.push(failureObj); } return failures; } + +function getCommandLineFromReport(report: TestReport) { + if ('testsuites' in report) { + return report.testsuites?.testsuite?.[0]?.$['command-line'] || ''; + } else { + return report.testsuite?.$['command-line'] || ''; + } +} diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_failures_to_file.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_failures_to_file.ts index ab54d7f60dfe5..d2c0fb705d1aa 100644 --- a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_failures_to_file.ts +++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_failures_to_file.ts @@ -170,14 +170,23 @@ export async function reportFailuresToFile(

${escape(failure.name)}

- Failures in tracked branches: ${ - failure.failureCount || 0 - } + ${ + failure.commandLine + ? `

+ Command Line: + 
${escape(failure.commandLine)}
+
` + : '' + } +
+ Failures in tracked branches: + ${failure.failureCount || 0} +
${ failure.githubIssue - ? `
${escape( - failure.githubIssue - )}` + ? `` : '' } diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/test_report.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/test_report.ts index e70aa44a2a088..ce817f90079e0 100644 --- a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/test_report.ts +++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/test_report.ts @@ -37,6 +37,8 @@ export interface TestSuite { skipped: string; /* optional JSON encoded metadata */ 'metadata-json'?: string; + /* the command that ran this suite */ + 'command-line'?: string; }; testcase?: TestCase[]; } @@ -51,6 +53,8 @@ export interface TestCase { time: string; /* optional JSON encoded metadata */ 'metadata-json'?: string; + /* the command that ran this suite */ + 'command-line'?: string; }; /* contents of system-out elements */ 'system-out'?: Array; diff --git a/packages/kbn-test/src/jest/junit_reporter/junit_reporter.ts b/packages/kbn-test/src/jest/junit_reporter/junit_reporter.ts index edb109eaa7000..ef6986183dd6c 100644 --- a/packages/kbn-test/src/jest/junit_reporter/junit_reporter.ts +++ b/packages/kbn-test/src/jest/junit_reporter/junit_reporter.ts @@ -17,6 +17,7 @@ import { AggregatedResult, Test, BaseReporter } from '@jest/reporters'; import { escapeCdata } from '../../mocha/xml'; import { getUniqueJunitReportPath } from '../../report_path'; +import { prettifyCommandLine } from '../../prettify_command_line'; interface ReporterOptions { reportName?: string; @@ -71,6 +72,7 @@ export default class JestJUnitReporter extends BaseReporter { tests: results.numTotalTests, failures: results.numFailedTests, skipped: results.numPendingTests, + 'command-line': prettifyCommandLine(process.argv), }); // top level test results are the files/suites @@ -83,6 +85,7 @@ export default class JestJUnitReporter extends BaseReporter { failures: suite.numFailingTests, skipped: suite.numPendingTests, file: suite.testFilePath, + 'command-line': prettifyCommandLine(process.argv), }); // nested in there are the tests in that file diff --git a/packages/kbn-test/src/mocha/junit_report_generation.js b/packages/kbn-test/src/mocha/junit_report_generation.js index 001fe79a38061..fff223b58cc7c 100644 --- a/packages/kbn-test/src/mocha/junit_report_generation.js +++ b/packages/kbn-test/src/mocha/junit_report_generation.js @@ -16,6 +16,7 @@ import { getUniqueJunitReportPath } from '../report_path'; import { getSnapshotOfRunnableLogs } from './log_cache'; import { escapeCdata } from '../..'; +import { prettifyCommandLine } from '../prettify_command_line'; const dateNow = Date.now.bind(Date); @@ -91,14 +92,25 @@ export function setupJUnitReportGeneration(runner, options = {}) { .filter((node) => node.pending || !results.find((result) => result.node === node)) .map((node) => ({ skipped: true, node })); - const builder = xmlBuilder.create( + const commandLine = prettifyCommandLine(process.argv); + + const root = xmlBuilder.create( 'testsuites', { encoding: 'utf-8' }, {}, { skipNullAttributes: true } ); - const testsuitesEl = builder.ele('testsuite', { + root.att({ + name: 'ftr', + time: getDuration(stats), + tests: allTests.length + failedHooks.length, + failures: failures.length, + skipped: skippedResults.length, + 'command-line': commandLine, + }); + + const testsuitesEl = root.ele('testsuite', { name: reportName, timestamp: new Date(stats.startTime).toISOString().slice(0, -5), time: getDuration(stats), @@ -106,6 +118,7 @@ export function setupJUnitReportGeneration(runner, options = {}) { failures: failures.length, skipped: skippedResults.length, 'metadata-json': JSON.stringify(metadata ?? {}), + 'command-line': commandLine, }); function addTestcaseEl(node) { @@ -134,7 +147,7 @@ export function setupJUnitReportGeneration(runner, options = {}) { }); const reportPath = getUniqueJunitReportPath(rootDirectory, reportName); - const reportXML = builder.end(); + const reportXML = root.end(); mkdirSync(dirname(reportPath), { recursive: true }); writeFileSync(reportPath, reportXML, 'utf8'); }); diff --git a/packages/kbn-test/src/mocha/junit_report_generation.test.js b/packages/kbn-test/src/mocha/junit_report_generation.test.js index ac23d91390ed9..26dd3dc28af7b 100644 --- a/packages/kbn-test/src/mocha/junit_report_generation.test.js +++ b/packages/kbn-test/src/mocha/junit_report_generation.test.js @@ -45,26 +45,25 @@ describe('dev/mocha/junit report generation', () => { // test case results are wrapped in expect(report).toEqual({ - testsuites: { + testsuites: expect.objectContaining({ testsuite: [report.testsuites.testsuite[0]], - }, + }), }); // the single element at the root contains summary data for all tests results const [testsuite] = report.testsuites.testsuite; expect(testsuite.$.time).toMatch(DURATION_REGEX); expect(testsuite.$.timestamp).toMatch(ISO_DATE_SEC_REGEX); - expect(testsuite).toEqual({ - $: { - failures: '2', - name: 'test', - skipped: '1', - tests: '4', - 'metadata-json': '{}', - time: testsuite.$.time, - timestamp: testsuite.$.timestamp, - }, - testcase: testsuite.testcase, + expect(testsuite.$).toEqual({ + 'command-line': + 'node scripts/jest --config=packages/kbn-test/jest.config.js --runInBand --coverage=false --passWithNoTests', + failures: '2', + name: 'test', + skipped: '1', + tests: '4', + 'metadata-json': '{}', + time: testsuite.$.time, + timestamp: testsuite.$.timestamp, }); // there are actually only three tests, but since the hook failed diff --git a/packages/kbn-test/src/prettify_command_line.ts b/packages/kbn-test/src/prettify_command_line.ts new file mode 100644 index 0000000000000..0f8f1eb75570b --- /dev/null +++ b/packages/kbn-test/src/prettify_command_line.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { execSync } from 'child_process'; +import * as path from 'path'; + +const kibanaRoot = execSync('git rev-parse --show-toplevel').toString().trim() || process.cwd(); + +export function prettifyCommandLine(args: string[]) { + let [executable, ...rest] = args; + if (executable.endsWith('node')) { + executable = 'node'; + } + rest = rest.map((arg) => path.relative(kibanaRoot, arg)); + + return [executable, ...rest].join(' '); +}