Skip to content

Commit

Permalink
[8.14] [CI] Display command on failure page (elastic#186999) (elastic…
Browse files Browse the repository at this point in the history
…#187351)

# Backport

This will backport the following commits from `main` to `8.14`:
- [[CI] Display command on failure page
(elastic#186999)](elastic#186999)

<!--- Backport version: 8.9.8 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Alex
Szabo","email":"[email protected]"},"sourceCommit":{"committedDate":"2024-07-02T09:45:46Z","message":"[CI]
Display command on failure page (elastic#186999)\n\n## Summary\r\nThis PR adds
the executed command line to the failures page.\r\nWe tweak the
reporters to export the executed command to the junit xmls,\r\nthen we
read those attributes after parsing the results.\r\n\r\nThe tests needed
some adjustment, because they're very brittle, and\r\ndon't seem to be
very accurate anymore.\r\n\r\n\r\nCloses:
https://github.com/elastic/kibana-operations/issues/127\r\n\r\nCheck out
the `[logs]` for the failed tests
here\r\n(ftr/jest/jest_integration):\r\nhttps://buildkite.com/elastic/kibana-pull-request/builds/218457","sha":"afec9eb0e2699ce24a3fa4d341433cda18372466","branchLabelMapping":{"^v8.15.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["Team:Operations","release_note:skip","backport:prev-minor","v8.15.0"],"number":186999,"url":"https://github.com/elastic/kibana/pull/186999","mergeCommit":{"message":"[CI]
Display command on failure page (elastic#186999)\n\n## Summary\r\nThis PR adds
the executed command line to the failures page.\r\nWe tweak the
reporters to export the executed command to the junit xmls,\r\nthen we
read those attributes after parsing the results.\r\n\r\nThe tests needed
some adjustment, because they're very brittle, and\r\ndon't seem to be
very accurate anymore.\r\n\r\n\r\nCloses:
https://github.com/elastic/kibana-operations/issues/127\r\n\r\nCheck out
the `[logs]` for the failed tests
here\r\n(ftr/jest/jest_integration):\r\nhttps://buildkite.com/elastic/kibana-pull-request/builds/218457","sha":"afec9eb0e2699ce24a3fa4d341433cda18372466"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v8.15.0","labelRegex":"^v8.15.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/186999","number":186999,"mergeCommit":{"message":"[CI]
Display command on failure page (elastic#186999)\n\n## Summary\r\nThis PR adds
the executed command line to the failures page.\r\nWe tweak the
reporters to export the executed command to the junit xmls,\r\nthen we
read those attributes after parsing the results.\r\n\r\nThe tests needed
some adjustment, because they're very brittle, and\r\ndon't seem to be
very accurate anymore.\r\n\r\n\r\nCloses:
https://github.com/elastic/kibana-operations/issues/127\r\n\r\nCheck out
the `[logs]` for the failed tests
here\r\n(ftr/jest/jest_integration):\r\nhttps://buildkite.com/elastic/kibana-pull-request/builds/218457","sha":"afec9eb0e2699ce24a3fa4d341433cda18372466"}}]}]
BACKPORT-->
  • Loading branch information
delanni authored Jul 2, 2024
1 parent 34a0100 commit c47097b
Show file tree
Hide file tree
Showing 12 changed files with 108 additions and 35 deletions.
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<testsuites>
<testsuite timestamp="2019-06-05T23:37:10" time="903.670" tests="129" failures="5" skipped="71">
<testsuites name="ftr" timestamp="2019-06-05T23:37:10" time="903.670" tests="129" failures="5" skipped="71" command-line="node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts">
<testsuite timestamp="2019-06-05T23:37:10" time="903.670" tests="129" failures="5" skipped="71" command-line="node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts">
<testcase name="maps app maps loaded from sample data ecommerce &quot;before all&quot; hook" classname="Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps/sample_data·js" time="154.378">
<system-out>
<![CDATA[[00:00:00] │
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<testsuites name="jest" timestamp="2019-06-07T03:36:23" time="781.292" tests="5487" skipped="9">
<testsuite name="x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts" timestamp="2019-06-07T03:42:21" time="14.504" tests="5" failures="1" skipped="0" file="/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts">
<testsuites name="jest" timestamp="2019-06-07T03:36:23" time="781.292" tests="5487" skipped="9" command-line="node scripts/jest --config some/jest/config.ts">
<testsuite name="x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts" timestamp="2019-06-07T03:42:21" time="14.504" tests="5" failures="1" skipped="0" file="/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts" command-line="node scripts/jest --config some/jest/config.ts">
<testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can start and end a process" time="1.316"/>
<testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can force kill the process if langServer can not exit" time="3.182"/>
<testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can reconnect if process died" time="7.060">
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<testsuites>
<testsuite timestamp="2019-06-13T23:29:36" time="30.739" tests="1444" failures="2" skipped="3">
<testsuites command-line="node scripts/functional_tests --config super-mocha-test.config.js">
<testsuite timestamp="2019-06-13T23:29:36" time="30.739" tests="1444" failures="2" skipped="3" command-line="node scripts/functional_tests --config super-mocha-test.config.js">
<testcase name="code in multiple nodes &quot;before all&quot; hook" classname="X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts" time="0.121">
<system-out>
<![CDATA[]]>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,8 @@ it('rewrites ftr reports with minimal changes', async () => {
+++ ftr.xml
@@ -1,53 +1,56 @@
‹?xml version="1.0" encoding="utf-8"?›
‹testsuites›
‹testsuite timestamp="2019-06-05T23:37:10" time="903.670" tests="129" failures="5" skipped="71"›
‹testsuites name="ftr" timestamp="2019-06-05T23:37:10" time="903.670" tests="129" failures="5" skipped="71" command-line="node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts"
‹testsuite timestamp="2019-06-05T23:37:10" time="903.670" tests="129" failures="5" skipped="71" command-line="node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts"
‹testcase name="maps app maps loaded from sample data ecommerce &quot;before all&quot; hook" classname="Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps/sample_data·js" time="154.378"›
- ‹system-out›
- ‹![CDATA[[00:00:00] │
Expand Down Expand Up @@ -155,7 +155,7 @@ it('rewrites jest reports with minimal changes', async () => {
--- jest.xml
+++ jest.xml
@@ -3,13 +3,17 @@
‹testsuite name="x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts" timestamp="2019-06-07T03:42:21" time="14.504" tests="5" failures="1" skipped="0" file="/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts"›
‹testsuite name="x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts" timestamp="2019-06-07T03:42:21" time="14.504" tests="5" failures="1" skipped="0" file="/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts" command-line="node scripts/jest --config some/jest/config.ts"
‹testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can start and end a process" time="1.316"/›
‹testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can force kill the process if langServer can not exit" time="3.182"/›
‹testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can reconnect if process died" time="7.060"›
Expand Down Expand Up @@ -203,8 +203,8 @@ it('rewrites mocha reports with minimal changes', async () => {
+++ mocha.xml
@@ -1,13 +1,16 @@
‹?xml version="1.0" encoding="utf-8"?›
‹testsuites›
‹testsuite timestamp="2019-06-13T23:29:36" time="30.739" tests="1444" failures="2" skipped="3"›
‹testsuites command-line="node scripts/functional_tests --config super-mocha-test.config.js"
‹testsuite timestamp="2019-06-13T23:29:36" time="30.739" tests="1444" failures="2" skipped="3" command-line="node scripts/functional_tests --config super-mocha-test.config.js"
‹testcase name="code in multiple nodes &quot;before all&quot; hook" classname="X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts" time="0.121"›
- ‹system-out›
- ‹![CDATA[]]›
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ it('discovers failures in ftr report', async () => {
Array [
Object {
"classname": "Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps/sample_data·js",
"commandLine": "node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts",
"failure": "
Error: retry.try timeout: TimeoutError: Waiting for element to be located By(css selector, [data-test-subj~=\\"layerTocActionsPanelToggleButtonRoad_Map_-_Bright\\"])
Wait timed out after 10055ms
Expand All @@ -37,6 +38,7 @@ it('discovers failures in ftr report', async () => {
},
Object {
"classname": "Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps",
"commandLine": "node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts",
"failure": "
{ NoSuchSessionError: This driver instance does not have a valid session ID (did you call WebDriver.quit()?) and may no longer be used.
at promise.finally (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/node_modules/selenium-webdriver/lib/webdriver.js:726:38)
Expand All @@ -56,6 +58,7 @@ it('discovers failures in ftr report', async () => {
},
Object {
"classname": "Firefox XPack UI Functional Tests.x-pack/test/functional/apps/machine_learning/anomaly_detection/saved_search_job·ts",
"commandLine": "node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts",
"failure": "{ NoSuchSessionError: Tried to run command without establishing a connection
at Object.throwDecodedError (/dev/shm/workspace/kibana/node_modules/selenium-webdriver/lib/error.js:550:15)
at parseHttpResponse (/dev/shm/workspace/kibana/node_modules/selenium-webdriver/lib/http.js:563:13)
Expand All @@ -76,6 +79,7 @@ it('discovers failures in jest report', async () => {
Array [
Object {
"classname": "X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp",
"commandLine": "node scripts/jest --config some/jest/config.ts",
"failure": "
TypeError: Cannot read property '0' of undefined
at Object.<anonymous>.test (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts:166:10)
Expand All @@ -95,6 +99,7 @@ it('discovers failures in mocha report', async () => {
Array [
Object {
"classname": "X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts",
"commandLine": "node scripts/functional_tests --config super-mocha-test.config.js",
"failure": "
Error: Unable to read artifact info from https://artifacts-api.elastic.co/v1/versions/8.0.0-SNAPSHOT/builds/latest/projects/elasticsearch: Service Temporarily Unavailable
<html>
Expand All @@ -117,6 +122,7 @@ it('discovers failures in mocha report', async () => {
},
Object {
"classname": "X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts",
"commandLine": "node scripts/functional_tests --config super-mocha-test.config.js",
"failure": "
TypeError: Cannot read property 'shutdown' of undefined
at Context.shutdown (plugins/code/server/__tests__/multi_node.ts:125:23)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ export type TestFailure = FailedTestCase['$'] & {
'system-out'?: string;
githubIssue?: string;
failureCount?: number;
commandLine?: string;
};

const getText = (node?: Array<string | { _: string }>) => {
Expand Down Expand Up @@ -71,19 +72,35 @@ const isLikelyIrrelevant = (name: string, failure: string) => {
export function getFailures(report: TestReport) {
const failures: TestFailure[] = [];

const commandLine = getCommandLineFromReport(report);

for (const testCase of makeFailedTestCaseIter(report)) {
const failure = getText(testCase.failure);
const likelyIrrelevant = isLikelyIrrelevant(testCase.$.name, failure);

failures.push({
const failureObj = {
// unwrap xml weirdness
...testCase.$,
// Strip ANSI color characters
failure,
likelyIrrelevant,
'system-out': getText(testCase['system-out']),
});
commandLine,
};

// cleaning up duplicates
delete failureObj['command-line'];

failures.push(failureObj);
}

return failures;
}

function getCommandLineFromReport(report: TestReport) {
if ('testsuites' in report) {
return report.testsuites?.testsuite?.[0]?.$['command-line'] || '';
} else {
return report.testsuite?.$['command-line'] || '';
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -170,14 +170,23 @@ export async function reportFailuresToFile(
<p><strong>${escape(failure.name)}</strong></p>
<p>
<small>
<strong>Failures in tracked branches</strong>: <span class="badge rounded-pill bg-danger">${
failure.failureCount || 0
}</span>
${
failure.commandLine
? `<div>
<strong>Command Line</strong>:
<pre>${escape(failure.commandLine)}</pre>
</div>`
: ''
}
<div>
<strong>Failures in tracked branches</strong>:
<span class="badge rounded-pill bg-danger">${failure.failureCount || 0}</span>
</div>
${
failure.githubIssue
? `<br /><a href="${escape(failure.githubIssue)}">${escape(
failure.githubIssue
)}</a>`
? `<div>
<a href="${escape(failure.githubIssue)}">${escape(failure.githubIssue)}</a>
</div>`
: ''
}
</small>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ export interface TestSuite {
skipped: string;
/* optional JSON encoded metadata */
'metadata-json'?: string;
/* the command that ran this suite */
'command-line'?: string;
};
testcase?: TestCase[];
}
Expand All @@ -51,6 +53,8 @@ export interface TestCase {
time: string;
/* optional JSON encoded metadata */
'metadata-json'?: string;
/* the command that ran this suite */
'command-line'?: string;
};
/* contents of system-out elements */
'system-out'?: Array<string | { _: string }>;
Expand Down
3 changes: 3 additions & 0 deletions packages/kbn-test/src/jest/junit_reporter/junit_reporter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import { AggregatedResult, Test, BaseReporter } from '@jest/reporters';

import { escapeCdata } from '../../mocha/xml';
import { getUniqueJunitReportPath } from '../../report_path';
import { prettifyCommandLine } from '../../prettify_command_line';

interface ReporterOptions {
reportName?: string;
Expand Down Expand Up @@ -71,6 +72,7 @@ export default class JestJUnitReporter extends BaseReporter {
tests: results.numTotalTests,
failures: results.numFailedTests,
skipped: results.numPendingTests,
'command-line': prettifyCommandLine(process.argv),
});

// top level test results are the files/suites
Expand All @@ -83,6 +85,7 @@ export default class JestJUnitReporter extends BaseReporter {
failures: suite.numFailingTests,
skipped: suite.numPendingTests,
file: suite.testFilePath,
'command-line': prettifyCommandLine(process.argv),
});

// nested in there are the tests in that file
Expand Down
19 changes: 16 additions & 3 deletions packages/kbn-test/src/mocha/junit_report_generation.js
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import { getUniqueJunitReportPath } from '../report_path';

import { getSnapshotOfRunnableLogs } from './log_cache';
import { escapeCdata } from '../..';
import { prettifyCommandLine } from '../prettify_command_line';

const dateNow = Date.now.bind(Date);

Expand Down Expand Up @@ -91,21 +92,33 @@ export function setupJUnitReportGeneration(runner, options = {}) {
.filter((node) => node.pending || !results.find((result) => result.node === node))
.map((node) => ({ skipped: true, node }));

const builder = xmlBuilder.create(
const commandLine = prettifyCommandLine(process.argv);

const root = xmlBuilder.create(
'testsuites',
{ encoding: 'utf-8' },
{},
{ skipNullAttributes: true }
);

const testsuitesEl = builder.ele('testsuite', {
root.att({
name: 'ftr',
time: getDuration(stats),
tests: allTests.length + failedHooks.length,
failures: failures.length,
skipped: skippedResults.length,
'command-line': commandLine,
});

const testsuitesEl = root.ele('testsuite', {
name: reportName,
timestamp: new Date(stats.startTime).toISOString().slice(0, -5),
time: getDuration(stats),
tests: allTests.length + failedHooks.length,
failures: failures.length,
skipped: skippedResults.length,
'metadata-json': JSON.stringify(metadata ?? {}),
'command-line': commandLine,
});

function addTestcaseEl(node) {
Expand Down Expand Up @@ -134,7 +147,7 @@ export function setupJUnitReportGeneration(runner, options = {}) {
});

const reportPath = getUniqueJunitReportPath(rootDirectory, reportName);
const reportXML = builder.end();
const reportXML = root.end();
mkdirSync(dirname(reportPath), { recursive: true });
writeFileSync(reportPath, reportXML, 'utf8');
});
Expand Down
25 changes: 12 additions & 13 deletions packages/kbn-test/src/mocha/junit_report_generation.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -45,26 +45,25 @@ describe('dev/mocha/junit report generation', () => {

// test case results are wrapped in <testsuites></testsuites>
expect(report).toEqual({
testsuites: {
testsuites: expect.objectContaining({
testsuite: [report.testsuites.testsuite[0]],
},
}),
});

// the single <testsuite> element at the root contains summary data for all tests results
const [testsuite] = report.testsuites.testsuite;
expect(testsuite.$.time).toMatch(DURATION_REGEX);
expect(testsuite.$.timestamp).toMatch(ISO_DATE_SEC_REGEX);
expect(testsuite).toEqual({
$: {
failures: '2',
name: 'test',
skipped: '1',
tests: '4',
'metadata-json': '{}',
time: testsuite.$.time,
timestamp: testsuite.$.timestamp,
},
testcase: testsuite.testcase,
expect(testsuite.$).toEqual({
'command-line':
'node scripts/jest --config=packages/kbn-test/jest.config.js --runInBand --coverage=false --passWithNoTests',
failures: '2',
name: 'test',
skipped: '1',
tests: '4',
'metadata-json': '{}',
time: testsuite.$.time,
timestamp: testsuite.$.timestamp,
});

// there are actually only three tests, but since the hook failed
Expand Down
22 changes: 22 additions & 0 deletions packages/kbn-test/src/prettify_command_line.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

import { execSync } from 'child_process';
import * as path from 'path';

const kibanaRoot = execSync('git rev-parse --show-toplevel').toString().trim() || process.cwd();

export function prettifyCommandLine(args: string[]) {
let [executable, ...rest] = args;
if (executable.endsWith('node')) {
executable = 'node';
}
rest = rest.map((arg) => path.relative(kibanaRoot, arg));

return [executable, ...rest].join(' ');
}

0 comments on commit c47097b

Please sign in to comment.