From fb2ecb203a2e0369ff0be2e7cec154327f0b0526 Mon Sep 17 00:00:00 2001 From: Kai Hao Date: Mon, 30 Aug 2021 15:16:18 +0800 Subject: [PATCH] Try reporting flaky tests to issues --- .github/report-flaky-tests/action.yml | 17 + .github/report-flaky-tests/index.js | 263 ++++++++++++++ .github/workflows/end2end-test.yml | 10 +- .github/workflows/flaky-tests.yml | 32 ++ package-lock.json | 323 ++++++++++++++++++ package.json | 2 + .../e2e-tests/config/flaky-tests-reporter.js | 94 +++++ .../e2e-tests/config/setup-test-framework.js | 7 + packages/e2e-tests/jest.config.js | 13 +- packages/e2e-tests/package.json | 1 + 10 files changed, 760 insertions(+), 2 deletions(-) create mode 100644 .github/report-flaky-tests/action.yml create mode 100644 .github/report-flaky-tests/index.js create mode 100644 .github/workflows/flaky-tests.yml create mode 100644 packages/e2e-tests/config/flaky-tests-reporter.js diff --git a/.github/report-flaky-tests/action.yml b/.github/report-flaky-tests/action.yml new file mode 100644 index 00000000000000..457889fc9ae783 --- /dev/null +++ b/.github/report-flaky-tests/action.yml @@ -0,0 +1,17 @@ +name: 'Report flaky tests' +description: 'Report flaky tests to GitHub issues' +inputs: + repo-token: + description: 'GitHub token' + required: true + label: + description: 'The flaky-test label name' + required: true + default: 'flaky-test' + artifact-name: + description: 'The name of the uploaded artifact' + required: true + default: 'flaky-tests-report' +runs: + using: 'node12' + main: 'index.js' diff --git a/.github/report-flaky-tests/index.js b/.github/report-flaky-tests/index.js new file mode 100644 index 00000000000000..38de95c81312d7 --- /dev/null +++ b/.github/report-flaky-tests/index.js @@ -0,0 +1,263 @@ +/** + * External dependencies + */ +const path = require( 'path' ); +const github = require( '@actions/github' ); +const core = require( '@actions/core' ); +const unzipper = require( 'unzipper' ); +const { formatResultsErrors } = require( 'jest-message-util' ); + +const TEST_RESULTS_LIST = { + open: ``, + close: ``, +}; +const TEST_RESULT = { + open: '', + close: '', +}; + +const metaData = { + render: ( json ) => ``, + get: ( str ) => { + const matched = str.match( // ); + if ( matched ) { + return JSON.parse( matched[ 1 ] ); + } + }, +}; + +( async function run() { + const token = core.getInput( 'repo-token', { required: true } ); + const label = core.getInput( 'label', { required: true } ); + const artifactName = core.getInput( 'artifact-name', { required: true } ); + + const octokit = github.getOctokit( token ); + + const flakyTests = await downloadReportFromArtifact( + octokit, + artifactName + ); + + if ( ! flakyTests ) { + return; + } + + const issues = await fetchAllIssuesLabeledFlaky( octokit, label ); + + for ( const flakyTest of flakyTests ) { + const { + title: testTitle, + path: testPath, + results: testResults, + } = flakyTest; + const issueTitle = getIssueTitle( testTitle ); + const reportedIssue = issues.find( + ( issue ) => issue.title === issueTitle + ); + const isTrunk = getHeadBranch() === 'trunk'; + let issue; + + if ( reportedIssue ) { + let body = reportedIssue.body; + const meta = metaData.get( body ); + + if ( isTrunk ) { + const headCommit = github.context.sha; + const baseCommit = meta.baseCommit || github.context.sha; + + try { + const { data } = await octokit.rest.repos.compareCommits( { + ...github.context.repo, + base: baseCommit, + head: headCommit, + per_page: 1, + } ); + + meta.failedTimes += testResults.length; + meta.totalCommits = data.total_commits + 1; + } catch ( err ) { + // It might be a deleted commit, + // treat the current commit as the base commit. + meta.baseCommit = headCommit; + meta.failedTimes = testResults.length; + meta.totalCommits = 1; + } + } + + // Reconstruct the body with the description + previous errors + new error. + body = + renderIssueDescription( { meta, testTitle, testPath } ) + + body.slice( + body.indexOf( TEST_RESULTS_LIST.open ), + body.indexOf( TEST_RESULTS_LIST.close ) + ) + + [ + renderTestErrorMessage( { testPath, testResults } ), + TEST_RESULTS_LIST.close, + ].join( '\n' ); + + const response = await octokit.rest.issues.update( { + ...github.context.repo, + issue_number: reportedIssue.number, + state: 'open', + body, + } ); + + issue = response.data; + } else { + const meta = isTrunk + ? { + failedTimes: testResults.length, + totalCommits: 1, + baseCommit: github.context.sha, + } + : { + failedTimes: 0, + totalCommits: 0, + }; + + const response = await octokit.rest.issues.create( { + ...github.context.repo, + title: issueTitle, + body: renderIssueBody( { + meta, + testTitle, + testPath, + testResults, + } ), + labels: [ label ], + // TODO: Maybe we can combine git blame to automatically assign issue to the original author? + } ); + + issue = response.data; + } + + core.info( `Reported flaky test to ${ issue.html_url }` ); + } +} )().catch( ( err ) => { + core.error( err ); +} ); + +async function fetchAllIssuesLabeledFlaky( octokit, label ) { + const issues = await octokit.paginate( 'GET /repos/{owner}/{repo}/issues', { + ...github.context.repo, + state: 'all', + labels: label, + } ); + + return issues; +} + +async function downloadReportFromArtifact( octokit, artifactName ) { + const { + data: { artifacts }, + } = await octokit.rest.actions.listWorkflowRunArtifacts( { + ...github.context.repo, + run_id: github.context.payload.workflow_run.id, + } ); + + const matchArtifact = artifacts.find( + ( artifact ) => artifact.name === artifactName + ); + + if ( ! matchArtifact ) { + // No flaky tests reported in this run. + return; + } + + const download = await octokit.rest.actions.downloadArtifact( { + ...github.context.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + } ); + + const { files } = await unzipper.Open.buffer( + Buffer.from( download.data ) + ); + const fileBuffers = await Promise.all( + files.map( ( file ) => file.buffer() ) + ); + const parsedFiles = fileBuffers.map( ( buffer ) => + JSON.parse( buffer.toString() ) + ); + + return parsedFiles; +} + +function getIssueTitle( testTitle ) { + return `[Flaky Test] ${ testTitle }`; +} + +function renderIssueBody( { meta, testTitle, testPath, testResults } ) { + return ( + renderIssueDescription( { meta, testTitle, testPath } ) + + renderTestResults( { testPath, testResults } ) + ); +} + +function renderIssueDescription( { meta, testTitle, testPath } ) { + return `${ metaData.render( meta ) } +**Flaky test detected. This is an auto-generated issue by GitHub Actions. Please do NOT edit this manually.** + +## Test title +${ testTitle } + +## Test path +\`${ testPath }\` + +## Flaky rate (_estimated_) +\`${ meta.failedTimes } / ${ meta.totalCommits + meta.failedTimes }\` runs + +## Errors +`; +} + +function renderTestResults( { testPath, testResults } ) { + return `${ TEST_RESULTS_LIST.open } +${ renderTestErrorMessage( { testPath, testResults } ) } +${ TEST_RESULTS_LIST.close } +`; +} + +function renderTestErrorMessage( { testPath, testResults } ) { + const date = new Date().toISOString(); + return `${ TEST_RESULT.open }
+ + + Test passed after ${ testResults.length } failed ${ + testResults.length === 0 ? 'attempt' : 'attempts' + } on ${ getHeadBranch() }. + + +\`\`\` +${ stripAnsi( + formatResultsErrors( + testResults, + { rootDir: path.join( process.cwd(), 'packages/e2e-tests' ) }, + {}, + testPath + ) +) } +\`\`\` +
${ TEST_RESULT.close }`; +} + +function getHeadBranch() { + return github.context.payload.workflow_run.head_branch; +} + +function getRunURL() { + return github.context.payload.workflow_run.html_url; +} + +function stripAnsi( string ) { + return string.replace( + new RegExp( + [ + '[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)', + '(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~]))', + ].join( '|' ), + 'g' + ) + ); +} diff --git a/.github/workflows/end2end-test.yml b/.github/workflows/end2end-test.yml index d42e10dc26098d..618d10f7db2b93 100644 --- a/.github/workflows/end2end-test.yml +++ b/.github/workflows/end2end-test.yml @@ -51,7 +51,15 @@ jobs: - name: Archive debug artifacts (screenshots, HTML snapshots) uses: actions/upload-artifact@e448a9b857ee2131e752b06002bf0e093c65e571 # v2.2.2 - if: always() + if: failure() with: name: failures-artifacts path: artifacts + + - name: Archive flaky tests report + uses: actions/upload-artifact@e448a9b857ee2131e752b06002bf0e093c65e571 # v2.2.2 + if: always() + with: + name: flaky-tests-report + path: flaky-tests + if-no-files-found: ignore diff --git a/.github/workflows/flaky-tests.yml b/.github/workflows/flaky-tests.yml new file mode 100644 index 00000000000000..4b7ab77379aeb1 --- /dev/null +++ b/.github/workflows/flaky-tests.yml @@ -0,0 +1,32 @@ +name: Report Flaky Tests + +on: + workflow_run: + workflows: ['End-to-End Tests'] + types: + - completed + +jobs: + report-to-issues: + name: Report to GitHub issues + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'success' }} + steps: + - uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f # v2.3.4 + + - name: Use desired version of NodeJS + uses: actions/setup-node@38d90ce44d5275ad62cc48384b3d8a58c500bb5f # v2.2.2 + with: + node-version: 14 + cache: npm + + - name: Npm install + run: | + npm ci + + - name: Report flaky tests + uses: ./.github/report-flaky-tests + with: + repo-token: '${{ secrets.GITHUB_TOKEN }}' + label: flaky-test + artifact-name: flaky-tests-report diff --git a/package-lock.json b/package-lock.json index 5e6609560bfb52..6ae24cd1e82162 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16802,6 +16802,12 @@ "integrity": "sha512-hkgzYF+qnIl8uTO8rmUSVSfQ8BIfMXC4yJAF4n8BE758YsKBZvFC4NumnAegj7KmylP0liEZNpb9RRGFMbFejA==", "dev": true }, + "@types/stack-utils": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.1.tgz", + "integrity": "sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw==", + "dev": true + }, "@types/tapable": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/@types/tapable/-/tapable-1.0.5.tgz", @@ -18397,6 +18403,7 @@ "@wordpress/url": "file:packages/url", "chalk": "^4.0.0", "expect-puppeteer": "^4.4.0", + "jest-message-util": "^27.0.6", "lodash": "^4.17.21", "puppeteer-testing-library": "^0.5.0", "uuid": "^8.3.0" @@ -28870,6 +28877,16 @@ "integrity": "sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ==", "dev": true }, + "binary": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", + "integrity": "sha1-n2BVO8XOjDOG87VTz/R0Yq3sqnk=", + "dev": true, + "requires": { + "buffers": "~0.1.1", + "chainsaw": "~0.1.0" + } + }, "binary-extensions": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", @@ -29409,12 +29426,24 @@ "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.0.tgz", "integrity": "sha512-c5mRlguI/Pe2dSZmpER62rSCu0ryKmWddzRYsuXc50U2/g8jMOulc31VZMa4mYx31U5xsmSOpDCgH88Vl9cDGQ==" }, + "buffer-indexof-polyfill": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.2.tgz", + "integrity": "sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A==", + "dev": true + }, "buffer-xor": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz", "integrity": "sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk=", "dev": true }, + "buffers": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", + "integrity": "sha1-skV5w77U1tOWru5tmorn9Ugqt7s=", + "dev": true + }, "builtin-modules": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz", @@ -29939,6 +29968,23 @@ "integrity": "sha512-Jt9tIBkRc9POUof7QA/VwWd+58fKkEEfI+/t1/eOlxKM7ZhrczNzMFefge7Ai+39y1pR/pP6cI19guHy3FSLmw==", "dev": true }, + "chainsaw": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", + "integrity": "sha1-XqtQsor+WAdNDVgpE4iCi15fvJg=", + "dev": true, + "requires": { + "traverse": ">=0.3.0 <0.4" + }, + "dependencies": { + "traverse": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", + "integrity": "sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk=", + "dev": true + } + } + }, "chalk": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.1.tgz", @@ -33237,6 +33283,15 @@ "integrity": "sha1-rOb/gIwc5mtX0ev5eXessCM0z8E=", "dev": true }, + "duplexer2": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", + "integrity": "sha1-ixLauHjA1p4+eJEFFmKjL8a93ME=", + "dev": true, + "requires": { + "readable-stream": "^2.0.2" + } + }, "duplexer3": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz", @@ -36925,6 +36980,43 @@ "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", "optional": true }, + "fstream": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.12.tgz", + "integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.2", + "inherits": "~2.0.0", + "mkdirp": ">=0.5 0", + "rimraf": "2" + }, + "dependencies": { + "glob": { + "version": "7.1.7", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz", + "integrity": "sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "dev": true, + "requires": { + "glob": "^7.1.3" + } + } + } + }, "function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", @@ -42202,6 +42294,182 @@ } } }, + "jest-message-util": { + "version": "27.0.6", + "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-27.0.6.tgz", + "integrity": "sha512-rBxIs2XK7rGy+zGxgi+UJKP6WqQ+KrBbD1YMj517HYN3v2BG66t3Xan3FWqYHKZwjdB700KiAJ+iES9a0M+ixw==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.12.13", + "@jest/types": "^27.0.6", + "@types/stack-utils": "^2.0.0", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.4", + "micromatch": "^4.0.4", + "pretty-format": "^27.0.6", + "slash": "^3.0.0", + "stack-utils": "^2.0.3" + }, + "dependencies": { + "@babel/code-frame": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.14.5.tgz", + "integrity": "sha512-9pzDqyc6OLDaqe+zbACgFkb6fKMNG6CObKpnYXChRsvYGyEdc7CA2BaqeOM+vOtCS5ndmJicPJhKAwYRI6UfFw==", + "dev": true, + "requires": { + "@babel/highlight": "^7.14.5" + } + }, + "@babel/helper-validator-identifier": { + "version": "7.14.9", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.14.9.tgz", + "integrity": "sha512-pQYxPY0UP6IHISRitNe8bsijHex4TWZXi2HwKVsjPiltzlhse2znVcm9Ace510VT1kxIHjGJCZZQBX2gJDbo0g==", + "dev": true + }, + "@babel/highlight": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.14.5.tgz", + "integrity": "sha512-qf9u2WFWVV0MppaL877j2dBtQIDgmidgjGk5VIMw3OadXvYaXn66U1BFlH2t4+t3i+8PhedppRv+i40ABzd+gg==", + "dev": true, + "requires": { + "@babel/helper-validator-identifier": "^7.14.5", + "chalk": "^2.0.0", + "js-tokens": "^4.0.0" + }, + "dependencies": { + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + } + } + }, + "@jest/types": { + "version": "27.0.6", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-27.0.6.tgz", + "integrity": "sha512-aSquT1qa9Pik26JK5/3rvnYb4bGtm1VFNesHKmNTwmPIgOrixvhL2ghIvFRNEpzy3gU+rUgjIF/KodbkFAl++g==", + "dev": true, + "requires": { + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^16.0.0", + "chalk": "^4.0.0" + } + }, + "@types/istanbul-reports": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.1.tgz", + "integrity": "sha512-c3mAZEuK0lvBp8tmuL74XRKn1+y2dcwOUpH7x4WrF6gk1GIgiluDRgMYQtw2OFcBvAJWlt6ASU3tSqxp0Uu0Aw==", + "dev": true, + "requires": { + "@types/istanbul-lib-report": "*" + } + }, + "@types/yargs": { + "version": "16.0.4", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-16.0.4.tgz", + "integrity": "sha512-T8Yc9wt/5LbJyCaLiHPReJa0kApcIgJ7Bn735GjItUfh08Z1pJvu8QZqb9s+mMvKV6WUQRV7K2R46YbjMXTTJw==", + "dev": true, + "requires": { + "@types/yargs-parser": "*" + } + }, + "ansi-regex": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "dev": true + }, + "braces": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "dev": true, + "requires": { + "fill-range": "^7.0.1" + } + }, + "fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "dev": true, + "requires": { + "to-regex-range": "^5.0.1" + } + }, + "graceful-fs": { + "version": "4.2.8", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.8.tgz", + "integrity": "sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg==", + "dev": true + }, + "is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true + }, + "micromatch": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.4.tgz", + "integrity": "sha512-pRmzw/XUcwXGpD9aI9q/0XOwLNygjETJ8y0ao0wdqprrzDa4YnxLcz7fQRZr8voh8V10kGhABbNcHVk5wHgWwg==", + "dev": true, + "requires": { + "braces": "^3.0.1", + "picomatch": "^2.2.3" + } + }, + "picomatch": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.0.tgz", + "integrity": "sha512-lY1Q/PiJGC2zOv/z391WOTD+Z02bCgsFfvxoXXf6h7kv9o+WmsmzYqrAwY63sNgOxE4xEdq0WyUnXfKeBrSvYw==", + "dev": true + }, + "pretty-format": { + "version": "27.0.6", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-27.0.6.tgz", + "integrity": "sha512-8tGD7gBIENgzqA+UBzObyWqQ5B778VIFZA/S66cclyd5YkFLYs2Js7gxDKf0MXtTc9zcS7t1xhdfcElJ3YIvkQ==", + "dev": true, + "requires": { + "@jest/types": "^27.0.6", + "ansi-regex": "^5.0.0", + "ansi-styles": "^5.0.0", + "react-is": "^17.0.1" + }, + "dependencies": { + "ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true + } + } + }, + "react-is": { + "version": "17.0.2", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", + "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==", + "dev": true + }, + "to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "requires": { + "is-number": "^7.0.0" + } + } + } + }, "jest-pnp-resolver": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.2.tgz", @@ -45152,6 +45420,12 @@ } } }, + "listenercount": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/listenercount/-/listenercount-1.0.1.tgz", + "integrity": "sha1-hMinKrWcRyUyFIDJdeZQg0LnCTc=", + "dev": true + }, "listr": { "version": "0.14.3", "resolved": "https://registry.npmjs.org/listr/-/listr-0.14.3.tgz", @@ -56561,6 +56835,23 @@ "integrity": "sha512-ji9qxRnOVfcuLDySj9qzhGSEFVobyt1kIOSkj1qZzYLzq7Tos/oUUWvotUPQLlrsidqsK6tBH89Bc9kL5zHA6w==", "dev": true }, + "stack-utils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.3.tgz", + "integrity": "sha512-gL//fkxfWUsIlFL2Tl42Cl6+HFALEaB1FU76I/Fy+oZjRreP7OPMXFlGbxM7NQsI0ZpUfw76sHnv0WNYuTb7Iw==", + "dev": true, + "requires": { + "escape-string-regexp": "^2.0.0" + }, + "dependencies": { + "escape-string-regexp": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", + "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==", + "dev": true + } + } + }, "stackframe": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/stackframe/-/stackframe-1.2.0.tgz", @@ -59827,6 +60118,38 @@ } } }, + "unzipper": { + "version": "0.10.11", + "resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.10.11.tgz", + "integrity": "sha512-+BrAq2oFqWod5IESRjL3S8baohbevGcVA+teAIOYWM3pDVdseogqbzhhvvmiyQrUNKFUnDMtELW3X8ykbyDCJw==", + "dev": true, + "requires": { + "big-integer": "^1.6.17", + "binary": "~0.3.0", + "bluebird": "~3.4.1", + "buffer-indexof-polyfill": "~1.0.0", + "duplexer2": "~0.1.4", + "fstream": "^1.0.12", + "graceful-fs": "^4.2.2", + "listenercount": "~1.0.1", + "readable-stream": "~2.3.6", + "setimmediate": "~1.0.4" + }, + "dependencies": { + "bluebird": { + "version": "3.4.7", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz", + "integrity": "sha1-9y12C+Cbf3bQjtj66Ysomo0F+rM=", + "dev": true + }, + "graceful-fs": { + "version": "4.2.8", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.8.tgz", + "integrity": "sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg==", + "dev": true + } + } + }, "upath": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/upath/-/upath-1.2.0.tgz", diff --git a/package.json b/package.json index 5aa265595e143b..e1d52a79f5548d 100644 --- a/package.json +++ b/package.json @@ -177,6 +177,7 @@ "inquirer": "7.1.0", "jest": "26.6.3", "jest-junit": "11.0.0", + "jest-message-util": "27.0.6", "jest-serializer-enzyme": "1.0.0", "jest-watch-typeahead": "0.6.1", "jsdom": "16.4.0", @@ -211,6 +212,7 @@ "terser-webpack-plugin": "5.1.4", "typescript": "4.1.3", "uglify-js": "3.13.7", + "unzipper": "0.10.11", "uuid": "8.3.0", "wd": "1.12.1", "webpack": "5.47.1", diff --git a/packages/e2e-tests/config/flaky-tests-reporter.js b/packages/e2e-tests/config/flaky-tests-reporter.js new file mode 100644 index 00000000000000..65b688ba0ac015 --- /dev/null +++ b/packages/e2e-tests/config/flaky-tests-reporter.js @@ -0,0 +1,94 @@ +/** + * A **flaky** test is defined as a test which passed after auto-retrying. + * - By default, all tests run once if they pass. + * - If a test fails, it will automatically re-run at most 2 times. + * - If it pass after retrying (below 2 times), then it's marked as **flaky** + * but displayed as **passed** in the original test suite. + * - If it fail all 3 times, then it's a **failed** test. + */ +/** + * External dependencies + */ +const fs = require( 'fs' ).promises; +const path = require( 'path' ); +const { formatResultsErrors } = require( 'jest-message-util' ); + +class FlakyTestsReporter { + constructor( globalConfig, options ) { + this._globalConfig = globalConfig; + this._options = options; + + this.failingTestCaseResults = new Map(); + } + + async onRunStart() { + try { + fs.mkdir( 'flaky-tests' ); + } catch ( err ) { + // Ignore the error if the directory already exists. + if ( err.code !== 'EEXIST' ) { + throw err; + } + } + } + + async onTestCaseResult( test, testCaseResult ) { + const testPath = path.relative( this._globalConfig.rootDir, test.path ); + const testTitle = testCaseResult.title; + + switch ( testCaseResult.status ) { + case 'failed': { + if ( ! this.failingTestCaseResults.has( testTitle ) ) { + this.failingTestCaseResults.set( testTitle, [] ); + } + this.failingTestCaseResults + .get( testTitle ) + .push( testCaseResult ); + break; + } + case 'passed': { + if ( this.failingTestCaseResults.has( testTitle ) ) { + const failingResults = this.failingTestCaseResults.get( + testTitle + ); + + await fs.writeFile( + `flaky-tests/${ testTitle }.json`, + JSON.stringify( { + title: testTitle, + path: testPath, + results: failingResults, + } ), + 'utf-8' + ); + + // Don't silence flaky error messages for debugging reason. + // eslint-disable-next-line no-console + console.error( + `Test passed after ${ failingResults.length } failed ${ + failingResults.length === 1 ? 'attempt' : 'attempts' + }:` + ); + // eslint-disable-next-line no-console + console.error( + formatResultsErrors( + failingResults, + this._globalConfig, + {}, + test.path + ) + ); + } + break; + } + default: + break; + } + } + + onRunComplete() { + this.failingTestCaseResults.clear(); + } +} + +module.exports = FlakyTestsReporter; diff --git a/packages/e2e-tests/config/setup-test-framework.js b/packages/e2e-tests/config/setup-test-framework.js index c7c3fe750c7c29..94bc54a8775760 100644 --- a/packages/e2e-tests/config/setup-test-framework.js +++ b/packages/e2e-tests/config/setup-test-framework.js @@ -68,6 +68,13 @@ const pageEvents = []; // The Jest timeout is increased because these tests are a bit slow jest.setTimeout( PUPPETEER_TIMEOUT || 100000 ); +// Retry failed tests at most 2 times in CI. +// This enables `flaky-tests-reporter` and `report-flaky-tests` GitHub action +// to mark test as flaky and automatically create a tracking issue about it. +if ( process.env.CI && process.env.GITHUB_EVENT_NAME !== 'pull_request' ) { + jest.retryTimes( 2 ); +} + async function setupBrowser() { await clearLocalStorage(); await setBrowserViewport( 'large' ); diff --git a/packages/e2e-tests/jest.config.js b/packages/e2e-tests/jest.config.js index 9311a0d621337c..77d312fe259f2c 100644 --- a/packages/e2e-tests/jest.config.js +++ b/packages/e2e-tests/jest.config.js @@ -1,5 +1,10 @@ +/** + * WordPress dependencies + */ +const baseConfig = require( '@wordpress/scripts/config/jest-e2e.config' ); + module.exports = { - ...require( '@wordpress/scripts/config/jest-e2e.config' ), + ...baseConfig, setupFiles: [ '/config/gutenberg-phase.js' ], setupFilesAfterEnv: [ '/config/setup-test-framework.js', @@ -12,4 +17,10 @@ module.exports = { '/node_modules/', 'e2e-tests/specs/performance/', ], + reporters: [ + ...baseConfig.reporters, + process.env.CI && + process.env.GITHUB_EVENT_NAME !== 'pull_request' && + '/config/flaky-tests-reporter.js', + ].filter( Boolean ), }; diff --git a/packages/e2e-tests/package.json b/packages/e2e-tests/package.json index 479f2331230e76..e19b013f0f4950 100644 --- a/packages/e2e-tests/package.json +++ b/packages/e2e-tests/package.json @@ -30,6 +30,7 @@ "@wordpress/url": "file:../url", "chalk": "^4.0.0", "expect-puppeteer": "^4.4.0", + "jest-message-util": "^27.0.6", "lodash": "^4.17.21", "puppeteer-testing-library": "^0.5.0", "uuid": "^8.3.0"