Skip to content

Commit

Permalink
Refactor performance tests artifacts handling (#48684)
Browse files Browse the repository at this point in the history
  • Loading branch information
@WunderBart
WunderBart authored Mar 28, 2023
1 parent 65cd782 commit 2cf161f
Show file tree
Hide file tree
Showing 12 changed files with 161 additions and 147 deletions.
20 changes: 11 additions & 9 deletions .github/workflows/performance.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ jobs:
name: Run performance tests
runs-on: ubuntu-latest
if: ${{ github.repository == 'WordPress/gutenberg' }}
env:
WP_ARTIFACTS_PATH: ${{ github.workspace }}/artifacts

steps:
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
Expand All @@ -39,13 +41,6 @@ jobs:
if: github.event_name == 'pull_request'
run: ./bin/plugin/cli.js perf $GITHUB_SHA trunk --tests-branch $GITHUB_SHA

- name: Store performance measurements
if: github.event_name == 'pull_request'
uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # v3.1.1
with:
name: perf-test-results
path: ./__test-results/*.json

- name: Compare performance with current WordPress Core and previous Gutenberg versions
if: github.event_name == 'release'
env:
Expand Down Expand Up @@ -80,6 +75,13 @@ jobs:
run: |
./bin/plugin/cli.js perf $(echo $BRANCHES | tr ',' ' ') --tests-branch $GITHUB_SHA --wp-version "$WP_VERSION"
- name: Archive performance results
if: success()
uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # v3.1.1
with:
name: performance-results
path: ${{ env.WP_ARTIFACTS_PATH }}/*.performance-results.json

- name: Publish performance results
if: github.event_name == 'push'
env:
Expand All @@ -90,8 +92,8 @@ jobs:
- name: Archive debug artifacts (screenshots, HTML snapshots)
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
if: always()
if: failure()
with:
name: failures-artifacts
path: ./__test-results/artifacts
path: ${{ env.WP_ARTIFACTS_PATH }}
if-no-files-found: ignore
11 changes: 7 additions & 4 deletions bin/log-performance-results.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,22 +10,25 @@ const [ token, branch, hash, baseHash, timestamp ] = process.argv.slice( 2 );

const resultsFiles = [
{
file: 'post-editor-performance-results.json',
file: 'post-editor.performance-results.json',
metricsPrefix: '',
},
{
file: 'front-end-block-theme-performance-results.json',
file: 'front-end-block-theme.performance-results.json',
metricsPrefix: 'block-theme-',
},
{
file: 'front-end-classic-theme-performance-results.json',
file: 'front-end-classic-theme.performance-results.json',
metricsPrefix: 'classic-theme-',
},
];

const performanceResults = resultsFiles.map( ( { file } ) =>
JSON.parse(
fs.readFileSync( path.join( __dirname, '../' + file ), 'utf8' )
fs.readFileSync(
path.join( process.env.WP_ARTIFACTS_PATH, file ),
'utf8'
)
)
);

Expand Down
113 changes: 66 additions & 47 deletions bin/plugin/commands/performance.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@ const {
} = require( '../lib/utils' );
const config = require( '../config' );

const ARTIFACTS_PATH =
process.env.WP_ARTIFACTS_PATH || path.join( process.cwd(), 'artifacts' );

/**
* @typedef WPPerformanceCommandOptions
*
Expand Down Expand Up @@ -193,38 +196,22 @@ function curateResults( testSuite, results ) {
* @return {Promise<WPPerformanceResults>} Performance results for the branch.
*/
async function runTestSuite( testSuite, performanceTestDirectory, runKey ) {
try {
await runShellScript(
`npm run test:performance -- packages/e2e-tests/specs/performance/${ testSuite }.test.js`,
performanceTestDirectory
);
} catch ( error ) {
fs.mkdirSync( './__test-results/artifacts', { recursive: true } );
const artifactsFolder = path.join(
performanceTestDirectory,
'artifacts/'
);
await runShellScript(
'cp -Rv ' + artifactsFolder + ' ' + './__test-results/artifacts/'
);

throw error;
}
const resultsFilename = `${ runKey }.performance-results.json`;

const resultsFile = path.join(
await runShellScript(
`npm run test:performance -- ${ testSuite }`,
performanceTestDirectory,
`packages/e2e-tests/specs/performance/${ testSuite }.test.results.json`
);
fs.mkdirSync( './__test-results', { recursive: true } );
fs.copyFileSync( resultsFile, `./__test-results/${ runKey }.results.json` );
const rawResults = await readJSONFile(
path.join(
performanceTestDirectory,
`packages/e2e-tests/specs/performance/${ testSuite }.test.results.json`
)
{
...process.env,
WP_ARTIFACTS_PATH: ARTIFACTS_PATH,
RESULTS_FILENAME: resultsFilename,
}
);

return curateResults( testSuite, rawResults );
return curateResults(
testSuite,
await readJSONFile( path.join( ARTIFACTS_PATH, resultsFilename ) )
);
}

/**
Expand Down Expand Up @@ -339,15 +326,41 @@ async function runPerformanceTests( branches, options ) {
buildPath
);

await runShellScript(
'cp ' +
path.resolve(
performanceTestDirectory,
'bin/plugin/utils/.wp-env.performance.json'
) +
' ' +
environmentDirectory +
'/.wp-env.json'
// Create the config file for the current env.
fs.writeFileSync(
path.join( environmentDirectory, '.wp-env.json' ),
JSON.stringify(
{
core: 'WordPress/WordPress',
plugins: [ path.join( environmentDirectory, 'plugin' ) ],
themes: [
path.join(
performanceTestDirectory,
'test/emptytheme'
),
'https://downloads.wordpress.org/theme/twentytwentyone.1.7.zip',
'https://downloads.wordpress.org/theme/twentytwentythree.1.0.zip',
],
env: {
tests: {
mappings: {
'wp-content/mu-plugins': path.join(
performanceTestDirectory,
'packages/e2e-tests/mu-plugins'
),
'wp-content/plugins/gutenberg-test-plugins':
path.join(
performanceTestDirectory,
'packages/e2e-tests/plugins'
),
},
},
},
},
null,
2
),
'utf8'
);

if ( options.wpVersion ) {
Expand Down Expand Up @@ -404,33 +417,39 @@ async function runPerformanceTests( branches, options ) {

/** @type {Record<string,Record<string, WPPerformanceResults>>} */
const results = {};
const wpEnvPath = path.join(
performanceTestDirectory,
'node_modules/.bin/wp-env'
);

for ( const testSuite of testSuites ) {
results[ testSuite ] = {};
/** @type {Array<Record<string, WPPerformanceResults>>} */
const rawResults = [];
// Alternate three times between branches.
for ( let i = 0; i < TEST_ROUNDS; i++ ) {
const roundInfo = `round ${ i + 1 } of ${ TEST_ROUNDS }`;
log( ` >> Suite: ${ testSuite } (${ roundInfo })` );
rawResults[ i ] = {};
for ( const branch of branches ) {
const sanitizedBranch = sanitizeBranchName( branch );
const runKey = `${ sanitizedBranch }_${ testSuite }_run-${ i }`;
const runKey = `${ testSuite }_${ sanitizedBranch }_run-${ i }`;
// @ts-ignore
const environmentDirectory = branchDirectories[ branch ];
log( ` >> Branch: ${ branch }, Suite: ${ testSuite }` );
log( ' >> Starting the environment.' );
log( ` >> Branch: ${ branch }` );
log( ' >> Starting the environment.' );
await runShellScript(
'../../tests/node_modules/.bin/wp-env start',
`${ wpEnvPath } start`,
environmentDirectory
);
log( ' >> Running the test.' );
log( ' >> Running the test.' );
rawResults[ i ][ branch ] = await runTestSuite(
testSuite,
performanceTestDirectory,
runKey
);
log( ' >> Stopping the environment' );
log( ' >> Stopping the environment' );
await runShellScript(
'../../tests/node_modules/.bin/wp-env stop',
`${ wpEnvPath } stop`,
environmentDirectory
);
}
Expand Down Expand Up @@ -493,9 +512,9 @@ async function runPerformanceTests( branches, options ) {
);
console.table( invertedResult );

const resultsFilename = testSuite + '-performance-results.json';
const resultsFilename = testSuite + '.performance-results.json';
fs.writeFileSync(
path.resolve( __dirname, '../../../', resultsFilename ),
path.join( ARTIFACTS_PATH, resultsFilename ),
JSON.stringify( results[ testSuite ], null, 2 )
);
}
Expand Down
17 changes: 0 additions & 17 deletions bin/plugin/utils/.wp-env.performance.json
View file

This file was deleted.

6 changes: 4 additions & 2 deletions packages/e2e-tests/config/performance-reporter.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,11 @@ const success = chalk.bold.green;

class PerformanceReporter {
onTestResult( test ) {
const dirname = path.dirname( test.path );
const basename = path.basename( test.path, '.js' );
const filepath = path.join( dirname, basename + '.results.json' );
const filepath = path.join(
process.env.WP_ARTIFACTS_PATH,
basename + '.performance-results.json'
);

if ( ! existsSync( filepath ) ) {
return;
Expand Down
15 changes: 5 additions & 10 deletions packages/e2e-tests/specs/performance/front-end-block-theme.test.js
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
/**
* External dependencies
* WordPress dependencies
*/
import { basename, join } from 'path';
import { writeFileSync } from 'fs';
import { activateTheme, createURL, logout } from '@wordpress/e2e-test-utils';

/**
* WordPress dependencies
* Internal dependencies
*/
import { activateTheme, createURL, logout } from '@wordpress/e2e-test-utils';
import { saveResultsFile } from './utils';

describe( 'Front End Performance', () => {
const results = {
Expand All @@ -22,12 +21,8 @@ describe( 'Front End Performance', () => {
} );

afterAll( async () => {
saveResultsFile( __filename, results );
await activateTheme( 'twentytwentyone' );
const resultsFilename = basename( __filename, '.js' ) + '.results.json';
writeFileSync(
join( __dirname, resultsFilename ),
JSON.stringify( results, null, 2 )
);
} );

it( 'Report TTFB, LCP, and LCP-TTFB', async () => {
Expand Down
15 changes: 5 additions & 10 deletions packages/e2e-tests/specs/performance/front-end-classic-theme.test.js
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
/**
* External dependencies
* WordPress dependencies
*/
import { basename, join } from 'path';
import { writeFileSync } from 'fs';
import { activateTheme, createURL, logout } from '@wordpress/e2e-test-utils';

/**
* WordPress dependencies
* Internal dependencies
*/
import { activateTheme, createURL, logout } from '@wordpress/e2e-test-utils';
import { saveResultsFile } from './utils';

describe( 'Front End Performance', () => {
const results = {
Expand All @@ -22,11 +21,7 @@ describe( 'Front End Performance', () => {
} );

afterAll( async () => {
const resultsFilename = basename( __filename, '.js' ) + '.results.json';
writeFileSync(
join( __dirname, resultsFilename ),
JSON.stringify( results, null, 2 )
);
saveResultsFile( __filename, results );
} );

it( 'Report TTFB, LCP, and LCP-TTFB', async () => {
Expand Down
Loading

1 comment on commit 2cf161f

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Flaky tests detected in 2cf161f.
Some tests passed with failed attempts. The failures may not be related to this commit but are still reported for visibility. See the documentation for more information.

🔍 Workflow run URL: https://github.com/WordPress/gutenberg/actions/runs/4543958902
📝 Reported issues:

Please sign in to comment.