Skip to content

Commit

Permalink
Make the performance CI job more stable (#33710)
Browse files Browse the repository at this point in the history
  • Loading branch information
@youknowriad
youknowriad authored Jul 27, 2021
1 parent d8fa500 commit 2166961
Show file tree
Hide file tree
Showing 2 changed files with 168 additions and 113 deletions.
253 changes: 148 additions & 105 deletions bin/plugin/commands/performance.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,12 +128,12 @@ async function setUpGitBranch( branch, environmentDirectory ) {
// changes after install).
await git.discardLocalChanges( environmentDirectory );

log( '>> Fetching the ' + formats.success( branch ) + ' branch' );
log( ' >> Fetching the ' + formats.success( branch ) + ' branch' );
await git.checkoutRemoteBranch( environmentDirectory, branch );

log( '>> Building the ' + formats.success( branch ) + ' branch' );
log( ' >> Building the ' + formats.success( branch ) + ' branch' );
await runShellScript(
'rm -rf node_modules packages/*/node_modules && npm install && npm run build',
'npm install && npm run build',
environmentDirectory
);
}
Expand All @@ -147,42 +147,17 @@ async function setUpGitBranch( branch, environmentDirectory ) {
* @return {Promise<WPPerformanceResults>} Performance results for the branch.
*/
async function runTestSuite( testSuite, performanceTestDirectory ) {
const results = [];
for ( let i = 0; i < 3; i++ ) {
await runShellScript(
`npm run test-performance -- packages/e2e-tests/specs/performance/${ testSuite }.test.js`,
performanceTestDirectory
);
const rawResults = await readJSONFile(
path.join(
performanceTestDirectory,
`packages/e2e-tests/specs/performance/${ testSuite }.test.results.json`
)
);
results.push( curateResults( rawResults ) );
}

const medians = mapValues(
{
load: results.map( ( r ) => r.load ),
type: results.map( ( r ) => r.type ),
minType: results.map( ( r ) => r.minType ),
maxType: results.map( ( r ) => r.maxType ),
focus: results.map( ( r ) => r.focus ),
minFocus: results.map( ( r ) => r.minFocus ),
maxFocus: results.map( ( r ) => r.maxFocus ),
inserterOpen: results.map( ( r ) => r.inserterOpen ),
minInserterOpen: results.map( ( r ) => r.minInserterOpen ),
maxInserterOpen: results.map( ( r ) => r.maxInserterOpen ),
inserterHover: results.map( ( r ) => r.inserterHover ),
minInserterHover: results.map( ( r ) => r.minInserterHover ),
maxInserterHover: results.map( ( r ) => r.maxInserterHover ),
},
median
await runShellScript(
`npm run test-performance -- packages/e2e-tests/specs/performance/${ testSuite }.test.js`,
performanceTestDirectory
);

// Format results as times.
return mapValues( medians, formatTime );
const rawResults = await readJSONFile(
path.join(
performanceTestDirectory,
`packages/e2e-tests/specs/performance/${ testSuite }.test.results.json`
)
);
return curateResults( rawResults );
}

/**
Expand All @@ -198,8 +173,8 @@ async function runPerformanceTests( branches, options ) {
}

log(
formats.title( '\n💃 Performance Tests 🕺\n\n' ),
'Welcome! This tool runs the performance tests on multiple branches and displays a comparison table.\n' +
formats.title( '\n💃 Performance Tests 🕺\n' ),
'\nWelcome! This tool runs the performance tests on multiple branches and displays a comparison table.\n' +
'In order to run the tests, the tool is going to load a WordPress environment on 8888 and 8889 ports.\n' +
'Make sure these ports are not used before continuing.\n'
);
Expand All @@ -208,12 +183,17 @@ async function runPerformanceTests( branches, options ) {
await askForConfirmation( 'Ready to go? ' );
}

log( '>> Cloning the repository' );
const performanceTestDirectory = await git.clone( config.gitRepositoryURL );

// 1- Preparing the tests directory.
log( '\n>> Preparing the tests directory' );
log( ' >> Cloning the repository' );
const baseDirectory = await git.clone( config.gitRepositoryURL );
const performanceTestDirectory = getRandomTemporaryPath();
await runShellScript(
'cp -R ' + baseDirectory + ' ' + performanceTestDirectory
);
if ( !! options.testsBranch ) {
log(
'>> Fetching the ' +
' >> Fetching the test branch: ' +
formats.success( options.testsBranch ) +
' branch'
);
Expand All @@ -222,85 +202,146 @@ async function runPerformanceTests( branches, options ) {
options.testsBranch
);
}

const environmentDirectory = getRandomTemporaryPath();
log(
'>> Perf Tests Directory : ' +
formats.success( performanceTestDirectory )
);
log(
'>> Environment Directory : ' + formats.success( environmentDirectory )
);

log( '>> Installing dependencies' );
// The build packages is necessary for the performance folder
log( ' >> Installing dependencies and building packages' );
await runShellScript(
'npm install && npm run build:packages',
performanceTestDirectory
);
await runShellScript(
'cp -R ' + performanceTestDirectory + ' ' + environmentDirectory
);

log( '>> Starting the WordPress environment' );
if ( options.wpVersion ) {
// In order to match the topology of ZIP files at wp.org, remap .0
// patch versions to major versions:
//
// 5.7 -> 5.7 (unchanged)
// 5.7.0 -> 5.7 (changed)
// 5.7.2 -> 5.7.2 (unchanged)
const zipVersion = options.wpVersion.replace( /^(\d+\.\d+).0/, '$1' );
const zipUrl = `https://wordpress.org/wordpress-${ zipVersion }.zip`;
// 2- Preparing the environment directories per branch.
log( '\n>> Preparing an environment directory per branch' );
const branchDirectories = {};
for ( const branch of branches ) {
log( ' >> Branch: ' + branch );
const environmentDirectory = getRandomTemporaryPath();
// @ts-ignore
branchDirectories[ branch ] = environmentDirectory;
await runShellScript(
'cp -R ' + baseDirectory + ' ' + environmentDirectory
);
await setUpGitBranch( branch, environmentDirectory );

log( `Using WordPress version ${ zipVersion }` );
if ( options.wpVersion ) {
// In order to match the topology of ZIP files at wp.org, remap .0
// patch versions to major versions:
//
// 5.7 -> 5.7 (unchanged)
// 5.7.0 -> 5.7 (changed)
// 5.7.2 -> 5.7.2 (unchanged)
const zipVersion = options.wpVersion.replace(
/^(\d+\.\d+).0/,
'$1'
);
const zipUrl = `https://wordpress.org/wordpress-${ zipVersion }.zip`;
log( ` Using WordPress version ${ zipVersion }` );

// Patch the environment's .wp-env.json config to use the specified WP
// version:
//
// {
// "core": "https://wordpress.org/wordpress-$VERSION.zip",
// ...
// }
const confPath = `${ environmentDirectory }/.wp-env.json`;
const conf = { ...readJSONFile( confPath ), core: zipUrl };
await fs.writeFileSync(
confPath,
JSON.stringify( conf, null, 2 ),
'utf8'
);
}
}

// Patch the environment's .wp-env.json config to use the specified WP
// version:
//
// {
// "core": "https://wordpress.org/wordpress-$VERSION.zip",
// ...
// }
const confPath = `${ environmentDirectory }/.wp-env.json`;
const conf = { ...readJSONFile( confPath ), core: zipUrl };
await fs.writeFileSync(
confPath,
JSON.stringify( conf, null, 2 ),
'utf8'
// 3- Printing the used folders.
log(
'\n>> Perf Tests Directory : ' +
formats.success( performanceTestDirectory )
);
for ( const branch of branches ) {
log(
'>> Environment Directory (' +
branch +
') : ' +
// @ts-ignore
formats.success( branchDirectories[ branch ] )
);
}
await runShellScript( 'npm run wp-env start', environmentDirectory );

// 4- Running the tests.
log( '\n>> Running the tests' );

const testSuites = [ 'post-editor', 'site-editor' ];

/** @type {Record<string,Record<string, WPPerformanceResults>>} */
let results = {};
for ( const branch of branches ) {
await setUpGitBranch( branch, environmentDirectory );
log(
'>> Running the test on the ' +
formats.success( branch ) +
' branch'
);
const results = {};
for ( const testSuite of testSuites ) {
results[ testSuite ] = {};
/** @type {Array<Record<string, WPPerformanceResults>>} */
const rawResults = [];
// Alternate three times between branches
for ( let i = 0; i < 3; i++ ) {
rawResults[ i ] = {};
for ( const branch of branches ) {
// @ts-ignore
const environmentDirectory = branchDirectories[ branch ];
log( ' >> Branch: ' + branch + ', Suite: ' + testSuite );
log( ' >> Starting the environment.' );
await runShellScript(
'npm run wp-env start',
environmentDirectory
);
log( ' >> Running the test.' );
rawResults[ i ][ branch ] = await runTestSuite(
testSuite,
performanceTestDirectory
);
log( ' >> Stopping the environment' );
await runShellScript(
'npm run wp-env stop',
environmentDirectory
);
}
}

for ( const testSuite of testSuites ) {
results = {
...results,
[ testSuite ]: {
...results[ testSuite ],
[ branch ]: await runTestSuite(
testSuite,
performanceTestDirectory
// Computing medians.
for ( const branch of branches ) {
const medians = mapValues(
{
load: rawResults.map( ( r ) => r[ branch ].load ),
type: rawResults.map( ( r ) => r[ branch ].type ),
minType: rawResults.map( ( r ) => r[ branch ].minType ),
maxType: rawResults.map( ( r ) => r[ branch ].maxType ),
focus: rawResults.map( ( r ) => r[ branch ].focus ),
minFocus: rawResults.map( ( r ) => r[ branch ].minFocus ),
maxFocus: rawResults.map( ( r ) => r[ branch ].maxFocus ),
inserterOpen: rawResults.map(
( r ) => r[ branch ].inserterOpen
),
minInserterOpen: rawResults.map(
( r ) => r[ branch ].minInserterOpen
),
maxInserterOpen: rawResults.map(
( r ) => r[ branch ].maxInserterOpen
),
inserterHover: rawResults.map(
( r ) => r[ branch ].inserterHover
),
minInserterHover: rawResults.map(
( r ) => r[ branch ].minInserterHover
),
maxInserterHover: rawResults.map(
( r ) => r[ branch ].maxInserterHover
),
},
};
median
);

// Format results as times.
results[ testSuite ][ branch ] = mapValues( medians, formatTime );
}
}

log( '>> Stopping the WordPress environment' );
await runShellScript( 'npm run wp-env stop', environmentDirectory );

// 5- Formatting the results.
log( '\n>> 🎉 Results.\n' );
for ( const testSuite of testSuites ) {
log( `\n>> ${ testSuite }\n` );
Expand All @@ -310,7 +351,9 @@ async function runPerformanceTests( branches, options ) {
Object.entries( results[ testSuite ] ).reduce(
( acc, [ key, val ] ) => {
for ( const entry of Object.keys( val ) ) {
if ( ! acc[ entry ] ) acc[ entry ] = {};
// @ts-ignore
if ( ! acc[ entry ] && isFinite( val[ entry ] ) )
acc[ entry ] = {};
// @ts-ignore
if ( isFinite( val[ entry ] ) ) {
// @ts-ignore
Expand Down
28 changes: 20 additions & 8 deletions bin/plugin/lib/utils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,26 @@ const { log, formats } = require( './logger' );
* @param {string=} cwd Working directory.
*/
function runShellScript( script, cwd ) {
childProcess.execSync( script, {
cwd,
env: {
NO_CHECKS: 'true',
PATH: process.env.PATH,
HOME: process.env.HOME,
},
stdio: [ 'inherit', 'ignore', 'inherit' ],
return new Promise( ( resolve, reject ) => {
childProcess.exec(
script,
{
cwd,
env: {
NO_CHECKS: 'true',
PATH: process.env.PATH,
HOME: process.env.HOME,
},
},
function ( error, _, stderr ) {
if ( error ) {
console.log( stderr );
reject( error );
} else {
resolve( true );
}
}
);
} );
}

Expand Down

0 comments on commit 2166961

Please sign in to comment.