Run performance tests in a matrix

.github/workflows/performance.yml

@@ -88,11 +88,17 @@ jobs:
   # - Ensure version-controlled files are not modified or deleted.
   # - Dispatch workflow run.
   performance:
-    name: Run performance tests
+    name: Run performance tests / ${{ matrix.locale }}) / ${{ matrix.theme }} ${{ matrix.memcached && ' with memcached' || '' }}
     runs-on: ubuntu-latest
     permissions:
       contents: read
     if: ${{ ( github.repository == 'WordPress/wordpress-develop' || github.event_name == 'pull_request' ) && ! contains( github.event.before, '00000000' ) }}
+    strategy:
+      fail-fast: false
+      matrix:
+        theme: [ 'twentytwentyone', 'twentytwentythree' ]
+        memcached: [ true, false ]
+        locale: [ 'en_US', 'de_DE' ]
 
     steps:
       - name: Configure environment variables
@@ -130,8 +136,9 @@ jobs:
         run: npm run build
 
       - name: Start Docker environment
-        run: |
-          npm run env:start
+        run: npm run env:start
+        env:
+          LOCAL_PHP_MEMCACHED: ${{ matrix.memcached }}
 
       - name: Log running Docker containers
         run: docker ps -a
@@ -162,11 +169,17 @@ jobs:
         run: |
           npm run env:cli -- rewrite structure '/%year%/%monthnum%/%postname%/' --path=/var/www/${{ env.LOCAL_DIR }}
 
-      - name: Install additional languages
+      - name: Configure theme
+        run: |
+          npm run env:cli -- theme activate ${{ matrix.theme }} --path=/var/www/${{ env.LOCAL_DIR }}
+
+      - name: Configure language
+        if: ${{ matrix.locale != 'en_US }}
         run: |
-          npm run env:cli -- language core install de_DE --path=/var/www/${{ env.LOCAL_DIR }}
-          npm run env:cli -- language plugin install de_DE --all --path=/var/www/${{ env.LOCAL_DIR }}
-          npm run env:cli -- language theme install de_DE --all --path=/var/www/${{ env.LOCAL_DIR }}
+          npm run env:cli -- language core install ${{ matrix.locale }} --path=/var/www/${{ env.LOCAL_DIR }}
+          npm run env:cli -- language plugin install ${{ matrix.locale }} --all --path=/var/www/${{ env.LOCAL_DIR }}
+          npm run env:cli -- language theme install ${{ matrix.locale }} --all --path=/var/www/${{ env.LOCAL_DIR }}
+          npm run env:cli -- site switch-language ${{ matrix.locale }}
 
       - name: Install MU plugin
         run: |
@@ -176,9 +189,6 @@ jobs:
       - name: Run performance tests (current commit)
         run: npm run test:performance
 
-      - name: Print performance tests results
-        run: node ./tests/performance/results.js
-
       - name: Check out target commit (target branch or previous commit)
         run: |
           if [[ -z "$TARGET_REF" ]]; then
@@ -194,16 +204,14 @@ jobs:
       - name: Build WordPress
         run: npm run build
 
+      - name: Flush cache
+        run: npm run env:cli -- cache flush --path=/var/www/${{ env.LOCAL_DIR }}
+
       - name: Run target performance tests (base/previous commit)
         env:
           TEST_RESULTS_PREFIX: before
         run: npm run test:performance
 
-      - name: Print target performance tests results
-        env:
-          TEST_RESULTS_PREFIX: before
-        run: node ./tests/performance/results.js
-
       - name: Reset to original commit
         run: git reset --hard $GITHUB_SHA
 
@@ -215,22 +223,17 @@ jobs:
           npm run env:cli -- core update --version=${{ env.BASE_TAG }} --force --path=/var/www/${{ env.LOCAL_DIR }}
           npm run env:cli -- core version --path=/var/www/${{ env.LOCAL_DIR }}
 
+      - name: Flush cache
+        run: npm run env:cli -- cache flush --path=/var/www/${{ env.LOCAL_DIR }}
+
       - name: Run baseline performance tests
         env:
           TEST_RESULTS_PREFIX: base
         run: npm run test:performance
 
-      - name: Print baseline performance tests results
-        env:
-          TEST_RESULTS_PREFIX: base
-        run: node ./tests/performance/results.js
-
       - name: Compare results with base
         run: node ./tests/performance/compare-results.js ${{ runner.temp }}/summary.md
 
-      - name: Add workflow summary
-        run: cat ${{ runner.temp }}/summary.md >> $GITHUB_STEP_SUMMARY
-
       - name: Set the base sha
         # Only needed when publishing results.
         if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}

tests/performance/compare-results.js

@@ -3,14 +3,19 @@
 /**
  * External dependencies.
  */
-const fs = require( 'node:fs' );
-const path = require( 'node:path' );
+const { readFileSync, readDirSync } = require( 'node:fs' );
+const { join, basename } = require( 'node:path' );
 
 /**
  * Internal dependencies
  */
 const { median } = require( './utils' );
 
+process.env.WP_ARTIFACTS_PATH ??= join( process.cwd(), 'artifacts' );
+
+const args = process.argv.slice( 2 );
+const summaryFile = args[ 0 ];
+
 /**
  * Parse test files into JSON objects.
  *
@@ -19,36 +24,25 @@ const { median } = require( './utils' );
  */
 const parseFile = ( fileName ) =>
 	JSON.parse(
-		fs.readFileSync( path.join( __dirname, '/specs/', fileName ), 'utf8' )
+		readFileSync( join( process.env.WP_ARTIFACTS_PATH, fileName ), 'utf8' )
 	);
 
-// The list of test suites to log.
-const testSuites = [
-	'admin',
-	'admin-l10n',
-	'home-block-theme',
-	'home-block-theme-l10n',
-	'home-classic-theme',
-	'home-classic-theme-l10n',
-];
-
-// The current commit's results.
-const testResults = Object.fromEntries(
-	testSuites
-		.filter( ( key ) => fs.existsSync( path.join( __dirname, '/specs/', `${ key }.test.results.json` ) ) )
-		.map( ( key ) => [ key, parseFile( `${ key }.test.results.json` ) ] )
-);
-
-// The previous commit's results.
-const prevResults = Object.fromEntries(
-	testSuites
-		.filter( ( key ) => fs.existsSync( path.join( __dirname, '/specs/', `before-${ key }.test.results.json` ) ) )
-		.map( ( key ) => [ key, parseFile( `before-${ key }.test.results.json` ) ] )
-);
+const testResults = {};
+const prevResults = {};
 
-const args = process.argv.slice( 2 );
+for ( const { name } of readDirSync( process.env.WP_ARTIFACTS_PATH ) ) {
+	if ( ! name.endsWith( '.results.json' ) ) {
+		continue;
+	}
 
-const summaryFile = args[ 0 ];
+	const testSuiteName = basename( name, '.results.json' );
+
+	if ( ! name.startsWith( 'before-' ) ) {
+		testResults[ testSuiteName ] = parseFile( name );
+	} else {
+		prevResults[ testSuiteName ] = parseFile( name );
+	}
+}
 
 /**
  * Formats an array of objects as a Markdown table.
@@ -115,16 +109,14 @@ function linkToSha(sha) {
 	return `[${sha.slice(0, 7)}](https://github.com/${repoName}/commit/${sha})`;
 }
 
-let summaryMarkdown = `# Performance Test Results\n\n`;
-
-if ( process.env.GITHUB_SHA ) {
-	summaryMarkdown += `🛎️ Performance test results for ${ linkToSha( process.env.GITHUB_SHA ) } are in!\n\n`;
-} else {
-	summaryMarkdown += `🛎️ Performance test results are in!\n\n`;
-}
+let summaryMarkdown = `## Performance Test Results\n\n`;
 
 if ( process.env.TARGET_SHA ) {
-	summaryMarkdown += `This compares the results from this commit with the ones from ${ linkToSha( process.env.TARGET_SHA ) }.\n\n`;
+	if ( process.env.GITHUB_SHA ) {
+		summaryMarkdown += `This compares the results from this commit (${ linkToSha( process.env.GITHUB_SHA ) }) with the ones from ${linkToSha(process.env.TARGET_SHA)}.\n\n`;
+	} else {
+		summaryMarkdown += `This compares the results from this commit with the ones from ${linkToSha(process.env.TARGET_SHA)}.\n\n`;
+	}
 }
 
 if ( process.env.GITHUB_SHA ) {
@@ -179,7 +171,7 @@ for ( const key of testSuites ) {
 	}
 
 	if ( rows.length > 0 ) {
-		summaryMarkdown += `## ${ title }\n\n`;
+		summaryMarkdown += `### ${ title }\n\n`;
 		summaryMarkdown += `${ formatAsMarkdownTable( rows ) }\n`;
 
 		console.log( title );
@@ -188,7 +180,7 @@ for ( const key of testSuites ) {
 }
 
 if ( summaryFile ) {
-	fs.writeFileSync(
+	writeFileSync(
 		summaryFile,
 		summaryMarkdown
 	);

tests/performance/config/performance-reporter.js

@@ -1,7 +1,7 @@
 /**
  * External dependencies
  */
-import { join, dirname, basename } from 'node:path';
+import { join, basename } from 'node:path';
 import { writeFileSync } from 'node:fs';
 
 /**
@@ -26,8 +26,8 @@ class PerformanceReporter {
 		if ( performanceResults?.body ) {
 			writeFileSync(
 				join(
-					dirname( test.location.file ),
-					getResultsFilename( basename( test.location.file, '.js' ) )
+					process.env.WP_ARTIFACTS_PATH,
+					getResultsFilename( basename( test.location.file, '.test.js' ) )
 				),
 				performanceResults.body.toString( 'utf-8' )
 			);

tests/performance/specs/admin.test.js

@@ -13,10 +13,6 @@ const results = {
 };
 
 test.describe( 'Admin', () => {
-	test.beforeAll( async ( { requestUtils } ) => {
-		await requestUtils.activateTheme( 'twentytwentyone' );
-	} );
-
 	test.afterAll( async ( {}, testInfo ) => {
 		await testInfo.attach( 'results', {
 			body: JSON.stringify( results, null, 2 ),