Boost: Refactor and add new e2e tests (#21819)

* WIP: Refactor and create new e2e tests

* WIP: Refactor and create new e2e tests

* Fix tests following recent refactoring migrating Jest to Playwright

* WIP: Refactor tests

* Fix coding standards for consistency

* WIP: Refactor and extend E2E tests

* Fix tests name

* Fix test name

* Reduce timeout

* Improve test assertion

* Add a E2E test for the render blocking js module

* Refactor the test for the Lazy images module

* Add a new prerequisite to install the jitm package before tests

* Change connection tests order

* Remove useless prerequisite for connection test

* Fix line break

* Improve build package command

* Fix test content

* Improve the modules common tests

* WIP: Add Critical CSS module related E2E tests

* Install required packages for testing inside the worfklow rather than as prerequisites.

* [not verified] Revert "Install required packages for testing inside the worfklow rather than as prerequisites."

This reverts commit fee8d30.

* Remove not needed prerequesite

* Refactor the specs directory structure and remove useless test

* Implement some common/general E2E tests

* Rename test description

* Remove useless export statements

* Fix logging message

* Make sure that the ensurePluginsState prerequisite is run only on a local site

* Remove the describe tests grouping as not required in the context of common tests

* Add missing page.close statements

* Removing the use of the TestContentPage in favor to the PostFrontEnd page

* Refactor to remove locator statements from the tests

* Remove a page close statement which may break tests

* Remove helper file not needed anymore

* Refactor the way to check if the site score is loading to make it more robust

* Refactor tests check

* Add back the page close statement

* Create a new common Themes page

* Create a new sidebar page method to navigate to the themes page

* Fix test description

* Fix test comment

* Implement a new Critical CSS related test

* Fix test description

* Relocate tests conment

* Add new Critical CSS related tests

* Update the README file

* Ensure a clean env pre-requisite before running test

* Increase timeout

* Fix typo

* Standardize the way new pages are created

* Fix typo

* Improve methods naming

* Don't setup load promises before navigation; Playwright doesn't work that way

* Ensure jetpack is active before ensuring it's connected

* Don't use .serial; tests don't rely on previous tests, so can continue after failure

* Use `test.describe.serial` for `Modules` test suit

* Add more apiEndpointsRegex

New endpoints to make waitForApiResponse to be effective for lazy-images and render-blocking-js status

Co-authored-by: Mark George <[email protected]>
Co-authored-by: Adnan Haque <[email protected]>

projects/plugins/boost/app/lib/class-cli.php

@@ -60,18 +60,14 @@ public function reset_settings() {
 	 *
 	 * ## EXAMPLES
 	 *
-	 * wp jetpack module activate critical-css
-	 * wp jetpack module deactivate critical-css
+	 * wp jetpack-boost module activate critical-css
+	 * wp jetpack-boost module deactivate critical-css
 	 *
 	 * @param array $args Command arguments.
 	 */
 	public function module( $args ) {
 		$action = isset( $args[0] ) ? $args[0] : null;
 
-		if ( ! $action ) {
-			\WP_CLI::error( __( 'Please specify a valid action.', 'jetpack-boost' ) );
-		}
-
 		$module_slug = null;
 
 		if ( isset( $args[1] ) ) {
@@ -83,7 +79,8 @@ public function module( $args ) {
 				);
 			}
 		} else {
-			\WP_CLI::error( __( 'Please specify a valid module.', 'jetpack-boost' ) );
+			/* translators: Placeholder is list of available modules. */
+			\WP_CLI::error( sprintf( __( 'Please specify a valid module. It should be one of %s', 'jetpack-boost' ), wp_json_encode( Jetpack_Boost::AVAILABLE_MODULES_DEFAULT ) ) );
 		}
 
 		switch ( $action ) {
@@ -113,4 +110,74 @@ private function set_module_status( $module_slug, $status ) {
 			sprintf( __( "'%1\$s' has been %2\$s.", 'jetpack-boost' ), $module_slug, $status_label )
 		);
 	}
+
+	/**
+	 * Manage Jetpack Boost connection
+	 *
+	 * ## OPTIONS
+	 *
+	 * <activate|deactivate|status>
+	 * : The action to take.
+	 * ---
+	 * options:
+	 *  - activate
+	 *  - deactivate
+	 *  - status
+	 * ---
+	 *
+	 * ## EXAMPLES
+	 *
+	 * wp jetpack-boost connection activate
+	 * wp jetpack-boost connection deactivate
+	 * wp jetpack-boost connection status
+	 *
+	 * @param array $args Command arguments.
+	 */
+	public function connection( $args ) {
+		$action = isset( $args[0] ) ? $args[0] : null;
+
+		switch ( $action ) {
+			case 'activate':
+				$result = $this->jetpack_boost->connection->register();
+				if ( true === $result ) {
+					\WP_CLI::success( __( 'Boost is connected to WP.com', 'jetpack-boost' ) );
+				} else {
+					\WP_CLI::Error( __( 'Boost could not be connected to WP.com', 'jetpack-boost' ) );
+				}
+				break;
+			case 'deactivate':
+				require_once ABSPATH . '/wp-admin/includes/plugin.php';
+
+				if ( is_plugin_active_for_network( JETPACK_BOOST_PATH ) ) {
+					$this->jetpack_boost->connection->deactivate_disconnect_network();
+				} else {
+					$this->jetpack_boost->connection->disconnect();
+				}
+
+				\WP_CLI::success( __( 'Boost is disconnected from WP.com', 'jetpack-boost' ) );
+				break;
+			case 'status':
+				$is_connected = $this->jetpack_boost->connection->is_connected();
+				if ( $is_connected ) {
+					\WP_CLI::line( 'connected' );
+				} else {
+					\WP_CLI::line( 'disconnected' );
+				}
+				break;
+		}
+	}
+
+	/**
+	 * Reset Jetpack Boost
+	 *
+	 * ## EXAMPLE
+	 *
+	 * wp jetpack-boost reset
+	 */
+	public function reset() {
+		$this->jetpack_boost->deactivate();
+		$this->jetpack_boost->uninstall();
+		$this->jetpack_boost->config()->reset();
+		\WP_CLI::success( 'Reset successfully' );
+	}
 }

projects/plugins/boost/changelog/add-boost-e2e-tests

@@ -0,0 +1,4 @@
+Significance: minor
+Type: added
+
+Refactor and add new core E2E tests

projects/plugins/boost/tests/e2e/README.md

@@ -26,9 +26,24 @@ To decrypt the config file (a8c only):
 
 Typically, the workflow is the same as the one described in the Jetpack E2E [documentation](../../../jetpack/tests/e2e/README.md). You can follow the same workflow but running the commands inside the Jetpack Boost E2E tests folder.
 
-However, Boost has some shortcuts to get the environment started and run all the tests by running the following commands from the root of the Jetpack Boost repository:
+However,below is a quick reminder of the critical steps to run the tests.
 
-- `pnpm test-e2e:start` - This will command will start the e2e testing environment and the tunnel.
+From the root of the repo (this has to be done only once or when pulling new changes):
+
+1. run `pnpm install` - This command will install the monorepo NPM dependencies. 
+2. run `jetpack build plugins/jetpack` - This command will install Jetpack NPM and Composer dependencies as well as building the asset files.
+3. run `jetpack build plugins/boost` - This command will install Jetpack Boost NPM and Composer dependencies as well as building the asset files.
+
+From the `projects/plugins/boost/tests/e2e` folder:
+
+4. run `pnpm install` - This will install the Jetpack Boost E2E tests NPM dependencies.
+5. run `pnpm run test-decrypt-config` - This command will decrypt the config and create/overwrite the local test config file . 
+6. run `pnpm run env-start && pnpm run tunnel-on` - This command will start the e2e testing environment and the tunnel. 
+7. run `pnpm run test-e2e` - This command will run the e2e tests.
+
+However, Boost has some shortcuts to get the environment started and run all the tests by running the following commands from the root of the Jetpack Boost folder:
+
+- `pnpm test-e2e:decrypt-config` - This command will decrypt the config and create/overwrite the local test config file .
+- `pnpm test-e2e:start` - This command will start the e2e testing environment and the tunnel.
 - `pnpm test-e2e:run` - This command will run the e2e tests.
 - `pnpm test-e2e:stop` - This command will stop the e2e testing environment.
-- `pnpm test-e2e:decrypt-config` - This command will decrypt the config and create/overwrite the local test config file .

projects/plugins/boost/tests/e2e/lib/env/prerequisites.js

@@ -1,10 +1,16 @@
 import logger from 'jetpack-e2e-commons/logger.cjs';
 import { execWpCommand } from 'jetpack-e2e-commons/helpers/utils-helper.cjs';
+
 import { expect } from '@playwright/test';
+import { JetpackBoostPage } from '../pages/index.js';
 
-export function boostPrerequisitesBuilder() {
+export function boostPrerequisitesBuilder( page ) {
 	const state = {
+		testPostTitles: [],
+		clean: undefined,
 		modules: { active: undefined, inactive: undefined },
+		connected: undefined,
+		jetpackDeactivated: undefined,
 	};
 
 	return {
@@ -16,15 +22,30 @@ export function boostPrerequisitesBuilder() {
 			state.modules.inactive = modules;
 			return this;
 		},
+		withConnection( shouldBeConnected ) {
+			state.connected = shouldBeConnected;
+			return this;
+		},
+		withTestContent( testPostTitles = [] ) {
+			state.testPostTitles = testPostTitles;
+			return this;
+		},
+		withCleanEnv() {
+			state.clean = true;
+			return this;
+		},
 		async build() {
-			await buildPrerequisites( state );
+			await buildPrerequisites( state, page );
 		},
 	};
 }
 
-async function buildPrerequisites( state ) {
+async function buildPrerequisites( state, page ) {
 	const functions = {
 		modules: () => ensureModulesState( state.modules ),
+		connected: () => ensureConnectedState( state.connected, page ),
+		testPostTitles: () => ensureTestPosts( state.testPostTitles ),
+		clean: () => ensureCleanState( state.clean ),
 	};
 
 	logger.prerequisites( JSON.stringify( state, null, 2 ) );
@@ -54,19 +75,91 @@ export async function ensureModulesState( modules ) {
 		logger.prerequisites( 'Cannot find list of modules to deactivate!' );
 	}
 }
-
-export async function activateModules( modulesList ) {
-	for ( const module of modulesList ) {
+export async function activateModules( modules ) {
+	for ( const module of modules ) {
 		logger.prerequisites( `Activating module ${ module }` );
 		const result = await execWpCommand( `jetpack-boost module activate ${ module }` );
 		expect( result ).toMatch( new RegExp( `Success: .* has been activated.`, 'i' ) );
 	}
 }
 
-export async function deactivateModules( modulesList ) {
-	for ( const module of modulesList ) {
+export async function deactivateModules( modules ) {
+	for ( const module of modules ) {
 		logger.prerequisites( `Deactivating module ${ module }` );
 		const result = await execWpCommand( `jetpack-boost module deactivate ${ module }` );
 		expect( result ).toMatch( new RegExp( `Success: .* has been deactivated.`, 'i' ) );
 	}
 }
+
+export async function ensureConnectedState( requiredConnected = undefined, page ) {
+	const isConnected = await checkIfConnected();
+
+	if ( requiredConnected && isConnected ) {
+		logger.prerequisites( 'Jetpack Boost is already connected, moving on' );
+	} else if ( requiredConnected && ! isConnected ) {
+		logger.prerequisites( 'Connecting Jetpack Boost' );
+		await connect( page );
+	} else if ( ! requiredConnected && isConnected ) {
+		logger.prerequisites( 'Disconnecting Jetpack Boost' );
+		await disconnect();
+	} else {
+		logger.prerequisites( 'Jetpack Boost is already disconnected, moving on' );
+	}
+}
+
+export async function connect( page ) {
+	logger.prerequisites( `Connecting Boost plugin to WP.com` );
+	// Boost cannot be connected to WP.com using the WP-CLI because the site is considered
+	// as a localhost site. The only solution is to do it via the site itself running under the localtunnel.
+	const jetpackBoostPage = await JetpackBoostPage.visit( page );
+	await jetpackBoostPage.connect();
+	await jetpackBoostPage.waitForApiResponse( 'connection' );
+	await jetpackBoostPage.isOverallScoreHeaderShown();
+}
+
+export async function disconnect() {
+	logger.prerequisites( `Disconnecting Boost plugin to WP.com` );
+	const cliCmd = 'jetpack-boost connection deactivate';
+	const result = await execWpCommand( cliCmd );
+	expect( result ).toEqual( 'Success: Boost is disconnected from WP.com' );
+}
+
+export async function checkIfConnected() {
+	const cliCmd = 'jetpack-boost connection status';
+	const result = await execWpCommand( cliCmd );
+	if ( typeof result !== 'object' ) {
+		return result === 'connected';
+	}
+	const txt = result.toString();
+	if ( txt.includes( "Error: 'jetpack-boost' is not a registered wp command" ) ) {
+		return false;
+	}
+	throw result;
+}
+
+async function ensureTestPosts( testPostTitles ) {
+	const testPostTitlesCommands = {
+		'Hello World with image':
+			"post create --post_status='publish' --post_title='Hello World with image' --post_content='<h1>Hello World with image</h1><div><p>This is just a test post with an image</p><img src=\"https://picsum.photos/seed/picsum/600/600\" alt=\"placeholder Image\"></div>'",
+		'Hello World with JavaScript':
+			'post create --post_status=\'publish\' --post_title=\'Hello World with JavaScript\' --post_content=\'<h1>Hello World with JavaScript</h1><div class="render-blocking-js"><script id="blockingScript">document.getElementById("testDiv").style.display = "block";</script></div><div id="testDiv" style="display: none">This is made visible by JavaScript</div>\'',
+	};
+	for ( const testPostTitle of testPostTitles ) {
+		if ( testPostTitle in testPostTitlesCommands ) {
+			const result = await execWpCommand( 'post list --fields=post_title' );
+			if ( result.includes( testPostTitle ) ) {
+				logger.prerequisites( 'The test content post already exists' );
+			} else {
+				logger.prerequisites( 'Creating test content post...' );
+				await execWpCommand( testPostTitlesCommands[ testPostTitle ] );
+			}
+		}
+	}
+}
+
+async function ensureCleanState( shouldReset ) {
+	if ( shouldReset ) {
+		logger.prerequisites( 'Resetting Jetpack Boost' );
+		await execWpCommand( 'jetpack-boost reset' );
+	}
+}

projects/plugins/boost/tests/e2e/lib/pages/index.js

@@ -0,0 +1 @@
+export { default as JetpackBoostPage } from './wp-admin/JetpackBoostPage.js';

projects/plugins/boost/tests/e2e/lib/pages/wp-admin/JetpackBoostPage.js

@@ -3,6 +3,8 @@ import { resolveSiteUrl } from 'jetpack-e2e-commons/helpers/utils-helper.cjs';
 
 const apiEndpointsRegex = {
 	'critical-css-status': /jetpack-boost\/v1\/module\/critical-css\/status/,
+	'lazy-images-status': /jetpack-boost\/v1\/module\/lazy-images\/status/,
+	'render-blocking-js-status': /jetpack-boost\/v1\/module\/render-blocking-js\/status/,
 	'speed-scores-update': /jetpack-boost\/v1\/speed-scores\/\w*\/update/,
 };
 
@@ -12,6 +14,27 @@ export default class JetpackBoostPage extends WpPage {
 		super( page, { expectedSelectors: [ '#jb-settings' ], url } );
 	}
 
+	async connect() {
+		const button = await this.page.$( '.jb-connection button' );
+		await button.click();
+	}
+
+	async isFreshlyConnected() {
+		await this.connect();
+		await this.waitForApiResponse( 'connection' );
+		return await this.isSiteScoreLoading();
+	}
+
+	async isOverallScoreHeaderShown() {
+		return await this.isElementVisible( '.jb-site-score' );
+	}
+
+	async isSiteScoreLoading() {
+		const selector = await this.waitForElementToBeVisible( '.jb-site-score' );
+		const classNames = await selector.getAttribute( 'class' );
+		return classNames.includes( 'loading' );
+	}
+
 	async waitForApiResponse( apiEndpointId ) {
 		await this.page.waitForResponse(
 			response =>
@@ -41,4 +64,37 @@ export default class JetpackBoostPage extends WpPage {
 		} );
 		return Number( await speedBar.$eval( '.jb-score-bar__score', e => e.textContent ) );
 	}
+
+	async isTheCriticalCssMetaInformationVisible() {
+		const selector = '.jb-critical-css__meta';
+		return this.page.isVisible( selector );
+	}
+
+	async waitForCriticalCssMetaInfoVisibility() {
+		const selector = '.jb-critical-css__meta';
+		return this.waitForElementToBeVisible( selector, 3 * 60 * 1000 );
+	}
+
+	async waitForCriticalCssGenerationProgressUIVisibility() {
+		const selector = '.jb-critical-css-progress';
+		return this.waitForElementToBeVisible( selector );
+	}
+
+	async isTheCriticalCssFailureMessageVisible() {
+		const selector = '.jb-critical-css__meta .failures';
+		return this.page.isVisible( selector );
+	}
+
+	async navigateToCriticalCSSAdvancedRecommendations() {
+		await this.page.click( 'text=Advanced Recommendations' );
+	}
+
+	async isCriticalCSSAdvancedRecommendationsVisible() {
+		const selector = '.jb-critical-css__advanced';
+		return this.waitForElementToBeVisible( selector );
+	}
+
+	async navigateToMainSettingsPage() {
+		await this.page.click( 'text=Go back' );
+	}
 }

projects/plugins/boost/tests/e2e/lib/setupTests.js

@@ -1,8 +1,11 @@
 import { chromium } from '@playwright/test';
 import { prerequisitesBuilder } from 'jetpack-e2e-commons/env/prerequisites.js';
+import { boostPrerequisitesBuilder } from './env/prerequisites.js';
 
 export default async function () {
 	const browser = await chromium.launch();
 	const page = await browser.newPage();
-	await prerequisitesBuilder( page ).withLoggedIn( true ).withConnection( true ).build();
+	await prerequisitesBuilder( page ).withLoggedIn( true ).withActivePlugins( [ 'boost' ] ).build();
+	await boostPrerequisitesBuilder( page ).withCleanEnv( true ).withConnection( true ).build();
+	await page.close();
 }