Migrate/e2e to playwright round one

.eslintrc.js

@@ -229,12 +229,39 @@ module.exports = {
 			extends: [ 'plugin:@wordpress/eslint-plugin/test-unit' ],
 		},
 		{
-			files: [ 'packages/e2e-test*/**/*.js' ],
+			files: [
+				'packages/e2e-tests/**/*.js',
+				'packages/e2e-test-utils/**/*.js',
+			],
 			extends: [ 'plugin:@wordpress/eslint-plugin/test-e2e' ],
 			rules: {
 				'jest/expect-expect': 'off',
 			},
 		},
+		{
+			files: [
+				'packages/e2e-tests-playwright/**/*.[tj]s',
+				'packages/e2e-test-utils-playwright/**/*.[tj]s',
+			],
+			extends: [ 'plugin:@wordpress/eslint-plugin/test-e2e-playwright' ],
+			rules: {
+				'@wordpress/no-global-active-element': 'off',
+				'@wordpress/no-global-get-selection': 'off',
+				'no-restricted-syntax': [
+					'error',
+					{
+						selector: 'CallExpression[callee.name="$"]',
+						message:
+							'`$` is discouraged, please use `locator` instead',
+					},
+					{
+						selector: 'CallExpression[callee.name="$$"]',
+						message:
+							'`$$` is discouraged, please use `locator` instead',
+					},
+				],
+			},
+		},
 		{
 			files: [ 'bin/**/*.js', 'packages/env/**' ],
 			rules: {

.github/report-flaky-tests/index.js

@@ -46,6 +46,7 @@ const metaData = {
 
 	for ( const flakyTest of flakyTests ) {
 		const {
+			runner: testRunner = 'jest-circus',
 			title: testTitle,
 			path: testPath,
 			results: testResults,
@@ -92,7 +93,11 @@ const metaData = {
 					body.indexOf( TEST_RESULTS_LIST.close )
 				) +
 				[
-					renderTestErrorMessage( { testPath, testResults } ),
+					renderTestErrorMessage( {
+						testRunner,
+						testPath,
+						testResults,
+					} ),
 					TEST_RESULTS_LIST.close,
 				].join( '\n' );
 
@@ -121,6 +126,7 @@ const metaData = {
 				title: issueTitle,
 				body: renderIssueBody( {
 					meta,
+					testRunner,
 					testTitle,
 					testPath,
 					testResults,
@@ -187,10 +193,16 @@ function getIssueTitle( testTitle ) {
 	return `[Flaky Test] ${ testTitle }`;
 }
 
-function renderIssueBody( { meta, testTitle, testPath, testResults } ) {
+function renderIssueBody( {
+	meta,
+	testRunner,
+	testTitle,
+	testPath,
+	testResults,
+} ) {
 	return (
 		renderIssueDescription( { meta, testTitle, testPath } ) +
-		renderTestResults( { testPath, testResults } )
+		renderTestResults( { testRunner, testPath, testResults } )
 	);
 }
 
@@ -211,14 +223,41 @@ ${ testTitle }
 `;
 }
 
-function renderTestResults( { testPath, testResults } ) {
+function renderTestResults( { testRunner, testPath, testResults } ) {
 	return `${ TEST_RESULTS_LIST.open }
-${ renderTestErrorMessage( { testPath, testResults } ) }
+${ renderTestErrorMessage( { testRunner, testPath, testResults } ) }
 ${ TEST_RESULTS_LIST.close }
 `;
 }
 
-function renderTestErrorMessage( { testPath, testResults } ) {
+function renderTestResults( { testRunner, testResults, testPath } ) {
+	switch ( testRunner ) {
+		case '@playwright/test': {
+			// Could do a slightly better formatting than this.
+			return stripAnsi(
+				testResults.map( ( result ) => result.error.stack ).join( '\n' )
+			);
+		}
+		case 'jest-circus':
+		default: {
+			return stripAnsi(
+				formatResultsErrors(
+					testResults,
+					{
+						rootDir: path.join(
+							process.cwd(),
+							'packages/e2e-tests'
+						),
+					},
+					{},
+					testPath
+				)
+			);
+		}
+	}
+}
+
+function renderTestErrorMessage( { testRunner, testPath, testResults } ) {
 	const date = new Date().toISOString();
 	// It will look something like this without formatting:
 	// ▶ [2021-08-31T16:15:19.875Z] Test passed after 2 failed attempts on trunk
@@ -231,14 +270,7 @@ function renderTestErrorMessage( { testPath, testResults } ) {
 </summary>
 
 \`\`\`
-${ stripAnsi(
-	formatResultsErrors(
-		testResults,
-		{ rootDir: path.join( process.cwd(), 'packages/e2e-tests' ) },
-		{},
-		testPath
-	)
-) }
+${ renderTestResults( { testRunner, testPath, testResults } ) }
 \`\`\`
 </details>${ TEST_RESULT.close }`;
 }

.github/workflows/end2end-test-playwright.yml

@@ -0,0 +1,68 @@
+name: End-to-End Tests Playwright
+
+on:
+    pull_request:
+    push:
+        branches:
+            - trunk
+            - 'release/**'
+            - 'wp/**'
+
+# Cancels all previous workflow runs for pull requests that have not completed.
+concurrency:
+    # The concurrency group contains the workflow name and the branch name for pull requests
+    # or the commit hash for any other events.
+    group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
+    cancel-in-progress: true
+
+jobs:
+    e2e:
+        name: E2E Tests
+        runs-on: ubuntu-latest
+        if: ${{ github.repository == 'WordPress/gutenberg' || github.event_name == 'pull_request' }}
+        strategy:
+            fail-fast: false
+            matrix:
+                node: ['14']
+
+        steps:
+            - uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f # v2.3.4
+
+            - name: Use desired version of NodeJS
+              uses: actions/setup-node@38d90ce44d5275ad62cc48384b3d8a58c500bb5f # v2.2.2
+              with:
+                  node-version: ${{ matrix.node }}
+                  cache: npm
+
+            - name: Npm install and build
+              run: |
+                  npm ci
+                  npm run build
+
+            - name: Install Playwright dependencies
+              run: |
+                  npx playwright install chromium --with-deps
+
+            - name: Install WordPress and start the server
+              run: |
+                  npm run wp-env start
+
+            - name: Run the tests
+              run: |
+                  npm run test-e2e:playwright
+
+            - name: Archive debug artifacts (screenshots, traces)
+              uses: actions/upload-artifact@e448a9b857ee2131e752b06002bf0e093c65e571 # v2.2.2
+              if: always()
+              with:
+                  name: failures-artifacts
+                  path: artifacts
+                  if-no-files-found: ignore
+
+            - name: Archive flaky tests report
+              uses: actions/upload-artifact@e448a9b857ee2131e752b06002bf0e093c65e571 # v2.2.2
+              if: always()
+              with:
+                  name: flaky-tests-report
+                  path: flaky-tests
+                  if-no-files-found: ignore

.github/workflows/flaky-tests.yml

@@ -2,6 +2,8 @@ name: Report Flaky Tests
 
 on:
     workflow_run:
+        # We should also add 'End-to-End Tests Playwright' here but that
+        # wil run this workflow whenever either one of them completes.
         workflows: ['End-to-End Tests']
         types:
             - completed

docs/manifest.json

@@ -1493,12 +1493,24 @@
 		"markdown_source": "../packages/dom/README.md",
 		"parent": "packages"
 	},
+	{
+		"title": "@wordpress/e2e-test-utils-playwright",
+		"slug": "packages-e2e-test-utils-playwright",
+		"markdown_source": "../packages/e2e-test-utils-playwright/README.md",
+		"parent": "packages"
+	},
 	{
 		"title": "@wordpress/e2e-test-utils",
 		"slug": "packages-e2e-test-utils",
 		"markdown_source": "../packages/e2e-test-utils/README.md",
 		"parent": "packages"
 	},
+	{
+		"title": "@wordpress/e2e-tests-playwright",
+		"slug": "packages-e2e-tests-playwright",
+		"markdown_source": "../packages/e2e-tests-playwright/README.md",
+		"parent": "packages"
+	},
 	{
 		"title": "@wordpress/e2e-tests",
 		"slug": "packages-e2e-tests",