From b6f43e0e725a10956a807238c68fd4a96fda1387 Mon Sep 17 00:00:00 2001
From: Liam DeBeasi <liamdebeasi@users.noreply.github.com>
Date: Thu, 10 Aug 2023 12:22:46 -0500
Subject: [PATCH] test(playwright): enable github reporter, test retries
 (#27961)

Issue number: N/A

---------

<!-- Please do not submit updates to dependencies unless it fixes an
issue. -->

<!-- Please try to limit your pull request to one type (bugfix, feature,
etc). Submit multiple pull requests if needed. -->

## What is the current behavior?
<!-- Please describe the current behavior that you are modifying. -->

The team would like to explore solutions for being informed of flaky
tests in a way that is not disruptive to our workflow. Currently, flaky
tests fail immediately which means we have to re-run them every time.
We'd like flaky tests to be automatically retried but also reported to
us so we can address them in a separate PR.

## What is the new behavior?
<!-- Please describe the behavior or changes that are being added by
this PR. -->

- Enables the Playwright GitHub reporter. This will report about flaky
tests on the PR if applicable as well as in the CI results.
- Enables test retries. Tests will be retried up to 2 times before
failing.
- Disables reporting slow tests in the GitHub reporter. Some of our
tests require gesture interaction which are inherently slow but
otherwise working as intended. We don't necessarily need to know about
these right now.
- Disables "maxFailures". Tests that can fail at most once are never
detected as flaky since they are never retried. As a result, we need to
disable this in order to have flaky tests be reported to us.

## Does this introduce a breaking change?

- [ ] Yes
- [x] No

<!-- If this introduces a breaking change, please describe the impact
and migration path for existing applications below. -->


## Other information

<!-- Any other information that is important to this PR such as
screenshots of how the component looks before and after the change. -->
---
 core/playwright.config.ts | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/core/playwright.config.ts b/core/playwright.config.ts
index eab43885880..24aa0b28aed 100644
--- a/core/playwright.config.ts
+++ b/core/playwright.config.ts
@@ -63,13 +63,16 @@ const config: PlaywrightTestConfig = {
   /* Fail the build on CI if you accidentally left test.only in the source code. */
   forbidOnly: !!process.env.CI,
   /* Fail fast on CI */
-  maxFailures: process.env.CI ? 1 : 0,
-  /* Flaky test should be either addressed or disabled until we can address them */
-  retries: 0,
+  maxFailures: 0,
+  retries: 2,
+  reportSlowTests: null,
   /* Opt out of parallel tests on CI. */
   workers: process.env.CI ? 1 : undefined,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-  reporter: 'html',
+  reporter: [
+    ['html'],
+    ['github']
+  ],
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
   use: {
     /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */