add basic e2e tests for Metro web Fast Refresh (expo#28237)

# Why

- It's important we keep Metro web working continuously. The behavior of
Fast Refresh is a bit of a black box, so having a test gives us a
clearer list of expected steps that must work.

<!--
Please describe the motivation for this PR, and link to relevant GitHub
issues, forums posts, or feature requests.
-->

# How

- I've added Playwright to the CLI and an additional script for running
playwright tests.
- The CI job is split out of the E2E CLI job so we can get faster
failures. Pinging @byCedric for advice on the structure and how we could
potentially reuse work between these jobs.

<!--
How did you build this feature or fix this bug and why?
-->

# Test Plan

- Running `yarn test:playwright` in `packages/@expo/cli` works as
expected locally.
- CI should run the playwright test continuously. We'll also be keeping
an eye on the timings to ensure this test runs fast and doesn't become
another burden on our merge process.

<!--
Please describe how you tested this change and how a reviewer could
reproduce your test, especially if this PR does not include automated
tests! If possible, please also provide terminal output and/or
screenshots demonstrating your test/reproduction.
-->

# Checklist

<!--
Please check the appropriate items below if they apply to your diff.
This is required for changes to Expo modules.
-->

- [ ] Documentation is up to date to reflect these changes (eg:
https://docs.expo.dev and README.md).
- [ ] Conforms with the [Documentation Writing Style
Guide](https://github.com/expo/expo/blob/main/guides/Expo%20Documentation%20Writing%20Style%20Guide.md)
- [ ] This diff will work correctly for `npx expo prebuild` & EAS Build
(eg: updated a module plugin).

.github/workflows/cli.yml

@@ -52,6 +52,9 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 100
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
       - name: ⬇️ Fetch commits from base branch
         run: git fetch origin ${{ github.event.before || github.base_ref || 'main' }}:${{ github.event.before || github.base_ref || 'main' }} --depth 100
         if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch'
@@ -86,3 +89,40 @@ jobs:
       #     status: ${{ job.status }}
       #     fields: job,message,ref,eventName,author,took
       #     author_name: Check packages
+  web:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: 👀 Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 100
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+      - name: ⬇️ Fetch commits from base branch
+        run: git fetch origin ${{ github.event.before || github.base_ref || 'main' }}:${{ github.event.before || github.base_ref || 'main' }} --depth 100
+        if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch'
+      - name: 🏗️ Setup Bun
+        uses: oven-sh/setup-bun@v1
+        with:
+          bun-version: latest
+      - name: ♻️ Restore caches
+        uses: ./.github/actions/expo-caches
+        id: expo-caches
+        with:
+          yarn-workspace: 'true'
+      - name: 🧶 Install node modules in root dir
+        if: steps.expo-caches.outputs.yarn-workspace-hit != 'true'
+        run: yarn install --frozen-lockfile
+      - name: Install Playwright Browsers
+        run: bun playwright install --with-deps
+        working-directory: packages/@expo/cli
+      - name: E2E (playwright) Test CLI
+        run: yarn test:playwright
+        working-directory: packages/@expo/cli
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: playwright-report
+          path: packages/@expo/cli/playwright-report/
+          retention-days: 30

apps/router-e2e/__e2e__/fast-refresh/app/_layout.tsx

@@ -0,0 +1,14 @@
+import { Slot } from 'expo-router';
+import Head from 'expo-router/head';
+
+export default function Layout() {
+  const LAYOUT_VALUE = 'TEST_VALUE';
+  return (
+    <>
+      <Head>
+        <meta name="expo-nested-layout" content={LAYOUT_VALUE} />
+      </Head>
+      <Slot />
+    </>
+  );
+}

apps/router-e2e/__e2e__/fast-refresh/app/index.tsx

@@ -0,0 +1,17 @@
+import React from 'react';
+
+export default function Page() {
+  const [index, setIndex] = React.useState(0);
+  // Do not change this value, it is used in tests
+  const input = 'ROUTE_VALUE';
+
+  return (
+    <>
+      <button data-testid="index-increment" onClick={() => setIndex((i) => i + 1)}>
+        increment
+      </button>
+      <div data-testid="index-count">{index}</div>
+      <div data-testid="index-text">{input}</div>
+    </>
+  );
+}

packages/@expo/cli/.gitignore

@@ -1 +1,4 @@
-/build/
+/build/
+/test-results/
+/playwright-report/
+/playwright/.cache/

packages/@expo/cli/e2e/__tests__/export/export-side-effects.ts

@@ -18,15 +18,15 @@ declare const process: {
 const originalForceColor = process.env.FORCE_COLOR;
 const originalCI = process.env.CI;
 
-function clearEnv() {
+export function clearEnv() {
   process.env.FORCE_COLOR = '0';
   process.env.CI = '1';
   process.env.EXPO_USE_PATH_ALIASES = '1';
   delete process.env.EXPO_USE_STATIC;
   delete process.env.EXPO_E2E_BASE_PATH;
 }
 
-function restoreEnv() {
+export function restoreEnv() {
   process.env.FORCE_COLOR = originalForceColor;
   process.env.CI = originalCI;
   delete process.env.EXPO_USE_PATH_ALIASES;

packages/@expo/cli/e2e/playwright/dev/fast-refresh.test.ts

@@ -0,0 +1,195 @@
+import { test, Page, WebSocket, expect } from '@playwright/test';
+import path from 'path';
+import fs from 'fs';
+
+import { clearEnv, restoreEnv } from '../../__tests__/export/export-side-effects';
+import { getRouterE2ERoot } from '../../__tests__/utils';
+import { ExpoStartCommand } from '../../utils/command-instance';
+
+test.beforeAll(() => clearEnv());
+test.afterAll(() => restoreEnv());
+
+const projectRoot = getRouterE2ERoot();
+const inputDir = 'fast-refresh';
+
+test.describe(inputDir, () => {
+  test.beforeAll(async () => {
+    // Could take 45s depending on how fast the bundler resolves
+    test.setTimeout(560 * 1000);
+  });
+
+  let expo: ExpoStartCommand;
+
+  test.beforeEach(async () => {
+    expo = new ExpoStartCommand(projectRoot, {
+      NODE_ENV: 'production',
+      EXPO_USE_STATIC: 'single',
+      E2E_ROUTER_JS_ENGINE: 'hermes',
+      E2E_ROUTER_SRC: inputDir,
+      E2E_ROUTER_ASYNC: 'development',
+
+      // Ensure CI is disabled otherwise the file watcher won't run.
+      CI: '0',
+    });
+  });
+
+  test.afterEach(async () => {
+    await expo.stopAsync();
+  });
+
+  const targetDirectory = path.join(projectRoot, '__e2e__/fast-refresh/app');
+  const indexFile = path.join(targetDirectory, 'index.tsx');
+
+  const mutateIndexFile = async (mutator: (contents: string) => string) => {
+    const indexContents = await fs.promises.readFile(indexFile, 'utf8');
+    await fs.promises.writeFile(indexFile, mutator(indexContents), 'utf8');
+  };
+
+  test.beforeAll(async () => {
+    // Ensure `const ROUTE_VALUE = 'ROUTE_VALUE_1';` -> `const ROUTE_VALUE = 'ROUTE_VALUE';` before starting
+    await mutateIndexFile((contents) => {
+      return contents.replace(/ROUTE_VALUE_[\d\w]+/g, 'ROUTE_VALUE');
+    });
+  });
+
+  test.afterAll(async () => {
+    // Ensure `const ROUTE_VALUE = 'ROUTE_VALUE_1';` -> `const ROUTE_VALUE = 'ROUTE_VALUE';` before starting
+    await mutateIndexFile((contents) => {
+      return contents.replace(/ROUTE_VALUE_[\d\w]+/g, 'ROUTE_VALUE');
+    });
+  });
+
+  test('updates with fast refresh', async ({ page }) => {
+    await expo.startAsync();
+    console.log('Server running:', expo.url);
+    await expo.fetchAsync('/');
+    console.log('Eagerly bundled JS');
+
+    // Navigate to the app
+    await page.goto(expo.url);
+
+    // Ensure the message socket connects (not related to HMR).
+
+    console.log('Waiting for /hot socket');
+
+    // Ensure the hot socket connects
+    const hotSocket = await raceOrFail(
+      waitForSocket(page, (ws) => ws.url().endsWith('/hot')),
+      // Should be really fast
+      500,
+      'HMR websocket on client took too long to connect.'
+    );
+
+    console.log('Found /hot socket');
+
+    // Order matters, message socket is set second.
+    await raceOrFail(
+      waitForSocket(page, (ws) => ws.url().endsWith('/message')),
+      500,
+      'Message socket on client took too long to connect.'
+    );
+
+    // Ensure the entry point is registered
+    await hotSocket.waitForEvent('framesent', {
+      predicate: makeHotPredicate((event) => {
+        return event.type === 'register-entrypoints' && !!event.entryPoints.length;
+      }),
+    });
+    // Observe the handshake with Metro
+    await hotSocket.waitForEvent('framereceived', {
+      predicate: makeHotPredicate((event) => {
+        return event.type === 'bundle-registered';
+      }),
+    });
+
+    // Ensure the initial state is correct
+    await expect(page.locator('[data-testid="index-count"]')).toHaveText('0');
+
+    // Trigger a state change by clicking a button, then check if the state is rendered to the screen.
+    page.locator('[data-testid="index-increment"]').click();
+    await expect(page.locator('[data-testid="index-count"]')).toHaveText('1');
+
+    // data-testid="index-text"
+    const test = page.locator('[data-testid="index-text"]');
+    await expect(test).toHaveText('ROUTE_VALUE');
+
+    // Now we'll modify the file and observe a fast refresh event...
+
+    // Use a changing value to prevent caching.
+    const nextValue = 'ROUTE_VALUE_' + Date.now();
+
+    // Ensure `const ROUTE_VALUE = 'ROUTE_VALUE_1';` -> `const ROUTE_VALUE = 'ROUTE_VALUE';` before starting
+    await mutateIndexFile((contents) => {
+      if (!contents.includes("'ROUTE_VALUE'")) {
+        throw new Error(`Expected to find 'ROUTE_VALUE' in the file`);
+      }
+      console.log('Emulate writing to a file');
+      return contents.replace(/ROUTE_VALUE/g, nextValue);
+    });
+
+    // Metro begins the HMR process
+    await raceOrFail(
+      hotSocket.waitForEvent('framereceived', {
+        predicate: makeHotPredicate((event) => {
+          return event.type === 'update-start';
+        }),
+      }),
+      1000,
+      'Metro took too long to detect the file change and start the HMR process.'
+    );
+
+    // Metro sends the HMR mutation
+    await hotSocket.waitForEvent('framereceived', {
+      predicate: makeHotPredicate((event) => {
+        return event.type === 'update' && !!event.body.modified.length;
+      }),
+    });
+
+    // Metro completes the HMR update
+    await hotSocket.waitForEvent('framereceived', {
+      predicate: makeHotPredicate((event) => {
+        return event.type === 'update-done';
+      }),
+    });
+
+    // Observe that our change has been rendered to the screen
+    await expect(page.locator('[data-testid="index-text"]')).toHaveText(nextValue);
+
+    // Ensure the state is preserved between updates
+    await expect(page.locator('[data-testid="index-count"]')).toHaveText('1');
+  });
+});
+
+function makeHotPredicate(predicate: (data: Record<string, any>) => boolean) {
+  return ({ payload }: { payload: string | Buffer }) => {
+    const event = JSON.parse(typeof payload === 'string' ? payload : payload.toString());
+    return predicate(event);
+  };
+}
+
+function waitForSocket(page: Page, matcher: (ws: WebSocket) => boolean) {
+  return new Promise<WebSocket>((res) => {
+    page.on('websocket', (ws) => {
+      if (matcher(ws)) {
+        res(ws);
+      }
+    });
+  });
+}
+
+export const raceOrFail = (promise: Promise<any>, timeout: number, message: string) =>
+  Promise.race([
+    // Wrap promise with profile logging
+    (async () => {
+      const start = Date.now();
+      const value = await promise;
+      const end = Date.now();
+      console.log('Resolved:', end - start + 'ms');
+      return value;
+    })(),
+    new Promise((resolve, reject) => {
+      setTimeout(() => {
+        reject(new Error(`Test was too slow (${timeout}ms): ${message}`));
+      }, timeout);
+    }),
+  ]);