test: extend tests to cover all playwright browsers (#7)

* test: extend tests
* test: attempt to stabilise the initial page interaction
* test: increase timeout and store recordings as artifacts
* revert: can't trust the browser/vo output

.github/workflows/test-voiceover.yml

@@ -12,6 +12,7 @@ jobs:
     strategy:
       matrix:
         os: [macos-11, macos-12]
+        browser: [chromium, firefox, webkit]
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v1
@@ -20,9 +21,12 @@ jobs:
       - name: Guidepup Setup
         uses: guidepup/[email protected]
       - run: yarn install --frozen-lockfile
-      - run: yarn test
+      - run: yarn pretest
+      - run: yarn test:${{ matrix.browser }}
       - uses: actions/upload-artifact@v2
         if: always()
         with:
           name: artifacts
-          path: ./test-results/**/*
+          path: |
+            **/test-results/**/*
+            **/recordings/**/*

example/playwright.config.ts → example/chromium.config.ts

@@ -1,17 +1,11 @@
 import { devices, PlaywrightTestConfig } from "@playwright/test";
-import { voConfig } from "../src";
 
 const config: PlaywrightTestConfig = {
-  ...voConfig,
   reportSlowTests: null,
   workers: 1,
-  timeout: 2 * 60 * 1000,
-  retries: 3,
+  timeout: 3 * 60 * 1000,
+  retries: 2,
   projects: [
-    {
-      name: "webkit",
-      use: { ...devices["Desktop Safari"], headless: false, video: "on" },
-    },
     {
       name: "chromium",
       use: { ...devices["Desktop Chrome"], headless: false, video: "on" },

example/firefox.config.ts

@@ -0,0 +1,16 @@
+import { devices, PlaywrightTestConfig } from "@playwright/test";
+
+const config: PlaywrightTestConfig = {
+  reportSlowTests: null,
+  workers: 1,
+  timeout: 3 * 60 * 1000,
+  retries: 2,
+  projects: [
+    {
+      name: "firefox",
+      use: { ...devices["Desktop Firefox"], headless: false, video: "on" },
+    },
+  ],
+};
+
+export default config;

example/tests/chromium/chromium.itemText.snapshot.json

@@ -0,0 +1,8 @@
+[
+  "guidepup/guidepup heading level 1",
+  "Latest commit heading level 2",
+  "Git stats heading level 2",
+  "Files heading level 2",
+  "README.md heading level 2",
+  "Guidepup heading level 1"
+]

example/tests/chromium/chromium.spec.ts

@@ -0,0 +1,42 @@
+import { expect } from "@playwright/test";
+import { headerNavigation } from "../headerNavigation";
+import itemTextSnapshot from "./chromium.itemText.snapshot.json";
+import { macOSRecord } from "@guidepup/guidepup";
+import spokenPhraseSnapshot from "./chromium.spokenPhrase.snapshot.json";
+import { voTest as test } from "../../../src";
+
+test.describe("Chromium Playwright VoiceOver", () => {
+  test("I can navigate the Guidepup Github page", async ({
+    page,
+    voiceOver,
+  }) => {
+    const stopRecording = macOSRecord(
+      `./recordings/playwright-voiceover-chromium-${+new Date()}.mov`
+    );
+
+    await headerNavigation({ page, voiceOver });
+
+    // Assert that we've ended up where we expected and what we were told on
+    // the way there is as expected.
+
+    const itemTextLog = await voiceOver.itemTextLog();
+    const spokenPhraseLog = await voiceOver.spokenPhraseLog();
+
+    console.log(JSON.stringify(itemTextLog, undefined, 2));
+    console.log(JSON.stringify(spokenPhraseLog, undefined, 2));
+
+    for (const expectedItem of itemTextSnapshot) {
+      expect(!!itemTextLog.find((log) => log.includes(expectedItem))).toBe(
+        true
+      );
+    }
+
+    for (const expectedPhrase of spokenPhraseSnapshot) {
+      expect(
+        !!spokenPhraseLog.find((log) => log.includes(expectedPhrase))
+      ).toBe(true);
+    }
+
+    stopRecording();
+  });
+});

example/tests/chromium/chromium.spokenPhrase.snapshot.json

@@ -0,0 +1,8 @@
+[
+  "heading level 1 guidepup/guidepup. You are currently on a heading level 1.",
+  "heading level 2 Latest commit. You are currently on a heading level 2.",
+  "heading level 2 Git stats. You are currently on a heading level 2.",
+  "heading level 2 Files. You are currently on a heading level 2.",
+  "heading level 2 link README.md. You are currently on a heading level 2.",
+  "heading level 1 Guidepup. You are currently on a heading level 1."
+]

example/tests/firefox/firefox.itemText.snapshot.json

@@ -0,0 +1,8 @@
+[
+  "guidepup/guidepup heading level 1",
+  "Latest commit heading level 2",
+  "Git stats heading level 2",
+  "Files heading level 2",
+  "README.md heading level 2",
+  "Guidepup heading level 1"
+]

example/tests/firefox/firefox.spec.ts

@@ -0,0 +1,42 @@
+import { expect } from "@playwright/test";
+import { headerNavigation } from "../headerNavigation";
+import itemTextSnapshot from "./firefox.itemText.snapshot.json";
+import { macOSRecord } from "@guidepup/guidepup";
+import spokenPhraseSnapshot from "./firefox.spokenPhrase.snapshot.json";
+import { voTest as test } from "../../../src";
+
+test.describe("Firefox Playwright VoiceOver", () => {
+  test("I can navigate the Guidepup Github page", async ({
+    page,
+    voiceOver,
+  }) => {
+    const stopRecording = macOSRecord(
+      `./recordings/playwright-voiceover-firefox-${+new Date()}.mov`
+    );
+
+    await headerNavigation({ page, voiceOver });
+
+    // Assert that we've ended up where we expected and what we were told on
+    // the way there is as expected.
+
+    const itemTextLog = await voiceOver.itemTextLog();
+    const spokenPhraseLog = await voiceOver.spokenPhraseLog();
+
+    console.log(JSON.stringify(itemTextLog, undefined, 2));
+    console.log(JSON.stringify(spokenPhraseLog, undefined, 2));
+
+    for (const expectedItem of itemTextSnapshot) {
+      expect(!!itemTextLog.find((log) => log.includes(expectedItem))).toBe(
+        true
+      );
+    }
+
+    for (const expectedPhrase of spokenPhraseSnapshot) {
+      expect(
+        !!spokenPhraseLog.find((log) => log.includes(expectedPhrase))
+      ).toBe(true);
+    }
+
+    stopRecording();
+  });
+});

example/tests/firefox/firefox.spokenPhrase.snapshot.json

@@ -0,0 +1,8 @@
+[
+  "heading level 1 guidepup/guidepup. You are currently on a heading level 1.",
+  "heading level 2 Latest commit. You are currently on a heading level 2.",
+  "heading level 2 Git stats. You are currently on a heading level 2.",
+  "heading level 2 Files. You are currently on a heading level 2.",
+  "heading level 2 link README.md. You are currently on a heading level 2.",
+  "heading level 1 Guidepup. You are currently on a heading level 1."
+]

example/tests/headerNavigation.ts

@@ -0,0 +1,23 @@
+import { expect } from "@playwright/test";
+
+async function delay(ms: number) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+export async function headerNavigation({ page, voiceOver }) {
+  // Navigate to Guidepup GitHub page 🎉
+  await page.goto("https://github.com/guidepup/guidepup", {
+    waitUntil: "domcontentloaded",
+  });
+
+  // Wait for page to be ready and interact 🙌
+  await expect(page.locator('header[role="banner"]')).toBeVisible();
+  await delay(500);
+
+  await voiceOver.interact();
+
+  // Move across the page menu to the Guidepup heading using VoiceOver 🔎
+  while ((await voiceOver.itemText()) !== "Guidepup heading level 1") {
+    await voiceOver.perform(voiceOver.keyboard.commands.findNextHeading);
+  }
+}

example/tests/webkit/webkit.itemText.snapshot.json

@@ -0,0 +1,8 @@
+[
+  "guidepup/guidepup heading level 1",
+  "Latest commit heading level 2",
+  "Git stats heading level 2",
+  "Files heading level 2",
+  "README.md heading level 2",
+  "Guidepup heading level 1"
+]

example/tests/webkit/webkit.spec.ts

@@ -0,0 +1,40 @@
+import { expect } from "@playwright/test";
+import { headerNavigation } from "../headerNavigation";
+import itemTextSnapshot from "./webkit.itemText.snapshot.json";
+import { macOSRecord } from "@guidepup/guidepup";
+import spokenPhraseSnapshot from "./webkit.spokenPhrase.snapshot.json";
+import { voTest as test } from "../../../src";
+
+test.describe("Webkit Playwright VoiceOver", () => {
+  test("I can navigate the Guidepup Github page", async ({
+    page,
+    voiceOver,
+  }) => {
+    const stopRecording = macOSRecord(
+      `./recordings/playwright-voiceover-webkit-${+new Date()}.mov`
+    );
+
+    await headerNavigation({ page, voiceOver });
+
+    // Assert that we've ended up where we expected and what we were told on
+    // the way there is as expected.
+
+    const itemTextLog = await voiceOver.itemTextLog();
+    const spokenPhraseLog = await voiceOver.spokenPhraseLog();
+
+    console.log(JSON.stringify(itemTextLog, undefined, 2));
+    console.log(JSON.stringify(spokenPhraseLog, undefined, 2));
+
+    for (const expectedItem of itemTextSnapshot) {
+      expect(!!itemTextLog.find((log) => log.includes(expectedItem))).toBe(true);
+    }
+
+    for (const expectedPhrase of spokenPhraseSnapshot) {
+      expect(!!spokenPhraseLog.find((log) => log.includes(expectedPhrase))).toBe(
+        true
+      );
+    }
+
+    stopRecording();
+  });
+});

example/tests/webkit/webkit.spokenPhrase.snapshot.json

@@ -0,0 +1,8 @@
+[
+  "heading level 1 guidepup/guidepup. You are currently on a heading level 1.",
+  "heading level 2 Latest commit. You are currently on a heading level 2.",
+  "heading level 2 Git stats. You are currently on a heading level 2.",
+  "heading level 2 Files. You are currently on a heading level 2.",
+  "heading level 2 link README.md. You are currently on a heading level 2.",
+  "heading level 1 Guidepup. You are currently on a heading level 1."
+]

example/webkit.config.ts

@@ -0,0 +1,16 @@
+import { devices, PlaywrightTestConfig } from "@playwright/test";
+
+const config: PlaywrightTestConfig = {
+  reportSlowTests: null,
+  workers: 1,
+  timeout: 3 * 60 * 1000,
+  retries: 2,
+  projects: [
+    {
+      name: "webkit",
+      use: { ...devices["Desktop Safari"], headless: false, video: "on" },
+    },
+  ],
+};
+
+export default config;