Add UI Playwright Tests to E2E Workflow

.github/workflows/e2e.yaml

@@ -61,6 +61,16 @@ jobs:
         - name: Install Python Deps
           run: python -m pip install "."
 
+        - name: Setup Node
+          uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
+          with:
+            node-version-file: 'src/leapfrogai_ui/package.json'
+
+        - name: Install UI/Playwright Dependencies
+          run: |
+            npm --prefix src/leapfrogai_ui ci
+            npx --prefix src/leapfrogai_ui playwright install
+
         - name: Setup UDS Environment
           uses: defenseunicorns/uds-common/.github/actions/setup@05f42bb3117b66ebef8c72ae050b34bce19385f5
           with:
@@ -71,9 +81,9 @@ jobs:
           run: |
             uds deploy k3d-core-slim-dev:0.22.2 --confirm
 
-          ##########
-          # Supabase
-          ##########
+        ##########
+        # Supabase
+        ##########
         - name: Deploy Supabase
           run: |
             make build-supabase LOCAL_VERSION=e2e-test
@@ -92,6 +102,16 @@ jobs:
             python -m pip install requests
             python -m pytest ./tests/e2e/test_supabase.py -v
 
+        ##########
+        # UI
+        ##########
+        - name: Deploy LFAI-UI
+          run: |
+            make build-ui LOCAL_VERSION=e2e-test
+            docker image prune -af
+            uds zarf package deploy packages/ui/zarf-package-leapfrogai-ui-amd64-e2e-test.tar.zst --confirm
+            rm packages/ui/zarf-package-leapfrogai-ui-amd64-e2e-test.tar.zst
+
         ##########
         # API
         ##########
@@ -107,6 +127,17 @@ jobs:
             python -m pip install requests
             python -m pytest ./tests/e2e/test_api.py -v
 
+        # Run the playwright UI tests using the deployed Supabase endpoint
+        - name: UI/API/Supabase E2E Playwright Tests
+          run: |
+            cp src/leapfrogai_ui/.env.example src/leapfrogai_ui/.env
+            TEST_ENV=CI PUBLIC_DISABLE_KEYCLOAK=true PUBLIC_SUPABASE_ANON_KEY=$ANON_KEY npm --prefix src/leapfrogai_ui run test:integration:ci
+
+        # The UI can be removed after the Playwright tests are finished
+        - name: Cleanup UI
+          run: |
+            uds zarf package remove leapfrogai-ui --confirm
+
         ##########
         # llama
         ##########

src/leapfrogai_ui/package.json

@@ -16,6 +16,7 @@
     "format": "prettier --write . && eslint . --fix",
     "test:integration": "playwright test",
     "test:integration:ui": "playwright test --ui",
+    "test:integration:ci": "playwright test tests/global.setup.ts tests/api.test.ts tests/api-keys.test.ts tests/header.test.ts tests/logout.test.ts",
     "test:unit": "vitest run",
     "test:unit:watch": "vitest",
     "prepare": "husky",

src/leapfrogai_ui/playwright.config.ts

@@ -39,7 +39,7 @@ const chromeConfig = {
 //   dependencies: ['setup']
 // };
 
-const config: PlaywrightTestConfig = {
+const defaultConfig: PlaywrightTestConfig = {
   // running more than 1 worker can cause flakiness due to test files being run at the same time in different browsers
   // (e.x. navigation history is incorrect)
   // Additionally, Leapfrog API is slow when attaching files to assistants, resulting in flaky tests
@@ -52,14 +52,35 @@ const config: PlaywrightTestConfig = {
       testMatch: /global\.teardown\.ts/
     },
     { ...chromeConfig }
-  ],
+  ]
+};
+
+// when in dev, create a local webserver
+const devConfig: PlaywrightTestConfig = {
   webServer: {
     command: 'npm run build && npm run preview',
     port: 4173,
     stderr: 'pipe'
   },
-  testDir: 'tests',
-  testMatch: /(.+\.)?(test|spec)\.[jt]s/
+  use: {
+    baseURL: 'http://localhost:4173'
+  }
+};
+
+// when e2e testing, use the deployed instance
+const CI_Config: PlaywrightTestConfig = {
+  use: {
+    baseURL: 'https://ai.uds.dev'
+  }
+};
+
+// get the environment type from command line. If none, set it to dev
+const environment = process.env.TEST_ENV || 'development';
+
+// config object with default configuration and environment specific configuration
+const config: PlaywrightTestConfig = {
+  ...defaultConfig,
+  ...(environment === 'CI' ? CI_Config : devConfig)
 };
 
 export default config;

src/leapfrogai_ui/testUtils/fakeData/index.ts

@@ -10,12 +10,12 @@ import type {
 } from 'openai/resources/beta/threads/messages';
 import { getUnixSeconds } from '../../src/lib/helpers/dates';
 import type { FileObject } from 'openai/resources/files';
-import type { Profile } from '$lib/types/profile';
+import type { Profile } from '../../src/lib/types/profile';
 import type { Session } from '@supabase/supabase-js';
 import type { Assistant } from 'openai/resources/beta/assistants';
 import type { VectorStore } from 'openai/resources/beta/vector-stores/index';
 import type { VectorStoreFile } from 'openai/resources/beta/vector-stores/files';
-import { type APIKeyRow, PERMISSIONS } from '$lib/types/apiKeys';
+import { type APIKeyRow, PERMISSIONS } from '../../src/lib/types/apiKeys';
 
 const todayOverride = new Date('2024-03-20T00:00');
 

src/leapfrogai_ui/tests/global.setup.ts

@@ -4,15 +4,24 @@ import * as OTPAuth from 'otpauth';
 const authFile = 'playwright/.auth/user.json';
 
 setup('authenticate', async ({ page }) => {
-  await page.goto('http://localhost:4173');
+  await page.goto('/'); // go to the home page
   if (process.env.PUBLIC_DISABLE_KEYCLOAK === 'true') {
-    // uses local supabase test users, logs in directly with Supabase, no Keycloak
-    await page.getByText('Already have an account? Sign In').click();
-    await page.getByPlaceholder('Your email address').click();
-    await page.getByPlaceholder('Your email address').fill('[email protected]');
-    await page.getByPlaceholder('Your password').click();
-    await page.getByPlaceholder('Your password').fill('password123');
-    await page.getByRole('button', { name: 'Sign In' }).click();
+    // when running in Github CI, create a new account because we don't have seed migrations
+    if (process.env.TEST_ENV === 'CI') {
+      await page.getByPlaceholder('Your email address').click();
+      await page.getByPlaceholder('Your email address').fill('[email protected]');
+      await page.getByPlaceholder('Your password').click();
+      await page.getByPlaceholder('Your password').fill('password123');
+      await page.getByRole('button', { name: 'Sign Up' }).click();
+    } else {
+      // uses local supabase test users, logs in directly with Supabase, no Keycloak
+      await page.getByText('Already have an account? Sign In').click();
+      await page.getByPlaceholder('Your email address').click();
+      await page.getByPlaceholder('Your email address').fill('[email protected]');
+      await page.getByPlaceholder('Your password').click();
+      await page.getByPlaceholder('Your password').fill('password123');
+      await page.getByRole('button', { name: 'Sign In' }).click();
+    }
   } else {
     // With Keycloak
     await page.getByRole('button', { name: 'Log In' }).click();
@@ -37,7 +46,7 @@ setup('authenticate', async ({ page }) => {
   //
   // Login flow sets cookies in the process of several redirects.
   // Wait for the final URL to ensure that the cookies are actually set.
-  await page.waitForURL('http://localhost:4173/chat');
+  await page.waitForURL('/chat');
 
   // Alternatively, you can wait until the page reaches a state where all cookies are set.
   //   await expect(page.getByRole('button', { name: 'View profile and more' })).toBeVisible();

src/leapfrogai_ui/tests/global.teardown.ts

@@ -4,13 +4,16 @@ import { deleteAllAssistants, deleteAssistantAvatars } from './helpers/assistant
 import { deleteAllTestThreadsWithApi } from './helpers/threadHelpers';
 import { deleteAllTestAPIKeys } from './helpers/apiHelpers';
 
-teardown('teardown', async ({ openAIClient }) => {
-  console.log('cleaning up...');
-  deleteAllGeneratedFixtureFiles();
-  await deleteAllTestFilesWithApi(openAIClient);
-  await deleteAllAssistants(openAIClient);
-  await deleteAllTestThreadsWithApi(openAIClient);
-  await deleteAssistantAvatars();
-  await deleteAllTestAPIKeys();
-  console.log('clean up complete');
-});
+// teardown not necessary in CI testing envs
+if (process.env.TEST_ENV !== 'CI') {
+  teardown('teardown', async ({ openAIClient }) => {
+    console.log('cleaning up...');
+    deleteAllGeneratedFixtureFiles();
+    await deleteAllTestFilesWithApi(openAIClient);
+    await deleteAllAssistants(openAIClient);
+    await deleteAllTestThreadsWithApi(openAIClient);
+    await deleteAssistantAvatars();
+    await deleteAllTestAPIKeys();
+    console.log('clean up complete');
+  });
+}

src/leapfrogai_ui/tests/helpers/assistantHelpers.ts

@@ -1,8 +1,8 @@
 import OpenAI from 'openai';
 import { expect, type Page } from '@playwright/test';
-import { getFakeAssistantInput } from '$testUtils/fakeData';
+import { getFakeAssistantInput } from '../../testUtils/fakeData';
 import type { AssistantCreateParams } from 'openai/resources/beta/assistants';
-import type { AssistantInput, LFAssistant } from '$lib/types/assistants';
+import type { AssistantInput, LFAssistant } from '../../src/lib/types/assistants';
 import { supabase } from './helpers';
 
 // Note - this will not apply the temperature slider value provided, it only clicks on the 0.5 increment

src/leapfrogai_ui/tests/logout.test.ts

@@ -6,7 +6,7 @@ test('it can log out', async ({ page }) => {
   await page.getByLabel('User').click();
   await page.getByLabel('Log Out').click();
 
-  await page.waitForURL('http://localhost:4173');
+  await page.waitForURL('/');
 
   if (process.env.PUBLIC_DISABLE_KEYCLOAK === 'true') {
     await expect(page.getByText('Sign In')).toBeVisible();

src/leapfrogai_ui/tests/rag.test.ts

@@ -1,5 +1,5 @@
 import { expect, test } from './fixtures';
-import { getFakeAssistantInput } from '$testUtils/fakeData';
+import { getFakeAssistantInput } from '../testUtils/fakeData';
 import { delay } from 'msw';
 import {
   createPDF,