diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index ee48128b6..7e9f1e8e1 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -61,6 +61,16 @@ jobs: - name: Install Python Deps run: python -m pip install "." + - name: Setup Node + uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2 + with: + node-version-file: 'src/leapfrogai_ui/package.json' + + - name: Install UI/Playwright Dependencies + run: | + npm --prefix src/leapfrogai_ui ci + npx --prefix src/leapfrogai_ui playwright install + - name: Setup UDS Environment uses: defenseunicorns/uds-common/.github/actions/setup@05f42bb3117b66ebef8c72ae050b34bce19385f5 with: @@ -71,9 +81,9 @@ jobs: run: | uds deploy k3d-core-slim-dev:0.22.2 --confirm - ########## - # Supabase - ########## + ########## + # Supabase + ########## - name: Deploy Supabase run: | make build-supabase LOCAL_VERSION=e2e-test @@ -92,6 +102,16 @@ jobs: python -m pip install requests python -m pytest ./tests/e2e/test_supabase.py -v + ########## + # UI + ########## + - name: Deploy LFAI-UI + run: | + make build-ui LOCAL_VERSION=e2e-test + docker image prune -af + uds zarf package deploy packages/ui/zarf-package-leapfrogai-ui-amd64-e2e-test.tar.zst --confirm + rm packages/ui/zarf-package-leapfrogai-ui-amd64-e2e-test.tar.zst + ########## # API ########## @@ -107,6 +127,17 @@ jobs: python -m pip install requests python -m pytest ./tests/e2e/test_api.py -v + # Run the playwright UI tests using the deployed Supabase endpoint + - name: UI/API/Supabase E2E Playwright Tests + run: | + cp src/leapfrogai_ui/.env.example src/leapfrogai_ui/.env + TEST_ENV=CI PUBLIC_DISABLE_KEYCLOAK=true PUBLIC_SUPABASE_ANON_KEY=$ANON_KEY npm --prefix src/leapfrogai_ui run test:integration:ci + + # The UI can be removed after the Playwright tests are finished + - name: Cleanup UI + run: | + uds zarf package remove leapfrogai-ui --confirm + ########## # llama ########## diff --git a/src/leapfrogai_ui/package.json b/src/leapfrogai_ui/package.json index b7264b976..a7c7446d8 100644 --- a/src/leapfrogai_ui/package.json +++ b/src/leapfrogai_ui/package.json @@ -16,6 +16,7 @@ "format": "prettier --write . && eslint . --fix", "test:integration": "playwright test", "test:integration:ui": "playwright test --ui", + "test:integration:ci": "playwright test tests/global.setup.ts tests/api.test.ts tests/api-keys.test.ts tests/header.test.ts tests/logout.test.ts", "test:unit": "vitest run", "test:unit:watch": "vitest", "prepare": "husky", diff --git a/src/leapfrogai_ui/playwright.config.ts b/src/leapfrogai_ui/playwright.config.ts index 985692680..eb599e436 100644 --- a/src/leapfrogai_ui/playwright.config.ts +++ b/src/leapfrogai_ui/playwright.config.ts @@ -39,7 +39,7 @@ const chromeConfig = { // dependencies: ['setup'] // }; -const config: PlaywrightTestConfig = { +const defaultConfig: PlaywrightTestConfig = { // running more than 1 worker can cause flakiness due to test files being run at the same time in different browsers // (e.x. navigation history is incorrect) // Additionally, Leapfrog API is slow when attaching files to assistants, resulting in flaky tests @@ -52,14 +52,35 @@ const config: PlaywrightTestConfig = { testMatch: /global\.teardown\.ts/ }, { ...chromeConfig } - ], + ] +}; + +// when in dev, create a local webserver +const devConfig: PlaywrightTestConfig = { webServer: { command: 'npm run build && npm run preview', port: 4173, stderr: 'pipe' }, - testDir: 'tests', - testMatch: /(.+\.)?(test|spec)\.[jt]s/ + use: { + baseURL: 'http://localhost:4173' + } +}; + +// when e2e testing, use the deployed instance +const CI_Config: PlaywrightTestConfig = { + use: { + baseURL: 'https://ai.uds.dev' + } +}; + +// get the environment type from command line. If none, set it to dev +const environment = process.env.TEST_ENV || 'development'; + +// config object with default configuration and environment specific configuration +const config: PlaywrightTestConfig = { + ...defaultConfig, + ...(environment === 'CI' ? CI_Config : devConfig) }; export default config; diff --git a/src/leapfrogai_ui/testUtils/fakeData/index.ts b/src/leapfrogai_ui/testUtils/fakeData/index.ts index 7760db90f..08f8ffb50 100644 --- a/src/leapfrogai_ui/testUtils/fakeData/index.ts +++ b/src/leapfrogai_ui/testUtils/fakeData/index.ts @@ -10,12 +10,12 @@ import type { } from 'openai/resources/beta/threads/messages'; import { getUnixSeconds } from '../../src/lib/helpers/dates'; import type { FileObject } from 'openai/resources/files'; -import type { Profile } from '$lib/types/profile'; +import type { Profile } from '../../src/lib/types/profile'; import type { Session } from '@supabase/supabase-js'; import type { Assistant } from 'openai/resources/beta/assistants'; import type { VectorStore } from 'openai/resources/beta/vector-stores/index'; import type { VectorStoreFile } from 'openai/resources/beta/vector-stores/files'; -import { type APIKeyRow, PERMISSIONS } from '$lib/types/apiKeys'; +import { type APIKeyRow, PERMISSIONS } from '../../src/lib/types/apiKeys'; const todayOverride = new Date('2024-03-20T00:00'); diff --git a/src/leapfrogai_ui/tests/global.setup.ts b/src/leapfrogai_ui/tests/global.setup.ts index 1f2557918..6d8a007fe 100644 --- a/src/leapfrogai_ui/tests/global.setup.ts +++ b/src/leapfrogai_ui/tests/global.setup.ts @@ -4,15 +4,24 @@ import * as OTPAuth from 'otpauth'; const authFile = 'playwright/.auth/user.json'; setup('authenticate', async ({ page }) => { - await page.goto('http://localhost:4173'); + await page.goto('/'); // go to the home page if (process.env.PUBLIC_DISABLE_KEYCLOAK === 'true') { - // uses local supabase test users, logs in directly with Supabase, no Keycloak - await page.getByText('Already have an account? Sign In').click(); - await page.getByPlaceholder('Your email address').click(); - await page.getByPlaceholder('Your email address').fill('user1@test.com'); - await page.getByPlaceholder('Your password').click(); - await page.getByPlaceholder('Your password').fill('password123'); - await page.getByRole('button', { name: 'Sign In' }).click(); + // when running in Github CI, create a new account because we don't have seed migrations + if (process.env.TEST_ENV === 'CI') { + await page.getByPlaceholder('Your email address').click(); + await page.getByPlaceholder('Your email address').fill('ci_user@test.com'); + await page.getByPlaceholder('Your password').click(); + await page.getByPlaceholder('Your password').fill('password123'); + await page.getByRole('button', { name: 'Sign Up' }).click(); + } else { + // uses local supabase test users, logs in directly with Supabase, no Keycloak + await page.getByText('Already have an account? Sign In').click(); + await page.getByPlaceholder('Your email address').click(); + await page.getByPlaceholder('Your email address').fill('user1@test.com'); + await page.getByPlaceholder('Your password').click(); + await page.getByPlaceholder('Your password').fill('password123'); + await page.getByRole('button', { name: 'Sign In' }).click(); + } } else { // With Keycloak await page.getByRole('button', { name: 'Log In' }).click(); @@ -37,7 +46,7 @@ setup('authenticate', async ({ page }) => { // // Login flow sets cookies in the process of several redirects. // Wait for the final URL to ensure that the cookies are actually set. - await page.waitForURL('http://localhost:4173/chat'); + await page.waitForURL('/chat'); // Alternatively, you can wait until the page reaches a state where all cookies are set. // await expect(page.getByRole('button', { name: 'View profile and more' })).toBeVisible(); diff --git a/src/leapfrogai_ui/tests/global.teardown.ts b/src/leapfrogai_ui/tests/global.teardown.ts index 144612124..715079bf5 100644 --- a/src/leapfrogai_ui/tests/global.teardown.ts +++ b/src/leapfrogai_ui/tests/global.teardown.ts @@ -4,13 +4,16 @@ import { deleteAllAssistants, deleteAssistantAvatars } from './helpers/assistant import { deleteAllTestThreadsWithApi } from './helpers/threadHelpers'; import { deleteAllTestAPIKeys } from './helpers/apiHelpers'; -teardown('teardown', async ({ openAIClient }) => { - console.log('cleaning up...'); - deleteAllGeneratedFixtureFiles(); - await deleteAllTestFilesWithApi(openAIClient); - await deleteAllAssistants(openAIClient); - await deleteAllTestThreadsWithApi(openAIClient); - await deleteAssistantAvatars(); - await deleteAllTestAPIKeys(); - console.log('clean up complete'); -}); +// teardown not necessary in CI testing envs +if (process.env.TEST_ENV !== 'CI') { + teardown('teardown', async ({ openAIClient }) => { + console.log('cleaning up...'); + deleteAllGeneratedFixtureFiles(); + await deleteAllTestFilesWithApi(openAIClient); + await deleteAllAssistants(openAIClient); + await deleteAllTestThreadsWithApi(openAIClient); + await deleteAssistantAvatars(); + await deleteAllTestAPIKeys(); + console.log('clean up complete'); + }); +} diff --git a/src/leapfrogai_ui/tests/helpers/assistantHelpers.ts b/src/leapfrogai_ui/tests/helpers/assistantHelpers.ts index 72ac51a96..51ac3ee6a 100644 --- a/src/leapfrogai_ui/tests/helpers/assistantHelpers.ts +++ b/src/leapfrogai_ui/tests/helpers/assistantHelpers.ts @@ -1,8 +1,8 @@ import OpenAI from 'openai'; import { expect, type Page } from '@playwright/test'; -import { getFakeAssistantInput } from '$testUtils/fakeData'; +import { getFakeAssistantInput } from '../../testUtils/fakeData'; import type { AssistantCreateParams } from 'openai/resources/beta/assistants'; -import type { AssistantInput, LFAssistant } from '$lib/types/assistants'; +import type { AssistantInput, LFAssistant } from '../../src/lib/types/assistants'; import { supabase } from './helpers'; // Note - this will not apply the temperature slider value provided, it only clicks on the 0.5 increment diff --git a/src/leapfrogai_ui/tests/logout.test.ts b/src/leapfrogai_ui/tests/logout.test.ts index ba3ad5887..b53dfe99b 100644 --- a/src/leapfrogai_ui/tests/logout.test.ts +++ b/src/leapfrogai_ui/tests/logout.test.ts @@ -6,7 +6,7 @@ test('it can log out', async ({ page }) => { await page.getByLabel('User').click(); await page.getByLabel('Log Out').click(); - await page.waitForURL('http://localhost:4173'); + await page.waitForURL('/'); if (process.env.PUBLIC_DISABLE_KEYCLOAK === 'true') { await expect(page.getByText('Sign In')).toBeVisible(); diff --git a/src/leapfrogai_ui/tests/rag.test.ts b/src/leapfrogai_ui/tests/rag.test.ts index 4c0e4fab5..0edeb9eca 100644 --- a/src/leapfrogai_ui/tests/rag.test.ts +++ b/src/leapfrogai_ui/tests/rag.test.ts @@ -1,5 +1,5 @@ import { expect, test } from './fixtures'; -import { getFakeAssistantInput } from '$testUtils/fakeData'; +import { getFakeAssistantInput } from '../testUtils/fakeData'; import { delay } from 'msw'; import { createPDF,