diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml index a66b034c23..1f92f21947 100644 --- a/.github/workflows/ui.yml +++ b/.github/workflows/ui.yml @@ -82,7 +82,7 @@ jobs: - name: 📥 Install node dependencies working-directory: ui - run: pnpm i --frozen-lockfile + run: pnpm i --frozen-lockfile --ignore-scripts - name: Install Playwright Browsers run: npx playwright install --with-deps @@ -97,7 +97,7 @@ jobs: run: pip install -e . - name: Run UI - run: evidently ui --port 8000 --demo-projects all --workspace test-workspace & + run: evidently ui --port 8000 --demo-projects bikes,reviews --workspace test-workspace & - name: Wait UI to be ready to test working-directory: ui/service diff --git a/README.md b/README.md index 288d6ae711..7069433344 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ You can get an **HTML report** (best for exploratory analysis and debugging), ** > This functionality is available from v0.4.0. -![Dashboard example](docs/images/evidently_ml_monitoring_main.png) +![Dashboard example](ui/service/tests/visual.spec.ts-snapshots/Demo-project---Bikes-Dashboard-1-chromium-linux.png) You can self-host an ML monitoring dashboard to visualize metrics and test results over time. This functionality sits on top of Reports and Test Suites. You must store their outputs as Evidently JSON `snapshots` that serve as a data source for the Evidently Monitoring UI. diff --git a/ui/service/playwright.config.ts b/ui/service/playwright.config.ts index 40f3dbfd25..0c5d797dab 100644 --- a/ui/service/playwright.config.ts +++ b/ui/service/playwright.config.ts @@ -10,6 +10,8 @@ import { defineConfig, devices } from '@playwright/test' * @see https://playwright.dev/docs/test-configuration */ export default defineConfig({ + timeout: 1000 * 15, + /* timeout 15 seconds */ testDir: './tests', /* Run tests in files in parallel */ fullyParallel: false, @@ -46,4 +48,7 @@ export default defineConfig({ // use: { ...devices['Desktop Safari'] } // } ] + // expect: { + // toHaveScreenshot: { maxDiffPixels: 2000 } + // } }) diff --git a/ui/service/tests/smoke.spec.ts b/ui/service/tests/smoke.spec.ts index 50b6aeb9ee..2a0d83d3cb 100644 --- a/ui/service/tests/smoke.spec.ts +++ b/ui/service/tests/smoke.spec.ts @@ -6,7 +6,8 @@ test('Has title', async ({ page }) => { await expect(page).toHaveTitle(/Evidently/) }) -test('Can view Snapshot', async ({ page }) => { +// We don't need this since visual testing enabled +test.skip('Can view Snapshot', async ({ page }) => { await page.goto('/') await page.getByRole('link', { name: 'Demo project - Bikes' }).click() await page.getByText('Bike Rental Demand Forecast').click() @@ -77,7 +78,8 @@ test('Download test suites', async ({ page }) => { } }) -test('We expect to see at least 3 plotly graphs', async ({ page }) => { +// We don't need this since visual testing enabled +test.skip('We expect to see at least 3 plotly graphs', async ({ page }) => { await page.goto('/') await page.getByRole('link', { name: 'Demo project - Bikes' }).click() for (let index = 0; index < 3; index++) { @@ -155,7 +157,8 @@ test('Altering project title and description', async ({ page }) => { await page.waitForLoadState('domcontentloaded') }) -test('Check header of ColumnSummaryMetric', async ({ page }) => { +// We don't need this since visual testing enabled +test.skip('Check header of ColumnSummaryMetric', async ({ page }) => { const expectedHeaderCells = ['', 'current', 'reference'] await page.goto('/') diff --git a/ui/service/tests/visual.spec.ts b/ui/service/tests/visual.spec.ts new file mode 100644 index 0000000000..ebfd6948fb --- /dev/null +++ b/ui/service/tests/visual.spec.ts @@ -0,0 +1,91 @@ +import { test, expect, Page } from '@playwright/test' + +const goToFirstSnapshotAndExpanSomeWidgets = async ({ + projectName, + page, + isTestSuite +}: { + page: Page + isTestSuite: boolean + projectName: string +}) => { + await page.getByRole('link', { name: projectName }).click() + await page.getByRole('tab', { name: isTestSuite ? 'Test Suites' : 'Reports' }).click() + await page.getByRole('button', { name: 'View' }).first().click() + await page.waitForLoadState('networkidle') + + const Details = page.getByRole('button', { name: 'Details' }) + await expect(Details.first()).toBeVisible() + + const DetailsCount = await Details.count() + + await Details.first().click() + await page.waitForLoadState('networkidle') + + if (DetailsCount > 1) { + await Details.nth(1).click() + await page.waitForLoadState('networkidle') + } + + if (DetailsCount > 2) { + await Details.last().click() + await page.waitForLoadState('networkidle') + } + + await page.waitForTimeout(1000) +} + +const VisualTestSnapshot = async ({ + page, + projectName, + isTestSuite +}: { + page: Page + projectName: string + isTestSuite: boolean +}) => { + await page.goto('/') + await goToFirstSnapshotAndExpanSomeWidgets({ + page, + projectName, + isTestSuite + }) + + await expect(page).toHaveScreenshot({ fullPage: true, maxDiffPixels: 1800 }) +} + +const VisualTestDashboard = async ({ page, projectName }: { page: Page; projectName: string }) => { + await page.goto('/') + await page.getByRole('link', { name: projectName }).click() + await page.waitForLoadState('networkidle') + await expect(page).toHaveScreenshot({ fullPage: true, maxDiffPixels: 800 }) +} + +const BikesDemoProjectName = 'Demo project - Bikes' +const ReviewsDemoProjectName = 'Demo project - Reviews' + +///////////////////// +/// Dashboards +///////////////////// +test(`${BikesDemoProjectName}: Dashboard`, async ({ page }) => { + await VisualTestDashboard({ page, projectName: BikesDemoProjectName }) +}) + +test(`${ReviewsDemoProjectName}: Dashboard`, async ({ page }) => { + await VisualTestDashboard({ page, projectName: ReviewsDemoProjectName }) +}) + +///////////////////// +/// Snapshots +///////////////////// +test(`${BikesDemoProjectName}: Report`, async ({ page }) => { + await VisualTestSnapshot({ page, projectName: BikesDemoProjectName, isTestSuite: false }) +}) + +test(`${BikesDemoProjectName}: Test Suite`, async ({ page }) => { + await VisualTestSnapshot({ page, projectName: BikesDemoProjectName, isTestSuite: true }) +}) + +test(`${ReviewsDemoProjectName}: Report`, async ({ page }) => { + await VisualTestSnapshot({ page, projectName: ReviewsDemoProjectName, isTestSuite: false }) +}) diff --git a/ui/service/tests/visual.spec.ts-snapshots/Demo-project---Bikes-Dashboard-1-chromium-linux.png b/ui/service/tests/visual.spec.ts-snapshots/Demo-project---Bikes-Dashboard-1-chromium-linux.png new file mode 100644 index 0000000000..c38477664f Binary files /dev/null and b/ui/service/tests/visual.spec.ts-snapshots/Demo-project---Bikes-Dashboard-1-chromium-linux.png differ diff --git a/ui/service/tests/visual.spec.ts-snapshots/Demo-project---Bikes-Report-1-chromium-linux.png b/ui/service/tests/visual.spec.ts-snapshots/Demo-project---Bikes-Report-1-chromium-linux.png new file mode 100644 index 0000000000..5ee16e3675 Binary files /dev/null and b/ui/service/tests/visual.spec.ts-snapshots/Demo-project---Bikes-Report-1-chromium-linux.png differ diff --git a/ui/service/tests/visual.spec.ts-snapshots/Demo-project---Bikes-Test-Suite-1-chromium-linux.png b/ui/service/tests/visual.spec.ts-snapshots/Demo-project---Bikes-Test-Suite-1-chromium-linux.png new file mode 100644 index 0000000000..1b546c342d Binary files /dev/null and b/ui/service/tests/visual.spec.ts-snapshots/Demo-project---Bikes-Test-Suite-1-chromium-linux.png differ diff --git a/ui/service/tests/visual.spec.ts-snapshots/Demo-project---Reviews-Dashboard-1-chromium-linux.png b/ui/service/tests/visual.spec.ts-snapshots/Demo-project---Reviews-Dashboard-1-chromium-linux.png new file mode 100644 index 0000000000..fad334b2fe Binary files /dev/null and b/ui/service/tests/visual.spec.ts-snapshots/Demo-project---Reviews-Dashboard-1-chromium-linux.png differ diff --git a/ui/service/tests/visual.spec.ts-snapshots/Demo-project---Reviews-Report-1-chromium-linux.png b/ui/service/tests/visual.spec.ts-snapshots/Demo-project---Reviews-Report-1-chromium-linux.png new file mode 100644 index 0000000000..4122e6419f Binary files /dev/null and b/ui/service/tests/visual.spec.ts-snapshots/Demo-project---Reviews-Report-1-chromium-linux.png differ