UI: visuail tests by playwright

.github/workflows/ui.yml

@@ -82,7 +82,7 @@ jobs:
 
       - name: 📥 Install node dependencies
         working-directory: ui
-        run: pnpm i --frozen-lockfile
+        run: pnpm i --frozen-lockfile --ignore-scripts
 
       - name: Install Playwright Browsers
         run: npx playwright install --with-deps
@@ -97,7 +97,7 @@ jobs:
         run: pip install -e .
 
       - name: Run UI
-        run: evidently ui --port 8000 --demo-projects all --workspace test-workspace &
+        run: evidently ui --port 8000 --demo-projects bikes,reviews --workspace test-workspace &
 
       - name: Wait UI to be ready to test
         working-directory: ui/service

README.md

@@ -50,7 +50,7 @@ You can get an **HTML report** (best for exploratory analysis and debugging), **
 
 > This functionality is available from v0.4.0. 
 
-![Dashboard example](docs/images/evidently_ml_monitoring_main.png)
+![Dashboard example](ui/service/tests/visual.spec.ts-snapshots/Demo-project---Bikes-Dashboard-1-chromium-linux.png)
 
 You can self-host an ML monitoring dashboard to visualize metrics and test results over time. This functionality sits on top of Reports and Test Suites. You must store their outputs as Evidently JSON `snapshots` that serve as a data source for the Evidently Monitoring UI.  
 

ui/service/playwright.config.ts

@@ -10,6 +10,8 @@ import { defineConfig, devices } from '@playwright/test'
  * @see https://playwright.dev/docs/test-configuration
  */
 export default defineConfig({
+  timeout: 1000 * 15,
+  /* timeout 15 seconds */
   testDir: './tests',
   /* Run tests in files in parallel */
   fullyParallel: false,
@@ -46,4 +48,7 @@ export default defineConfig({
     //   use: { ...devices['Desktop Safari'] }
     // }
   ]
+  // expect: {
+  //   toHaveScreenshot: { maxDiffPixels: 2000 }
+  // }
 })

ui/service/tests/smoke.spec.ts

@@ -6,7 +6,8 @@ test('Has title', async ({ page }) => {
   await expect(page).toHaveTitle(/Evidently/)
 })
 
-test('Can view Snapshot', async ({ page }) => {
+// We don't need this since visual testing enabled
+test.skip('Can view Snapshot', async ({ page }) => {
   await page.goto('/')
   await page.getByRole('link', { name: 'Demo project - Bikes' }).click()
   await page.getByText('Bike Rental Demand Forecast').click()
@@ -77,7 +78,8 @@ test('Download test suites', async ({ page }) => {
   }
 })
 
-test('We expect to see at least 3 plotly graphs', async ({ page }) => {
+// We don't need this since visual testing enabled
+test.skip('We expect to see at least 3 plotly graphs', async ({ page }) => {
   await page.goto('/')
   await page.getByRole('link', { name: 'Demo project - Bikes' }).click()
   for (let index = 0; index < 3; index++) {
@@ -155,7 +157,8 @@ test('Altering project title and description', async ({ page }) => {
   await page.waitForLoadState('domcontentloaded')
 })
 
-test('Check header of ColumnSummaryMetric', async ({ page }) => {
+// We don't need this since visual testing enabled
+test.skip('Check header of ColumnSummaryMetric', async ({ page }) => {
   const expectedHeaderCells = ['', 'current', 'reference']
 
   await page.goto('/')

ui/service/tests/visual.spec.ts

@@ -0,0 +1,91 @@
+import { test, expect, Page } from '@playwright/test'
+
+const goToFirstSnapshotAndExpanSomeWidgets = async ({
+  projectName,
+  page,
+  isTestSuite
+}: {
+  page: Page
+  isTestSuite: boolean
+  projectName: string
+}) => {
+  await page.getByRole('link', { name: projectName }).click()
+  await page.getByRole('tab', { name: isTestSuite ? 'Test Suites' : 'Reports' }).click()
+  await page.getByRole('button', { name: 'View' }).first().click()
+  await page.waitForLoadState('networkidle')
+
+  const Details = page.getByRole('button', { name: 'Details' })
+  await expect(Details.first()).toBeVisible()
+
+  const DetailsCount = await Details.count()
+
+  await Details.first().click()
+  await page.waitForLoadState('networkidle')
+
+  if (DetailsCount > 1) {
+    await Details.nth(1).click()
+    await page.waitForLoadState('networkidle')
+  }
+
+  if (DetailsCount > 2) {
+    await Details.last().click()
+    await page.waitForLoadState('networkidle')
+  }
+
+  await page.waitForTimeout(1000)
+}
+
+const VisualTestSnapshot = async ({
+  page,
+  projectName,
+  isTestSuite
+}: {
+  page: Page
+  projectName: string
+  isTestSuite: boolean
+}) => {
+  await page.goto('/')
+  await goToFirstSnapshotAndExpanSomeWidgets({
+    page,
+    projectName,
+    isTestSuite
+  })
+
+  await expect(page).toHaveScreenshot({ fullPage: true, maxDiffPixels: 1800 })
+}
+
+const VisualTestDashboard = async ({ page, projectName }: { page: Page; projectName: string }) => {
+  await page.goto('/')
+  await page.getByRole('link', { name: projectName }).click()
+  await page.waitForLoadState('networkidle')
+  await expect(page).toHaveScreenshot({ fullPage: true, maxDiffPixels: 800 })
+}
+
+const BikesDemoProjectName = 'Demo project - Bikes'
+const ReviewsDemoProjectName = 'Demo project - Reviews'
+
+/////////////////////
+///   Dashboards
+/////////////////////
+test(`${BikesDemoProjectName}: Dashboard`, async ({ page }) => {
+  await VisualTestDashboard({ page, projectName: BikesDemoProjectName })
+})
+
+test(`${ReviewsDemoProjectName}: Dashboard`, async ({ page }) => {
+  await VisualTestDashboard({ page, projectName: ReviewsDemoProjectName })
+})
+
+/////////////////////
+///   Snapshots
+/////////////////////
+test(`${BikesDemoProjectName}: Report`, async ({ page }) => {
+  await VisualTestSnapshot({ page, projectName: BikesDemoProjectName, isTestSuite: false })
+})
+
+test(`${BikesDemoProjectName}: Test Suite`, async ({ page }) => {
+  await VisualTestSnapshot({ page, projectName: BikesDemoProjectName, isTestSuite: true })
+})
+
+test(`${ReviewsDemoProjectName}: Report`, async ({ page }) => {
+  await VisualTestSnapshot({ page, projectName: ReviewsDemoProjectName, isTestSuite: false })
+})