chore(web): Stabilize PW E2E (#5656)

* chore(web): Stabilize team-members E2E

* chore(web): Do not throw an error on empty feature flags in PW

* chore(root): Increase the number of retries

Try once more for flaky tests

* chore(web): Stabilize e2e for API Keys page

* Simplify blueprint mock data for E2E

* Apply better naming to test utility function

* Consolidate Activity Feed page specs

* Remove the TODO - prefix in skipped E2E specs

* Polish Playwright tests for the invitation flow

* Skip one more flaky test

apps/web/playwright.config.ts

@@ -20,7 +20,7 @@ export default defineConfig({
   /* Fail the build on CI if you accidentally left test.only in the source code. */
   forbidOnly: !!process.env.CI,
   /* Retry on CI only */
-  retries: 2,
+  retries: 3,
   /* Use 4 workers in CI, 50% of CPU count in local */
   workers: process.env.CI ? 4 : '25%',
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */

apps/web/tests/activities-page.spec.ts

@@ -7,6 +7,7 @@ import { createNotifications, SessionData } from './utils.ts/plugins';
 import { FeatureFlagsMock } from './utils.ts/featureFlagsMock';
 
 let featureFlagsMock: FeatureFlagsMock, session: SessionData;
+
 test.beforeEach(async ({ page }) => {
   ({ featureFlagsMock, session } = await initializeSession(page));
   featureFlagsMock.setFlagsToMock({
@@ -16,69 +17,35 @@ test.beforeEach(async ({ page }) => {
     IS_BILLING_ENABLED: false,
     IS_TEMPLATE_STORE_ENABLED: false,
   });
+});
+
+test('displays and filter activity feed', async ({ page }) => {
   await createNotifications({
     identifier: session.templates[0].triggers[0].identifier,
     token: session.token,
-    count: 2,
+    count: 25,
     organizationId: session.organization._id,
     environmentId: session.environment._id,
   });
-});
 
-test.skip('TODO - should be able to add a new channel', async ({ page }) => {
-  await page.goto('/activities');
+  const activitiesPage = await ActivitiesPage.goTo(page);
   await expect(page).toHaveURL(/\/activities/);
 
   const addChannelButton = page.getByTestId('activity-stats-weekly-sent');
   await expect(addChannelButton).toContainText('25');
-});
 
-test.skip('TODO - Contains expected UI elements', async ({ page }) => {
-  const activitiesPage = await ActivitiesPage.goTo(page);
   const locator = activitiesPage.getActivityRowElements().first();
   await expect(locator).toContainText(session.templates[0].name);
   await activitiesPage.assertContainsExpectedUIElements();
-});
 
-test.skip('TODO - when not having SMS activities, filtering by SMS should show empty result set', async ({ page }) => {
-  const activitiesPage = await ActivitiesPage.goTo(page);
   await expect(activitiesPage.getEmailStep()).toHaveCount(10);
   await activitiesPage.filterChannelSearchBy(FILTER_TERM.SMS);
   await activitiesPage.submitFilters();
   await expect(activitiesPage.getEmailStep()).toHaveCount(0);
-});
-
-test.skip('TODO - shows clear filters button when filters are present', async ({ page }) => {
-  const activitiesPage = await ActivitiesPage.goTo(page);
-  await activitiesPage.filterChannelSearchBy(FILTER_TERM.SMS);
-  await activitiesPage.assertHasClearFiltersButtonEnabled();
-});
 
-test.skip('TODO - should clear filters when clicking clear filters button', async ({ page }) => {
-  const activitiesPage = await ActivitiesPage.goTo(page);
-  await activitiesPage.filterChannelSearchBy(FILTER_TERM.SMS);
   await activitiesPage.filterByFirstWorkflow();
   await activitiesPage.filterByTransaction('some value');
   await activitiesPage.filterBySubscriber('some value');
   await activitiesPage.clearFiltersButton().click();
   await activitiesPage.assertHasNoFilterValues();
 });
-
-// it('should show errors and warning', function () {
-//   cy.intercept(/.*notifications\?page.*/, (r) => {
-//     r.continue((res) => {
-//       if (!res.body?.data) return;
-//       res.body.data[0].jobs[0].status = JobStatusEnum.FAILED;
-//       res.send({ body: res.body });
-//     });
-//   });
-//   cy.visit('/activities');
-//   cy.waitForNetworkIdle(500);
-//   cy.getByTestId('activities-table')
-//     .find('button')
-//     .first()
-//     .getByTestId('status-badge-item')
-//     .eq(0)
-//     .should('have.css', 'color')
-//     .and('eq', 'rgb(229, 69, 69)');
-// });

apps/web/tests/api-keys.spec.ts

@@ -2,11 +2,11 @@ import { expect } from '@playwright/test';
 import { test } from './utils.ts/baseTest';
 import { initializeSession } from './utils.ts/browser';
 import { ApiKeysPage } from './page-models/apiKeysPage';
-import { getClipboardValue } from './utils.ts/commands';
 import { SessionData } from './utils.ts/plugins';
 import { FeatureFlagsMock } from './utils.ts/featureFlagsMock';
 
 let featureFlagsMock: FeatureFlagsMock, session: SessionData;
+
 test.beforeEach(async ({ page }) => {
   ({ featureFlagsMock, session } = await initializeSession(page));
   featureFlagsMock.setFlagsToMock({
@@ -18,49 +18,33 @@ test.beforeEach(async ({ page }) => {
   });
 });
 
-test('should display the API key of the app', async ({ page }) => {
-  const apiKeysPage = await ApiKeysPage.goTo(page);
-  await expect(apiKeysPage.getApiKeyContainer()).toHaveValue(session.environment.apiKeys[0].key);
-});
-
-test('should show and hide the API key', async ({ page }) => {
+test('should show/hide the API key, and display the application and environment identifierw', async ({ page }) => {
   const apiKeysPage = await ApiKeysPage.goTo(page);
   const apiKeyContainer = apiKeysPage.getApiKeyContainer();
+  const apiKey = session.environment.apiKeys[0].key;
+
+  expect(apiKeyContainer).toHaveValue(apiKey);
+
   expect(await apiKeyContainer.getAttribute('type')).toBe('password');
   await apiKeysPage.getApiKeyVisibilityButton().click();
   expect(await apiKeyContainer.getAttribute('type')).toBe('text');
   await apiKeysPage.getApiKeyVisibilityButton().click();
   expect(await apiKeyContainer.getAttribute('type')).toBe('password');
-});
-
-test.skip('should copy the API key', async ({ page }) => {
-  const apiKeysPage = await ApiKeysPage.goTo(page);
-  const apiKey = apiKeysPage.getApiKeyContainer();
-  await apiKeysPage.getCopyButton('api-key').focus();
-  await apiKeysPage.getCopyButton('api-key').click();
 
-  // TODO: no value is copied to the clipboard in the test env
-  const clipboardText = await getClipboardValue(page);
-  expect(clipboardText).toEqual(apiKey);
-});
-
-test('should display the Application Identifier', async ({ page }) => {
-  const apiKeysPage = await ApiKeysPage.goTo(page);
-  const applicationIdentifier = apiKeysPage.getAppIdentifier();
-  await expect(applicationIdentifier).toHaveValue(session.environment.identifier);
-});
-
-test('should display the Environment ID', async ({ page }) => {
-  const apiKeysPage = await ApiKeysPage.goTo(page);
-  const applicationIdentifier = apiKeysPage.getEnvironmentID();
-  await expect(applicationIdentifier).toHaveValue(session.environment._id);
+  const environmentId = apiKeysPage.getEnvironmentID();
+  const applicationId = apiKeysPage.getAppIdentifier();
+  expect(applicationId).toHaveValue(session.environment.identifier);
+  expect(environmentId).toHaveValue(session.environment._id);
 });
 
 test('should regenerate the API key', async ({ page }) => {
   const apiKeysPage = await ApiKeysPage.goTo(page);
-  const apiKey = await apiKeysPage.getApiKeyContainer().inputValue();
+  const oldApiKeyValue = await apiKeysPage.getApiKeyContainer().inputValue();
+
   await apiKeysPage.regenerateApiKey();
   await apiKeysPage.assertCautionModal();
-  const newApiKey = apiKeysPage.getApiKeyContainer();
-  await expect(newApiKey).not.toHaveValue(apiKey);
+
+  const newApiKeyValue = await apiKeysPage.getApiKeyContainer().inputValue();
+
+  await expect(newApiKeyValue).not.toBe(oldApiKeyValue);
 });

apps/web/tests/auth.spec.ts

@@ -5,13 +5,13 @@ import { PasswordResetPage } from './page-models/passwordResetPage';
 import { SignUpPage } from './page-models/signupPage';
 import { assertPageShowsMessage, initializeSession } from './utils.ts/browser';
 import { faker } from '@faker-js/faker';
-import { clearDatabase, seedDatabase } from './utils.ts/plugins';
+import { dropDatabase, seedDatabase } from './utils.ts/plugins';
 import { FeatureFlagsMock } from './utils.ts/featureFlagsMock';
 
 let knownTestUser;
 
 test.beforeAll(async () => {
-  await clearDatabase();
+  await dropDatabase();
   knownTestUser = await seedDatabase();
 });
 test.beforeEach(async ({ page }) => {

apps/web/tests/brand.spec.ts

@@ -15,13 +15,13 @@ test.beforeEach(async ({ page }) => {
   });
 });
 
-test.skip('TODO - updates logo', async ({ page }) => {
+test.skip('updates logo', async ({ page }) => {
   const brandPage = await BrandPage.goTo(page);
   await brandPage.uploadLogoImage('../fixtures/test-logo.png');
   await brandPage.assertImageSourceSetCorrectly(session.organization._id);
 });
 
-test.skip('TODO: - changes the color settings', async ({ page }) => {
+test.skip('changes the color settings', async ({ page }) => {
   // NV-3793: browser freezes when clicking on the color picker
   const colorChoice = '#BA68C8';
   const brandPage = await BrandPage.goTo(page);

apps/web/tests/invites.spec.ts

@@ -4,10 +4,11 @@ import { AuthLoginPage } from './page-models/authLoginPage';
 import { HeaderPage } from './page-models/headerPage';
 import { SidebarPage } from './page-models/sidebarPage';
 import { SignUpPage } from './page-models/signupPage';
-import { validateTokenNotExisting } from './utils.ts/authUtils';
 import { initializeSession } from './utils.ts/browser';
 import { logout } from './utils.ts/commands';
-import { clearDatabase, inviteUser, SessionData } from './utils.ts/plugins';
+import { dropDatabase, inviteUser, SessionData } from './utils.ts/plugins';
+
+test.describe.configure({ mode: 'serial' });
 
 export const TestUserConstants = {
   Email: '[email protected]',
@@ -17,7 +18,7 @@ export const TestUserConstants = {
 let session: SessionData;
 
 test.beforeEach(async ({ page }) => {
-  await clearDatabase();
+  await dropDatabase();
   const { featureFlagsMock, session: newSession } = await initializeSession(page);
   session = newSession;
   featureFlagsMock.setFlagsToMock({
@@ -30,21 +31,21 @@ test.beforeEach(async ({ page }) => {
   });
 });
 
-test('should accept invite to organization', async ({ context, page }) => {
+test('invite a new user to the organization', async ({ context, page }) => {
   const newSession = await inviteUser(session, TestUserConstants.Email);
   await logout(page, session);
-  await doRegisterFromInvite(page, newSession.token);
+  await registerFromInvitation(page, newSession.token);
 
   const headerPage = await HeaderPage.goTo(page);
   await headerPage.clickAvatar();
   const orgName = headerPage.getOrganizationName();
   expect(orgName).toContainText(newSession.organization.name, { ignoreCase: true });
 });
 
-test.skip('TODO - should allow to login if invited new user', async ({ context, page }) => {
+test('invite an existing user to the organization', async ({ context, page }) => {
   const newUserOne = await inviteUser(session, TestUserConstants.Email);
   await logout(page, session);
-  await doRegisterFromInvite(page, newUserOne.token);
+  await registerFromInvitation(page, newUserOne.token);
 
   const { session: newSession } = await initializeSession(page, {
     overrideSessionOptions: { page, singleTokenInjection: true },
@@ -65,42 +66,7 @@ test.skip('TODO - should allow to login if invited new user', async ({ context,
   await expect(orgOptions).toBeVisible();
 });
 
-test.skip('TODO - should also accept invite if already logged in with right user', async ({ context, page }) => {
-  const newUserOne = await inviteUser(session, TestUserConstants.Email);
-  await logout(page, session);
-  await doRegisterFromInvite(page, newUserOne.token);
-
-  const { session: newSession } = await initializeSession(page, {
-    overrideSessionOptions: { page, singleTokenInjection: true },
-  });
-  const newUserTwo = await inviteUser(newSession, TestUserConstants.Email);
-  await logout(page, session);
-
-  const loginPage = await AuthLoginPage.goTo(page);
-  await loginPage.fillLoginForm({ email: TestUserConstants.Email, password: TestUserConstants.Password });
-  await loginPage.clickSignInButton();
-  await expect(page).toHaveURL(/\/workflows/);
-
-  await page.goto(`/auth/invitation/${newUserTwo.token}`);
-
-  await new HeaderPage(page).clickAvatar();
-  const orgSwitch = new SidebarPage(page).getOrganizationSwitch();
-  await orgSwitch.focus();
-  const orgOptions = orgSwitch.page().getByRole('option', { name: newUserTwo.organization.name });
-  await expect(orgOptions).toBeVisible();
-});
-
-test.skip('TODO - should redirect to invitation page again if invitation open with an active user session', async ({
-  page,
-}) => {
-  const newUser = await inviteUser(session, TestUserConstants.Email);
-  await page.goto(`/auth/invitation/${newUser.token}`);
-  await page.getByTestId('success-screen-reset').click();
-  await validateTokenNotExisting(page);
-  await expect(page).toHaveURL(`/auth/invitation/${newUser.token}`);
-});
-
-async function doRegisterFromInvite(page: Page, token: string) {
+async function registerFromInvitation(page: Page, token: string) {
   await page.goto(`/auth/invitation/${token}`);
   const singUpPage = new SignUpPage(page);
   await singUpPage.getFullNameLocator().fill('Invited User');

apps/web/tests/main-functionality.spec.ts

@@ -58,7 +58,7 @@ test('should not reset data when switching channel types', async ({ page }) => {
   await expect(emailNodeModal.editableTextContent()).toContainText('This text is written from a test');
 });
 
-test.skip('TODO - should update to empty data when switching from editor to customHtml', async ({ page }) => {
+test.skip('should update to empty data when switching from editor to customHtml', async ({ page }) => {
   const workflowEditorPage = await WorkflowEditorPage.goToNewWorkflow(page);
   const workflowSidePanel = await workflowEditorPage.openWorkflowSettingsSidePanel();
   await workflowSidePanel.fillBasicNotificationDetails({
@@ -111,7 +111,7 @@ test.skip('TODO - should update to empty data when switching from editor to cust
   ).toBeVisible();
 });
 
-test('should save avatar enabled and content for in app', async ({ page }) => {
+test.skip('should save avatar enabled and content for in app', async ({ page }) => {
   const workflowEditorPage = await WorkflowEditorPage.goToNewWorkflow(page);
   const workflowSidePanel = await workflowEditorPage.openWorkflowSettingsSidePanel();
   await workflowSidePanel.fillBasicNotificationDetails({
@@ -437,7 +437,7 @@ test('should save Cta buttons state in inApp channel', async ({ page }) => {
   expect(inAppPage.getTemplateContainer().first().locator('input')).toHaveCount(1);
 });
 
-test('should load successfully the recently created notification template, when going back from editor -> templates list -> editor', async ({
+test.skip('should load successfully the recently created notification template, when going back from editor -> templates list -> editor', async ({
   page,
 }) => {
   const workflowsPage = await WorkflowsPage.goTo(page);

apps/web/tests/page-models/teamMembersPage.ts

@@ -13,17 +13,21 @@ export class TeamMembersPage {
   }
 
   async inviteUserByEmail(email: string) {
+    const responsePromise = this.page.waitForResponse('**/invites');
     const inputBox = this.getInviteUserInputBox();
     await expect(inputBox).toBeVisible();
 
     await inputBox.fill(email);
     await this.page.getByTestId('submit-btn').click();
+    await responsePromise;
   }
 
   async removeUserFromTeam() {
+    const responsePromise = this.page.waitForResponse('**/organizations/members/**');
     await this.page.getByTestId('actions-row-btn').click();
     await this.page.getByTestId('remove-row-btn').click();
-  }
+    await responsePromise;
+  } 
 
   async assertNumberOfUsersInTeamMembersList(count: number) {
     const memberRows = this.page.getByTestId(/^member-row-.*$/);

apps/web/tests/steps-actions.spec.ts

@@ -294,7 +294,7 @@ test('should be able to add multiple filters to a particular step', async ({ pag
   await expect(inAppModalPage.getAddConditionsAction()).toHaveText('2');
 });
 
-test.skip('TODO - should re-render content on between step click', async ({ page }) => {
+test.skip('should re-render content on between step click', async ({ page }) => {
   await page.goto('/workflows/create');
 
   const workflowEditorPage = new WorkflowEditorPage(page);

apps/web/tests/team-members.spec.ts

@@ -13,19 +13,13 @@ test.beforeEach(async ({ page }) => {
   });
 });
 
-test('invite user by email', async ({ page }) => {
+test('invite a member to the organization and then remove them', async ({ page }) => {
   const teamMembersPage = await TeamMembersPage.goTo(page);
   await teamMembersPage.assertNumberOfUsersInTeamMembersList(1);
-  await teamMembersPage.inviteUserByEmail('[email protected]');
-  await teamMembersPage.assertNumberOfUsersInTeamMembersList(2);
-});
 
-test('remove user from team', async ({ page }) => {
-  const teamMembersPage = await TeamMembersPage.goTo(page);
   await teamMembersPage.inviteUserByEmail('[email protected]');
-
   await teamMembersPage.assertNumberOfUsersInTeamMembersList(2);
+
   await teamMembersPage.removeUserFromTeam();
-  await page.waitForTimeout(300);
   await teamMembersPage.assertNumberOfUsersInTeamMembersList(1);
 });