V15 QA updated flaky E2E tests (#17845)

* Updated tests

* More cleanup

* Removed log

* Bumped version

* Update tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Content/ContentWithMultiURLPicker.spec.ts

* Update tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Settings/DocumentType/DocumentTypeDesignTab.spec.ts

* Update tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Settings/DocumentType/DocumentTypeDesignTab.spec.ts

* Update tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Users/Permissions/UserGroup/DefaultPermissionsInContent.spec.ts

tests/Umbraco.Tests.AcceptanceTest/package.json

@@ -21,7 +21,7 @@
   },
   "dependencies": {
     "@umbraco/json-models-builders": "^2.0.26",
-    "@umbraco/playwright-testhelpers": "^15.0.6",
+    "@umbraco/playwright-testhelpers": "^15.0.7",
     "camelize": "^1.0.0",
     "dotenv": "^16.3.1",
     "node-fetch": "^2.6.7"

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Content/ContentInfoTab.spec.ts

@@ -33,6 +33,7 @@ test('can see correct information when published', async ({umbracoApi, umbracoUi
   await umbracoUi.content.clickSaveAndPublishButton();
 
   // Assert
+  await umbracoUi.content.isSuccessNotificationVisible();
   await umbracoUi.waitForTimeout(2000);
   const contentData = await umbracoApi.document.getByName(contentName);
   await umbracoUi.content.doesIdHaveText(contentData.id);
@@ -106,6 +107,7 @@ test('can change template', async ({umbracoApi, umbracoUi}) => {
   await umbracoUi.content.clickSaveButton();
 
   // Assert
+  await umbracoUi.content.isSuccessNotificationVisible();
   const contentData = await umbracoApi.document.getByName(contentName);
   expect(contentData.template.id).toBe(secondTemplateId);
 

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Content/ContentWithContentPicker.spec.ts

@@ -120,6 +120,7 @@ test('can choose start node for the content picker in the content', async ({umbr
   // Clean
   await umbracoApi.dataType.ensureNameNotExists(customDataTypeName);
   await umbracoApi.document.ensureNameNotExists(childContentPickerName);
+  await umbracoApi.documentType.ensureNameNotExists(childContentPickerDocumentTypeName);
 });
 
 test.skip('can ignore user start node for the content picker in the content', async ({umbracoApi, umbracoUi}) => {
@@ -150,6 +151,7 @@ test.skip('can ignore user start node for the content picker in the content', as
   // Clean
   await umbracoApi.dataType.ensureNameNotExists(customDataTypeName);
   await umbracoApi.document.ensureNameNotExists(childContentPickerName);
+  await umbracoApi.documentType.ensureNameNotExists(childContentPickerDocumentTypeName);
 });
 
 test('can remove content picker in the content', async ({umbracoApi, umbracoUi}) => {

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Content/ContentWithListViewContent.spec.ts

@@ -103,6 +103,7 @@ test('can publish content with a child in the list', async ({umbracoApi, umbraco
 
   // Act
   await umbracoUi.content.clickSaveAndPublishButton();
+  await umbracoUi.content.doesSuccessNotificationsHaveCount(2);
   await umbracoUi.content.goToContentInListViewWithName(childContentName);
   await umbracoUi.content.clickContainerSaveAndPublishButton();
 

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Content/ContentWithMultiURLPicker.spec.ts

@@ -17,7 +17,7 @@ test.afterEach(async ({umbracoApi}) => {
   await umbracoApi.documentType.ensureNameNotExists(documentTypeName);
 });
 
-test('can create content with the document link', {tag: '@smoke'}, async ({page, umbracoApi, umbracoUi}) => {
+test('can create content with the document link', {tag: '@smoke'}, async ({umbracoApi, umbracoUi}) => {
   // Arrange
   const expectedState = 'Draft';
   const dataTypeData = await umbracoApi.dataType.getByName(dataTypeName);
@@ -27,7 +27,6 @@ test('can create content with the document link', {tag: '@smoke'}, async ({page,
   const documentTypeForLinkedDocumentId = await umbracoApi.documentType.createDefaultDocumentTypeWithAllowAsRoot(documentTypeForLinkedDocumentName);
   const linkedDocumentName = 'LinkedDocument';
   const linkedDocumentId = await umbracoApi.document.createDefaultDocument(linkedDocumentName, documentTypeForLinkedDocumentId);
-  await umbracoUi.waitForTimeout(2000);
   await umbracoApi.document.publish(linkedDocumentId);
   await umbracoUi.goToBackOffice();
   await umbracoUi.content.goToSection(ConstantHelper.sections.content);

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Content/CultureAndHostnames.spec.ts

@@ -40,6 +40,7 @@ test('can add a culture', {tag: '@smoke'}, async ({umbracoApi, umbracoUi}) => {
   await umbracoUi.content.clickSaveModalButton();
 
   // Assert
+  await umbracoUi.content.isSuccessNotificationVisible();
   await umbracoUi.waitForTimeout(2000);
   const domainsData = await umbracoApi.document.getDomains(contentId);
   expect(domainsData.defaultIsoCode).toEqual(isoCode);
@@ -56,6 +57,7 @@ test('can add a domain', {tag: '@smoke'}, async ({umbracoApi, umbracoUi}) => {
   await umbracoUi.content.clickSaveModalButton();
 
   // Assert
+  await umbracoUi.content.isSuccessNotificationVisible();
   const domainsData = await umbracoApi.document.getDomains(contentId);
   expect(domainsData.domains.length).toBe(1);
   expect(domainsData.domains[0].domainName).toEqual(domainName);
@@ -76,6 +78,7 @@ test('can update culture and hostname', async ({umbracoApi, umbracoUi}) => {
   await umbracoUi.content.clickSaveModalButton();
 
   // Assert
+  await umbracoUi.content.isSuccessNotificationVisible();
   domainsData = await umbracoApi.document.getDomains(contentId);
   expect(domainsData.domains[0].domainName).toEqual(updatedDomainName);
   expect(domainsData.domains[0].isoCode).toEqual(isoCode);
@@ -94,6 +97,7 @@ test('can delete culture and hostname', async ({umbracoApi, umbracoUi}) => {
   await umbracoUi.content.clickSaveModalButton();
 
   // Assert
+  await umbracoUi.content.isSuccessNotificationVisible();
   domainsData = await umbracoApi.document.getDomains(contentId);
   expect(domainsData.domains.length).toBe(0);
 });
@@ -120,6 +124,7 @@ test('can add culture and hostname for multiple languages', async ({umbracoApi,
   await umbracoUi.waitForTimeout(500);
 
   // Assert
+  await umbracoUi.content.isSuccessNotificationVisible();
   const domainsData = await umbracoApi.document.getDomains(contentId);
   expect(domainsData.domains.length).toBe(2);
   expect(domainsData.domains[0].domainName).toEqual(domainName);

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/DataType/ApprovedColor.spec.ts

@@ -3,7 +3,7 @@ import {expect} from "@playwright/test";
 
 const dataTypeName = 'Approved Color';
 let dataTypeDefaultData = null;
-let dataTypeData = null;  
+let dataTypeData = null;
 const colorValue = 'ffffff';
 const colorLabel = '';
 
@@ -38,6 +38,7 @@ test('can include label', async ({umbracoApi, umbracoUi}) => {
   await umbracoUi.dataType.clickSaveButton();
 
   // Assert
+  await umbracoUi.dataType.isSuccessNotificationVisible();
   dataTypeData = await umbracoApi.dataType.getByName(dataTypeName);
   expect(dataTypeData.values).toEqual(expectedDataTypeValues);
 });
@@ -66,11 +67,12 @@ test('can add color', async ({umbracoApi, umbracoUi}) => {
   await umbracoUi.dataType.clickSaveButton();
 
   // Assert
+  await umbracoUi.dataType.isSuccessNotificationVisible();
   dataTypeData = await umbracoApi.dataType.getByName(dataTypeName);
   expect(dataTypeData.values).toEqual(expectedDataTypeValues);
 });
 
-test('can remove color', async ({umbracoApi, umbracoUi}) => {  
+test('can remove color', async ({umbracoApi, umbracoUi}) => {
   // Arrange
   const removedDataTypeValues = [
     {
@@ -94,6 +96,7 @@ test('can remove color', async ({umbracoApi, umbracoUi}) => {
   await umbracoUi.dataType.clickSaveButton();
 
   // Assert
+  await umbracoUi.dataType.isSuccessNotificationVisible();
   dataTypeData = await umbracoApi.dataType.getByName(dataTypeName);
   expect(dataTypeData.values).toEqual([]);
 });

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/DataType/BlockGrid/BlockGridEditor.spec.ts

@@ -186,7 +186,7 @@ test('can add multiple blocks to a group in a block grid editor', async ({umbrac
   await umbracoApi.documentType.ensureNameNotExists(secondElementName);
 });
 
-test('can delete a block in a group from a block grid editor', {tag: '@smoke'}, async ({umbracoApi, umbracoUi}) => {
+test('can remove a block in a group from a block grid editor', {tag: '@smoke'}, async ({umbracoApi, umbracoUi}) => {
   // Arrange
   const textStringData = await umbracoApi.dataType.getByName(dataTypeName);
   const elementTypeId = await umbracoApi.documentType.createDefaultElementType(elementTypeName, groupName, dataTypeName, textStringData.id);

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/DataType/BlockListEditor/BlockListBlocks.spec.ts

@@ -147,6 +147,9 @@ test('can add a settings model to a block', {tag: '@smoke'}, async ({umbracoApi,
   // Assert
   await umbracoUi.dataType.isSuccessNotificationVisible();
   expect(await umbracoApi.dataType.doesBlockEditorContainBlocksWithSettingsTypeIds(blockListEditorName, [settingsElementTypeId])).toBeTruthy();
+
+  // Clean
+  await umbracoApi.documentType.ensureNameNotExists(secondElementName);
 });
 
 test('can remove a settings model from a block', async ({umbracoApi, umbracoUi}) => {
@@ -169,6 +172,9 @@ test('can remove a settings model from a block', async ({umbracoApi, umbracoUi})
   // Assert
   await umbracoUi.dataType.isSuccessNotificationVisible();
   expect(await umbracoApi.dataType.doesBlockEditorContainBlocksWithSettingsTypeIds(blockListEditorName, [settingsElementTypeId])).toBeFalsy();
+
+  // Clean
+  await umbracoApi.documentType.ensureNameNotExists(secondElementName);
 });
 
 test('can add a background color to a block', async ({umbracoApi, umbracoUi}) => {
@@ -408,11 +414,11 @@ test('can disable hide content editor in a block', async ({umbracoApi, umbracoUi
 });
 
 // TODO: Thumbnails are not showing in the UI
-test.skip('can add a thumbnail to a block ', {tag: '@smoke'}, async ({page, umbracoApi, umbracoUi}) => {
+test.skip('can add a thumbnail to a block ', {tag: '@smoke'}, async ({umbracoApi, umbracoUi}) => {
 
 });
 
 // TODO: Thumbnails are not showing in the UI
-test.skip('can remove a thumbnail to a block ', {tag: '@smoke'}, async ({page, umbracoApi, umbracoUi}) => {
+test.skip('can remove a thumbnail to a block ', {tag: '@smoke'}, async ({umbracoApi, umbracoUi}) => {
 
-});
+});

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/DataType/Textarea.spec.ts

@@ -14,8 +14,8 @@ test.beforeEach(async ({umbracoUi, umbracoApi}) => {
 
 test.afterEach(async ({umbracoApi}) => {
   if (dataTypeDefaultData !== null) {
-    await umbracoApi.dataType.update(dataTypeDefaultData.id, dataTypeDefaultData);   
-  }   
+    await umbracoApi.dataType.update(dataTypeDefaultData.id, dataTypeDefaultData);
+  }
 });
 
 test('can update maximum allowed characters value', async ({umbracoApi, umbracoUi}) => {
@@ -31,6 +31,7 @@ test('can update maximum allowed characters value', async ({umbracoApi, umbracoU
   await umbracoUi.dataType.clickSaveButton();
 
   // Assert
+  await umbracoUi.dataType.isSuccessNotificationVisible();
   dataTypeData = await umbracoApi.dataType.getByName(dataTypeName);
   expect(dataTypeData.values).toContainEqual(expectedDataTypeValues);
 });
@@ -48,6 +49,7 @@ test('can update number of rows value', async ({umbracoApi, umbracoUi}) => {
   await umbracoUi.dataType.clickSaveButton();
 
   // Assert
+  await umbracoUi.dataType.isSuccessNotificationVisible();
   dataTypeData = await umbracoApi.dataType.getByName(dataTypeName);
   expect(dataTypeData.values).toContainEqual(expectedDataTypeValues);
 });
@@ -65,6 +67,7 @@ test('can update min height (pixels) value', async ({umbracoApi, umbracoUi}) =>
   await umbracoUi.dataType.clickSaveButton();
 
   // Assert
+  await umbracoUi.dataType.isSuccessNotificationVisible();
   dataTypeData = await umbracoApi.dataType.getByName(dataTypeName);
   expect(dataTypeData.values).toContainEqual(expectedDataTypeValues);
 });
@@ -82,6 +85,7 @@ test('can update max height (pixels) value', async ({umbracoApi, umbracoUi}) =>
   await umbracoUi.dataType.clickSaveButton();
 
   // Assert
+  await umbracoUi.dataType.isSuccessNotificationVisible();
   dataTypeData = await umbracoApi.dataType.getByName(dataTypeName);
   expect(dataTypeData.values).toContainEqual(expectedDataTypeValues);
 });

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/DataType/Tiptap.spec.ts

@@ -200,8 +200,8 @@ test('can enable ignore user start nodes', async ({umbracoApi, umbracoUi}) => {
 
   // Assert
   await umbracoUi.dataType.doesSuccessNotificationHaveText(NotificationConstantHelper.success.saved);
-  const tiptapData = await umbracoApi.dataType.getByName(tipTapName);
-  expect(tiptapData.values).toContainEqual(expectedTiptapValues);
+  const tipTapData = await umbracoApi.dataType.getByName(tipTapName);
+  expect(tipTapData.values).toContainEqual(expectedTiptapValues);
 });
 
 test('can delete toolbar group', async ({umbracoApi, umbracoUi}) => {
@@ -264,4 +264,4 @@ test('can disable extensions item', async ({umbracoApi, umbracoUi}) => {
   const extensionsValue = tipTapData.values.find(value => value.alias === 'extensions');
   expect(extensionsValue.value.length).toBe(extensionsCount - 1);
   expect(extensionsValue.value).not.toContain(extensionItemName);
-});
+});

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/DataType/TrueFalse.spec.ts

@@ -14,8 +14,8 @@ test.beforeEach(async ({umbracoUi, umbracoApi}) => {
 
 test.afterEach(async ({umbracoApi}) => {
   if (dataTypeDefaultData !== null) {
-    await umbracoApi.dataType.update(dataTypeDefaultData.id, dataTypeDefaultData);   
-  }   
+    await umbracoApi.dataType.update(dataTypeDefaultData.id, dataTypeDefaultData);
+  }
 });
 
 test('can update initial state', async ({umbracoApi, umbracoUi}) => {
@@ -63,6 +63,7 @@ test('can update label on', async ({umbracoApi, umbracoUi}) => {
   await umbracoUi.dataType.clickSaveButton();
 
   // Assert
+  await umbracoUi.dataType.isSuccessNotificationVisible();
   dataTypeData = await umbracoApi.dataType.getByName(dataTypeName);
   expect(dataTypeData.values).toContainEqual(expectedDataTypeValues);
 });
@@ -80,6 +81,7 @@ test('can update label off', async ({umbracoApi, umbracoUi}) => {
   await umbracoUi.dataType.clickSaveButton();
 
   // Assert
+  await umbracoUi.dataType.isSuccessNotificationVisible();
   dataTypeData = await umbracoApi.dataType.getByName(dataTypeName);
   expect(dataTypeData.values).toContainEqual(expectedDataTypeValues);
 });

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/DataType/Upload.spec.ts

@@ -40,6 +40,7 @@ for (const uploadType of uploadTypes) {
       await umbracoUi.dataType.clickSaveButton();
 
       // Assert
+      await umbracoUi.dataType.isSuccessNotificationVisible();
       dataTypeData = await umbracoApi.dataType.getByName(uploadType);
       expect(dataTypeData.values).toEqual(expectedDataTypeValues);
     });
@@ -64,6 +65,7 @@ for (const uploadType of uploadTypes) {
       await umbracoUi.dataType.clickSaveButton();
 
       // Assert
+      await umbracoUi.dataType.isSuccessNotificationVisible();
       dataTypeData = await umbracoApi.dataType.getByName(uploadType);
       expect(dataTypeData.values).toEqual([]);
     });

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Dictionary/Dictionary.spec.ts

@@ -23,8 +23,8 @@ test('can create a dictionary item', async ({umbracoApi, umbracoUi}) => {
   await umbracoUi.dictionary.clickSaveButton();
 
   // Assert
-  expect(await umbracoApi.dictionary.doesNameExist(dictionaryName)).toBeTruthy();
   await umbracoUi.dictionary.doesSuccessNotificationHaveText(NotificationConstantHelper.success.created);
+  expect(await umbracoApi.dictionary.doesNameExist(dictionaryName)).toBeTruthy();
   await umbracoUi.dictionary.clickLeftArrowButton();
   // Verify the dictionary item displays in the tree and in the list
   await umbracoUi.dictionary.isDictionaryTreeItemVisible(dictionaryName);

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Media/Media.spec.ts

@@ -40,7 +40,7 @@ test('can rename a media file', async ({umbracoApi, umbracoUi}) => {
 
   // Arrange
   await umbracoUi.waitForTimeout(1000);
-  await umbracoUi.media.clickLabelWithName(wrongMediaFileName, true, true);
+  await umbracoUi.media.clickLabelWithName(wrongMediaFileName, true);
   await umbracoUi.media.enterMediaItemName(mediaFileName);
   await umbracoUi.media.clickSaveButton();
 
@@ -66,7 +66,6 @@ for (const mediaFileType of mediaFileTypes) {
     await umbracoUi.media.goToSection(ConstantHelper.sections.media);
 
     // Act
-    await umbracoUi.waitForTimeout(1000);
     await umbracoUi.media.clickCreateMediaWithType(mediaFileType.fileName);
     await umbracoUi.media.enterMediaItemName(mediaFileType.fileName);
     await umbracoUi.media.uploadFile('./fixtures/mediaLibrary/' + mediaFileType.filePath);

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Settings/DocumentType/DocumentTypeDesignTab.spec.ts

@@ -84,7 +84,7 @@ test('can delete a group in a document type', {tag: '@smoke'}, async ({umbracoAp
 
   // Act
   await umbracoUi.documentType.goToDocumentType(documentTypeName);
-  await umbracoUi.documentType.deleteGroup(groupName, true);
+  await umbracoUi.documentType.deleteGroup(groupName);
   await umbracoUi.documentType.clickConfirmToDeleteButton();
   await umbracoUi.documentType.clickSaveButton();
 
@@ -429,4 +429,4 @@ test('can set appearance to label on top for a property in a document type', asy
   await umbracoUi.documentType.isSuccessNotificationVisible();
   const documentTypeData = await umbracoApi.documentType.getByName(documentTypeName);
   expect(documentTypeData.properties[0].appearance.labelOnTop).toBeTruthy();
-});
+});

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Settings/MediaType/MediaTypeDesignTab.spec.ts

@@ -175,7 +175,7 @@ test('can delete a group in a media type', {tag: '@smoke'}, async ({umbracoApi,
 
   // Act
   await umbracoUi.mediaType.goToMediaType(mediaTypeName);
-  await umbracoUi.mediaType.deleteGroup(groupName, true);
+  await umbracoUi.mediaType.deleteGroup(groupName);
   await umbracoUi.mediaType.clickConfirmToDeleteButton();
   await umbracoUi.mediaType.clickSaveButton();
 
@@ -328,7 +328,6 @@ test('can reorder properties in a media type', async ({umbracoApi, umbracoUi}) =
   const dragFromLocator = umbracoUi.mediaType.getTextLocatorWithName(dataTypeNameTwo);
   const dragToLocator = umbracoUi.mediaType.getTextLocatorWithName(dataTypeName);
   await umbracoUi.mediaType.dragAndDrop(dragFromLocator, dragToLocator);
-  await umbracoUi.waitForTimeout(200);
   await umbracoUi.mediaType.clickIAmDoneReorderingButton();
   await umbracoUi.mediaType.clickSaveButton();
 

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Settings/PartialView/PartialView.spec.ts

@@ -50,7 +50,6 @@ test('can create a partial view from snippet', async ({umbracoApi, umbracoUi}) =
   await umbracoUi.partialView.doesSuccessNotificationHaveText(NotificationConstantHelper.success.created);
   expect(await umbracoApi.partialView.doesExist(partialViewFileName)).toBeTruthy();
   const partialViewData = await umbracoApi.partialView.getByName(partialViewFileName);
-  console.log(partialViewData.content);
 
   switch (process.platform) {
     case 'win32':
@@ -180,7 +179,6 @@ test('can use query builder with Where statement for a partial view', async ({um
 
   // Act
   await umbracoUi.partialView.openPartialViewAtRoot(partialViewFileName);
-  await umbracoUi.waitForTimeout(500);
   await umbracoUi.partialView.addQueryBuilderWithWhereStatement(propertyAliasValue, operatorValue, constrainValue);
   // Verify that the code is shown
   await umbracoUi.partialView.isQueryBuilderCodeShown(expectedCode);