V11: add test flakyness reporting (#13386)

* update json-models-builders * Add reporting to all before each loops * Add commitId to env for pipelines * Test reports correctly * Bumb testhelper version * Update playwright-testhelpers version * Bumb playwright test-helper version * Bumb playwright-testhelpers version * Add AgentOs to yaml env * Bumb playwright-testhelper version * Try different AgentOS * Revert "Try different AgentOS" This reverts commit fc90620. * Revert logo test * Add missing semicolons * Update package-lock.json * Add reports to Blocklist tests Co-authored-by: Zeegaan <[email protected]>
umbraco · Nov 17, 2022 · 22bcc9b · 22bcc9b
1 parent 9b13a26
commit 22bcc9b
Show file tree

Hide file tree

Showing 38 changed files with 86 additions and 53 deletions.
diff --git a/build/azure-pipelines.yml b/build/azure-pipelines.yml
@@ -452,6 +452,8 @@ stages:
             workingDirectory: tests/Umbraco.Tests.AcceptanceTest
             env:
               CI: true
+              CommitId: $(Build.SourceVersion)
+              AgentOs: $(Agent.OS)
           - pwsh: |
               docker logs $(dockerImageName) > $(Build.ArtifactStagingDirectory)/playwright.log 2>&1
               docker stop $(dockerImageName)

diff --git a/tests/Umbraco.Tests.AcceptanceTest/package-lock.json b/tests/Umbraco.Tests.AcceptanceTest/package-lock.json
diff --git a/tests/Umbraco.Tests.AcceptanceTest/package.json b/tests/Umbraco.Tests.AcceptanceTest/package.json
@@ -20,7 +20,7 @@
   },
   "dependencies": {
     "@umbraco/json-models-builders": "^1.0.2",
-    "@umbraco/playwright-testhelpers": "^1.0.12",
+    "@umbraco/playwright-testhelpers": "^1.0.18",
     "camelize": "^1.0.0",
     "faker": "^4.1.0",
     "form-data": "^4.0.0",

diff --git a/...o.Tests.AcceptanceTest/tests/DefaultConfig/BlockListEditor/blockListEditorContent.spec.ts b/...o.Tests.AcceptanceTest/tests/DefaultConfig/BlockListEditor/blockListEditorContent.spec.ts
@@ -14,7 +14,8 @@ test.describe('BlockListEditorContent', () => {
   // Won't work if I use the to alias for the elementAlias
   const elementAlias = 'testElement';
 
-  test.beforeEach(async ({page, umbracoApi, umbracoUi}) => {
+  test.beforeEach(async ({page, umbracoApi, umbracoUi}, testInfo) => {
+    await umbracoApi.report.report(testInfo);
     await umbracoApi.login();
     await umbracoApi.documentTypes.ensureNameNotExists(documentName);
     await umbracoApi.documentTypes.ensureNameNotExists(elementName);

diff --git a/....Tests.AcceptanceTest/tests/DefaultConfig/BlockListEditor/blockListEditorDataType.spec.ts b/....Tests.AcceptanceTest/tests/DefaultConfig/BlockListEditor/blockListEditorDataType.spec.ts
@@ -10,7 +10,8 @@ test.describe('BlockListEditorDataType', () => {
 
   const elementAlias = AliasHelper.toAlias(elementName);
 
-  test.beforeEach(async ({page, umbracoApi, umbracoUi}) => {
+  test.beforeEach(async ({page, umbracoApi, umbracoUi}, testInfo) => {
+    await umbracoApi.report.report(testInfo);
     await umbracoApi.login();
     await umbracoApi.documentTypes.ensureNameNotExists(documentName);
     await umbracoApi.dataTypes.ensureNameNotExists(blockListName);

diff --git a/....Tests.AcceptanceTest/tests/DefaultConfig/BlockListEditor/blockListEditorDocument.spec.ts b/....Tests.AcceptanceTest/tests/DefaultConfig/BlockListEditor/blockListEditorDocument.spec.ts
@@ -11,7 +11,8 @@ test.describe('BlockListEditorDocument', () => {
   const blockListAlias = AliasHelper.toAlias(blockListName);
   const elementAlias = AliasHelper.toAlias(elementName);
 
-  test.beforeEach(async ({page, umbracoApi, umbracoUi}) => {
+  test.beforeEach(async ({page, umbracoApi, umbracoUi}, testInfo) => {
+    await umbracoApi.report.report(testInfo);
     await umbracoApi.login();
     await umbracoApi.documentTypes.ensureNameNotExists(documentName);
     await umbracoApi.dataTypes.ensureNameNotExists(blockListName);

diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Branding/logo.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Branding/logo.spec.ts
@@ -3,10 +3,8 @@ import { expect } from "@playwright/test";
 
 test.describe('Umbraco Logo Information', () => {
 
-  test.beforeEach(async ({ page, umbracoApi }) => {
-    // TODO: REMOVE THIS WHEN SQLITE IS FIXED
-    // Wait so we don't bombard the API
-    await page.waitForTimeout(1000);
+  test.beforeEach(async ({ page, umbracoApi }, testInfo) => {
+    await umbracoApi.report.report(testInfo);
     await umbracoApi.login();
   });
 

diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Content/content.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Content/content.spec.ts
@@ -10,7 +10,8 @@ import {
 
 test.describe('Content tests', () => {
 
-  test.beforeEach(async ({page, umbracoApi}) => {
+  test.beforeEach(async ({ page, umbracoApi }, testInfo) => {
+    await umbracoApi.report.report(testInfo);
     await umbracoApi.login();
   });
 

diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Content/recycleBin.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Content/recycleBin.spec.ts
@@ -4,7 +4,8 @@ import {ContentBuilder, DocumentTypeBuilder} from "@umbraco/json-models-builders
 
 test.describe('Recycle bin', () => {
 
-  test.beforeEach(async ({page, umbracoApi}) => {
+  test.beforeEach(async ({ page, umbracoApi }, testInfo) => {
+    await umbracoApi.report.report(testInfo);
     await umbracoApi.login();
   });
 

diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Content/routing.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Content/routing.spec.ts
@@ -11,7 +11,8 @@ test.describe('Routing', () => {
   const grandChildNodeName = "Grandchild";
   const rootDocTypeName = "Test document type";
 
-  test.beforeEach(async ({page, umbracoApi}) => {
+  test.beforeEach(async ({ page, umbracoApi }, testInfo) => {
+    await umbracoApi.report.report(testInfo);
     await umbracoApi.login();
     await umbracoApi.content.deleteAllContent();
     await umbracoApi.documentTypes.ensureNameNotExists(rootDocTypeName);

diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/DataTypes/dataTypes.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/DataTypes/dataTypes.spec.ts
@@ -8,7 +8,8 @@ import {
 
 test.describe('DataTypes', () => {
 
-  test.beforeEach(async ({page, umbracoApi}) => {
+  test.beforeEach(async ({ page, umbracoApi }, testInfo) => {
+    await umbracoApi.report.report(testInfo);
     await umbracoApi.login();
   });
 

diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/DataTypes/textBoxVariation.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/DataTypes/textBoxVariation.spec.ts
@@ -4,7 +4,8 @@ import {ContentBuilder, DocumentTypeBuilder, DomainBuilder} from "@umbraco/json-
 
 test.describe('Vary by culture for TextBox', () => {
 
-    test.beforeEach(async ({page, umbracoApi, umbracoUi}) => {
+    test.beforeEach(async ({ page, umbracoApi }, testInfo) => {
+        await umbracoApi.report.report(testInfo);
         await umbracoApi.login();
     });
 

diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/HelpPanel/systemInformation.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/HelpPanel/systemInformation.spec.ts
@@ -5,7 +5,8 @@ test.describe('System Information', () => {
   const enCulture = "en-US";
   const dkCulture = "da-DK";
 
-  test.beforeEach(async ({page, umbracoApi}) => {
+  test.beforeEach(async ({ page, umbracoApi }, testInfo) => {
+    await umbracoApi.report.report(testInfo);
     await umbracoApi.login();
     await umbracoApi.users.setCurrentLanguage(enCulture);
   });

diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Languages/languages.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Languages/languages.spec.ts
@@ -3,7 +3,8 @@ import {expect} from "@playwright/test";
 
 test.describe('Languages', () => {
 
-  test.beforeEach(async ({page, umbracoApi}) => {
+  test.beforeEach(async ({ page, umbracoApi }, testInfo) => {
+    await umbracoApi.report.report(testInfo);
     await umbracoApi.login();
   });
 

diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Login/login.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Login/login.spec.ts
@@ -1,7 +1,10 @@
-import { test, expect } from '@playwright/test';
+import {expect } from '@playwright/test';
+import {test} from "@umbraco/playwright-testhelpers";
+
 test.describe('Login', () => {
 
-    test.beforeEach(async ({ page }) => {
+    test.beforeEach(async ({ page, umbracoApi }, testInfo) => {
+      await umbracoApi.report.report(testInfo);
       await page.goto(process.env.URL + '/umbraco');
     });
     test('Login with correct username and password', async ({page}) => {

diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Media/mediaFiles.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Media/mediaFiles.spec.ts
@@ -3,7 +3,8 @@ import {ConstantHelper, test} from '@umbraco/playwright-testhelpers';
 
 test.describe('media File Types', () => {
 
-    test.beforeEach(async ({page, umbracoApi, umbracoUi}) => {
+    test.beforeEach(async ({page, umbracoApi, umbracoUi}, testInfo) => {
+        await umbracoApi.report.report(testInfo);
         await umbracoApi.login();
         await umbracoUi.goToSection(ConstantHelper.sections.media);
         await umbracoApi.media.deleteAllMedia();

diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Media/mediaSection.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Media/mediaSection.spec.ts
@@ -3,7 +3,8 @@ import {ConstantHelper, test} from '@umbraco/playwright-testhelpers';
 
 test.describe('Media', () => {
 
-    test.beforeEach(async ({page, umbracoApi, umbracoUi}) => {
+    test.beforeEach(async ({page, umbracoApi, umbracoUi}, testInfo) => {
+        await umbracoApi.report.report(testInfo);
         await umbracoApi.login();
         await umbracoUi.goToSection(ConstantHelper.sections.media);
         await umbracoApi.media.deleteAllMedia()

diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Members/memberGroups.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Members/memberGroups.spec.ts
@@ -2,7 +2,8 @@ import {ConstantHelper, test} from '@umbraco/playwright-testhelpers';
 
 test.describe('Packages', () => {
 
-  test.beforeEach(async ({page, umbracoApi}) => {
+  test.beforeEach(async ({ page, umbracoApi }, testInfo) => {
+    await umbracoApi.report.report(testInfo);
     await umbracoApi.login();
   });
 

diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Members/members.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Members/members.spec.ts
@@ -2,7 +2,8 @@ import {ConstantHelper, test} from '@umbraco/playwright-testhelpers';
 
 test.describe('Packages', () => {
 
-  test.beforeEach(async ({page, umbracoApi}) => {
+  test.beforeEach(async ({ page, umbracoApi }, testInfo) => {
+    await umbracoApi.report.report(testInfo);
     await umbracoApi.login();
   });
 

diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/ModelsBuilder/modelsbuilder.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/ModelsBuilder/modelsbuilder.spec.ts
@@ -6,7 +6,8 @@ import {
 
 test.describe('Modelsbuilder tests', () => {
 
-  test.beforeEach(async ({page, umbracoApi}) => {
+  test.beforeEach(async ({ page, umbracoApi }, testInfo) => {
+    await umbracoApi.report.report(testInfo);
     await umbracoApi.login();
   });
 

diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Packages/packages.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Packages/packages.spec.ts
@@ -7,7 +7,8 @@ test.describe('Packages', () => {
   const packageName = "TestPackage";
   const rootDocTypeName = "Test document type";
   const nodeName = "1) Home";
-  test.beforeEach(async ({page, umbracoApi}) => {
+  test.beforeEach(async ({ page, umbracoApi }, testInfo) => {
+    await umbracoApi.report.report(testInfo);
     await umbracoApi.login();
   });
 

diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Settings/dataType.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Settings/dataType.spec.ts
@@ -3,8 +3,9 @@ import {expect} from "@playwright/test";
 import {LabelDataTypeBuilder} from "@umbraco/json-models-builders";
 
 test.describe('Data Types', () => {
-
-  test.beforeEach(async ({page, umbracoApi}) => {
+
+  test.beforeEach(async ({ page, umbracoApi }, testInfo) => {
+    await umbracoApi.report.report(testInfo);
     await umbracoApi.login();
   });
 

diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Settings/documentTypes.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Settings/documentTypes.spec.ts
@@ -3,7 +3,8 @@ import {ConstantHelper, test} from '@umbraco/playwright-testhelpers';
 import {DocumentTypeBuilder} from "@umbraco/json-models-builders";
 
 test.describe('Document types', () => {
-  test.beforeEach(async ({ page, umbracoApi }) => {
+  test.beforeEach(async ({ page, umbracoApi }, testInfo) => {
+    await umbracoApi.report.report(testInfo);
     await umbracoApi.login();
   });
 

diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Settings/languages.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Settings/languages.spec.ts
@@ -2,7 +2,8 @@ import {test, ApiHelpers, UiHelpers, ConstantHelper} from '@umbraco/playwright-t
 import {expect} from "@playwright/test";
 
 test.describe('Languages', () => {
-  test.beforeEach(async ({ page, umbracoApi }) => {
+  test.beforeEach(async ({ page, umbracoApi }, testInfo) => {
+    await umbracoApi.report.report(testInfo);
     await umbracoApi.login();
   });
 

diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Settings/macro.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Settings/macro.spec.ts
@@ -3,7 +3,8 @@ import {PartialViewMacroBuilder} from "@umbraco/json-models-builders";
 
 test.describe('Macros', () => {
 
-  test.beforeEach(async ({ page, umbracoApi }) => {
+  test.beforeEach(async ({ page, umbracoApi }, testInfo) => {
+    await umbracoApi.report.report(testInfo);
     await umbracoApi.login();
   });
 

diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Settings/mediaTypes.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Settings/mediaTypes.ts
@@ -3,7 +3,8 @@ import {expect} from "@playwright/test";
 
 test.describe('Media types', () => {
 
-  test.beforeEach(async ({ page, umbracoApi }) => {
+  test.beforeEach(async ({ page, umbracoApi }, testInfo) => {
+    await umbracoApi.report.report(testInfo);
     await umbracoApi.login();
   });
 

diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Settings/memberTypes.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Settings/memberTypes.spec.ts
@@ -1,7 +1,8 @@
 import {ConstantHelper, test} from '@umbraco/playwright-testhelpers';
 
 test.describe('Member Types', () => {
-  test.beforeEach(async ({page, umbracoApi}) => {
+  test.beforeEach(async ({ page, umbracoApi }, testInfo) => {
+    await umbracoApi.report.report(testInfo);
     await umbracoApi.login();
   });
 

diff --git a/...s/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Settings/partialViewMacroFiles.spec.ts b/...s/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Settings/partialViewMacroFiles.spec.ts
@@ -4,7 +4,8 @@ import {PartialViewMacroBuilder} from "@umbraco/json-models-builders";
 
 test.describe('Partial View Macro Files', () => {
 
-    test.beforeEach(async ({page, umbracoApi}) => {
+    test.beforeEach(async ({ page, umbracoApi }, testInfo) => {
+        await umbracoApi.report.report(testInfo);
         await umbracoApi.login();
     });
 

diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Settings/partialViews.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Settings/partialViews.spec.ts
@@ -4,7 +4,8 @@ import {PartialViewBuilder} from "@umbraco/json-models-builders";
 
 test.describe('Partial Views', () => {
 
-  test.beforeEach(async ({page, umbracoApi}) => {
+  test.beforeEach(async ({ page, umbracoApi }, testInfo) => {
+    await umbracoApi.report.report(testInfo);
     await umbracoApi.login();
   });
 

diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Settings/relationTypes.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Settings/relationTypes.spec.ts
@@ -3,7 +3,8 @@ import {ConstantHelper, test} from '@umbraco/playwright-testhelpers';
 
 test.describe('Relation Types', () => {
 
-  test.beforeEach(async ({page, umbracoApi}) => {
+  test.beforeEach(async ({ page, umbracoApi }, testInfo) => {
+    await umbracoApi.report.report(testInfo);
     await umbracoApi.login();
   });
 

diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Settings/scripts.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Settings/scripts.spec.ts
@@ -3,7 +3,8 @@ import {expect} from "@playwright/test";
 import {ScriptBuilder} from "@umbraco/json-models-builders";
 
 test.describe('Scripts', () => {
-  test.beforeEach(async ({ page, umbracoApi }) => {
+  test.beforeEach(async ({ page, umbracoApi }, testInfo) => {
+    await umbracoApi.report.report(testInfo);
     await umbracoApi.login();
   });
 

diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Settings/stylesheets.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Settings/stylesheets.spec.ts
@@ -7,7 +7,8 @@ test.describe('Stylesheets', () => {
   const name = "TestStylesheet";
   const fileName = name + ".css";
 
-  test.beforeEach(async ({page, umbracoApi}) => {
+  test.beforeEach(async ({ page, umbracoApi }, testInfo) => {
+    await umbracoApi.report.report(testInfo);
     await umbracoApi.login();
   });
 

diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Settings/templates.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Settings/templates.spec.ts
@@ -3,7 +3,8 @@ import {ConstantHelper, test} from '@umbraco/playwright-testhelpers';
 import {TemplateBuilder} from "@umbraco/json-models-builders";
 
 test.describe('Templates', () => {
-  test.beforeEach(async ({page, umbracoApi}) => {
+  test.beforeEach(async ({ page, umbracoApi }, testInfo) => {
+    await umbracoApi.report.report(testInfo);
     await umbracoApi.login();
   });
 

diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Tabs/tabs.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Tabs/tabs.spec.ts
@@ -6,8 +6,9 @@ const tabsDocTypeName = 'Tabs Test Document';
 const tabsDocTypeAlias = AliasHelper.toAlias(tabsDocTypeName);
 
 test.describe('Tabs', () => {
-
-  test.beforeEach(async ({ umbracoApi, page }) => {
+
+  test.beforeEach(async ({ page, umbracoApi }, testInfo) => {
+    await umbracoApi.report.report(testInfo);
     await umbracoApi.login();
   });
 

diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Tour/tours.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Tour/tours.spec.ts
@@ -3,7 +3,8 @@ import {test} from '@umbraco/playwright-testhelpers';
 
 test.describe('Tours', () => {
   const timeout = 60000;
-  test.beforeEach(async ({page, umbracoApi}) => {
+  test.beforeEach(async ({ page, umbracoApi }, testInfo) => {
+    await umbracoApi.report.report(testInfo);
     await umbracoApi.login();
     await resetTourData(umbracoApi);
   });

diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Translation/translation.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Translation/translation.spec.ts
@@ -2,10 +2,8 @@ import {ConstantHelper, test} from '@umbraco/playwright-testhelpers';
 
 test.describe('Translation', () => {
 
-  test.beforeEach(async ({page, umbracoApi}) => {
-    // TODO: REMOVE THIS WHEN SQLITE IS FIXED
-    // Wait so we don't bombard the API
-    await page.waitForTimeout(1000);
+  test.beforeEach(async ({ page, umbracoApi }, testInfo) => {
+    await umbracoApi.report.report(testInfo);
     await umbracoApi.login();
   });