Merge pull request #362 from Lombiq/issue/OSOE-838

OSOE-838: Updating Atata v3.1.0, and other dependencies to latest too, reliability improvements

Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj

@@ -3,9 +3,6 @@
   <PropertyGroup>
     <TargetFramework>net8.0</TargetFramework>
     <IsPackable>false</IsPackable>
-    <!-- Eliminates problem with dotnet publish caused by duplicate .htmlvalidate.json file. See
-         https://docs.microsoft.com/en-us/dotnet/core/compatibility/sdk/6.0/duplicate-files-in-output. -->
-    <ErrorOnDuplicatePublishOutputFiles>false</ErrorOnDuplicatePublishOutputFiles>
   </PropertyGroup>
 
   <ItemGroup>
@@ -16,7 +13,7 @@
 
   <ItemGroup>
     <Content Include=".htmlvalidate.json">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
     <Content Include="Tests\CustomZapAutomationFrameworkPlan.yml">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
@@ -28,7 +25,7 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.7.2" />
-    <PackageReference Include="xunit.runner.visualstudio" Version="2.5.1">
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.8.0">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>

Lombiq.Tests.UI.Samples/Tests/InteractiveModeTests.cs

@@ -44,8 +44,8 @@ public Task SampleTest() =>
             });
 
     // This test checks if interactive mode works by opening it in one thread, and then clicking it away in a different
-    // thread. This ensures that the new tab correctly appears with the clickable "Continue Test" button, and then
-    // disappears once it's clicked.
+    // thread. Two threads are necessary because interactive mode stops test execution on its current thread, so we
+    // wouldn't be able to end it from within a test.
     [Fact]
     public Task EnteringInteractiveModeShouldWait() =>
         ExecuteTestAfterSetupAsync(
@@ -59,7 +59,7 @@ await Task.WhenAll(
                     {
                         // Ensure that the interactive mode polls for status at least once, so the arbitrary waiting
                         // actually works in a real testing scenario.
-                        await Task.Delay(1000);
+                        await Task.Delay(5000);
 
                         await context.ClickReliablyOnAsync(By.ClassName("interactive__continue"));
                     }));

Lombiq.Tests.UI.Samples/UITestBase.cs

@@ -79,7 +79,7 @@ protected override Task ExecuteTestAsync(
                 // disable it with the below config). With this, you can make sure that the HTML markup the app
                 // generates (also from content items) is valid. While the default settings for HTML validation are most
                 // possibly suitable for your projects, check out the HtmlValidationConfiguration class for what else
-                // you can configure. We've also added a .htmlvalidate.json file (note the Content Build
+                // you can configure. We've also added a custom .htmlvalidate.json file (note the Content Build
                 // Action) to further configure it.
                 ////configuration.HtmlValidationConfiguration.RunHtmlValidationAssertionOnAllPageChanges = false;
 

Lombiq.Tests.UI/Extensions/ElementRetrievalUITestContextExtensions.cs

@@ -11,7 +11,7 @@
 namespace Lombiq.Tests.UI.Extensions;
 
 /// <summary>
-/// Extension methods to retrieve elements using Atata helpers. See the Atata docs ( <see
+/// Extension methods to retrieve elements using Atata helpers. See the Atata docs (<see
 /// href="https://github.com/atata-framework/atata-webdriverextras#usage"/>) for more information on what you can do
 /// with these.
 /// </summary>
@@ -146,5 +146,5 @@ private static ExtendedSearchContext<IWebDriver> CreateSearchContext(this UITest
         new(
             context.Driver,
             context.Configuration.TimeoutConfiguration.RetryTimeout,
-            context.Configuration.TimeoutConfiguration.RetryTimeout);
+            context.Configuration.TimeoutConfiguration.RetryInterval);
 }

Lombiq.Tests.UI/Extensions/ExtendedLoggingExtensions.cs

@@ -148,15 +148,15 @@ private static async Task<TResult> ExecuteSectionAsync<TResult>(
             var notLast = i < StabilityRetryCount - 1;
             try
             {
-                // This is somewhat risky. ILogManager is not thread-safe and uses as stack to keep track of sections, so if
-                // multiple sections are started in concurrent threads, the result will be incorrect. This shouldn't be too much
-                // of an issue for now though since tests, while async, are single-threaded.
-                context.Scope.AtataContext.Log.Start(section);
-                context.Scope.AtataContext.Log.Info("Log section {0} started.", section.Message);
-                var result = await functionAsync();
-                context.Scope.AtataContext.Log.Info("Log section {0} ended.", section.Message);
-                context.Scope.AtataContext.Log.EndSection();
-                return result;
+                return await context.Scope.AtataContext.Log.ExecuteSectionAsync(
+                    section,
+                    async () =>
+                    {
+                        context.Scope.AtataContext.Log.Info($"Log section {section.Message} started.");
+                        var result = await functionAsync();
+                        context.Scope.AtataContext.Log.Info($"Log section {section.Message} ended.");
+                        return result;
+                    });
             }
             catch (StaleElementReferenceException) when (notLast)
             {
@@ -171,7 +171,5 @@ private static async Task<TResult> ExecuteSectionAsync<TResult>(
     private static void LogStaleElementReferenceExceptionRetry(UITestContext context, int tryIndex) =>
         context.Scope.AtataContext.Log.Info(
             "The operation in the log section failed with StaleElementReferenceException but will be retried. This " +
-            "is try number {0} out of {1}.",
-            tryIndex + 1,
-            StabilityRetryCount);
+            $"is try number {(tryIndex + 1).ToTechnicalString()} out of {StabilityRetryCount}.");
 }

Lombiq.Tests.UI/Extensions/FormUITestContextExtensions.cs

@@ -108,6 +108,8 @@ public static void FillInMonacoEditor(
         string editorId,
         string text)
     {
+        WaitForMonacoEditor(context, editorId);
+
         var script = $@"
             monaco.editor.getEditors().find((element) =>
                 element.getContainerDomNode().id == {JsonConvert.SerializeObject(editorId)}).setValue({JsonConvert.SerializeObject(text)});";
@@ -122,6 +124,8 @@ public static string GetMonacoEditorText(
         this UITestContext context,
         string editorId)
     {
+        WaitForMonacoEditor(context, editorId);
+
         var script = $@"
             return monaco.editor.getEditors().find((element) =>
                 element.getContainerDomNode().id == {JsonConvert.SerializeObject(editorId)}).getValue();";
@@ -375,4 +379,7 @@ private static IWebElement TryFillElement(UITestContext context, By by, string t
 
         return context.Driver.TryFillElement(element, text);
     }
+
+    private static void WaitForMonacoEditor(UITestContext context, string editorId) =>
+        context.Get(By.CssSelector($"#{editorId} .monaco-editor"));
 }

Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs

@@ -34,7 +34,8 @@ public static async Task<IEnumerable<string>> GetErrorsAsync(this HtmlValidation
 
     public static string GetParsedErrorMessageString(IEnumerable<JsonHtmlValidationError> errors) =>
         string.Join(
-            '\n', errors.Select(error =>
+            '\n',
+            errors.Select(error =>
                 $"{error.Line.ToString(CultureInfo.InvariantCulture)}:{error.Column.ToString(CultureInfo.InvariantCulture)} - " +
                 $"{error.Message} - " +
                 $"{error.RuleId}"));

Lombiq.Tests.UI/Lombiq.Tests.UI.csproj

@@ -59,31 +59,30 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Atata" Version="2.9.0" />
-    <PackageReference Include="Atata.Bootstrap" Version="2.1.0" />
-    <PackageReference Include="Atata.HtmlValidation" Version="2.4.0" />
-    <PackageReference Include="Atata.WebDriverExtras" Version="2.2.0" />
+    <PackageReference Include="Atata" Version="3.1.0" />
+    <PackageReference Include="Atata.Bootstrap" Version="3.0.0" />
+    <PackageReference Include="Atata.HtmlValidation" Version="3.0.0" />
+    <PackageReference Include="Atata.WebDriverExtras" Version="3.0.0" />
     <PackageReference Include="Atata.WebDriverSetup" Version="2.10.0" />
-    <PackageReference Include="Azure.Storage.Blobs" Version="12.18.0" />
+    <PackageReference Include="Azure.Storage.Blobs" Version="12.19.1" />
     <PackageReference Include="Ben.Demystifier" Version="0.4.1" />
-    <PackageReference Include="Codeuctivity.ImageSharpCompare" Version="3.0.183" />
+    <PackageReference Include="Codeuctivity.ImageSharpCompare" Version="4.0.258" />
     <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="8.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
-    <PackageReference Include="Microsoft.SqlServer.DACFx" Version="162.0.52" />
-    <PackageReference Include="Microsoft.SqlServer.SqlManagementObjects" Version="170.18.0" />
+    <PackageReference Include="Microsoft.SqlServer.DACFx" Version="162.2.111" />
+    <PackageReference Include="Microsoft.SqlServer.SqlManagementObjects" Version="171.30.0" />
     <PackageReference Include="Mono.Posix.NETStandard" Version="1.0.0" />
     <PackageReference Include="OrchardCore.ContentFields" Version="1.8.2" />
     <PackageReference Include="OrchardCore.Logging.NLog" Version="1.8.2" />
     <PackageReference Include="OrchardCore.Abstractions" Version="1.8.2" />
     <PackageReference Include="OrchardCore.Recipes.Core" Version="1.8.2" />
     <PackageReference Include="Refit.HttpClientFactory" Version="7.0.0" />
-    <PackageReference Include="Sarif.Sdk" Version="4.3.7" />
-    <PackageReference Include="Selenium.Axe" Version="4.0.14" />
+    <PackageReference Include="Sarif.Sdk" Version="4.5.4" />
+    <PackageReference Include="Selenium.Axe" Version="4.0.17" />
     <PackageReference Include="Shouldly" Version="4.2.1" />
     <PackageReference Include="System.Linq.Async" Version="6.0.1" />
-    <PackageReference Include="WebDriverManager" Version="2.16.2" />
-    <PackageReference Include="xunit" Version="2.5.1" />
-    <PackageReference Include="YamlDotNet" Version="13.7.1" />
+    <PackageReference Include="xunit" Version="2.8.0" />
+    <PackageReference Include="YamlDotNet" Version="15.1.4" />
   </ItemGroup>
 
   <ItemGroup>

Lombiq.Tests.UI/MonkeyTesting/MonkeyTester.cs

@@ -26,61 +26,49 @@ internal MonkeyTester(UITestContext context, MonkeyTestingOptions options = null
         _randomizer = new NonSecurityRandomizer(_options.BaseRandomSeed);
     }
 
-    internal async Task TestOnePageAsync(int? randomSeed = null)
-    {
-        Log.Start(new LogSection("Execute monkey testing against one page"));
-
-        try
-        {
-            WriteOptionsToLog();
-
-            var pageTestInfo = GetCurrentPageTestInfo();
-
-            if (randomSeed is null) await TestCurrentPageAsync(pageTestInfo);
-            else await TestCurrentPageWithRandomSeedAsync(pageTestInfo, randomSeed.Value);
-        }
-        finally
-        {
-            Log.EndSection();
-        }
-    }
+    internal Task TestOnePageAsync(int? randomSeed = null) =>
+        Log.ExecuteSectionAsync(
+            new LogSection("Execute monkey testing against one page"),
+            async () =>
+            {
+                WriteOptionsToLog();
 
-    internal async Task TestRecursivelyAsync()
-    {
-        Log.Start(new LogSection($"Execute monkey testing recursively"));
+                var pageTestInfo = GetCurrentPageTestInfo();
 
-        try
-        {
-            WriteOptionsToLog();
-
-            var pageTestInfo = GetCurrentPageTestInfo();
-            await TestCurrentPageAsync(pageTestInfo);
+                if (randomSeed is null) await TestCurrentPageAsync(pageTestInfo);
+                else await TestCurrentPageWithRandomSeedAsync(pageTestInfo, randomSeed.Value);
+            });
 
-            while (true)
+    internal Task TestRecursivelyAsync() =>
+        Log.ExecuteSectionAsync(
+            new LogSection($"Execute monkey testing recursively"),
+            async () =>
             {
-                pageTestInfo = GetCurrentPageTestInfo();
+                WriteOptionsToLog();
 
-                if (CanTestPage(pageTestInfo))
-                {
-                    await TestCurrentPageAsync(pageTestInfo);
-                }
-                else if (TryGetAvailablePageToTest(out var availablePageToTest))
-                {
-                    await _context.GoToAbsoluteUrlAsync(availablePageToTest.Url);
+                var pageTestInfo = GetCurrentPageTestInfo();
+                await TestCurrentPageAsync(pageTestInfo);
 
-                    await TestCurrentPageAsync(availablePageToTest);
-                }
-                else
+                while (true)
                 {
-                    return;
+                    pageTestInfo = GetCurrentPageTestInfo();
+
+                    if (CanTestPage(pageTestInfo))
+                    {
+                        await TestCurrentPageAsync(pageTestInfo);
+                    }
+                    else if (TryGetAvailablePageToTest(out var availablePageToTest))
+                    {
+                        await _context.GoToAbsoluteUrlAsync(availablePageToTest.Url);
+
+                        await TestCurrentPageAsync(availablePageToTest);
+                    }
+                    else
+                    {
+                        return;
+                    }
                 }
-            }
-        }
-        finally
-        {
-            Log.EndSection();
-        }
-    }
+            });
 
     private void WriteOptionsToLog() =>
         Log.Trace(@$"Monkey testing options:

Lombiq.Tests.UI/Pages/OrchardCoreAdminPage.cs

@@ -12,9 +12,9 @@ public abstract class OrchardCoreAdminPage<TOwner> : Page<TOwner>
 
     public ControlList<AlertMessage<TOwner>, TOwner> AlertMessages { get; private set; }
 
-    public TOwner ShouldStayOnAdminPage() => AdminMenu.Should.Exist();
+    public TOwner ShouldStayOnAdminPage() => AdminMenu.Should.BePresent();
 
-    public TOwner ShouldLeaveAdminPage() => AdminMenu.Should.Not.Exist();
+    public TOwner ShouldLeaveAdminPage() => AdminMenu.Should.Not.BePresent();
 
     protected override void OnVerify()
     {

Lombiq.Tests.UI/Services/AtataFactory.cs

@@ -17,11 +17,13 @@ public class AtataConfiguration
 public static class AtataFactory
 {
     public static async Task<AtataScope> StartAtataScopeAsync(
+        string contextId,
         ITestOutputHelper testOutputHelper,
         Uri baseUri,
         OrchardCoreUITestExecutorConfiguration configuration)
     {
-        AtataContext.ModeOfCurrent = AtataContextModeOfCurrent.AsyncLocal;
+        AtataContext.GlobalProperties.ModeOfCurrent = AtataContextModeOfCurrent.AsyncLocal;
+        AtataContext.GlobalProperties.UseUtcTimeZone();
 
         // Since Atata 2.0 the default visibility option is Visibility.Any, these lines restore it to the 1.x behavior.
         AtataContext.GlobalConfiguration.UseDefaultControlVisibility(Visibility.Visible);
@@ -37,8 +39,8 @@ public static async Task<AtataScope> StartAtataScopeAsync(
             .UseTestName(configuration.AtataConfiguration.TestName)
             .UseBaseRetryTimeout(timeoutConfiguration.RetryTimeout)
             .UseBaseRetryInterval(timeoutConfiguration.RetryInterval)
-            .UseUtcTimeZone()
-            .PageSnapshots.UseCdpOrPageSourceStrategy(); // #spell-check-ignore-line
+            .PageSnapshots.UseCdpOrPageSourceStrategy() // #spell-check-ignore-line
+            .UseArtifactsPathTemplate(contextId); // Necessary to prevent long paths, an issue under Windows.
 
         builder.LogConsumers.AddDebug();
         builder.LogConsumers.Add(new TestOutputLogConsumer(testOutputHelper));

Lombiq.Tests.UI/Services/OrchardCoreInstance.cs

@@ -180,7 +180,8 @@ await _configuration.BeforeAppStart
 
     private async Task StopOrchardAppAsync()
     {
-        _reverseProxy.DetachConnectionProvider();
+        _reverseProxy?.DetachConnectionProvider();
+
         if (_orchardApplication == null) return;
 
         _testOutputHelper.WriteLineTimestampedAndDebug("Attempting to stop the Orchard Core instance.");

Lombiq.Tests.UI/Services/UITestExecutionSession.cs

@@ -656,7 +656,7 @@ Task UITestingBeforeAppStartHandlerAsync(string contentRootPath, InstanceCommand
             _configuration.Events.AfterPageChange += TakeScreenshotIfEnabledAsync;
         }
 
-        var atataScope = await AtataFactory.StartAtataScopeAsync(_testOutputHelper, uri, _configuration);
+        var atataScope = await AtataFactory.StartAtataScopeAsync(contextId, _testOutputHelper, uri, _configuration);
 
         return new UITestContext(
             contextId,

Lombiq.Tests.UI/Services/UITestExecutor.cs

@@ -125,11 +125,26 @@ private static string PrepareDumpFolder(
     {
         var dumpConfiguration = configuration.FailureDumpConfiguration;
         var dumpFolderNameBase = testManifest.Name;
-        if (dumpConfiguration.UseShortNames && dumpFolderNameBase.Contains('(', StringComparison.Ordinal))
+        if (dumpConfiguration.UseShortNames)
         {
-            var dumpFolderNameBeginningIndex =
-                dumpFolderNameBase[..dumpFolderNameBase.IndexOf('(', StringComparison.Ordinal)].LastIndexOf('.') + 1;
-            dumpFolderNameBase = dumpFolderNameBase[dumpFolderNameBeginningIndex..];
+            if (dumpFolderNameBase.Contains('(', StringComparison.Ordinal))
+            {
+                // The test uses parameters and is thus in the
+                // "Lombiq.Tests.UI.Samples.Tests.BasicTests.AnonymousHomePageShouldExist(browser: Chrome)" format.
+                var dumpFolderNameBeginningIndex =
+                    dumpFolderNameBase[..dumpFolderNameBase.IndexOf('(', StringComparison.Ordinal)].LastIndexOf('.') + 1;
+                dumpFolderNameBase = dumpFolderNameBase[dumpFolderNameBeginningIndex..];
+            }
+            else
+            {
+                // The test doesn't use parameters and is thus in the
+                // "Lombiq.Tests.UI.Samples.Tests.BasicTests.AnonymousHomePageShouldExist" format.
+                var dumpFolderNameBeginningIndex = dumpFolderNameBase.LastIndexOf('.') + 1;
+                dumpFolderNameBase = dumpFolderNameBase[dumpFolderNameBeginningIndex..];
+            }
+
+            // Can't use string.GetHasCode() because that varies between executions.
+            dumpFolderNameBase += "-" + Sha256Helper.ComputeHash(testManifest.Name);
         }
 
         dumpFolderNameBase = dumpFolderNameBase.MakeFileSystemFriendly();
@@ -159,9 +174,12 @@ private static string PrepareDumpFolder(
             var openingBracketIndex = dumpFolderNameBase.IndexOf('(', StringComparison.Ordinal);
             var closingBracketIndex = dumpFolderNameBase.LastIndexOf(')');
 
+            // Only adding a hash of the parameters if the hash of the test's full name is not already there due to
+            // path shortening above.
             // Can't use string.GetHasCode() because that varies between executions.
-            var hashedParameters = Sha256Helper
-                .ComputeHash(dumpFolderNameBase[(openingBracketIndex + 1)..(closingBracketIndex + 1)]);
+            var hashedParameters = dumpConfiguration.UseShortNames
+                ? string.Empty
+                : Sha256Helper.ComputeHash(dumpFolderNameBase[(openingBracketIndex + 1)..(closingBracketIndex + 1)]);
 
             dumpFolderNameBase =
                 dumpFolderNameBase[0..(openingBracketIndex + 1)] +

Lombiq.Tests.UI/Services/UITestExecutorFailureDumpConfiguration.cs

@@ -6,8 +6,9 @@ public class UITestExecutorFailureDumpConfiguration
 {
     /// <summary>
     /// Gets or sets a value indicating whether the subfolder of each test's dumps will use a shortened name, only
-    /// containing the name of the test method, without the name of the test class and its namespace. This is to
-    /// overcome the 260 character path length limitations on Windows. Defaults to <see langword="true"/> on Windows.
+    /// containing the name of the test method suffixed with the test name's hash to make it unique, without the name of
+    /// the test class and its namespace. This is to overcome the 260 character path length limitations on Windows.
+    /// Defaults to <see langword="true"/> on Windows.
     /// </summary>
     public bool UseShortNames { get; set; } = OperatingSystem.IsWindows();