OSOE-838: Updating Atata v3.1.0, and other dependencies to latest too, reliability improvements

Lombiq.Tests.UI.Samples/samples.htmlvalidate.json → Lombiq.Tests.UI.Samples/.htmlvalidate.json

@@ -1,11 +1,12 @@
 {
   "extends": [
-    "./.htmlvalidate.json"
+    "./default.htmlvalidate.json"
   ],
 
   "rules": {
     "long-title": "off",
     "no-dup-class": "off"
   },
+
   "root": true
 }

Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj

@@ -6,13 +6,14 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <None Remove=".htmlvalidate.json" />
     <None Remove="Tests\CustomZapAutomationFrameworkPlan.yml" />
     <None Remove="xunit.runner.json" />
   </ItemGroup>
 
   <ItemGroup>
-    <Content Include="samples.htmlvalidate.json">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    <Content Include=".htmlvalidate.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
     <Content Include="Tests\CustomZapAutomationFrameworkPlan.yml">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>

Lombiq.Tests.UI.Samples/UITestBase.cs

@@ -4,7 +4,6 @@
 using Lombiq.Tests.UI.Services;
 using Shouldly;
 using System;
-using System.IO;
 using System.Threading.Tasks;
 using Xunit.Abstractions;
 
@@ -84,13 +83,6 @@ protected override Task ExecuteTestAsync(
                 // Action) to further configure it.
                 ////configuration.HtmlValidationConfiguration.RunHtmlValidationAssertionOnAllPageChanges = false;
 
-                // This is to make sure that the HTML validation config file in this project takes effect. Only needed
-                // until we fix https://github.com/Lombiq/UI-Testing-Toolbox/issues/359.
-                configuration.HtmlValidationConfiguration.HtmlValidationOptions =
-                    configuration.HtmlValidationConfiguration.HtmlValidationOptions
-                        .CloneWith(validationOptions => validationOptions.ConfigPath =
-                            Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "samples.htmlvalidate.json")); // #spell-check-ignore-line
-
                 // The UI Testing Toolbox can run several checks for the app even if you don't add explicit
                 // assertions: By default, the Orchard logs and the browser logs (where e.g. JavaScript errors show
                 // up) are checked and if there are any errors, the test will fail. You can also enable the checking of

Lombiq.Tests.UI.Tests.UI/TestCases/TimeoutTestCases.cs

@@ -0,0 +1,25 @@
+using Lombiq.Tests.UI.Services;
+using Shouldly;
+
+namespace Lombiq.Tests.UI.Tests.UI.TestCases;
+
+public static class TimeoutTestCases
+{
+    public static Task TestRunTimeoutShouldThrowAsync(
+        ExecuteTestAfterSetupAsync executeTestAfterSetupAsync,
+        Browser browser = default) =>
+        Should.ThrowAsync(
+            async () => await executeTestAfterSetupAsync(
+                context => Task.Delay(TimeSpan.FromSeconds(1)),
+                browser,
+                configuration =>
+                {
+                    configuration.HtmlValidationConfiguration.RunHtmlValidationAssertionOnAllPageChanges = false;
+                    configuration.MaxRetryCount = 0;
+
+                    configuration.TimeoutConfiguration.TestRunTimeout = TimeSpan.FromMilliseconds(10);
+
+                    return Task.CompletedTask;
+                }),
+            typeof(TimeoutException));
+}

Lombiq.Tests.UI/Docs/Configuration.md

@@ -59,31 +59,42 @@ Recommendations and notes for such configuration:
 
 ### HTML validation configuration
 
-If you want to change some HTML validation rules from only a few specific tests, you can create a custom _.htmlvalidate.json_ file (e.g. _TestName.htmlvalidate.json_). It can also extend the default one, only overriding the configuration necessary. For example:
+If you want to change some HTML validation rules from only a few specific tests, you can create a custom _.htmlvalidate.json_ file (e.g. _TestName.htmlvalidate.json_). This should extend the [default.htmlvalidate.json](../default.htmlvalidate.json) file (which is always copied into the build directory) by setting the value of `"extends"` to a relative path pointing to it and declaring `"root": true`. For example:
 
 ```json
 {
-  "extends": [
-    "./.htmlvalidate.json"
-  ],
+    "extends": [
+        "./default.htmlvalidate.json"
+    ],
 
-  "rules": {
-    "element-required-attributes": "off",
-    "no-implicit-button-type": "off"
-  },
+    "rules": {
+        "element-required-attributes": "off",
+        "no-implicit-button-type": "off"
+    },
 
-  "root":  true
+    "root": true
 }
 ```
 
-Then you can change the configuration to use that:
+You can also create a completely standalone config file too, without any inheritance, but we'd recommend against that.
+
+You can change the configuration to use the above file as follows:
+
+```cs
+changeConfiguration: configuration =>
+    configuration.HtmlValidationConfiguration.HtmlValidationOptions.ConfigPath =
+        Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "TestName.htmlvalidate.json");
+```
+
+Though if the file is in the base directory like above, then it can be simplified using the `WithRelativeConfigPath(params string[] pathSegments)` method:
 
 ```cs
- changeConfiguration: configuration => 
-    configuration.HtmlValidationConfiguration.HtmlValidationOptions.SetLocalConfigFile("TestName.htmlvalidate.json");
+changeConfiguration: configuration => configuration.HtmlValidationConfiguration.WithRelativeConfigPath("TestName.htmlvalidate.json");
 ```
 
-Make sure to also include the `root` attribute and set it to `true` inside the custom _.htmlvalidate.json_ file and include it in the test project like this:
+If you want to do this for all tests in the project, just put an _.htmlvalidate.json_ file into the project root and it will be picked up without further configuration.
+
+Include it in the test project like this:
 
 ```xml
   <ItemGroup>

Lombiq.Tests.UI/Extensions/ElementRetrievalUITestContextExtensions.cs

@@ -11,7 +11,7 @@
 namespace Lombiq.Tests.UI.Extensions;
 
 /// <summary>
-/// Extension methods to retrieve elements using Atata helpers. See the Atata docs ( <see
+/// Extension methods to retrieve elements using Atata helpers. See the Atata docs (<see
 /// href="https://github.com/atata-framework/atata-webdriverextras#usage"/>) for more information on what you can do
 /// with these.
 /// </summary>
@@ -146,5 +146,5 @@ private static ExtendedSearchContext<IWebDriver> CreateSearchContext(this UITest
         new(
             context.Driver,
             context.Configuration.TimeoutConfiguration.RetryTimeout,
-            context.Configuration.TimeoutConfiguration.RetryTimeout);
+            context.Configuration.TimeoutConfiguration.RetryInterval);
 }

Lombiq.Tests.UI/Extensions/FormUITestContextExtensions.cs

@@ -108,8 +108,7 @@ public static void FillInMonacoEditor(
         string editorId,
         string text)
     {
-        // Waiting for the editor to load.
-        context.Get(By.CssSelector($"#{editorId} .monaco-editor"));
+        WaitForMonacoEditor(context, editorId);
 
         var script = $@"
             monaco.editor.getEditors().find((element) =>
@@ -125,6 +124,8 @@ public static string GetMonacoEditorText(
         this UITestContext context,
         string editorId)
     {
+        WaitForMonacoEditor(context, editorId);
+
         var script = $@"
             return monaco.editor.getEditors().find((element) =>
                 element.getContainerDomNode().id == {JsonConvert.SerializeObject(editorId)}).getValue();";
@@ -378,4 +379,7 @@ private static IWebElement TryFillElement(UITestContext context, By by, string t
 
         return context.Driver.TryFillElement(element, text);
     }
+
+    private static void WaitForMonacoEditor(UITestContext context, string editorId) =>
+        context.Get(By.CssSelector($"#{editorId} .monaco-editor"));
 }

Lombiq.Tests.UI/Lombiq.Tests.UI.csproj

@@ -40,7 +40,7 @@
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
       <PackageCopyToOutput>true</PackageCopyToOutput>
     </Content>
-    <Content Include=".htmlvalidate.json">
+    <Content Include="default.htmlvalidate.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
       <PackageCopyToOutput>true</PackageCopyToOutput>
     </Content>

Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs

@@ -2,6 +2,7 @@
 using Lombiq.Tests.UI.Helpers;
 using Shouldly;
 using System;
+using System.IO;
 using System.Threading.Tasks;
 
 namespace Lombiq.Tests.UI.Services;
@@ -33,6 +34,13 @@ public class HtmlValidationConfiguration
         // This is necessary so no long folder names will be generated, see:
         // https://github.com/atata-framework/atata-htmlvalidation/issues/5
         WorkingDirectory = "HtmlValidationTemp",
+        // If a consuming project adds a ".htmlvalidate.json" config file then use it, otherwise fall back to the
+        // "default.htmlvalidate.json" which always exists because Lombiq.Tests.UI copies it into the directory during
+        // build.
+        ConfigPath =
+            Path.Combine(AppDomain.CurrentDomain.BaseDirectory, ".htmlvalidate.json") is { } rootConfiguration && File.Exists(rootConfiguration)
+            ? rootConfiguration
+            : Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "default.htmlvalidate.json"),
     };
 
     /// <summary>
@@ -61,6 +69,22 @@ public class HtmlValidationConfiguration
     public Predicate<UITestContext> HtmlValidationAndAssertionOnPageChangeRule { get; set; } =
         EnableOnValidatablePagesHtmlValidationAndAssertionOnPageChangeRule;
 
+    /// <summary>
+    /// Updates the <see cref="HtmlValidationOptions"/>.<see cref="HtmlValidationOptions.ConfigPath"/> with a path
+    /// relative to the <see cref="AppDomain.BaseDirectory"/> of the <see cref="AppDomain.CurrentDomain"/> (i.e. the
+    /// build directory).
+    /// </summary>
+    /// <param name="pathSegments">
+    /// Directory and file names which are joined together using <see cref="Path.Combine(string[])"/>.
+    /// </param>
+    public HtmlValidationConfiguration WithRelativeConfigPath(params string[] pathSegments)
+    {
+        string[] path = [AppDomain.CurrentDomain.BaseDirectory, .. pathSegments];
+        HtmlValidationOptions.ConfigPath = Path.Combine(path);
+
+        return this;
+    }
+
     public static readonly Func<HtmlValidationResult, Task> AssertHtmlValidationOutputIsEmptyAsync =
         validationResult =>
         {

Lombiq.Tests.UI/Services/TimeoutConfiguration.cs

@@ -4,6 +4,14 @@ namespace Lombiq.Tests.UI.Services;
 
 public class TimeoutConfiguration
 {
+    private static readonly TimeoutConfiguration _default = new()
+    {
+        RetryTimeout = GetTimeoutConfiguration(nameof(RetryTimeout), 10),
+        RetryInterval = GetTimeoutConfiguration(nameof(RetryInterval), 500, useMilliseconds: true),
+        PageLoadTimeout = GetTimeoutConfiguration(nameof(PageLoadTimeout), 180),
+        TestRunTimeout = GetTimeoutConfiguration(nameof(TestRunTimeout), 600),
+    };
+
     /// <summary>
     /// Gets or sets how long to wait for an operation to finish when it's retried. Defaults to 10s. Higher,
     /// paradoxically, is usually less safe.
@@ -27,13 +35,27 @@ public class TimeoutConfiguration
     /// </summary>
     public TimeSpan PageLoadTimeout { get; set; }
 
-    public static readonly TimeoutConfiguration Default = new()
+    /// <summary>
+    /// Gets or sets how long an individual test can run to prevent hanging indefinitely. Defaults to 10 minutes.
+    /// </summary>
+    public TimeSpan TestRunTimeout { get; set; }
+
+    /// <summary>
+    /// Gets a copy of the timeout configuration derived from the values in the <see cref="TestConfigurationManager"/>.
+    /// </summary>
+    public static TimeoutConfiguration Default => new()
     {
-        RetryTimeout = TimeSpan
-            .FromSeconds(TestConfigurationManager.GetIntConfiguration("TimeoutConfiguration:RetryTimeoutSeconds", 10)),
-        RetryInterval = TimeSpan
-            .FromMilliseconds(TestConfigurationManager.GetIntConfiguration("TimeoutConfiguration:RetryIntervalMillisecondSeconds", 500)),
-        PageLoadTimeout = TimeSpan
-            .FromSeconds(TestConfigurationManager.GetIntConfiguration("TimeoutConfiguration:PageLoadTimeoutSeconds", 180)),
+        RetryTimeout = _default.RetryTimeout,
+        RetryInterval = _default.RetryInterval,
+        PageLoadTimeout = _default.PageLoadTimeout,
+        TestRunTimeout = _default.TestRunTimeout,
     };
+
+    private static TimeSpan GetTimeoutConfiguration(string name, int defaultValue, bool useMilliseconds = false)
+    {
+        var suffix = useMilliseconds ? "Milliseconds" : "Seconds";
+        var key = $"{nameof(TimeoutConfiguration)}:{name}{suffix}";
+        var value = TestConfigurationManager.GetIntConfiguration(key, defaultValue);
+        return useMilliseconds ? TimeSpan.FromMilliseconds(value) : TimeSpan.FromSeconds(value);
+    }
 }

Lombiq.Tests.UI/UITestBase.cs

@@ -21,6 +21,8 @@ protected async Task ExecuteOrchardCoreTestAsync(
         OrchardCoreUITestExecutorConfiguration configuration)
     {
         var originalTestOutputHelper = _testOutputHelper;
+        var timeout = configuration.TimeoutConfiguration.TestRunTimeout;
+
         Action afterTest = null;
         if (configuration.ExtendGitHubActionsOutput &&
             configuration.GitHubActionsOutputConfiguration.EnablePerTestOutputGrouping &&
@@ -33,7 +35,23 @@ protected async Task ExecuteOrchardCoreTestAsync(
 
         try
         {
-            await UITestExecutor.ExecuteOrchardCoreTestAsync(webApplicationInstanceFactory, testManifest, configuration);
+            var testTask = UITestExecutor.ExecuteOrchardCoreTestAsync(
+                webApplicationInstanceFactory,
+                testManifest,
+                configuration);
+            var timeoutTask = Task.Delay(timeout);
+
+            await Task.WhenAny(testTask, timeoutTask);
+
+            if (timeoutTask.IsCompleted)
+            {
+                throw new TimeoutException($"The time allotted for the test ({timeout}) was exceeded.");
+            }
+
+            // Since the timeout task is not yet completed but the Task.WhenAny has finished, the test task is done in
+            // some way. So it's safe to await it here. It's also necessary to cleanly propagate any exceptions that may
+            // have been thrown inside it.
+            await testTask;
         }
         finally
         {

Lombiq.Tests.UI/.htmlvalidate.json → Lombiq.Tests.UI/default.htmlvalidate.json

@@ -5,8 +5,9 @@
 
   "rules": {
     "attribute-boolean-style": "off",
-    "no-trailing-whitespace": "off",
+    "form-dup-name": "off",
     "no-inline-style": "off",
+    "no-trailing-whitespace": "off",
     "wcag/h30": "off",
     "wcag/h32": "off",
     "wcag/h36": "off",