From 32cb00aaf2d1aaa5ab18f4910300a7ee0fa73a9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Sun, 13 Oct 2024 19:58:56 +0200 Subject: [PATCH 01/51] Empty setup. --- .../Helpers/SetupHelpers.cs | 10 +++++++++ .../Tests/FrontendTests.cs | 22 +++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs diff --git a/Lombiq.Tests.UI.Samples/Helpers/SetupHelpers.cs b/Lombiq.Tests.UI.Samples/Helpers/SetupHelpers.cs index fe4ff9f32..019b4c429 100644 --- a/Lombiq.Tests.UI.Samples/Helpers/SetupHelpers.cs +++ b/Lombiq.Tests.UI.Samples/Helpers/SetupHelpers.cs @@ -2,6 +2,7 @@ using Lombiq.Tests.UI.Constants; using Lombiq.Tests.UI.Extensions; using Lombiq.Tests.UI.Pages; +using Lombiq.Tests.UI.Samples.Tests; using Lombiq.Tests.UI.Services; using OpenQA.Selenium; using System; @@ -68,6 +69,15 @@ public static async Task RunAutoSetupAsync(UITestContext context) return context.GetCurrentUri(); } + /// + /// This helper contains additional configuration used to set up a separate frontend server, used in . + /// + public static async Task ConfigureFrontendSetupAsync(OrchardCoreUITestExecutorConfiguration configuration) + { + + } + private static void AssertSetupSuccessful(UITestContext context) { try diff --git a/Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs b/Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs new file mode 100644 index 000000000..10dc445c9 --- /dev/null +++ b/Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs @@ -0,0 +1,22 @@ +using Lombiq.Tests.UI.Samples.Helpers; +using System.Threading.Tasks; +using Xunit; +using Xunit.Abstractions; + +namespace Lombiq.Tests.UI.Samples.Tests; + +public class FrontendTests : UITestBase +{ + public FrontendTests(ITestOutputHelper testOutputHelper) + : base(testOutputHelper) + { + } + + [Fact] + public Task ServerSideErrorOnLoadedPageShouldHaltTest() => + ExecuteTestAfterSetupAsync( + async context => + { + }, + SetupHelpers.ConfigureFrontendSetupAsync); +} From fe2c4335392970f15507ad597640503ff3b123c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Fri, 15 Nov 2024 00:35:49 +0100 Subject: [PATCH 02/51] Simplify DirectoryPaths methods. --- Lombiq.Tests.UI/Constants/DirectoryPaths.cs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Lombiq.Tests.UI/Constants/DirectoryPaths.cs b/Lombiq.Tests.UI/Constants/DirectoryPaths.cs index d7b3b426b..a218388b3 100644 --- a/Lombiq.Tests.UI/Constants/DirectoryPaths.cs +++ b/Lombiq.Tests.UI/Constants/DirectoryPaths.cs @@ -10,13 +10,10 @@ public static class DirectoryPaths public const string Screenshots = nameof(Screenshots); public static string GetTempDirectoryPath(params string[] subDirectoryNames) => - Path.Combine( - Path.Combine(Environment.CurrentDirectory, Temp), - Path.Combine(subDirectoryNames)); + Path.Combine([Environment.CurrentDirectory, Temp, ..subDirectoryNames]); public static string GetTempSubDirectoryPath(string contextId, params string[] subDirectoryNames) => - GetTempDirectoryPath( - Path.Combine(contextId, Path.Combine(subDirectoryNames))); + GetTempDirectoryPath([contextId, ..subDirectoryNames]); public static string GetScreenshotsDirectoryPath(string contextId) => GetTempSubDirectoryPath(contextId, Screenshots); From 0752d4d480edfc901e9a1be81517d09c5555b8d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Fri, 15 Nov 2024 00:49:38 +0100 Subject: [PATCH 03/51] Move GetTempSubDirectoryPath and GetScreenshotsDirectoryPath into UITestContext where they make more sense. --- Lombiq.Tests.UI/Constants/DirectoryPaths.cs | 3 +++ .../AccessibilityCheckingUITestContextExtensions.cs | 2 +- Lombiq.Tests.UI/SecurityScanning/ZapManager.cs | 2 +- Lombiq.Tests.UI/Services/OrchardCoreInstance.cs | 2 +- Lombiq.Tests.UI/Services/UITestContext.cs | 13 +++++++++++++ Lombiq.Tests.UI/Services/UITestExecutionSession.cs | 8 ++++---- 6 files changed, 23 insertions(+), 7 deletions(-) diff --git a/Lombiq.Tests.UI/Constants/DirectoryPaths.cs b/Lombiq.Tests.UI/Constants/DirectoryPaths.cs index a218388b3..141030f79 100644 --- a/Lombiq.Tests.UI/Constants/DirectoryPaths.cs +++ b/Lombiq.Tests.UI/Constants/DirectoryPaths.cs @@ -1,3 +1,4 @@ +using Lombiq.Tests.UI.Services; using System; using System.IO; @@ -12,9 +13,11 @@ public static class DirectoryPaths public static string GetTempDirectoryPath(params string[] subDirectoryNames) => Path.Combine([Environment.CurrentDirectory, Temp, ..subDirectoryNames]); + [Obsolete($"Use {nameof(UITestContext)}.{nameof(UITestContext.GetTempSubDirectoryPath)}() instead.")] public static string GetTempSubDirectoryPath(string contextId, params string[] subDirectoryNames) => GetTempDirectoryPath([contextId, ..subDirectoryNames]); + [Obsolete($"Use {nameof(UITestContext)}.{nameof(UITestContext.ScreenshotsDirectoryPath)} instead.")] public static string GetScreenshotsDirectoryPath(string contextId) => GetTempSubDirectoryPath(contextId, Screenshots); } diff --git a/Lombiq.Tests.UI/Extensions/AccessibilityCheckingUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/AccessibilityCheckingUITestContextExtensions.cs index e166268b9..d6674f694 100644 --- a/Lombiq.Tests.UI/Extensions/AccessibilityCheckingUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/AccessibilityCheckingUITestContextExtensions.cs @@ -47,7 +47,7 @@ public static void AssertAccessibility( if (accessibilityConfiguration.CreateReportAlways) { var reportDirectoryPath = DirectoryHelper.CreateEnumeratedDirectory( - DirectoryPaths.GetTempSubDirectoryPath(context.Id, "AxeHtmlReport")); + context.GetTempSubDirectoryPath("AxeHtmlReport")); var reportPath = Path.Combine( reportDirectoryPath, diff --git a/Lombiq.Tests.UI/SecurityScanning/ZapManager.cs b/Lombiq.Tests.UI/SecurityScanning/ZapManager.cs index 9e7ec222e..9f592730e 100644 --- a/Lombiq.Tests.UI/SecurityScanning/ZapManager.cs +++ b/Lombiq.Tests.UI/SecurityScanning/ZapManager.cs @@ -80,7 +80,7 @@ public async Task RunSecurityScanAsync( // Each attempt will have it's own "ZapN" directory inside the temp, starting with "Zap1". var mountedDirectoryPath = DirectoryHelper.CreateEnumeratedDirectory( - DirectoryPaths.GetTempSubDirectoryPath(context.Id, "Zap")); + context.GetTempSubDirectoryPath("Zap")); var reportsDirectoryPath = Path.Combine(mountedDirectoryPath, _zapReportsDirectoryName); Directory.CreateDirectory(reportsDirectoryPath); diff --git a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs index e8f856a80..6076b0e24 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs @@ -148,7 +148,7 @@ public async ValueTask DisposeAsync() private void CreateContentRootFolder() { - _contentRootPath = DirectoryPaths.GetTempSubDirectoryPath(_contextId, "App"); + _contentRootPath = DirectoryPaths.GetTempDirectoryPath(_contextId, "App"); Directory.CreateDirectory(_contentRootPath); _testOutputHelper.WriteLineTimestampedAndDebug("Content root path was created: {0}", _contentRootPath); } diff --git a/Lombiq.Tests.UI/Services/UITestContext.cs b/Lombiq.Tests.UI/Services/UITestContext.cs index f5ba24a8a..f93d5ad97 100644 --- a/Lombiq.Tests.UI/Services/UITestContext.cs +++ b/Lombiq.Tests.UI/Services/UITestContext.cs @@ -139,6 +139,12 @@ public class UITestContext /// public string AdminUrlPrefix { get; set; } = "/Admin"; + /// + /// Gets the absolute path of the subdirectory inside the current test + /// instance's directory. + /// + public string ScreenshotsDirectoryPath => GetTempSubDirectoryPath(DirectoryPaths.Screenshots); + // This is a central context object, we need the data to be passed in the constructor. #pragma warning disable S107 // Methods should not have too many parameters public UITestContext( @@ -276,6 +282,13 @@ public void SwitchCurrentTenant(string tenantName, string urlPrefix) Scope.BaseUri = new Uri(Scope.BaseUri, "/" + UrlPrefix + (string.IsNullOrEmpty(UrlPrefix) ? string.Empty : "/")); } + /// + /// Returns the subdirectory described by inside the current test instance's + /// directory. + /// + public string GetTempSubDirectoryPath(params string[] subDirectoryNames) => + DirectoryPaths.GetTempDirectoryPath([Id, ..subDirectoryNames]); + private bool IsAlert() { // If there's an alert (which can happen mostly after a click but also after navigating) then all other driver diff --git a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs index 8729cd023..568dd8f55 100644 --- a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs +++ b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs @@ -182,7 +182,7 @@ private async ValueTask ShutdownAsync() { try { - DirectoryHelper.SafelyDeleteDirectoryIfExists(DirectoryPaths.GetTempSubDirectoryPath(contextId)); + DirectoryHelper.SafelyDeleteDirectoryIfExists(DirectoryPaths.GetTempDirectoryPath(contextId)); } catch (Exception ex) when (GitHubHelper.IsGitHubEnvironment) { @@ -591,7 +591,7 @@ private async Task CreateContextAsync(Uri testStartRelativeUri) { var contextId = Guid.NewGuid().ToString(); - FileSystemHelper.EnsureDirectoryExists(DirectoryPaths.GetTempSubDirectoryPath(contextId)); + FileSystemHelper.EnsureDirectoryExists(DirectoryPaths.GetTempDirectoryPath(contextId)); SqlServerRunningContext sqlServerContext = null; AzureBlobStorageRunningContext azureBlobStorageContext = null; @@ -827,7 +827,7 @@ private Task TakeScreenshotIfEnabledAsync(UITestContext context) { if (_context == null || !_dumpConfiguration.CaptureScreenshots || !_context.IsBrowserRunning) return Task.CompletedTask; - var screenshotsPath = DirectoryPaths.GetScreenshotsDirectoryPath(_context.Id); + var screenshotsPath = _context.ScreenshotsDirectoryPath; FileSystemHelper.EnsureDirectoryExists(screenshotsPath); try @@ -853,7 +853,7 @@ private async Task CreateScreenshotsDumpAsync(string debugInformationPath) { await TakeScreenshotIfEnabledAsync(_context); - var screenshotsSourcePath = DirectoryPaths.GetScreenshotsDirectoryPath(_context.Id); + var screenshotsSourcePath = _context.ScreenshotsDirectoryPath; if (Directory.Exists(screenshotsSourcePath)) { var screenshotsDestinationPath = Path.Combine(debugInformationPath, DirectoryPaths.Screenshots); From 54133b075485226e6f626e2eebf281c69adaa672 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Fri, 15 Nov 2024 00:50:10 +0100 Subject: [PATCH 04/51] Add URI version of UITestContext.SwitchCurrentTenant(). --- Lombiq.Tests.UI/Services/UITestContext.cs | 31 ++++++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/Lombiq.Tests.UI/Services/UITestContext.cs b/Lombiq.Tests.UI/Services/UITestContext.cs index f93d5ad97..a69112bd2 100644 --- a/Lombiq.Tests.UI/Services/UITestContext.cs +++ b/Lombiq.Tests.UI/Services/UITestContext.cs @@ -1,8 +1,10 @@ +using Lombiq.Tests.UI.Constants; using Lombiq.Tests.UI.Exceptions; using Lombiq.Tests.UI.Extensions; using Lombiq.Tests.UI.Models; using Lombiq.Tests.UI.SecurityScanning; using OpenQA.Selenium; +using OrchardCore.Environment.Shell; using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; @@ -271,17 +273,38 @@ public async Task TriggerAfterPageChangeEventAndRefreshAtataContextAsync() /// /// Changes the current tenant context to the provided one. Note that this doesn't navigate the browser. /// - /// The technical name of the tenant to change to. + /// + /// The technical name of the tenant to change to. If , then is used instead. + /// /// - /// The URL prefix configured for the tenant. It should neither start nor end with a slash. + /// The URL prefix configured for the tenant. Any leading or trailing slashes are trimmed out. /// public void SwitchCurrentTenant(string tenantName, string urlPrefix) { - TenantName = tenantName; - UrlPrefix = urlPrefix; + TenantName = tenantName ?? ShellSettings.DefaultShellName; + UrlPrefix = urlPrefix.Trim('/'); Scope.BaseUri = new Uri(Scope.BaseUri, "/" + UrlPrefix + (string.IsNullOrEmpty(UrlPrefix) ? string.Empty : "/")); } + /// + /// Changes the current tenant context to the provided one. Note that this doesn't navigate the browser. + /// + /// + /// The technical name of the tenant to change to. If , then is used instead. + /// + /// + /// The new value of ./ Additionally, is + /// set to its (without leading or trailing slashes). + /// + public void SwitchCurrentTenant(string tenantName, Uri baseUri) + { + TenantName = tenantName ?? ShellSettings.DefaultShellName; + UrlPrefix = baseUri.AbsolutePath.Trim('/'); + Scope.BaseUri = baseUri; + } + /// /// Returns the subdirectory described by inside the current test instance's /// directory. From 2250d59c07e325a773f9377363fccc230a5afb03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Fri, 15 Nov 2024 01:45:47 +0100 Subject: [PATCH 05/51] Migrate, refactor and document extensions. --- ...reUITestExecutorConfigurationExtensions.cs | 34 +++++++++++ .../FrontendUITestContextExtensions.cs | 60 ++++++++++++++++++- 2 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 Lombiq.Tests.UI/Extensions/FrontendOrchardCoreUITestExecutorConfigurationExtensions.cs diff --git a/Lombiq.Tests.UI/Extensions/FrontendOrchardCoreUITestExecutorConfigurationExtensions.cs b/Lombiq.Tests.UI/Extensions/FrontendOrchardCoreUITestExecutorConfigurationExtensions.cs new file mode 100644 index 000000000..da9be02d0 --- /dev/null +++ b/Lombiq.Tests.UI/Extensions/FrontendOrchardCoreUITestExecutorConfigurationExtensions.cs @@ -0,0 +1,34 @@ +using Lombiq.Tests.UI.Services; +using System; + +namespace Lombiq.Tests.UI.Extensions; + +public static class FrontendOrchardCoreUITestExecutorConfigurationExtensions +{ + private const string BackendUri = nameof(BackendUri); + private const string FrontendUri = nameof(FrontendUri); + + /// + /// Returns the start URLs for the Vite frontend and the Orchard Core backend from the . + /// + public static (Uri FrontendUri, Uri BackendUri) GetFrontendAndBackendUris( + this OrchardCoreUITestExecutorConfiguration configuration) => + ( + (Uri)configuration.CustomConfiguration[FrontendUri], + (Uri)configuration.CustomConfiguration[BackendUri] + ); + + /// + /// Updates the by storing the Vite + /// frontend and the Orchard Core backend URLs as instances. + /// + public static void SetFrontendAndBackendUris( + this OrchardCoreUITestExecutorConfiguration configuration, + string frontendUrl, + string backendUrl) + { + configuration.CustomConfiguration[FrontendUri] = new Uri(frontendUrl); + configuration.CustomConfiguration[BackendUri] = new Uri(backendUrl); + } +} diff --git a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs index 7966deb1a..d535edd51 100644 --- a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs @@ -13,6 +13,34 @@ namespace Lombiq.Tests.UI.Extensions; public static class FrontendUITestContextExtensions { + public const string FrontendPseudoTenantName = "!Frontend"; + + /// + /// Navigates to the backend returned by and presents it as + /// switching to the default tenant. + /// + /// + /// If the backend URL has not been initialized to something else (e.g. using a custom URL prefix), this is + /// equivalent to using . Even so, this method should be + /// used for clarity when applicable. + /// + public static void SwitchToBackend(this UITestContext context) => + context.SwitchCurrentTenant( + tenantName: null, + context.Configuration.GetFrontendAndBackendUris().BackendUri); + + /// + /// Navigates to the frontend returned by and presents it as + /// switching to a tenant named which is not a real Orchard Core tenant so + /// this information can only be used for information. + /// + public static void SwitchToFrontend(this UITestContext context) => + context.SwitchCurrentTenant( + FrontendPseudoTenantName, + context.Configuration.GetFrontendAndBackendUris().FrontendUri); + public static string GetDriverPath(this UITestContext context) { if (context.Driver is not WebDriver { CommandExecutor: DriverServiceCommandExecutor executor }) @@ -51,7 +79,7 @@ await Cli.Wrap(command) scriptPath, context.GetDriverPath(), context.Driver.Url, - DirectoryPaths.GetTempSubDirectoryPath(context.Id), + context.GetTempSubDirectoryPath(), ]) .WithStandardOutputPipe(pipe) .WithStandardErrorPipe(pipe) @@ -65,4 +93,34 @@ await Cli.Wrap(command) throw; } } + + /// + /// Creates a blank Node.js project in the current test session's directory and + /// installs the provided NPM using pnpm. + /// + public static async Task SetupNodeDependenciesAsync(this UITestContext context, ITestOutputHelper helper, params string[] dependencies) + { + var workingDirectory = context.GetTempSubDirectoryPath(); + var projectFilePath = Path.Join(workingDirectory, "package.json"); + + if (!Directory.Exists(projectFilePath)) + { + await File.WriteAllTextAsync(projectFilePath, "{ \"private\": true }"); + } + + var pipe = helper.ToPipeTarget(nameof(SetupNodeSeleniumAsync)); + await Cli.Wrap("pnpm") + .WithArguments(["install", ..dependencies]) + .WithStandardOutputPipe(pipe) + .WithStandardErrorPipe(pipe) + .WithWorkingDirectory(workingDirectory) + .ExecuteAsync(); + } + + /// + /// Creates a blank Node.js project in the current test session's directory, then + /// installs selenium-webdriver and any additional NPM dependencies using pnpm. + /// + public static Task SetupNodeSeleniumAsync(this UITestContext context, ITestOutputHelper helper, params string[] otherDependencies) => + context.SetupNodeDependenciesAsync(helper, ["selenium-webdriver", ..otherDependencies]); } From 51e74ffc6335b3307428fd061b501d29e0cc168f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Fri, 15 Nov 2024 04:03:02 +0100 Subject: [PATCH 06/51] FrontendUITestBase --- Lombiq.Tests.UI.Samples/FrontendUITestBase.cs | 88 +++++++++++++++++++ Lombiq.Tests.UI/Services/FrontendServer.cs | 23 ++++- 2 files changed, 108 insertions(+), 3 deletions(-) create mode 100644 Lombiq.Tests.UI.Samples/FrontendUITestBase.cs diff --git a/Lombiq.Tests.UI.Samples/FrontendUITestBase.cs b/Lombiq.Tests.UI.Samples/FrontendUITestBase.cs new file mode 100644 index 000000000..8a2595fac --- /dev/null +++ b/Lombiq.Tests.UI.Samples/FrontendUITestBase.cs @@ -0,0 +1,88 @@ +using Lombiq.Tests.UI.Extensions; +using Lombiq.Tests.UI.Services; +using System; +using System.Diagnostics.CodeAnalysis; +using System.Threading.Tasks; +using Xunit.Abstractions; + +namespace Lombiq.Tests.UI.Samples; + +// Sometimes Orchard Core is used in a headless manner, as a web API server that the visitors reach through a web +// frontend (for example a Vue or React single page application). In this scenario you can test the API directly and +// test the frontend with a dummy backend separately, but that will only get you so far. You'll want some tests that +// ensure the two work together. To make this happen, you need some custom logic to initialize the frontend process +// after setup, with a unique port number to avoid clashes. Also the frontend may need the backend URL, whose port is +// already randomized with every test. +public abstract class FrontendUITestBase : UITestBase +{ + protected FrontendUITestBase(ITestOutputHelper testOutputHelper) + : base(testOutputHelper) + { + } + + /// + /// Executes a UI test where the frontend is served by a separate process. + /// + /// Additional test configurations. + /// Thrown if the server failed to set up. + [SuppressMessage("Style", "IDE0055:Fix formatting", Justification = "Needed for more readable comments.")] + [SuppressMessage("StyleCop.CSharp.ReadabilityRules", "SA1114:Parameter list should follow declaration", Justification = "Same.")] + protected Task ExecuteFrontendTestAfterSetupAsync( + Func testAsync, + Browser browser, + Func changeConfigurationAsync = null) => + ExecuteTestAfterSetupAsync( + context => + { + context.SwitchToFrontend(); + return testAsync(context); + }, + browser, + configuration => + { + // The FrontendServer instance manages the test configuration by registering event handlers for the + // application startup and stop. It also initializes the default frontend and backend URLs. Below the + // "name" is the label you will find in the test application logs in front of all lines that come from + // this process. The process's arguments can be set with the arguments array, but likely you'll want to + // set pass in the frontend and backend ports too which you can do in the "configureCommand" function. + new FrontendServer(name: "http-server (test frontend)", configuration, _testOutputHelper) + .Configure( + // Here we use NPX which executes an NPM package without locally installing it, to avoid having + // to maintain a separate binary or script just for this sample. + program: "npx", + arguments: null, + // This function lets you edit the command dynamically before the process is created. This is + // needed to add the frontend and backend URLs or port numbers to the process, e.g. as arguments + // or environment variables. Since we set the arguments here, the parameter above is left null. + // If necessary, this is also where you'd set the program's working directory. + configureCommand: (frontendCommand, _) => + { + // You can also get this from the second optional OrchardCoreAppStartContext parameter, but + // then you have to refer to context.Url.Port, which is less readable. The values in the + // configuration has been initialized shortly before this is called, so why not use them? + var backendPort = configuration.GetFrontendAndBackendUris().FrontendUri.Port.ToTechnicalString(); + + // The backend URL contains a unique reserved port number that's guaranteed to be available + // during this test just as much as any Orchard Core instance, because it's coming from the + // same pool of numbers. + return frontendCommand.WithArguments(["--yes", "http-server", "--port", backendPort]); + }, + // When this function is not null, test setup will call it on each output line and wait for it + // to return true. This can be used to look for an output that only appears when the frontend + // server is done with its own startup. + checkProgramReady: (line, _) => line?.Contains("Hit CTRL-C to stop the server") == true, + // You can use this async function to perform additional tasks right before the FrontendServer's + // BeforeAppStart event handler is finished. You can also edit the Orchard Core instance's + // startup command here. We don't need it right now. + thenAsync: _ => Task.CompletedTask, + // Don't start up the server during setup and snapshot restore. You'd rarely want that, so we + // have prepared a static method to generate a callback for this parameter. + skipStartup: FrontendServer.SkipDuringSetupAndRestore(configuration), + // Since we call NPX, the process may start with downloading from the network. So we allow some + // grace period here, but it's unlikely that it would take too long unless the program is stuck + // or frozen somehow. In that case it's better to fail than wait forever. + startupTimeout: TimeSpan.FromMinutes(3)); + + return changeConfigurationAsync.InvokeFuncAsync(configuration); + }); +} diff --git a/Lombiq.Tests.UI/Services/FrontendServer.cs b/Lombiq.Tests.UI/Services/FrontendServer.cs index 44d08a69d..91e5fb050 100644 --- a/Lombiq.Tests.UI/Services/FrontendServer.cs +++ b/Lombiq.Tests.UI/Services/FrontendServer.cs @@ -2,6 +2,7 @@ using CliWrap; using Lombiq.HelpfulLibraries.Common.Utilities; +using Lombiq.Tests.UI.Extensions; using Lombiq.Tests.UI.Models; using System; using System.Collections.Generic; @@ -67,11 +68,15 @@ public void Configure( ArgumentNullException.ThrowIfNull(program); skipStartup ??= _ => false; - var cli = Cli.Wrap(program).WithArguments(arguments ?? []); - _configuration.OrchardCoreConfiguration.BeforeAppStart += async (orchardContext, orchardArguments) => { + var cli = Cli + .Wrap(program) + .WithArguments(arguments ?? []) + .WithWorkingDirectory(orchardContext.ContentRootPath); + var frontendPort = await orchardContext.PortLeaseManager.LeaseAvailableRandomPortAsync(); + var backendPort = orchardContext.Url.Port; var context = new Context( orchardContext.ContentRootPath, orchardContext.Url, @@ -79,6 +84,11 @@ public void Configure( frontendPort, orchardArguments); + // Initialize the default frontend and backend URLs. This may be customized in thenAsync. + _configuration.SetFrontendAndBackendUris( + frontendUrl: "https://localhost:" + frontendPort.ToTechnicalString(), + backendUrl: "https://localhost:" + backendPort.ToTechnicalString()); + var cancellationTokenSource = new CancellationTokenSource(); var waitCompletionSource = new TaskCompletionSource(); var execute = !skipStartup(context); @@ -108,7 +118,7 @@ public void Configure( if (waiting) await WaitForStartupAsync(cliTask, waitCompletionSource.Task, startupTimeout); - _configuration.CustomConfiguration[GetKey(context.Url.Port)] = new FrontendServerContext + _configuration.CustomConfiguration[GetKey(backendPort)] = new FrontendServerContext { Port = frontendPort, Task = cliTask, @@ -160,6 +170,13 @@ private static async Task WaitForStartupAsync(Task mainTask, Task waitTask, Time } } + /// + /// Returns a checker function that can be passed to 's skipStartup parameter to skip + /// execution during setup and snapshot restore. + /// + public static Func SkipDuringSetupAndRestore(OrchardCoreUITestExecutorConfiguration configuration) => + _ => configuration.OrchardCoreConfiguration.StartCount > 2; + public record Context( string ContentRootPath, Uri Url, From 144e8a9c87c3fc950ba9797ceb9a305ccfb47bf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Fri, 15 Nov 2024 14:57:11 +0100 Subject: [PATCH 07/51] Add training sections. --- Lombiq.Tests.UI.Samples/Readme.md | 2 ++ Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs | 3 +++ Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs | 13 +++++++++++++ Lombiq.Tests.UI.Samples/Tests/ShiftTimeTests.cs | 1 + 4 files changed, 19 insertions(+) create mode 100644 Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs diff --git a/Lombiq.Tests.UI.Samples/Readme.md b/Lombiq.Tests.UI.Samples/Readme.md index 1689b2204..e23e1bce0 100644 --- a/Lombiq.Tests.UI.Samples/Readme.md +++ b/Lombiq.Tests.UI.Samples/Readme.md @@ -31,6 +31,8 @@ For general details about and on using the Toolbox see the [root Readme](../Read - [Security scanning](Tests/SecurityScanningTests.cs) - [Testing remote apps](Tests/RemoteTests.cs) - [Testing time-dependent functionality](Tests/ShiftTimeTests.cs) +- [Test headless Orchard Core with a frontend subprocess](FrontendUITestBase.cs) +- [Executing tests written in Javascript](Tests/JavascriptTests.cs) ## Adding new tutorials diff --git a/Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs b/Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs index 10dc445c9..64a9315c2 100644 --- a/Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs @@ -20,3 +20,6 @@ public Task ServerSideErrorOnLoadedPageShouldHaltTest() => }, SetupHelpers.ConfigureFrontendSetupAsync); } + +// END OF TRAINING SECTION: Test headless Orchard Core with a frontend subprocess. +// NEXT STATION: Head over to Tests/JavascriptTests.cs. diff --git a/Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs b/Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs new file mode 100644 index 000000000..dee54e8c6 --- /dev/null +++ b/Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs @@ -0,0 +1,13 @@ +using Xunit.Abstractions; + +namespace Lombiq.Tests.UI.Samples.Tests; + +public class JavascriptTests : RemoteUITestBase +{ + public JavascriptTests(ITestOutputHelper testOutputHelper) + : base(testOutputHelper) + { + } +} + +// END OF TRAINING SECTION: Executing tests written in Javascript. diff --git a/Lombiq.Tests.UI.Samples/Tests/ShiftTimeTests.cs b/Lombiq.Tests.UI.Samples/Tests/ShiftTimeTests.cs index 45905c91d..78bf1e167 100644 --- a/Lombiq.Tests.UI.Samples/Tests/ShiftTimeTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/ShiftTimeTests.cs @@ -61,3 +61,4 @@ private static async Task GetNowAsync(UITestContext context) } // END OF TRAINING SECTION: Testing time-dependent functionality. +// NEXT STATION: Head over to FrontendUITestBase.cs. From f7225ce72f68780b74d24fca66a90c42d8af94df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Fri, 15 Nov 2024 16:45:28 +0100 Subject: [PATCH 08/51] Add the C# part of JS tests. --- .../Lombiq.Tests.UI.Samples.csproj | 3 ++ .../Tests/ExampleJavascriptTestShouldWork.mjs | 0 .../Tests/JavascriptTests.cs | 52 ++++++++++++++++++- .../FrontendUITestContextExtensions.cs | 37 +++++++++++-- 4 files changed, 88 insertions(+), 4 deletions(-) create mode 100644 Lombiq.Tests.UI.Samples/Tests/ExampleJavascriptTestShouldWork.mjs diff --git a/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj b/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj index d7c7fce29..67c2c4f17 100644 --- a/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj +++ b/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj @@ -9,6 +9,9 @@ + + PreserveNewest + diff --git a/Lombiq.Tests.UI.Samples/Tests/ExampleJavascriptTestShouldWork.mjs b/Lombiq.Tests.UI.Samples/Tests/ExampleJavascriptTestShouldWork.mjs new file mode 100644 index 000000000..e69de29bb diff --git a/Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs b/Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs index dee54e8c6..333765040 100644 --- a/Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs @@ -1,13 +1,63 @@ +using Lombiq.Tests.UI.Extensions; +using Lombiq.Tests.UI.Services.GitHub; +using System.IO; +using System.Threading.Tasks; +using Xunit; using Xunit.Abstractions; namespace Lombiq.Tests.UI.Samples.Tests; -public class JavascriptTests : RemoteUITestBase +// Suppose you want to write UI tests in Javascript. Why would you want to do that? Unlikely if you are an Orchard Core +// developer, but what if the person responsible for writing the tests is not? In the previous training section we +// discussed using a separate frontend server, with mention of technologies using Node.js. In that case the frontend +// developers may be more familiar with Javascript so it makes sense to write and debug the tests in Node.js so they +// don't have to learn different tools and tech stacks just to create some UI tests. +public class JavascriptTests : UITestBase { public JavascriptTests(ITestOutputHelper testOutputHelper) : base(testOutputHelper) { } + + [Fact] + public Task ExampleJavascriptTestShouldWork() + { + // Don't forget to mark the script files as "Copy if newer", so they are available to the test. + var scriptPath = Path.Join("Tests", nameof(ExampleJavascriptTestShouldWork) + ".mjs"); + + // Set up the JS dependencies in the test's temp directory to ensure there are no clashes, then run the script. + return ExecuteTestAfterSetupAsync(context => context + .SetupSeleniumAndExecuteJavascriptTestAsync(scriptPath, _testOutputHelper)); + } + + // To best debug the Javascript code, you may want to set up the site and then invoke node manually. This is not a + // real test, but it sets up the site in interactive mode (see Tests/InteractiveModeTests.cs for more) with + // information how to start up test script from your GUI. It's an example of some tooling that can improve the test + // developer's workflow. + [Fact] + public Task Sandbox() + { + // This "test" will wait indefinitely, so it's important to skip it in CI. + if (GitHubHelper.IsGitHubEnvironment) return Task.CompletedTask; + + return ExecuteTestAfterSetupAsync( + async context => + { + var driverPath = context.GetDriverPath(); + var tempPath = context.GetTempSubDirectoryPath(); + + await context.SwitchToInteractiveAsync( + $"To start a Javascript test, open a command line terminal at \"{tempPath}\": and type the " + + $"following command: node --inspect ../../sandbox.js {driverPath} " + + $"{context.Driver.Url}"); + }, + configuration => + { + // Since this is an interactive "test", make sure the browser is always displayed. + configuration.BrowserConfiguration.Headless = false; + return Task.CompletedTask; + }); + } } // END OF TRAINING SECTION: Executing tests written in Javascript. diff --git a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs index d535edd51..5d66f63a7 100644 --- a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs @@ -66,7 +66,8 @@ public static string GetDriverPath(this UITestContext context) public static async Task ExecuteJavascriptTestAsync( this UITestContext context, string scriptPath, - ITestOutputHelper testOutputHelper) + ITestOutputHelper testOutputHelper, + string workingDirectory = null) { const string command = "node"; var pipe = testOutputHelper.ToPipeTarget($"{nameof(ExecuteJavascriptTestAsync)}({command})"); @@ -83,6 +84,7 @@ await Cli.Wrap(command) ]) .WithStandardOutputPipe(pipe) .WithStandardErrorPipe(pipe) + .WithWorkingDirectory(workingDirectory ?? Environment.CurrentDirectory) .ExecuteAsync(); } catch @@ -94,11 +96,36 @@ await Cli.Wrap(command) } } + /// + /// Sets up the Javascript dependencies using and then runs the script in the + /// same temp directory. + /// + /// + /// The path of the Javascript file to execute with node. Before passing it to , it's transformed into a relative path based on the temp directory to + /// conserve path length because long paths can be a problem in some operating systems. + /// + public static async Task SetupSeleniumAndExecuteJavascriptTestAsync( + this UITestContext context, + string scriptPath, + ITestOutputHelper testOutputHelper, + params string[] otherDependencies) + { + var workingDirectory = await context.SetupNodeSeleniumAsync(testOutputHelper, otherDependencies); + var relativePath = Path.GetRelativePath(workingDirectory, scriptPath); + + await context.ExecuteJavascriptTestAsync(relativePath, testOutputHelper, workingDirectory); + } + /// /// Creates a blank Node.js project in the current test session's directory and /// installs the provided NPM using pnpm. /// - public static async Task SetupNodeDependenciesAsync(this UITestContext context, ITestOutputHelper helper, params string[] dependencies) + /// The path of the directory where the project is set up. + public static async Task SetupNodeDependenciesAsync( + this UITestContext context, + ITestOutputHelper helper, + params string[] dependencies) { var workingDirectory = context.GetTempSubDirectoryPath(); var projectFilePath = Path.Join(workingDirectory, "package.json"); @@ -121,6 +148,10 @@ await Cli.Wrap("pnpm") /// Creates a blank Node.js project in the current test session's directory, then /// installs selenium-webdriver and any additional NPM dependencies using pnpm. /// - public static Task SetupNodeSeleniumAsync(this UITestContext context, ITestOutputHelper helper, params string[] otherDependencies) => + /// The path of the directory where the project is set up. + public static Task SetupNodeSeleniumAsync( + this UITestContext context, + ITestOutputHelper helper, + params string[] otherDependencies) => context.SetupNodeDependenciesAsync(helper, ["selenium-webdriver", ..otherDependencies]); } From 48e7c318e7a3e8ed1492a6f20a77768882951ec5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Fri, 15 Nov 2024 22:08:13 +0100 Subject: [PATCH 09/51] Fix missing return value. --- Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs index 5d66f63a7..79bcb42d2 100644 --- a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs @@ -142,6 +142,8 @@ await Cli.Wrap("pnpm") .WithStandardErrorPipe(pipe) .WithWorkingDirectory(workingDirectory) .ExecuteAsync(); + + return workingDirectory; } /// From b848f0f9f0cd903343df643ee171bad1bfd2016b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Fri, 15 Nov 2024 23:52:05 +0100 Subject: [PATCH 10/51] Frontend test and various fixes. --- Lombiq.Tests.UI.Samples/FrontendUITestBase.cs | 36 ++++++++++++------- .../Helpers/SetupHelpers.cs | 9 ----- .../Tests/FrontendTests.cs | 29 ++++++++++++--- ...reUITestExecutorConfigurationExtensions.cs | 7 ++-- 4 files changed, 52 insertions(+), 29 deletions(-) diff --git a/Lombiq.Tests.UI.Samples/FrontendUITestBase.cs b/Lombiq.Tests.UI.Samples/FrontendUITestBase.cs index 8a2595fac..418f3a62e 100644 --- a/Lombiq.Tests.UI.Samples/FrontendUITestBase.cs +++ b/Lombiq.Tests.UI.Samples/FrontendUITestBase.cs @@ -23,8 +23,6 @@ protected FrontendUITestBase(ITestOutputHelper testOutputHelper) /// /// Executes a UI test where the frontend is served by a separate process. /// - /// Additional test configurations. - /// Thrown if the server failed to set up. [SuppressMessage("Style", "IDE0055:Fix formatting", Justification = "Needed for more readable comments.")] [SuppressMessage("StyleCop.CSharp.ReadabilityRules", "SA1114:Parameter list should follow declaration", Justification = "Same.")] protected Task ExecuteFrontendTestAfterSetupAsync( @@ -32,10 +30,15 @@ protected Task ExecuteFrontendTestAfterSetupAsync( Browser browser, Func changeConfigurationAsync = null) => ExecuteTestAfterSetupAsync( - context => + async context => { + // Before executing provided test, we switch to the frontend URL, as if we switched to a different + // tenant, then actually navigate to the frontend so the tests won't start in the backend home page + // that's probably not used anyway. context.SwitchToFrontend(); - return testAsync(context); + await context.GoToHomePageAsync(); + + await testAsync(context); }, browser, configuration => @@ -55,17 +58,24 @@ protected Task ExecuteFrontendTestAfterSetupAsync( // needed to add the frontend and backend URLs or port numbers to the process, e.g. as arguments // or environment variables. Since we set the arguments here, the parameter above is left null. // If necessary, this is also where you'd set the program's working directory. - configureCommand: (frontendCommand, _) => + configureCommand: (frontendCommand, context) => { - // You can also get this from the second optional OrchardCoreAppStartContext parameter, but - // then you have to refer to context.Url.Port, which is less readable. The values in the - // configuration has been initialized shortly before this is called, so why not use them? - var backendPort = configuration.GetFrontendAndBackendUris().FrontendUri.Port.ToTechnicalString(); + // You can also get this from the configuration using configuration + // .GetFrontendAndBackendUris().FrontendUri.Port. The values in the configuration has been + // initialized shortly before this function is called. but it's not worth it unless you need + // both frontend and backend URLS. The frontend port is a unique, reserved number that's + // guaranteed to be available during this test just as much as any Orchard Core instance, + // because it's coming from the same pool of numbers. + var port = context.FrontendPort.ToTechnicalString(); + + // Here we configure NPX to automatically download http-server without prompting (--yes) and + // use the provided port number. Since this server uses HTTP instead of HTTPS, you have to + // set the frontend URL too. The backend URL is not changed, so pass null to leave it as-is. + configuration.SetFrontendAndBackendUris( + frontendUrl: $"http://localhost:" + port, + backendUrl: null); + return frontendCommand.WithArguments(["--yes", "http-server", "--port", port]); - // The backend URL contains a unique reserved port number that's guaranteed to be available - // during this test just as much as any Orchard Core instance, because it's coming from the - // same pool of numbers. - return frontendCommand.WithArguments(["--yes", "http-server", "--port", backendPort]); }, // When this function is not null, test setup will call it on each output line and wait for it // to return true. This can be used to look for an output that only appears when the frontend diff --git a/Lombiq.Tests.UI.Samples/Helpers/SetupHelpers.cs b/Lombiq.Tests.UI.Samples/Helpers/SetupHelpers.cs index 019b4c429..24c8ed4a6 100644 --- a/Lombiq.Tests.UI.Samples/Helpers/SetupHelpers.cs +++ b/Lombiq.Tests.UI.Samples/Helpers/SetupHelpers.cs @@ -69,15 +69,6 @@ public static async Task RunAutoSetupAsync(UITestContext context) return context.GetCurrentUri(); } - /// - /// This helper contains additional configuration used to set up a separate frontend server, used in . - /// - public static async Task ConfigureFrontendSetupAsync(OrchardCoreUITestExecutorConfiguration configuration) - { - - } - private static void AssertSetupSuccessful(UITestContext context) { try diff --git a/Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs b/Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs index 64a9315c2..50755e38f 100644 --- a/Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs @@ -1,24 +1,45 @@ -using Lombiq.Tests.UI.Samples.Helpers; +using Lombiq.Tests.UI.Extensions; +using OpenQA.Selenium; using System.Threading.Tasks; using Xunit; using Xunit.Abstractions; namespace Lombiq.Tests.UI.Samples.Tests; -public class FrontendTests : UITestBase +public class FrontendTests : FrontendUITestBase { public FrontendTests(ITestOutputHelper testOutputHelper) : base(testOutputHelper) { } + // The interesting details were in FrontendUITestBase, here we just show that you can freely interact with the pages + // served by frontend server the same way as usual. In this case we have an HTTP file server, so you can navigate + // these directories and files. If we had a large client application that interacts with the headless OC in the + // backend, we would be able to do something more interesting but that's outside the scope of this demo. [Fact] public Task ServerSideErrorOnLoadedPageShouldHaltTest() => - ExecuteTestAfterSetupAsync( + ExecuteFrontendTestAfterSetupAsync( async context => { + // Don't forget that if you want to interact with the frontend manually, you can just switch the context + // back to the back end and use the interactive mode extension method. + //// context.SwitchToBackend(); + //// await context.SwitchToInteractiveAsync(); + + await context.ClickReliablyOnAsync(By.LinkText("App_Data/")); + await context.ClickReliablyOnAsync(By.LinkText("Sites/")); + await context.ClickReliablyOnAsync(By.LinkText("Default/")); + await context.ClickReliablyOnAsync(By.LinkText("DataProtection-Keys/")); + await context.ClickReliablyOnAsync(By.XPath("//td/a[contains(@href, '.xml')]")); }, - SetupHelpers.ConfigureFrontendSetupAsync); + browser: default, + configuration => + { + // Since this server is not our code, we should disable HTML validation. + configuration.HtmlValidationConfiguration.RunHtmlValidationAssertionOnAllPageChanges = false; + return Task.CompletedTask; + }); } // END OF TRAINING SECTION: Test headless Orchard Core with a frontend subprocess. diff --git a/Lombiq.Tests.UI/Extensions/FrontendOrchardCoreUITestExecutorConfigurationExtensions.cs b/Lombiq.Tests.UI/Extensions/FrontendOrchardCoreUITestExecutorConfigurationExtensions.cs index da9be02d0..98631b336 100644 --- a/Lombiq.Tests.UI/Extensions/FrontendOrchardCoreUITestExecutorConfigurationExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/FrontendOrchardCoreUITestExecutorConfigurationExtensions.cs @@ -21,14 +21,15 @@ public static (Uri FrontendUri, Uri BackendUri) GetFrontendAndBackendUris( /// /// Updates the by storing the Vite - /// frontend and the Orchard Core backend URLs as instances. + /// frontend and the Orchard Core backend URLs as instances. If either parameter is null, that + /// value is not changed. /// public static void SetFrontendAndBackendUris( this OrchardCoreUITestExecutorConfiguration configuration, string frontendUrl, string backendUrl) { - configuration.CustomConfiguration[FrontendUri] = new Uri(frontendUrl); - configuration.CustomConfiguration[BackendUri] = new Uri(backendUrl); + if (frontendUrl != null) configuration.CustomConfiguration[FrontendUri] = new Uri(frontendUrl); + if (backendUrl != null) configuration.CustomConfiguration[BackendUri] = new Uri(backendUrl); } } From 4748218d36f1e9a7ea8ec788a5b15d6fba69c6bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Sat, 16 Nov 2024 17:39:01 +0100 Subject: [PATCH 11/51] Fix test name. --- Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs b/Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs index 50755e38f..8add905bf 100644 --- a/Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs @@ -18,7 +18,7 @@ public FrontendTests(ITestOutputHelper testOutputHelper) // these directories and files. If we had a large client application that interacts with the headless OC in the // backend, we would be able to do something more interesting but that's outside the scope of this demo. [Fact] - public Task ServerSideErrorOnLoadedPageShouldHaltTest() => + public Task FrontendServerShouldStartWithTest() => ExecuteFrontendTestAfterSetupAsync( async context => { From abd3a569d7b6331e647cdb2d08c929a914f6806c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Sat, 16 Nov 2024 17:43:30 +0100 Subject: [PATCH 12/51] Cleanup and organization. --- Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj | 4 +--- Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs | 6 ++++-- .../Tests/{ExampleJavascriptTestShouldWork.mjs => test.mjs} | 0 3 files changed, 5 insertions(+), 5 deletions(-) rename Lombiq.Tests.UI.Samples/Tests/{ExampleJavascriptTestShouldWork.mjs => test.mjs} (100%) diff --git a/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj b/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj index 67c2c4f17..4a97b0f27 100644 --- a/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj +++ b/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj @@ -9,9 +9,7 @@ - - PreserveNewest - + diff --git a/Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs b/Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs index 333765040..f2ee27d5c 100644 --- a/Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs @@ -22,8 +22,10 @@ public JavascriptTests(ITestOutputHelper testOutputHelper) [Fact] public Task ExampleJavascriptTestShouldWork() { - // Don't forget to mark the script files as "Copy if newer", so they are available to the test. - var scriptPath = Path.Join("Tests", nameof(ExampleJavascriptTestShouldWork) + ".mjs"); + // Don't forget to mark the script files as "Copy if newer", so they are available to the + // test. It's best to include something like the following in your csproj file: + // + var scriptPath = Path.Join("Tests", "test.mjs"); // Set up the JS dependencies in the test's temp directory to ensure there are no clashes, then run the script. return ExecuteTestAfterSetupAsync(context => context diff --git a/Lombiq.Tests.UI.Samples/Tests/ExampleJavascriptTestShouldWork.mjs b/Lombiq.Tests.UI.Samples/Tests/test.mjs similarity index 100% rename from Lombiq.Tests.UI.Samples/Tests/ExampleJavascriptTestShouldWork.mjs rename to Lombiq.Tests.UI.Samples/Tests/test.mjs From 32362c92a3eaa9f58838ba07b5728ccbeeeeda32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Sat, 16 Nov 2024 22:42:31 +0100 Subject: [PATCH 13/51] Add SwitchToInteractiveWithJavascriptTestInfoAsync --- .../Tests/JavascriptTests.cs | 10 ++-- .../FrontendUITestContextExtensions.cs | 47 ++++++++++++++++--- 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs b/Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs index f2ee27d5c..ab598b1ca 100644 --- a/Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs @@ -45,13 +45,9 @@ public Task Sandbox() return ExecuteTestAfterSetupAsync( async context => { - var driverPath = context.GetDriverPath(); - var tempPath = context.GetTempSubDirectoryPath(); - - await context.SwitchToInteractiveAsync( - $"To start a Javascript test, open a command line terminal at \"{tempPath}\": and type the " + - $"following command: node --inspect ../../sandbox.js {driverPath} " + - $"{context.Driver.Url}"); + var scriptPath = Path.Join("Tests", "test.mjs"); + var workingDirectory = await context.SetupNodeSeleniumAsync(_testOutputHelper); + await context.SwitchToInteractiveWithJavascriptTestInfoAsync(scriptPath, workingDirectory); }, configuration => { diff --git a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs index 79bcb42d2..e37ab3301 100644 --- a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs @@ -57,6 +57,26 @@ public static string GetDriverPath(this UITestContext context) return Path.Join(service.DriverServicePath, service.DriverServiceExecutableName); } + private static (string WorkingDirectory, string[] Arguments) GetExecuteJavascriptTestPats( + this UITestContext context, + string scriptPath, + string workingDirectory) + { + workingDirectory = Path.GetFullPath(workingDirectory ?? Environment.CurrentDirectory); + + var relativeScriptPath = Path.GetRelativePath(workingDirectory, scriptPath); + var arguments = new[] + { + "--inspect", + relativeScriptPath.Length < scriptPath.Length ? relativeScriptPath : scriptPath, + context.GetDriverPath(), + context.Driver.Url, + context.GetTempSubDirectoryPath(), + }; + + return (workingDirectory, arguments); + } + /// /// Executes the provided file via node with command line arguments containing the necessary information for /// Selenium JS to take over the browser. @@ -71,17 +91,12 @@ public static async Task ExecuteJavascriptTestAsync( { const string command = "node"; var pipe = testOutputHelper.ToPipeTarget($"{nameof(ExecuteJavascriptTestAsync)}({command})"); + (workingDirectory, var arguments) = context.GetExecuteJavascriptTestPats(scriptPath, workingDirectory); try { await Cli.Wrap(command) - .WithArguments([ - "--inspect", - scriptPath, - context.GetDriverPath(), - context.Driver.Url, - context.GetTempSubDirectoryPath(), - ]) + .WithArguments(arguments) .WithStandardOutputPipe(pipe) .WithStandardErrorPipe(pipe) .WithWorkingDirectory(workingDirectory ?? Environment.CurrentDirectory) @@ -96,6 +111,24 @@ await Cli.Wrap(command) } } + /// + /// Invokes with a custom notification + /// message that contains instructions to invoke the Javascript test manually with node. + /// + /// The relative or absolute path pointing to the test script file. + /// The path where the test script should be executed, will be converted to absolute. + public static Task SwitchToInteractiveWithJavascriptTestInfoAsync( + this UITestContext context, + string scriptPath, + string workingDirectory = null) + { + (workingDirectory, var arguments) = context.GetExecuteJavascriptTestPats(scriptPath, workingDirectory); + + return context.SwitchToInteractiveAsync( + $"To start a Javascript test, open a command line terminal at \"{workingDirectory}\": and type the " + + $"following command: node {string.Join(' ', arguments)}"); + } + /// /// Sets up the Javascript dependencies using and then runs the script in the /// same temp directory. From 08098cbf6d76c80c915607e2c0b310ceb635691e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Sat, 16 Nov 2024 22:42:44 +0100 Subject: [PATCH 14/51] Clean up comments. --- .../Tests/JavascriptTests.cs | 25 +++++++++++-------- Lombiq.Tests.UI.Samples/Tests/test.mjs | 3 +++ 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs b/Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs index ab598b1ca..c662b87a5 100644 --- a/Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs @@ -19,18 +19,21 @@ public JavascriptTests(ITestOutputHelper testOutputHelper) { } + // Using this approach you only have to write minimal C# boilerplate, which you can see below. [Fact] - public Task ExampleJavascriptTestShouldWork() - { - // Don't forget to mark the script files as "Copy if newer", so they are available to the - // test. It's best to include something like the following in your csproj file: - // - var scriptPath = Path.Join("Tests", "test.mjs"); + public Task ExampleJavascriptTestShouldWork() => + ExecuteTestAfterSetupAsync(context => + { + // Don't forget to mark the script files as "Copy if newer", so they are available to the test. It's best to + // include something like the following in your csproj file: + // + var scriptPath = Path.Join("Tests", "test.mjs"); - // Set up the JS dependencies in the test's temp directory to ensure there are no clashes, then run the script. - return ExecuteTestAfterSetupAsync(context => context - .SetupSeleniumAndExecuteJavascriptTestAsync(scriptPath, _testOutputHelper)); - } + // Set up the JS dependencies in the test's temp directory to ensure there are no clashes, then run the + // script. This method has an additional parameter to list further NPM dependencies beyond + // "selenium-webdriver", if the script requires it. We will check out this script file in the next station. + return context.SetupSeleniumAndExecuteJavascriptTestAsync(scriptPath, _testOutputHelper); + }); // To best debug the Javascript code, you may want to set up the site and then invoke node manually. This is not a // real test, but it sets up the site in interactive mode (see Tests/InteractiveModeTests.cs for more) with @@ -58,4 +61,4 @@ public Task Sandbox() } } -// END OF TRAINING SECTION: Executing tests written in Javascript. +// NEXT STATION: Head over to Tests/test.mjs. diff --git a/Lombiq.Tests.UI.Samples/Tests/test.mjs b/Lombiq.Tests.UI.Samples/Tests/test.mjs index e69de29bb..33e839827 100644 --- a/Lombiq.Tests.UI.Samples/Tests/test.mjs +++ b/Lombiq.Tests.UI.Samples/Tests/test.mjs @@ -0,0 +1,3 @@ + + +// END OF TRAINING SECTION: Executing tests written in Javascript. From de2cafd384f75acce11fda1eac6fc6a852168007 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Sun, 17 Nov 2024 00:01:19 +0100 Subject: [PATCH 15/51] Copy ui-testing-toolkit.mjs from OFFI. --- Lombiq.Tests.UI/Lombiq.Tests.UI.csproj | 3 + Lombiq.Tests.UI/ui-testing-toolkit.mjs | 132 +++++++++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 Lombiq.Tests.UI/ui-testing-toolkit.mjs diff --git a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj index 3fc27d706..17bb0c3e6 100644 --- a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj +++ b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj @@ -28,6 +28,9 @@ + + PreserveNewest + diff --git a/Lombiq.Tests.UI/ui-testing-toolkit.mjs b/Lombiq.Tests.UI/ui-testing-toolkit.mjs new file mode 100644 index 000000000..4d0a69e3e --- /dev/null +++ b/Lombiq.Tests.UI/ui-testing-toolkit.mjs @@ -0,0 +1,132 @@ +import path from 'path'; +import process from 'process'; +import { By, WebDriver, WebElement } from 'selenium-webdriver'; +import chrome from 'selenium-webdriver/chrome.js'; +import { writeFile } from 'node:fs/promises' + +async function _logSource(driver) { + const html = await driver.getPageSource(); + console.log('HTML:', html.replace(/\s*[\n\r]+\s*/g, ' ')); +} + +async function _takeScreenshot(driver, file){ + let image = await driver.takeScreenshot() + await writeFile(file, image, 'base64') +} + +/** + * Verifies that the provided element's inner text contains the provided text. + * @param {WebElement} element The web element whose inner text is examined. + * @param {string} text The expected inner text fragment. + * @returns {Promise} Success when the element text contains the expected string, rejection if it does not or if + * element is null or empty. + */ +async function shouldContainText(element, text) { + if (!element) { + throw new Error('The element is missing.'); + } + + if (element.then) { + element = await element; + } + + const actualText = (await element.getText())?.trim(); + + if (actualText?.includes(text) !== true) { + const url = await element.getDriver().getCurrentUrl(); + throw new Error( + `Expected element at ${url} to contain text "${text}", but it does not. (Actual text: ${actualText})`); + } +} + +/** + * Navigates the browser to the given URL and then verifies that the page has loaded with non-empty content. + * @param {WebDriver} driver The driver whose current tab should be navigated. + * @param {string} url The target URL. + * @param {number} maxAttempts The maximum number of attempts. If exceeded, an error is thrown. + * @returns {Promise} Success when a non-empty page has been reached. + */ +async function navigate(driver, url, maxAttempts = 10) { + for (let i = 0; i < maxAttempts; i++) { + await driver.navigate().to(url); + + await driver.wait(() => driver + .executeScript('return document.readyState') + .then((readyState) => readyState === 'complete')); + + try { + if ((await driver.findElement(By.xpath('//body')).getAttribute('innerHTML'))?.trim()) { + console.log(`Successfully reached ${url}.`); + return; + } + } + catch (exception) { + // Nothing to do here, let's try again. + } + } + + throw new Error(`Failed to navigate to a non-empty page at ${url} in ${maxAttempts} attempts.`) +} + +/** + * Executes a test by connecting to an existing web driver using the information in the command line arguments. + * @param {function(WebDriver, string):Promise} test + * @param {function(chrome.Options):chrome.Options} configureOptions Update the configuration before the driver is built. + * @returns {Promise} Success if the driver is created and the test has run to completion. + */ +async function runTest(test, configureOptions = null) { + const args = process.argv.slice(2); + if (args.length !== 3) throw new Error('Usage: node script.js driverPath startUrl tempDirectory'); + const [driverPath, startUrl, tempDirectory] = args; + + let options = new chrome.Options() + .addArguments('disable-dev-shm-usage') + .addArguments('unsafely-disable-devtools-self-xss-warnings') + .addArguments('disable-search-engine-choice-screen') + .addArguments('--lang=en-US') + .addArguments('disable-accelerated-2d-canvas') + .addArguments('disable-gpu') + .addArguments('force-color-profile=sRGB') + .addArguments('force-device-scale-factor=1') + .addArguments('high-dpi-support=1') + .addArguments('disable-smooth-scrolling') + .addArguments('ignore-certificate-errors') + .addArguments('--ignore-certificate-errors') + .addArguments('--no-sandbox') + ; + + if (process.env.GITHUB_ENV) options = options.addArguments('headless'); + if (configureOptions) options = configureOptions(options) ?? options; + + const service = new chrome.ServiceBuilder(driverPath).build(); + const driver = chrome.Driver.createSession(options, service); + await driver.manage().setTimeouts({ implicit: 10000 }); + + try { + await navigate(driver, startUrl); + + await test(driver, startUrl); + } + catch (exception) { + // Write out some context, doesn't matter if these fail. + try { console.log('Title:', await driver.getTitle()); } catch (error) { console.error(error); } + try { console.log('URL:', await driver.getCurrentUrl()); } catch (error) { console.error(error); } + try { await _logSource(driver); } catch (error) { console.error(error); } + + const screenshotPath = path.join(tempDirectory, 'Screenshots', 'error.png'); + console.log(`Writing screenshot to ${screenshotPath}...`); + await _takeScreenshot(driver, screenshotPath); + console.log('Done.') + + throw exception; + } + finally { + await driver.close(); + } +} + +export { + runTest, + shouldContainText, + navigate, +}; From 9b9eb9c3d0545efddb95768ec3589e4745dd3697 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Sun, 17 Nov 2024 00:01:25 +0100 Subject: [PATCH 16/51] typo --- Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs index e37ab3301..d908bbbfc 100644 --- a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs @@ -125,7 +125,7 @@ public static Task SwitchToInteractiveWithJavascriptTestInfoAsync( (workingDirectory, var arguments) = context.GetExecuteJavascriptTestPats(scriptPath, workingDirectory); return context.SwitchToInteractiveAsync( - $"To start a Javascript test, open a command line terminal at \"{workingDirectory}\": and type the " + + $"To start a Javascript test, open a command line terminal at \"{workingDirectory}\" and type the " + $"following command: node {string.Join(' ', arguments)}"); } From 60577e397c46e7b53816133470092e9b08f44bec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Sun, 17 Nov 2024 01:02:40 +0100 Subject: [PATCH 17/51] bug fix --- Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs index d908bbbfc..d3e6fce1e 100644 --- a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs @@ -64,11 +64,13 @@ private static (string WorkingDirectory, string[] Arguments) GetExecuteJavascrip { workingDirectory = Path.GetFullPath(workingDirectory ?? Environment.CurrentDirectory); + var absoluteScriptPath = Path.GetFullPath(scriptPath); var relativeScriptPath = Path.GetRelativePath(workingDirectory, scriptPath); + var arguments = new[] { "--inspect", - relativeScriptPath.Length < scriptPath.Length ? relativeScriptPath : scriptPath, + absoluteScriptPath.Length < relativeScriptPath.Length ? absoluteScriptPath : relativeScriptPath, context.GetDriverPath(), context.Driver.Url, context.GetTempSubDirectoryPath(), From 392249263151a55fa2dc5251a7111577ce5b4f6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Sun, 17 Nov 2024 01:12:21 +0100 Subject: [PATCH 18/51] Fix missing message in interactive mode. --- Lombiq.Tests.UI.Shortcuts/Views/InteractiveMode/Index.cshtml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Lombiq.Tests.UI.Shortcuts/Views/InteractiveMode/Index.cshtml b/Lombiq.Tests.UI.Shortcuts/Views/InteractiveMode/Index.cshtml index 6875d7d6a..de03ca1ea 100644 --- a/Lombiq.Tests.UI.Shortcuts/Views/InteractiveMode/Index.cshtml +++ b/Lombiq.Tests.UI.Shortcuts/Views/InteractiveMode/Index.cshtml @@ -27,6 +27,11 @@ width: 30vw; } + #messages { + z-index: 999999999; + position: fixed; + } + .message { width: 100%; } From 10314e964f0543a301fe5be39ae346eb2ffc03fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Sun, 17 Nov 2024 01:43:52 +0100 Subject: [PATCH 19/51] Shorter paths. --- .../FrontendUITestContextExtensions.cs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs index d3e6fce1e..c283c860d 100644 --- a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs @@ -62,18 +62,22 @@ private static (string WorkingDirectory, string[] Arguments) GetExecuteJavascrip string scriptPath, string workingDirectory) { - workingDirectory = Path.GetFullPath(workingDirectory ?? Environment.CurrentDirectory); + static string GetShorterPath(string basePath, string path) + { + var absolute = Path.GetFullPath(path); + var relative = Path.GetRelativePath(basePath, path); + return absolute.Length < relative.Length ? absolute : relative; + } - var absoluteScriptPath = Path.GetFullPath(scriptPath); - var relativeScriptPath = Path.GetRelativePath(workingDirectory, scriptPath); + workingDirectory = Path.GetFullPath(workingDirectory ?? Environment.CurrentDirectory); var arguments = new[] { "--inspect", - absoluteScriptPath.Length < relativeScriptPath.Length ? absoluteScriptPath : relativeScriptPath, - context.GetDriverPath(), + GetShorterPath(workingDirectory, scriptPath), + GetShorterPath(workingDirectory, context.GetDriverPath()), context.Driver.Url, - context.GetTempSubDirectoryPath(), + GetShorterPath(workingDirectory, context.GetTempSubDirectoryPath()), }; return (workingDirectory, arguments); From 21462ee30c24648752b2a4608234b9d03ea48fd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Sun, 17 Nov 2024 02:15:17 +0100 Subject: [PATCH 20/51] Bug fix. --- .../Tests/JavascriptTests.cs | 11 +++++--- .../FrontendUITestContextExtensions.cs | 28 +++++++++++-------- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs b/Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs index c662b87a5..0b0265cdd 100644 --- a/Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs @@ -27,12 +27,13 @@ public Task ExampleJavascriptTestShouldWork() => // Don't forget to mark the script files as "Copy if newer", so they are available to the test. It's best to // include something like the following in your csproj file: // - var scriptPath = Path.Join("Tests", "test.mjs"); + var workingDirectory = "Tests"; + var scriptPath = Path.Join(workingDirectory, "test.mjs"); // Set up the JS dependencies in the test's temp directory to ensure there are no clashes, then run the // script. This method has an additional parameter to list further NPM dependencies beyond // "selenium-webdriver", if the script requires it. We will check out this script file in the next station. - return context.SetupSeleniumAndExecuteJavascriptTestAsync(scriptPath, _testOutputHelper); + return context.SetupSeleniumAndExecuteJavascriptTestAsync(_testOutputHelper, scriptPath, workingDirectory); }); // To best debug the Javascript code, you may want to set up the site and then invoke node manually. This is not a @@ -48,8 +49,10 @@ public Task Sandbox() return ExecuteTestAfterSetupAsync( async context => { - var scriptPath = Path.Join("Tests", "test.mjs"); - var workingDirectory = await context.SetupNodeSeleniumAsync(_testOutputHelper); + var workingDirectory = "Tests"; + var scriptPath = Path.Join(workingDirectory, "test.mjs"); + + await context.SetupNodeSeleniumAsync(_testOutputHelper, workingDirectory); await context.SwitchToInteractiveWithJavascriptTestInfoAsync(scriptPath, workingDirectory); }, configuration => diff --git a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs index c283c860d..fd4100f7a 100644 --- a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs @@ -146,11 +146,12 @@ public static Task SwitchToInteractiveWithJavascriptTestInfoAsync( /// public static async Task SetupSeleniumAndExecuteJavascriptTestAsync( this UITestContext context, - string scriptPath, ITestOutputHelper testOutputHelper, + string scriptPath, + string workingDirectory, params string[] otherDependencies) { - var workingDirectory = await context.SetupNodeSeleniumAsync(testOutputHelper, otherDependencies); + await context.SetupNodeSeleniumAsync(testOutputHelper, workingDirectory, otherDependencies); var relativePath = Path.GetRelativePath(workingDirectory, scriptPath); await context.ExecuteJavascriptTestAsync(relativePath, testOutputHelper, workingDirectory); @@ -160,13 +161,12 @@ public static async Task SetupSeleniumAndExecuteJavascriptTestAsync( /// Creates a blank Node.js project in the current test session's directory and /// installs the provided NPM using pnpm. /// - /// The path of the directory where the project is set up. - public static async Task SetupNodeDependenciesAsync( + public static async Task SetupNodeDependenciesAsync( this UITestContext context, ITestOutputHelper helper, + string workingDirectory, params string[] dependencies) { - var workingDirectory = context.GetTempSubDirectoryPath(); var projectFilePath = Path.Join(workingDirectory, "package.json"); if (!Directory.Exists(projectFilePath)) @@ -181,18 +181,24 @@ await Cli.Wrap("pnpm") .WithStandardErrorPipe(pipe) .WithWorkingDirectory(workingDirectory) .ExecuteAsync(); - - return workingDirectory; } /// /// Creates a blank Node.js project in the current test session's directory, then /// installs selenium-webdriver and any additional NPM dependencies using pnpm. /// - /// The path of the directory where the project is set up. - public static Task SetupNodeSeleniumAsync( + public static Task SetupNodeSeleniumAsync( this UITestContext context, ITestOutputHelper helper, - params string[] otherDependencies) => - context.SetupNodeDependenciesAsync(helper, ["selenium-webdriver", ..otherDependencies]); + string workingDirectory, + params string[] otherDependencies) + { + const string uiTestingToolkitScript = "ui-testing-toolkit.mjs"; + if (File.Exists(uiTestingToolkitScript)) + { + File.Copy(uiTestingToolkitScript, Path.Join(workingDirectory, uiTestingToolkitScript)); + } + + return context.SetupNodeDependenciesAsync(helper, workingDirectory, ["selenium-webdriver", ..otherDependencies]); + } } From ab3a38faf548fcb5d0405567f2f79457701e720f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Sun, 17 Nov 2024 02:15:45 +0100 Subject: [PATCH 21/51] Correct await order. --- Lombiq.Tests.UI/ui-testing-toolkit.mjs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lombiq.Tests.UI/ui-testing-toolkit.mjs b/Lombiq.Tests.UI/ui-testing-toolkit.mjs index 4d0a69e3e..d0d5d943e 100644 --- a/Lombiq.Tests.UI/ui-testing-toolkit.mjs +++ b/Lombiq.Tests.UI/ui-testing-toolkit.mjs @@ -22,12 +22,12 @@ async function _takeScreenshot(driver, file){ * element is null or empty. */ async function shouldContainText(element, text) { - if (!element) { - throw new Error('The element is missing.'); + if (element?.then) { + element = await element; } - if (element.then) { - element = await element; + if (!element) { + throw new Error('The element is missing.'); } const actualText = (await element.getText())?.trim(); From 1b3978dede9425e06ddec244e0e41e9907340b0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Sun, 17 Nov 2024 02:15:51 +0100 Subject: [PATCH 22/51] Add JS test. --- Lombiq.Tests.UI.Samples/Tests/test.mjs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/Lombiq.Tests.UI.Samples/Tests/test.mjs b/Lombiq.Tests.UI.Samples/Tests/test.mjs index 33e839827..52fe3d969 100644 --- a/Lombiq.Tests.UI.Samples/Tests/test.mjs +++ b/Lombiq.Tests.UI.Samples/Tests/test.mjs @@ -1,3 +1,26 @@ +import { By, until } from 'selenium-webdriver'; +// This dependency is copied into the build directory by Lombiq.Tests.UI. +import { runTest, shouldContainText, navigate } from './ui-testing-toolkit.mjs'; + +// This function automatically handles the command line arguments and sets up a Chrome driver. +await runTest(async (driver, startUrl) => { + // Inside you can use all normal Selenium Javascript code, e.g.: + // - https://www.selenium.dev/selenium/docs/api/javascript/WebDriver.html + // - https://www.selenium.dev/selenium/docs/api/javascript/By.html + await driver.findElement(By.xpath("//a[@href = '/blog/post-1']")).click(); + + // We also included a shortcut function to safely check text content. + await shouldContainText( + await driver.findElement(By.tagName("h1")), + "Man must explore, and this is exploration at its greatest"); + await shouldContainText( + await driver.findElement(By.className("field-name-blog-post-subtitle")), + "Problems look mighty small from 150 miles up"); + + // And another one to navigate and safely wait for the page to load. + await navigate(driver, startUrl); + await driver.findElement(By.xpath("id('footer')//a[@href='https://lombiq.com/']")); +}); // END OF TRAINING SECTION: Executing tests written in Javascript. From 10e383f785eaf4b112bf9901559175c21b3d29a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Sun, 17 Nov 2024 02:36:52 +0100 Subject: [PATCH 23/51] bug fix --- .../Extensions/FrontendUITestContextExtensions.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs index fd4100f7a..6cb736ea1 100644 --- a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs @@ -152,9 +152,7 @@ public static async Task SetupSeleniumAndExecuteJavascriptTestAsync( params string[] otherDependencies) { await context.SetupNodeSeleniumAsync(testOutputHelper, workingDirectory, otherDependencies); - var relativePath = Path.GetRelativePath(workingDirectory, scriptPath); - - await context.ExecuteJavascriptTestAsync(relativePath, testOutputHelper, workingDirectory); + await context.ExecuteJavascriptTestAsync(scriptPath, testOutputHelper, workingDirectory); } /// @@ -196,7 +194,7 @@ public static Task SetupNodeSeleniumAsync( const string uiTestingToolkitScript = "ui-testing-toolkit.mjs"; if (File.Exists(uiTestingToolkitScript)) { - File.Copy(uiTestingToolkitScript, Path.Join(workingDirectory, uiTestingToolkitScript)); + File.Copy(uiTestingToolkitScript, Path.Join(workingDirectory, uiTestingToolkitScript), overwrite: true); } return context.SetupNodeDependenciesAsync(helper, workingDirectory, ["selenium-webdriver", ..otherDependencies]); From 685867cedaa8c35466c72b667faea0f334211930 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Sun, 17 Nov 2024 03:08:47 +0100 Subject: [PATCH 24/51] Organize ExecuteJavascriptTestAsync parameters. --- .../Extensions/FrontendUITestContextExtensions.cs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs index 6cb736ea1..8d71c2c12 100644 --- a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs @@ -83,16 +83,24 @@ static string GetShorterPath(string basePath, string path) return (workingDirectory, arguments); } + [Obsolete($"Use the overload where the first parameter is {nameof(ITestOutputHelper)}.")] + public static Task ExecuteJavascriptTestAsync( + this UITestContext context, + string scriptPath, + ITestOutputHelper testOutputHelper) => + context.ExecuteJavascriptTestAsync(testOutputHelper, scriptPath); + /// /// Executes the provided file via node with command line arguments containing the necessary information for /// Selenium JS to take over the browser. /// - /// The Javascript source file to execute using node. /// Needed to redirect the node output into the test logs. + /// The Javascript source file to execute using node. + /// The working directory where node is executed from. public static async Task ExecuteJavascriptTestAsync( this UITestContext context, - string scriptPath, ITestOutputHelper testOutputHelper, + string scriptPath, string workingDirectory = null) { const string command = "node"; @@ -152,7 +160,7 @@ public static async Task SetupSeleniumAndExecuteJavascriptTestAsync( params string[] otherDependencies) { await context.SetupNodeSeleniumAsync(testOutputHelper, workingDirectory, otherDependencies); - await context.ExecuteJavascriptTestAsync(scriptPath, testOutputHelper, workingDirectory); + await context.ExecuteJavascriptTestAsync(testOutputHelper, scriptPath, workingDirectory); } /// From dd3f4d752d19c8a72940aec4c7d6528cc6861209 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Sun, 17 Nov 2024 03:11:31 +0100 Subject: [PATCH 25/51] Fix formatting. --- Lombiq.Tests.UI/Constants/DirectoryPaths.cs | 4 ++-- .../AccessibilityCheckingUITestContextExtensions.cs | 1 - Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs | 4 ++-- Lombiq.Tests.UI/SecurityScanning/ZapManager.cs | 3 +-- Lombiq.Tests.UI/Services/UITestContext.cs | 2 +- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Lombiq.Tests.UI/Constants/DirectoryPaths.cs b/Lombiq.Tests.UI/Constants/DirectoryPaths.cs index 141030f79..4c3f769f7 100644 --- a/Lombiq.Tests.UI/Constants/DirectoryPaths.cs +++ b/Lombiq.Tests.UI/Constants/DirectoryPaths.cs @@ -11,11 +11,11 @@ public static class DirectoryPaths public const string Screenshots = nameof(Screenshots); public static string GetTempDirectoryPath(params string[] subDirectoryNames) => - Path.Combine([Environment.CurrentDirectory, Temp, ..subDirectoryNames]); + Path.Combine([Environment.CurrentDirectory, Temp, .. subDirectoryNames]); [Obsolete($"Use {nameof(UITestContext)}.{nameof(UITestContext.GetTempSubDirectoryPath)}() instead.")] public static string GetTempSubDirectoryPath(string contextId, params string[] subDirectoryNames) => - GetTempDirectoryPath([contextId, ..subDirectoryNames]); + GetTempDirectoryPath([contextId, .. subDirectoryNames]); [Obsolete($"Use {nameof(UITestContext)}.{nameof(UITestContext.ScreenshotsDirectoryPath)} instead.")] public static string GetScreenshotsDirectoryPath(string contextId) => diff --git a/Lombiq.Tests.UI/Extensions/AccessibilityCheckingUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/AccessibilityCheckingUITestContextExtensions.cs index d6674f694..20080b951 100644 --- a/Lombiq.Tests.UI/Extensions/AccessibilityCheckingUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/AccessibilityCheckingUITestContextExtensions.cs @@ -1,6 +1,5 @@ using Deque.AxeCore.Commons; using Deque.AxeCore.Selenium; -using Lombiq.Tests.UI.Constants; using Lombiq.Tests.UI.Exceptions; using Lombiq.Tests.UI.Helpers; using Lombiq.Tests.UI.Services; diff --git a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs index 8d71c2c12..2f643e0a5 100644 --- a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs @@ -182,7 +182,7 @@ public static async Task SetupNodeDependenciesAsync( var pipe = helper.ToPipeTarget(nameof(SetupNodeSeleniumAsync)); await Cli.Wrap("pnpm") - .WithArguments(["install", ..dependencies]) + .WithArguments(["install", .. dependencies]) .WithStandardOutputPipe(pipe) .WithStandardErrorPipe(pipe) .WithWorkingDirectory(workingDirectory) @@ -205,6 +205,6 @@ public static Task SetupNodeSeleniumAsync( File.Copy(uiTestingToolkitScript, Path.Join(workingDirectory, uiTestingToolkitScript), overwrite: true); } - return context.SetupNodeDependenciesAsync(helper, workingDirectory, ["selenium-webdriver", ..otherDependencies]); + return context.SetupNodeDependenciesAsync(helper, workingDirectory, ["selenium-webdriver", .. otherDependencies]); } } diff --git a/Lombiq.Tests.UI/SecurityScanning/ZapManager.cs b/Lombiq.Tests.UI/SecurityScanning/ZapManager.cs index 9f592730e..520449d0b 100644 --- a/Lombiq.Tests.UI/SecurityScanning/ZapManager.cs +++ b/Lombiq.Tests.UI/SecurityScanning/ZapManager.cs @@ -1,6 +1,5 @@ using CliWrap; using Lombiq.HelpfulLibraries.Cli; -using Lombiq.Tests.UI.Constants; using Lombiq.Tests.UI.Helpers; using Lombiq.Tests.UI.Services; using Lombiq.Tests.UI.Services.GitHub; @@ -78,7 +77,7 @@ public async Task RunSecurityScanAsync( automationFrameworkYamlPath = AutomationFrameworkPlanPaths.BaselinePlanPath; } - // Each attempt will have it's own "ZapN" directory inside the temp, starting with "Zap1". + // Each attempt will have its own "ZapN" directory inside the temp, starting with "Zap1". var mountedDirectoryPath = DirectoryHelper.CreateEnumeratedDirectory( context.GetTempSubDirectoryPath("Zap")); var reportsDirectoryPath = Path.Combine(mountedDirectoryPath, _zapReportsDirectoryName); diff --git a/Lombiq.Tests.UI/Services/UITestContext.cs b/Lombiq.Tests.UI/Services/UITestContext.cs index a69112bd2..e3577e23c 100644 --- a/Lombiq.Tests.UI/Services/UITestContext.cs +++ b/Lombiq.Tests.UI/Services/UITestContext.cs @@ -310,7 +310,7 @@ public void SwitchCurrentTenant(string tenantName, Uri baseUri) /// directory. /// public string GetTempSubDirectoryPath(params string[] subDirectoryNames) => - DirectoryPaths.GetTempDirectoryPath([Id, ..subDirectoryNames]); + DirectoryPaths.GetTempDirectoryPath([Id, .. subDirectoryNames]); private bool IsAlert() { From 402398fa7eb22bb0fa6355316c6130a205e15910 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Sun, 17 Nov 2024 03:14:49 +0100 Subject: [PATCH 26/51] Fix error CS0419: Ambiguous reference in cref attribute --- .../Extensions/FrontendUITestContextExtensions.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs index 2f643e0a5..dad46ac07 100644 --- a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs @@ -149,8 +149,9 @@ public static Task SwitchToInteractiveWithJavascriptTestInfoAsync( /// /// /// The path of the Javascript file to execute with node. Before passing it to , it's transformed into a relative path based on the temp directory to - /// conserve path length because long paths can be a problem in some operating systems. + /// cref="ExecuteJavascriptTestAsync(UITestContext,ITestOutputHelper,string,string)"/>, it's transformed into a + /// relative path based on the temp directory to conserve path length because long paths can be a problem in some + /// operating systems. /// public static async Task SetupSeleniumAndExecuteJavascriptTestAsync( this UITestContext context, From 18541188ec2f6f21f3889edea7c4bcb32dbaf564 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Sun, 17 Nov 2024 03:24:53 +0100 Subject: [PATCH 27/51] Code cleanup. --- Lombiq.Tests.UI.Samples/FrontendUITestBase.cs | 1 - Lombiq.Tests.UI.Samples/Helpers/SetupHelpers.cs | 1 - 2 files changed, 2 deletions(-) diff --git a/Lombiq.Tests.UI.Samples/FrontendUITestBase.cs b/Lombiq.Tests.UI.Samples/FrontendUITestBase.cs index 418f3a62e..861dc5988 100644 --- a/Lombiq.Tests.UI.Samples/FrontendUITestBase.cs +++ b/Lombiq.Tests.UI.Samples/FrontendUITestBase.cs @@ -75,7 +75,6 @@ protected Task ExecuteFrontendTestAfterSetupAsync( frontendUrl: $"http://localhost:" + port, backendUrl: null); return frontendCommand.WithArguments(["--yes", "http-server", "--port", port]); - }, // When this function is not null, test setup will call it on each output line and wait for it // to return true. This can be used to look for an output that only appears when the frontend diff --git a/Lombiq.Tests.UI.Samples/Helpers/SetupHelpers.cs b/Lombiq.Tests.UI.Samples/Helpers/SetupHelpers.cs index 24c8ed4a6..fe4ff9f32 100644 --- a/Lombiq.Tests.UI.Samples/Helpers/SetupHelpers.cs +++ b/Lombiq.Tests.UI.Samples/Helpers/SetupHelpers.cs @@ -2,7 +2,6 @@ using Lombiq.Tests.UI.Constants; using Lombiq.Tests.UI.Extensions; using Lombiq.Tests.UI.Pages; -using Lombiq.Tests.UI.Samples.Tests; using Lombiq.Tests.UI.Services; using OpenQA.Selenium; using System; From bdb345aa3710e59b52ce149c161b1eff6fa36e1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sun, 17 Nov 2024 22:00:58 +0100 Subject: [PATCH 28/51] Skipping Edge tests --- .../Tests/BasicVisualVerificationTests.cs | 3 ++- Lombiq.Tests.UI.Samples/Tests/MultiBrowserTests.cs | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Lombiq.Tests.UI.Samples/Tests/BasicVisualVerificationTests.cs b/Lombiq.Tests.UI.Samples/Tests/BasicVisualVerificationTests.cs index a2372dec0..d62bf0592 100644 --- a/Lombiq.Tests.UI.Samples/Tests/BasicVisualVerificationTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/BasicVisualVerificationTests.cs @@ -55,7 +55,8 @@ public Task VerifyBlogImage() => // is the different rendering of text on each platform, but it can occur between different Linux distributions too. // Here: https://pandasauce.org/post/linux-fonts/ you can find a good summary about this from 2019, but still valid // in 2022. - [Theory, Chrome, Edge] + // Temporarily not running Edge until https://github.com/atata-framework/atata-webdriversetup/issues/16 is fixed. + [Theory, Chrome] public Task VerifyNavbar(Browser browser) => ExecuteTestAfterSetupAsync( context => diff --git a/Lombiq.Tests.UI.Samples/Tests/MultiBrowserTests.cs b/Lombiq.Tests.UI.Samples/Tests/MultiBrowserTests.cs index c410f92be..7dc7eec6f 100644 --- a/Lombiq.Tests.UI.Samples/Tests/MultiBrowserTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/MultiBrowserTests.cs @@ -23,7 +23,9 @@ public MultiBrowserTests(ITestOutputHelper testOutputHelper) // First, let's see a test using Edge. While the default browser is Chrome if you don't set anything, all // ExecuteTest* methods can also accept a browser, if you want to use a different one. - [Fact] +#pragma warning disable xUnit1004 // Test methods should not be skipped + [Fact(Skip = "Temporarily not running Edge until https://github.com/atata-framework/atata-webdriversetup/issues/16 is fixed.")] +#pragma warning restore xUnit1004 // Test methods should not be skipped public Task AnonymousHomePageShouldExistWithEdge() => ExecuteTestAfterSetupAsync(NavbarIsCorrect, Browser.Edge); @@ -31,7 +33,8 @@ public Task AnonymousHomePageShouldExistWithEdge() => // tests. [Chrome] and [Edge] are input parameters of the test, and thus in effect, you have now two tests: // AnonymousHomePageShouldExistMultiBrowser once with Chrome, and once with Edge. See here for more info: // https://andrewlock.net/creating-parameterised-tests-in-xunit-with-inlinedata-classdata-and-memberdata/. - [Theory, Chrome, Edge] + // Temporarily not running Edge until https://github.com/atata-framework/atata-webdriversetup/issues/16 is fixed. + [Theory, Chrome] public Task AnonymousHomePageShouldExistMultiBrowser(Browser browser) => ExecuteTestAfterSetupAsync(NavbarIsCorrect, browser); From 9b4f2260d67e0b6b9b1c6ecee359d8c88fd6bf37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Mon, 18 Nov 2024 00:27:45 +0100 Subject: [PATCH 29/51] Grammar --- Lombiq.Tests.UI.Samples/FrontendUITestBase.cs | 4 ++-- Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs | 2 +- Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs | 8 ++++---- ...endOrchardCoreUITestExecutorConfigurationExtensions.cs | 4 ++-- .../Extensions/FrontendUITestContextExtensions.cs | 3 ++- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/Lombiq.Tests.UI.Samples/FrontendUITestBase.cs b/Lombiq.Tests.UI.Samples/FrontendUITestBase.cs index 861dc5988..91996ebbe 100644 --- a/Lombiq.Tests.UI.Samples/FrontendUITestBase.cs +++ b/Lombiq.Tests.UI.Samples/FrontendUITestBase.cs @@ -61,8 +61,8 @@ protected Task ExecuteFrontendTestAfterSetupAsync( configureCommand: (frontendCommand, context) => { // You can also get this from the configuration using configuration - // .GetFrontendAndBackendUris().FrontendUri.Port. The values in the configuration has been - // initialized shortly before this function is called. but it's not worth it unless you need + // .GetFrontendAndBackendUris().FrontendUri.Port. The values in the configuration have been + // initialized shortly before this function is called, but it's not worth it unless you need // both frontend and backend URLS. The frontend port is a unique, reserved number that's // guaranteed to be available during this test just as much as any Orchard Core instance, // because it's coming from the same pool of numbers. diff --git a/Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs b/Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs index 8add905bf..f599b5cb9 100644 --- a/Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs @@ -13,7 +13,7 @@ public FrontendTests(ITestOutputHelper testOutputHelper) { } - // The interesting details were in FrontendUITestBase, here we just show that you can freely interact with the pages + // The interesting details are in FrontendUITestBase, here we just show that you can freely interact with the pages // served by frontend server the same way as usual. In this case we have an HTTP file server, so you can navigate // these directories and files. If we had a large client application that interacts with the headless OC in the // backend, we would be able to do something more interesting but that's outside the scope of this demo. diff --git a/Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs b/Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs index 0b0265cdd..cf0c4b00c 100644 --- a/Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs @@ -7,8 +7,8 @@ namespace Lombiq.Tests.UI.Samples.Tests; -// Suppose you want to write UI tests in Javascript. Why would you want to do that? Unlikely if you are an Orchard Core -// developer, but what if the person responsible for writing the tests is not? In the previous training section we +// Let's suppose you want to write UI tests in Javascript. Why would you want to do that? Unlikely if you are an Orchard +// Core developer, but what if the person responsible for writing the tests is not? In the previous training section we // discussed using a separate frontend server, with mention of technologies using Node.js. In that case the frontend // developers may be more familiar with Javascript so it makes sense to write and debug the tests in Node.js so they // don't have to learn different tools and tech stacks just to create some UI tests. @@ -38,8 +38,8 @@ public Task ExampleJavascriptTestShouldWork() => // To best debug the Javascript code, you may want to set up the site and then invoke node manually. This is not a // real test, but it sets up the site in interactive mode (see Tests/InteractiveModeTests.cs for more) with - // information how to start up test script from your GUI. It's an example of some tooling that can improve the test - // developer's workflow. + // information on how to start up test scripts from your GUI. It's an example of some tooling that can improve the + // test developer's workflow. [Fact] public Task Sandbox() { diff --git a/Lombiq.Tests.UI/Extensions/FrontendOrchardCoreUITestExecutorConfigurationExtensions.cs b/Lombiq.Tests.UI/Extensions/FrontendOrchardCoreUITestExecutorConfigurationExtensions.cs index 98631b336..1a6ae5ced 100644 --- a/Lombiq.Tests.UI/Extensions/FrontendOrchardCoreUITestExecutorConfigurationExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/FrontendOrchardCoreUITestExecutorConfigurationExtensions.cs @@ -21,8 +21,8 @@ public static (Uri FrontendUri, Uri BackendUri) GetFrontendAndBackendUris( /// /// Updates the by storing the Vite - /// frontend and the Orchard Core backend URLs as instances. If either parameter is null, that - /// value is not changed. + /// frontend and the Orchard Core backend URLs as instances. If either parameter is , that value is not changed. /// public static void SetFrontendAndBackendUris( this OrchardCoreUITestExecutorConfiguration configuration, diff --git a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs index dad46ac07..8d4e3bdf9 100644 --- a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs @@ -18,7 +18,7 @@ public static class FrontendUITestContextExtensions /// /// Navigates to the backend returned by and presents it as - /// switching to the default tenant. + /// switching to the Default tenant. /// /// /// If the backend URL has not been initialized to something else (e.g. using a custom URL prefix), this is @@ -178,6 +178,7 @@ public static async Task SetupNodeDependenciesAsync( if (!Directory.Exists(projectFilePath)) { + // lang=json await File.WriteAllTextAsync(projectFilePath, "{ \"private\": true }"); } From 8af94acf5fff1f7c09cf8a2d383fee0085481d9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Mon, 18 Nov 2024 04:03:32 +0100 Subject: [PATCH 30/51] Additional note about FrontendUITestBase. --- Lombiq.Tests.UI.Samples/FrontendUITestBase.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lombiq.Tests.UI.Samples/FrontendUITestBase.cs b/Lombiq.Tests.UI.Samples/FrontendUITestBase.cs index 91996ebbe..ed7ee3546 100644 --- a/Lombiq.Tests.UI.Samples/FrontendUITestBase.cs +++ b/Lombiq.Tests.UI.Samples/FrontendUITestBase.cs @@ -13,6 +13,9 @@ namespace Lombiq.Tests.UI.Samples; // ensure the two work together. To make this happen, you need some custom logic to initialize the frontend process // after setup, with a unique port number to avoid clashes. Also the frontend may need the backend URL, whose port is // already randomized with every test. +// In this base class we define a custom "setup and test" method. Creating such a method is a good practice so you don't +// have to configure the FrontendServer over and over again in every test. We placed this method into its own +// intermediate abstract class for better organization, but you can also put it into your UITestBase as well. public abstract class FrontendUITestBase : UITestBase { protected FrontendUITestBase(ITestOutputHelper testOutputHelper) From ed7480dd688047a3aca3fb6d4bac49ad90ffafe9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Mon, 18 Nov 2024 04:07:42 +0100 Subject: [PATCH 31/51] Everywhere it should be spelled "JavaScript". --- .../Tests/FrontendTests.cs | 2 +- ...{JavascriptTests.cs => JavaScriptTests.cs} | 16 +++++----- .../FrontendUITestContextExtensions.cs | 32 +++++++++---------- 3 files changed, 25 insertions(+), 25 deletions(-) rename Lombiq.Tests.UI.Samples/Tests/{JavascriptTests.cs => JavaScriptTests.cs} (85%) diff --git a/Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs b/Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs index f599b5cb9..6136f7f01 100644 --- a/Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs @@ -43,4 +43,4 @@ public Task FrontendServerShouldStartWithTest() => } // END OF TRAINING SECTION: Test headless Orchard Core with a frontend subprocess. -// NEXT STATION: Head over to Tests/JavascriptTests.cs. +// NEXT STATION: Head over to Tests/JavaScriptTests.cs. diff --git a/Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs b/Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.cs similarity index 85% rename from Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs rename to Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.cs index cf0c4b00c..e3156a25f 100644 --- a/Lombiq.Tests.UI.Samples/Tests/JavascriptTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.cs @@ -7,21 +7,21 @@ namespace Lombiq.Tests.UI.Samples.Tests; -// Let's suppose you want to write UI tests in Javascript. Why would you want to do that? Unlikely if you are an Orchard +// Let's suppose you want to write UI tests in JavaScript. Why would you want to do that? Unlikely if you are an Orchard // Core developer, but what if the person responsible for writing the tests is not? In the previous training section we // discussed using a separate frontend server, with mention of technologies using Node.js. In that case the frontend -// developers may be more familiar with Javascript so it makes sense to write and debug the tests in Node.js so they +// developers may be more familiar with JavaScript so it makes sense to write and debug the tests in Node.js so they // don't have to learn different tools and tech stacks just to create some UI tests. -public class JavascriptTests : UITestBase +public class JavaScriptTests : UITestBase { - public JavascriptTests(ITestOutputHelper testOutputHelper) + public JavaScriptTests(ITestOutputHelper testOutputHelper) : base(testOutputHelper) { } // Using this approach you only have to write minimal C# boilerplate, which you can see below. [Fact] - public Task ExampleJavascriptTestShouldWork() => + public Task ExampleJavaScriptTestShouldWork() => ExecuteTestAfterSetupAsync(context => { // Don't forget to mark the script files as "Copy if newer", so they are available to the test. It's best to @@ -33,10 +33,10 @@ public Task ExampleJavascriptTestShouldWork() => // Set up the JS dependencies in the test's temp directory to ensure there are no clashes, then run the // script. This method has an additional parameter to list further NPM dependencies beyond // "selenium-webdriver", if the script requires it. We will check out this script file in the next station. - return context.SetupSeleniumAndExecuteJavascriptTestAsync(_testOutputHelper, scriptPath, workingDirectory); + return context.SetupSeleniumAndExecuteJavaScriptTestAsync(_testOutputHelper, scriptPath, workingDirectory); }); - // To best debug the Javascript code, you may want to set up the site and then invoke node manually. This is not a + // To best debug the JavaScript code, you may want to set up the site and then invoke node manually. This is not a // real test, but it sets up the site in interactive mode (see Tests/InteractiveModeTests.cs for more) with // information on how to start up test scripts from your GUI. It's an example of some tooling that can improve the // test developer's workflow. @@ -53,7 +53,7 @@ public Task Sandbox() var scriptPath = Path.Join(workingDirectory, "test.mjs"); await context.SetupNodeSeleniumAsync(_testOutputHelper, workingDirectory); - await context.SwitchToInteractiveWithJavascriptTestInfoAsync(scriptPath, workingDirectory); + await context.SwitchToInteractiveWithJavaScriptTestInfoAsync(scriptPath, workingDirectory); }, configuration => { diff --git a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs index 8d4e3bdf9..6fd8e09a7 100644 --- a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs @@ -57,7 +57,7 @@ public static string GetDriverPath(this UITestContext context) return Path.Join(service.DriverServicePath, service.DriverServiceExecutableName); } - private static (string WorkingDirectory, string[] Arguments) GetExecuteJavascriptTestPats( + private static (string WorkingDirectory, string[] Arguments) GetExecuteJavaScriptTestPats( this UITestContext context, string scriptPath, string workingDirectory) @@ -84,28 +84,28 @@ static string GetShorterPath(string basePath, string path) } [Obsolete($"Use the overload where the first parameter is {nameof(ITestOutputHelper)}.")] - public static Task ExecuteJavascriptTestAsync( + public static Task ExecuteJavaScriptTestAsync( this UITestContext context, string scriptPath, ITestOutputHelper testOutputHelper) => - context.ExecuteJavascriptTestAsync(testOutputHelper, scriptPath); + context.ExecuteJavaScriptTestAsync(testOutputHelper, scriptPath); /// /// Executes the provided file via node with command line arguments containing the necessary information for /// Selenium JS to take over the browser. /// /// Needed to redirect the node output into the test logs. - /// The Javascript source file to execute using node. + /// The JavaScript source file to execute using node. /// The working directory where node is executed from. - public static async Task ExecuteJavascriptTestAsync( + public static async Task ExecuteJavaScriptTestAsync( this UITestContext context, ITestOutputHelper testOutputHelper, string scriptPath, string workingDirectory = null) { const string command = "node"; - var pipe = testOutputHelper.ToPipeTarget($"{nameof(ExecuteJavascriptTestAsync)}({command})"); - (workingDirectory, var arguments) = context.GetExecuteJavascriptTestPats(scriptPath, workingDirectory); + var pipe = testOutputHelper.ToPipeTarget($"{nameof(ExecuteJavaScriptTestAsync)}({command})"); + (workingDirectory, var arguments) = context.GetExecuteJavaScriptTestPats(scriptPath, workingDirectory); try { @@ -127,33 +127,33 @@ await Cli.Wrap(command) /// /// Invokes with a custom notification - /// message that contains instructions to invoke the Javascript test manually with node. + /// message that contains instructions to invoke the JavaScript test manually with node. /// /// The relative or absolute path pointing to the test script file. /// The path where the test script should be executed, will be converted to absolute. - public static Task SwitchToInteractiveWithJavascriptTestInfoAsync( + public static Task SwitchToInteractiveWithJavaScriptTestInfoAsync( this UITestContext context, string scriptPath, string workingDirectory = null) { - (workingDirectory, var arguments) = context.GetExecuteJavascriptTestPats(scriptPath, workingDirectory); + (workingDirectory, var arguments) = context.GetExecuteJavaScriptTestPats(scriptPath, workingDirectory); return context.SwitchToInteractiveAsync( - $"To start a Javascript test, open a command line terminal at \"{workingDirectory}\" and type the " + + $"To start a JavaScript test, open a command line terminal at \"{workingDirectory}\" and type the " + $"following command: node {string.Join(' ', arguments)}"); } /// - /// Sets up the Javascript dependencies using and then runs the script in the + /// Sets up the JavaScript dependencies using and then runs the script in the /// same temp directory. /// /// - /// The path of the Javascript file to execute with node. Before passing it to , it's transformed into a + /// The path of the JavaScript file to execute with node. Before passing it to , it's transformed into a /// relative path based on the temp directory to conserve path length because long paths can be a problem in some /// operating systems. /// - public static async Task SetupSeleniumAndExecuteJavascriptTestAsync( + public static async Task SetupSeleniumAndExecuteJavaScriptTestAsync( this UITestContext context, ITestOutputHelper testOutputHelper, string scriptPath, @@ -161,7 +161,7 @@ public static async Task SetupSeleniumAndExecuteJavascriptTestAsync( params string[] otherDependencies) { await context.SetupNodeSeleniumAsync(testOutputHelper, workingDirectory, otherDependencies); - await context.ExecuteJavascriptTestAsync(testOutputHelper, scriptPath, workingDirectory); + await context.ExecuteJavaScriptTestAsync(testOutputHelper, scriptPath, workingDirectory); } /// From 674f9a99bd2148903f1be9f985ef1ceaf54f3dcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Mon, 18 Nov 2024 04:16:03 +0100 Subject: [PATCH 32/51] Remove leftover references to Vite. --- ...endOrchardCoreUITestExecutorConfigurationExtensions.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lombiq.Tests.UI/Extensions/FrontendOrchardCoreUITestExecutorConfigurationExtensions.cs b/Lombiq.Tests.UI/Extensions/FrontendOrchardCoreUITestExecutorConfigurationExtensions.cs index 1a6ae5ced..8c412051a 100644 --- a/Lombiq.Tests.UI/Extensions/FrontendOrchardCoreUITestExecutorConfigurationExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/FrontendOrchardCoreUITestExecutorConfigurationExtensions.cs @@ -9,7 +9,7 @@ public static class FrontendOrchardCoreUITestExecutorConfigurationExtensions private const string FrontendUri = nameof(FrontendUri); /// - /// Returns the start URLs for the Vite frontend and the Orchard Core backend from the . /// public static (Uri FrontendUri, Uri BackendUri) GetFrontendAndBackendUris( @@ -20,9 +20,9 @@ public static (Uri FrontendUri, Uri BackendUri) GetFrontendAndBackendUris( ); /// - /// Updates the by storing the Vite - /// frontend and the Orchard Core backend URLs as instances. If either parameter is , that value is not changed. + /// Updates the by storing the frontend and + /// the Orchard Core backend URLs as instances. If either parameter is , + /// that value is not changed. /// public static void SetFrontendAndBackendUris( this OrchardCoreUITestExecutorConfiguration configuration, From 19d309c58ea0dbcdb7a20abfb62c0ed3d7b44e62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Mon, 18 Nov 2024 04:20:44 +0100 Subject: [PATCH 33/51] Fix SwitchToFrontend doc. --- .../Extensions/FrontendUITestContextExtensions.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs index 6fd8e09a7..a52c160f5 100644 --- a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs @@ -3,6 +3,7 @@ using Lombiq.Tests.UI.Services; using OpenQA.Selenium; using OpenQA.Selenium.Remote; +using OrchardCore.Environment.Shell.Scope; using System; using System.IO; using System.Reflection; @@ -33,8 +34,9 @@ public static void SwitchToBackend(this UITestContext context) => /// /// Navigates to the frontend returned by and presents it as - /// switching to a tenant named which is not a real Orchard Core tenant so - /// this information can only be used for information. + /// switching to a tenant named . This is not a real Orchard Core tenant, so + /// this name can only be used for information (for example can't be used with ). /// public static void SwitchToFrontend(this UITestContext context) => context.SwitchCurrentTenant( From 04376220b63d8e17eb65253f1888f0a602e9f40f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Mon, 18 Nov 2024 04:46:07 +0100 Subject: [PATCH 34/51] Pass browser type into ExecuteJavaScriptTestAsync and throw in ui-testing-toolkit.mjs if it's not Chrome. --- .../Extensions/FrontendUITestContextExtensions.cs | 10 +++++++--- Lombiq.Tests.UI/ui-testing-toolkit.mjs | 6 ++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs index a52c160f5..5b90f55ae 100644 --- a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs @@ -59,7 +59,7 @@ public static string GetDriverPath(this UITestContext context) return Path.Join(service.DriverServicePath, service.DriverServiceExecutableName); } - private static (string WorkingDirectory, string[] Arguments) GetExecuteJavaScriptTestPats( + private static (string WorkingDirectory, string[] Arguments) GetExecuteJavaScriptTestPaths( this UITestContext context, string scriptPath, string workingDirectory) @@ -73,6 +73,9 @@ static string GetShorterPath(string basePath, string path) workingDirectory = Path.GetFullPath(workingDirectory ?? Environment.CurrentDirectory); + var browser = context.Configuration.BrowserConfiguration.Browser; + if (browser == Browser.None) browser = default; + var arguments = new[] { "--inspect", @@ -80,6 +83,7 @@ static string GetShorterPath(string basePath, string path) GetShorterPath(workingDirectory, context.GetDriverPath()), context.Driver.Url, GetShorterPath(workingDirectory, context.GetTempSubDirectoryPath()), + browser.ToString(), }; return (workingDirectory, arguments); @@ -107,7 +111,7 @@ public static async Task ExecuteJavaScriptTestAsync( { const string command = "node"; var pipe = testOutputHelper.ToPipeTarget($"{nameof(ExecuteJavaScriptTestAsync)}({command})"); - (workingDirectory, var arguments) = context.GetExecuteJavaScriptTestPats(scriptPath, workingDirectory); + (workingDirectory, var arguments) = context.GetExecuteJavaScriptTestPaths(scriptPath, workingDirectory); try { @@ -138,7 +142,7 @@ public static Task SwitchToInteractiveWithJavaScriptTestInfoAsync( string scriptPath, string workingDirectory = null) { - (workingDirectory, var arguments) = context.GetExecuteJavaScriptTestPats(scriptPath, workingDirectory); + (workingDirectory, var arguments) = context.GetExecuteJavaScriptTestPaths(scriptPath, workingDirectory); return context.SwitchToInteractiveAsync( $"To start a JavaScript test, open a command line terminal at \"{workingDirectory}\" and type the " + diff --git a/Lombiq.Tests.UI/ui-testing-toolkit.mjs b/Lombiq.Tests.UI/ui-testing-toolkit.mjs index d0d5d943e..2b7d76a6c 100644 --- a/Lombiq.Tests.UI/ui-testing-toolkit.mjs +++ b/Lombiq.Tests.UI/ui-testing-toolkit.mjs @@ -76,8 +76,10 @@ async function navigate(driver, url, maxAttempts = 10) { */ async function runTest(test, configureOptions = null) { const args = process.argv.slice(2); - if (args.length !== 3) throw new Error('Usage: node script.js driverPath startUrl tempDirectory'); - const [driverPath, startUrl, tempDirectory] = args; + if (args.length !== 3) throw new Error('Usage: node script.js driverPath startUrl tempDirectory browserName'); + const [driverPath, startUrl, tempDirectory, browserName] = args; + + if (browserName !== 'Chrome') throw new Error("Only Chrome is supported at this time"); let options = new chrome.Options() .addArguments('disable-dev-shm-usage') From b256028a494282cbcca36f07f8a6feb481e5f209 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Mon, 18 Nov 2024 04:55:45 +0100 Subject: [PATCH 35/51] Fix error CP0002 --- .../Extensions/FrontendUITestContextExtensions.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs index 5b90f55ae..77b011917 100644 --- a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs @@ -89,8 +89,9 @@ static string GetShorterPath(string basePath, string path) return (workingDirectory, arguments); } - [Obsolete($"Use the overload where the first parameter is {nameof(ITestOutputHelper)}.")] - public static Task ExecuteJavaScriptTestAsync( + // This uses a different casing of "JavaScript" to avoid breaking backwards compatibility. + [Obsolete($"Use {nameof(ExecuteJavaScriptTestAsync)} instead.")] + public static Task ExecuteJavascriptTestAsync( this UITestContext context, string scriptPath, ITestOutputHelper testOutputHelper) => From 667c55ddc30d2178475bb8e83072f39648ce6251 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Mon, 18 Nov 2024 12:14:27 +0100 Subject: [PATCH 36/51] Rewrite frontend stop code to use forceful cancellation as fallback. --- Lombiq.Tests.UI/Services/FrontendServer.cs | 23 +++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/Lombiq.Tests.UI/Services/FrontendServer.cs b/Lombiq.Tests.UI/Services/FrontendServer.cs index 91e5fb050..8d3b4cf4f 100644 --- a/Lombiq.Tests.UI/Services/FrontendServer.cs +++ b/Lombiq.Tests.UI/Services/FrontendServer.cs @@ -6,6 +6,7 @@ using Lombiq.Tests.UI.Models; using System; using System.Collections.Generic; +using System.Diagnostics; using System.Threading; using System.Threading.Tasks; using Xunit.Abstractions; @@ -89,7 +90,8 @@ public void Configure( frontendUrl: "https://localhost:" + frontendPort.ToTechnicalString(), backendUrl: "https://localhost:" + backendPort.ToTechnicalString()); - var cancellationTokenSource = new CancellationTokenSource(); + var gracefulCancellation = new CancellationTokenSource(); + var forcefulCancellation = new CancellationTokenSource(); var waitCompletionSource = new TaskCompletionSource(); var execute = !skipStartup(context); var waiting = execute && checkProgramReady != null; @@ -114,7 +116,7 @@ public void Configure( var cliTask = cli .WithStandardOutputPipe(pipe) .WithStandardErrorPipe(pipe) - .ExecuteAsync(cancellationTokenSource.Token); + .ExecuteAsync(forcefulCancellation.Token, gracefulCancellation.Token); if (waiting) await WaitForStartupAsync(cliTask, waitCompletionSource.Task, startupTimeout); @@ -124,11 +126,18 @@ public void Configure( Task = cliTask, StopAsync = async () => { - // This cancellation token forcefully closes the frontend server (i.e. SIGTERM, Ctrl+C), which is - // the only way to shut down most of these servers anyway. For this reason there is no need to await - // the task, and trying to do so would throw OperationCanceledException. - await cancellationTokenSource.CancelAsync(); - cancellationTokenSource.Dispose(); + // Attempt to close the process with an interrupt signal (SIGINT, same as hitting Ctrl+C), which is + // the only way to shut down most of these servers as they have an infinite main loop. If that + // fails within a minute, the process is killed forcefully (SIGTERM). + await gracefulCancellation.CancelAsync(); + forcefulCancellation.CancelAfter(TimeSpan.FromMinutes(1)); + + // Using Task.WhenAny() without unwrapping its output avoids the OperationCanceledException, which + // is not relevant in disposal code. + await Task.WhenAny(cliTask); + + gracefulCancellation.Dispose(); + forcefulCancellation.Dispose(); await context.PortLeaseManager.StopLeaseAsync(frontendPort); }, From 6592c4ffa9ad8e9a995f498419c6e485ca436fa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Mon, 18 Nov 2024 12:14:46 +0100 Subject: [PATCH 37/51] unusing --- Lombiq.Tests.UI/Services/FrontendServer.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/Lombiq.Tests.UI/Services/FrontendServer.cs b/Lombiq.Tests.UI/Services/FrontendServer.cs index 8d3b4cf4f..8703a84e1 100644 --- a/Lombiq.Tests.UI/Services/FrontendServer.cs +++ b/Lombiq.Tests.UI/Services/FrontendServer.cs @@ -6,7 +6,6 @@ using Lombiq.Tests.UI.Models; using System; using System.Collections.Generic; -using System.Diagnostics; using System.Threading; using System.Threading.Tasks; using Xunit.Abstractions; From 07c4c99a98e5f79b57a1458b4469f8302ecf8c56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Mon, 18 Nov 2024 19:34:08 +0100 Subject: [PATCH 38/51] Update Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Zoltán Lehóczky --- Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs b/Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs index 6136f7f01..709936896 100644 --- a/Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs @@ -23,7 +23,7 @@ public Task FrontendServerShouldStartWithTest() => async context => { // Don't forget that if you want to interact with the frontend manually, you can just switch the context - // back to the back end and use the interactive mode extension method. + // back to the backend and use the interactive mode extension method. //// context.SwitchToBackend(); //// await context.SwitchToInteractiveAsync(); From f18e0cbb76a6812c61a4642ab3a686bdaf546181 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Mon, 18 Nov 2024 19:47:44 +0100 Subject: [PATCH 39/51] rename --- Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.cs | 6 +++--- .../Tests/{test.mjs => JavaScriptTests.msj} | 0 2 files changed, 3 insertions(+), 3 deletions(-) rename Lombiq.Tests.UI.Samples/Tests/{test.mjs => JavaScriptTests.msj} (100%) diff --git a/Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.cs b/Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.cs index e3156a25f..55175cfe4 100644 --- a/Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.cs @@ -28,7 +28,7 @@ public Task ExampleJavaScriptTestShouldWork() => // include something like the following in your csproj file: // var workingDirectory = "Tests"; - var scriptPath = Path.Join(workingDirectory, "test.mjs"); + var scriptPath = Path.Join(workingDirectory, "JavaScriptTests.msj"); // Set up the JS dependencies in the test's temp directory to ensure there are no clashes, then run the // script. This method has an additional parameter to list further NPM dependencies beyond @@ -50,7 +50,7 @@ public Task Sandbox() async context => { var workingDirectory = "Tests"; - var scriptPath = Path.Join(workingDirectory, "test.mjs"); + var scriptPath = Path.Join(workingDirectory, "JavaScriptTests.msj"); await context.SetupNodeSeleniumAsync(_testOutputHelper, workingDirectory); await context.SwitchToInteractiveWithJavaScriptTestInfoAsync(scriptPath, workingDirectory); @@ -64,4 +64,4 @@ public Task Sandbox() } } -// NEXT STATION: Head over to Tests/test.mjs. +// NEXT STATION: Head over to Tests/JavaScriptTests.msj. diff --git a/Lombiq.Tests.UI.Samples/Tests/test.mjs b/Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.msj similarity index 100% rename from Lombiq.Tests.UI.Samples/Tests/test.mjs rename to Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.msj From 17caf83202c36639f08f7c2145dbb82ec3fda8a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Mon, 18 Nov 2024 20:37:54 +0100 Subject: [PATCH 40/51] EnsureValidOrchardCoreTenantScope --- .../Tests/FrontendTests.cs | 5 +- ...reUITestExecutorConfigurationExtensions.cs | 5 +- .../ShortcutsUITestContextExtensions.cs | 79 +++++++++++++++---- 3 files changed, 70 insertions(+), 19 deletions(-) diff --git a/Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs b/Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs index 709936896..02db66b47 100644 --- a/Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs @@ -22,9 +22,8 @@ public Task FrontendServerShouldStartWithTest() => ExecuteFrontendTestAfterSetupAsync( async context => { - // Don't forget that if you want to interact with the frontend manually, you can just switch the context - // back to the backend and use the interactive mode extension method. - //// context.SwitchToBackend(); + // Don't forget that if you want to interact with the frontend manually, you can use the interactive + // mode extension method. //// await context.SwitchToInteractiveAsync(); await context.ClickReliablyOnAsync(By.LinkText("App_Data/")); diff --git a/Lombiq.Tests.UI/Extensions/FrontendOrchardCoreUITestExecutorConfigurationExtensions.cs b/Lombiq.Tests.UI/Extensions/FrontendOrchardCoreUITestExecutorConfigurationExtensions.cs index 8c412051a..722753229 100644 --- a/Lombiq.Tests.UI/Extensions/FrontendOrchardCoreUITestExecutorConfigurationExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/FrontendOrchardCoreUITestExecutorConfigurationExtensions.cs @@ -1,5 +1,6 @@ using Lombiq.Tests.UI.Services; using System; +using System.Collections.Generic; namespace Lombiq.Tests.UI.Extensions; @@ -15,8 +16,8 @@ public static class FrontendOrchardCoreUITestExecutorConfigurationExtensions public static (Uri FrontendUri, Uri BackendUri) GetFrontendAndBackendUris( this OrchardCoreUITestExecutorConfiguration configuration) => ( - (Uri)configuration.CustomConfiguration[FrontendUri], - (Uri)configuration.CustomConfiguration[BackendUri] + configuration.CustomConfiguration.GetMaybe(FrontendUri) as Uri, + configuration.CustomConfiguration.GetMaybe(BackendUri) as Uri ); /// diff --git a/Lombiq.Tests.UI/Extensions/ShortcutsUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/ShortcutsUITestContextExtensions.cs index e82597740..99a717bf6 100644 --- a/Lombiq.Tests.UI/Extensions/ShortcutsUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/ShortcutsUITestContextExtensions.cs @@ -67,8 +67,11 @@ public static class ShortcutsUITestContextExtensions /// anything else happening on the login page. The target app needs to have Lombiq.Tests.UI.Shortcuts /// enabled. /// - public static Task SignInDirectlyAsync(this UITestContext context, string userName = DefaultUser.UserName) => - context.GoToAsync(controller => controller.SignInDirectly(userName)); + public static Task SignInDirectlyAsync(this UITestContext context, string userName = DefaultUser.UserName) + { + context.EnsureValidOrchardCoreTenantScope(); + return context.GoToAsync(controller => controller.SignInDirectly(userName)); + } /// /// Authenticates the client with the default user account and navigates to the given URL. Note that this will @@ -78,8 +81,8 @@ public static Task SignInDirectlyAsync(this UITestContext context, string userNa public static Task SignInDirectlyAndGoToRelativeUrlAsync( this UITestContext context, string relativeUrl, - bool onlyIfNotAlreadyThere = true) - => context.SignInDirectlyAndGoToRelativeUrlAsync(DefaultUser.UserName, relativeUrl, onlyIfNotAlreadyThere); + bool onlyIfNotAlreadyThere = true) => + context.SignInDirectlyAndGoToRelativeUrlAsync(DefaultUser.UserName, relativeUrl, onlyIfNotAlreadyThere); /// /// Authenticates the client with the given user account and navigates to the given URL. Note that this will execute @@ -100,8 +103,11 @@ public static async Task SignInDirectlyAndGoToRelativeUrlAsync( /// Signs the client out. Note that this will execute a direct sign in without anything else happening on the logoff /// page. The target app needs to have Lombiq.Tests.UI.Shortcuts enabled. /// - public static Task SignOutDirectlyAsync(this UITestContext context) => - context.GoToAsync(controller => controller.SignOutDirectly()); + public static Task SignOutDirectlyAsync(this UITestContext context) + { + context.EnsureValidOrchardCoreTenantScope(); + return context.GoToAsync(controller => controller.SignOutDirectly()); + } /// /// Retrieves the currently authenticated user's name, if any. The target app needs to have @@ -110,6 +116,7 @@ public static Task SignOutDirectlyAsync(this UITestContext context) => /// The currently authenticated user's name, empty or null string if the user is anonymous. public static async Task GetCurrentUserNameAsync(this UITestContext context) { + context.EnsureValidOrchardCoreTenantScope(); await context.GoToAsync(controller => controller.Index()); var userNameContainer = context.GetText(By.CssSelector("pre")); if (userNameContainer == "Unauthenticated") return string.Empty; @@ -306,6 +313,7 @@ public static Task DisableFeatureDirectlyAsync( /// public static async Task ExecuteAndAssertTestFeatureToggleAsync(this UITestContext context) { + context.EnsureValidOrchardCoreTenantScope(); await context.EnableFeatureDirectlyAsync(FeatureToggleTestBench); await context.GoToRelativeUrlAsync(FeatureToggleTestBenchUrl); context.Scope.Driver.PageSource.ShouldContain("The Feature Toggle Test Bench worked."); @@ -329,6 +337,7 @@ public static async Task PurgeMediaCacheDirectlyAsync(this UITestContext context await context.EnableFeatureDirectlyAsync(MediaCachePurge); } + context.EnsureValidOrchardCoreTenantScope(); await context.GoToAsync(controller => controller.PurgeMediaCacheDirectly()); if (toggleTheFeature) @@ -401,11 +410,16 @@ await recipeEnvironmentProviders /// Navigates to a page whose action method throws . This causes ASP.NET Core /// to display an error page. /// - public static Task GoToErrorPageDirectlyAsync(this UITestContext context) => - context.GoToAsync(controller => controller.Index()); + public static Task GoToErrorPageDirectlyAsync(this UITestContext context) + { + context.EnsureValidOrchardCoreTenantScope(); + return context.GoToAsync(controller => controller.Index()); + } private static IShortcutsApi GetApi(this UITestContext context) { + context.EnsureValidOrchardCoreTenantScope(); + // If there is a subdirectory-like URL prefix (e.g. for tenants) in the scope base URI, the requests will have // double slashes that results in 404 error. So the trailing slash has to be trimmed out. var baseUri = new Uri(context.Scope.BaseUri.ToString().TrimEnd('/')); @@ -627,6 +641,8 @@ await UsingScopeAsync( /// public static async Task SwitchToInteractiveAsync(this UITestContext context, string notificationHtml = null) { + context.EnsureValidOrchardCoreTenantScope(); + InteractiveModeHasBeenUsed = true; await context.EnterInteractiveModeAsync(notificationHtml); await context.WaitInteractiveModeAsync(); @@ -683,14 +699,20 @@ private static Task UsingScopeAsync( UITestContext context, Func execute, string tenant, - bool activateShell) => - context.Application.UsingScopeAsync(execute, tenant ?? context.TenantName, activateShell); + bool activateShell) + { + tenant ??= context.TenantName; + if (tenant.StartsWith('!')) tenant = ShellSettings.DefaultShellName; + + return context.Application.UsingScopeAsync(execute, tenant, activateShell); + } /// /// Places the provided into a recipe and executes it with JSON Import. /// public static async Task ExecuteJsonRecipeAsync(this UITestContext context, params object[] steps) { + context.EnsureValidOrchardCoreTenantScope(); await context.GoToAdminRelativeUrlAsync("/DeploymentPlan/Import/Json"); var json = JsonSerializer.Serialize(new { steps }); @@ -725,13 +747,42 @@ public static Task ExecuteJsonRecipeSiteSettingAsync(this UITestContext conte /// Sets the time shift to a specific value. If both and are /// provided, then the values are added together. /// - public static Task SetShiftTimeAsync(this UITestContext context, double days = 0, double seconds = 0) => - context.GoToAsync(controller => controller.Set(days, seconds)); + public static Task SetShiftTimeAsync(this UITestContext context, double days = 0, double seconds = 0) + { + context.EnsureValidOrchardCoreTenantScope(); + return context.GoToAsync(controller => controller.Set(days, seconds)); + } /// /// Adds the specified value to the time shift. If both and are /// provided, then the values for both are added. Negative values are supported as well. /// - public static Task AddShiftTimeAsync(this UITestContext context, double days = 0, double seconds = 0) => - context.GoToAsync(controller => controller.Set(days, seconds)); + public static Task AddShiftTimeAsync(this UITestContext context, double days = 0, double seconds = 0) + { + context.EnsureValidOrchardCoreTenantScope(); + return context.GoToAsync(controller => controller.Set(days, seconds)); + } + + /// + /// Switches the current tenant to if it's not a real Orchard Core + /// tenant but some other technical scope. + /// + /// + /// Real Orchard Core tenant names can't contain the ! character. So if the starts with it, this indicates we are in some other non-OC scope, like the + /// frontend of . + /// + public static void EnsureValidOrchardCoreTenantScope(this UITestContext context) + { + if (!context.TenantName.StartsWith('!')) return; + + if (context.Configuration.GetFrontendAndBackendUris().BackendUri is { }) + { + context.SwitchToBackend(); + } + else + { + context.SwitchCurrentTenantToDefault(); + } + } } From f7cf0aefba75253c9c3178ebcbe0020ac9630121 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Tue, 19 Nov 2024 00:52:16 +0100 Subject: [PATCH 41/51] Fix typo. --- Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.cs | 2 +- .../Tests/{JavaScriptTests.msj => JavaScriptTests.mjs} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename Lombiq.Tests.UI.Samples/Tests/{JavaScriptTests.msj => JavaScriptTests.mjs} (93%) diff --git a/Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.cs b/Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.cs index 55175cfe4..28f806867 100644 --- a/Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.cs @@ -64,4 +64,4 @@ public Task Sandbox() } } -// NEXT STATION: Head over to Tests/JavaScriptTests.msj. +// NEXT STATION: Head over to Tests/JavaScriptTests.mjs. diff --git a/Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.msj b/Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.mjs similarity index 93% rename from Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.msj rename to Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.mjs index 52fe3d969..b80c76273 100644 --- a/Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.msj +++ b/Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.mjs @@ -1,7 +1,7 @@ import { By, until } from 'selenium-webdriver'; // This dependency is copied into the build directory by Lombiq.Tests.UI. -import { runTest, shouldContainText, navigate } from './ui-testing-toolkit.mjs'; +import { runTest, shouldContainText, navigate } from '../ui-testing-toolkit.mjs'; // This function automatically handles the command line arguments and sets up a Chrome driver. await runTest(async (driver, startUrl) => { From ae5b7c32361d96cf90c86978fc99c4500e24e72e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Tue, 19 Nov 2024 00:56:23 +0100 Subject: [PATCH 42/51] Add SandboxAfterSetupAsync --- Lombiq.Tests.UI/OrchardCoreUITestBase.cs | 28 ++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/Lombiq.Tests.UI/OrchardCoreUITestBase.cs b/Lombiq.Tests.UI/OrchardCoreUITestBase.cs index db3188e03..14305f26d 100644 --- a/Lombiq.Tests.UI/OrchardCoreUITestBase.cs +++ b/Lombiq.Tests.UI/OrchardCoreUITestBase.cs @@ -4,6 +4,7 @@ using Lombiq.Tests.UI.Helpers; using Lombiq.Tests.UI.Models; using Lombiq.Tests.UI.Services; +using Lombiq.Tests.UI.Services.GitHub; using SixLabors.ImageSharp; using System; using System.Threading.Tasks; @@ -386,4 +387,31 @@ await ExecuteOrchardCoreTestAsync( testManifest, configuration); } + + /// + /// Starts a "test" with configuration adjustments to make it more suitable for interactive exploration of the UI + /// testing setup. In a GitHub Actions environment this method does nothing. + /// + protected Task SandboxAfterSetupAsync( + Func testAsync = null, + Browser browser = default, + Func changeConfigurationAsync = null) + { + // This "test" will wait indefinitely, so it's important to skip it in CI. + if (GitHubHelper.IsGitHubEnvironment) return Task.CompletedTask; + + testAsync ??= context => context.SwitchToInteractiveAsync(); + + return ExecuteTestAfterSetupAsync( + testAsync, + browser, + configuration => + { + // Since this made for human interaction, make sure the browser is always displayed and disable retries. + configuration.BrowserConfiguration.Headless = false; + configuration.MaxRetryCount = 0; + + return changeConfigurationAsync == null ? Task.CompletedTask : changeConfigurationAsync(configuration); + }); + } } From b01363ec6d4cd5c7a03904f576da652c799c4caf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Tue, 19 Nov 2024 00:56:47 +0100 Subject: [PATCH 43/51] Rewrite to start from working directory. --- .../Tests/JavaScriptTests.cs | 37 +++++-------------- .../FrontendUITestContextExtensions.cs | 13 +++++-- Lombiq.Tests.UI/ui-testing-toolkit.mjs | 2 +- 3 files changed, 20 insertions(+), 32 deletions(-) diff --git a/Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.cs b/Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.cs index 28f806867..8e1cc2077 100644 --- a/Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.cs @@ -1,5 +1,4 @@ using Lombiq.Tests.UI.Extensions; -using Lombiq.Tests.UI.Services.GitHub; using System.IO; using System.Threading.Tasks; using Xunit; @@ -24,16 +23,15 @@ public JavaScriptTests(ITestOutputHelper testOutputHelper) public Task ExampleJavaScriptTestShouldWork() => ExecuteTestAfterSetupAsync(context => { - // Don't forget to mark the script files as "Copy if newer", so they are available to the test. It's best to - // include something like the following in your csproj file: + // Don't forget to mark the script files as "Copy if newer", so they are available to the test. If you + // include something like the following in your csproj file, then you only have to do this once: // - var workingDirectory = "Tests"; - var scriptPath = Path.Join(workingDirectory, "JavaScriptTests.msj"); + var scriptPath = Path.Join("Tests", "JavaScriptTests.mjs"); // Set up the JS dependencies in the test's temp directory to ensure there are no clashes, then run the // script. This method has an additional parameter to list further NPM dependencies beyond // "selenium-webdriver", if the script requires it. We will check out this script file in the next station. - return context.SetupSeleniumAndExecuteJavaScriptTestAsync(_testOutputHelper, scriptPath, workingDirectory); + return context.SetupSeleniumAndExecuteJavaScriptTestAsync(_testOutputHelper, scriptPath); }); // To best debug the JavaScript code, you may want to set up the site and then invoke node manually. This is not a @@ -41,27 +39,12 @@ public Task ExampleJavaScriptTestShouldWork() => // information on how to start up test scripts from your GUI. It's an example of some tooling that can improve the // test developer's workflow. [Fact] - public Task Sandbox() - { - // This "test" will wait indefinitely, so it's important to skip it in CI. - if (GitHubHelper.IsGitHubEnvironment) return Task.CompletedTask; - - return ExecuteTestAfterSetupAsync( - async context => - { - var workingDirectory = "Tests"; - var scriptPath = Path.Join(workingDirectory, "JavaScriptTests.msj"); - - await context.SetupNodeSeleniumAsync(_testOutputHelper, workingDirectory); - await context.SwitchToInteractiveWithJavaScriptTestInfoAsync(scriptPath, workingDirectory); - }, - configuration => - { - // Since this is an interactive "test", make sure the browser is always displayed. - configuration.BrowserConfiguration.Headless = false; - return Task.CompletedTask; - }); - } + public Task Sandbox() => + SandboxAfterSetupAsync(async context => + { + await context.SetupNodeSeleniumAsync(_testOutputHelper); + await context.SwitchToInteractiveWithJavaScriptTestInfoAsync(Path.Join("Tests", "JavaScriptTests.mjs")); + }); } // NEXT STATION: Head over to Tests/JavaScriptTests.mjs. diff --git a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs index 77b011917..b37371b36 100644 --- a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs @@ -164,7 +164,7 @@ public static async Task SetupSeleniumAndExecuteJavaScriptTestAsync( this UITestContext context, ITestOutputHelper testOutputHelper, string scriptPath, - string workingDirectory, + string workingDirectory = null, params string[] otherDependencies) { await context.SetupNodeSeleniumAsync(testOutputHelper, workingDirectory, otherDependencies); @@ -205,13 +205,18 @@ await Cli.Wrap("pnpm") public static Task SetupNodeSeleniumAsync( this UITestContext context, ITestOutputHelper helper, - string workingDirectory, + string workingDirectory = null, params string[] otherDependencies) { + workingDirectory ??= Environment.CurrentDirectory; + + // First, copy out helper script if the working directory isn't already the current directory. const string uiTestingToolkitScript = "ui-testing-toolkit.mjs"; - if (File.Exists(uiTestingToolkitScript)) + var copyFrom = Path.GetFullPath(uiTestingToolkitScript); + var copyTo = Path.GetFullPath(Path.Join(workingDirectory, uiTestingToolkitScript)); + if (copyFrom != copyTo && File.Exists(uiTestingToolkitScript)) { - File.Copy(uiTestingToolkitScript, Path.Join(workingDirectory, uiTestingToolkitScript), overwrite: true); + File.Copy(copyFrom, copyTo, overwrite: true); } return context.SetupNodeDependenciesAsync(helper, workingDirectory, ["selenium-webdriver", .. otherDependencies]); diff --git a/Lombiq.Tests.UI/ui-testing-toolkit.mjs b/Lombiq.Tests.UI/ui-testing-toolkit.mjs index 2b7d76a6c..3b9df69bf 100644 --- a/Lombiq.Tests.UI/ui-testing-toolkit.mjs +++ b/Lombiq.Tests.UI/ui-testing-toolkit.mjs @@ -76,7 +76,7 @@ async function navigate(driver, url, maxAttempts = 10) { */ async function runTest(test, configureOptions = null) { const args = process.argv.slice(2); - if (args.length !== 3) throw new Error('Usage: node script.js driverPath startUrl tempDirectory browserName'); + if (args.length !== 4) throw new Error('Usage: node script.js driverPath startUrl tempDirectory browserName'); const [driverPath, startUrl, tempDirectory, browserName] = args; if (browserName !== 'Chrome') throw new Error("Only Chrome is supported at this time"); From 2b6d92caa9388785ad6763df254339b4faf88e8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Tue, 19 Nov 2024 01:43:35 +0100 Subject: [PATCH 44/51] Pass all arguments from the C# driver. --- .../FrontendUITestContextExtensions.cs | 6 ++++ .../Services/BrowserConfiguration.cs | 6 ++++ Lombiq.Tests.UI/Services/WebDriverFactory.cs | 11 +++++++ Lombiq.Tests.UI/ui-testing-toolkit.mjs | 29 +++++++++---------- 4 files changed, 37 insertions(+), 15 deletions(-) diff --git a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs index b37371b36..77a6e34ca 100644 --- a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs @@ -7,6 +7,7 @@ using System; using System.IO; using System.Reflection; +using System.Text.Json; using System.Threading.Tasks; using Xunit.Abstractions; @@ -116,6 +117,11 @@ public static async Task ExecuteJavaScriptTestAsync( try { + var browserArguments = context.Configuration.BrowserConfiguration.Arguments; + await File.WriteAllTextAsync( + context.GetTempSubDirectoryPath("BrowserArguments.json"), + JsonSerializer.Serialize(browserArguments)); + await Cli.Wrap(command) .WithArguments(arguments) .WithStandardOutputPipe(pipe) diff --git a/Lombiq.Tests.UI/Services/BrowserConfiguration.cs b/Lombiq.Tests.UI/Services/BrowserConfiguration.cs index 6a058e1aa..e7cedaa3a 100644 --- a/Lombiq.Tests.UI/Services/BrowserConfiguration.cs +++ b/Lombiq.Tests.UI/Services/BrowserConfiguration.cs @@ -2,6 +2,7 @@ using Lombiq.Tests.UI.Models; using SixLabors.ImageSharp; using System; +using System.Collections.Generic; using System.Globalization; namespace Lombiq.Tests.UI.Services; @@ -43,4 +44,9 @@ public class BrowserConfiguration /// Gets or sets the fake camera video source information. See: . /// public FakeBrowserVideoSource FakeVideoSource { get; set; } + + /// + /// Gets a list of command line arguments that were passed to the driver during the driver instance creation. + /// + public IList Arguments { get; } = []; } diff --git a/Lombiq.Tests.UI/Services/WebDriverFactory.cs b/Lombiq.Tests.UI/Services/WebDriverFactory.cs index db376e545..f65de85e5 100644 --- a/Lombiq.Tests.UI/Services/WebDriverFactory.cs +++ b/Lombiq.Tests.UI/Services/WebDriverFactory.cs @@ -8,8 +8,10 @@ using OpenQA.Selenium.Firefox; using OpenQA.Selenium.IE; using System; +using System.Collections.Generic; using System.IO; using System.Linq; +using System.Reflection; using System.Threading.Tasks; namespace Lombiq.Tests.UI.Services; @@ -55,6 +57,8 @@ Task> CreateDriverInnerAsync(string driverPath = null) => // Helps with misconfigured hosts. if (chromeConfig.Service.HostName == "localhost") chromeConfig.Service.HostName = "127.0.0.1"; + configuration.Arguments.SetItems(chromeConfig.Options.Arguments); + return new ChromeDriver(chromeConfig.Service, chromeConfig.Options, pageLoadTimeout) .SetCommonTimeouts(pageLoadTimeout); }); @@ -89,6 +93,8 @@ public static Task> CreateEdgeDriverAsync(BrowserConfiguration var service = EdgeDriverService.CreateDefaultService(); service.SuppressInitialDiagnosticInformation = true; + configuration.Arguments.SetItems(options.Arguments); + return () => new EdgeDriver(service, options).SetCommonTimeouts(pageLoadTimeout); }); @@ -112,6 +118,11 @@ public static Task> CreateFirefoxDriverAsync(BrowserConfigur configuration.BrowserOptionsConfigurator?.Invoke(options); + var arguments = typeof(FirefoxOptions) + .GetField("firefoxArguments", BindingFlags.Instance | BindingFlags.NonPublic)? + .GetValue(options) as IList ?? []; + configuration.Arguments.SetItems(arguments); + return new FirefoxDriver(options).SetCommonTimeouts(pageLoadTimeout); })); diff --git a/Lombiq.Tests.UI/ui-testing-toolkit.mjs b/Lombiq.Tests.UI/ui-testing-toolkit.mjs index 3b9df69bf..be096f58c 100644 --- a/Lombiq.Tests.UI/ui-testing-toolkit.mjs +++ b/Lombiq.Tests.UI/ui-testing-toolkit.mjs @@ -1,9 +1,14 @@ import path from 'path'; import process from 'process'; +import fs from 'node:fs/promises'; import { By, WebDriver, WebElement } from 'selenium-webdriver'; import chrome from 'selenium-webdriver/chrome.js'; import { writeFile } from 'node:fs/promises' +function _exists(path) { + return fs.stat(path).then(() => true, () => false); +} + async function _logSource(driver) { const html = await driver.getPageSource(); console.log('HTML:', html.replace(/\s*[\n\r]+\s*/g, ' ')); @@ -81,21 +86,15 @@ async function runTest(test, configureOptions = null) { if (browserName !== 'Chrome') throw new Error("Only Chrome is supported at this time"); - let options = new chrome.Options() - .addArguments('disable-dev-shm-usage') - .addArguments('unsafely-disable-devtools-self-xss-warnings') - .addArguments('disable-search-engine-choice-screen') - .addArguments('--lang=en-US') - .addArguments('disable-accelerated-2d-canvas') - .addArguments('disable-gpu') - .addArguments('force-color-profile=sRGB') - .addArguments('force-device-scale-factor=1') - .addArguments('high-dpi-support=1') - .addArguments('disable-smooth-scrolling') - .addArguments('ignore-certificate-errors') - .addArguments('--ignore-certificate-errors') - .addArguments('--no-sandbox') - ; + let options = new chrome.Options(); + + const argumentsPath = path.join(tempDirectory, 'BrowserArguments.json'); + if (await _exists(argumentsPath)) { + JSON.parse(await fs.readFile(argumentsPath, { encoding: 'utf8' })) + .forEach(argument => { + options = options.addArguments(argument); + }); + } if (process.env.GITHUB_ENV) options = options.addArguments('headless'); if (configureOptions) options = configureOptions(options) ?? options; From 70ac4caa3e627124b45e3e57299d41c086c73620 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Tue, 19 Nov 2024 01:58:53 +0100 Subject: [PATCH 45/51] spelling --- Lombiq.Tests.UI/Services/WebDriverFactory.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/Services/WebDriverFactory.cs b/Lombiq.Tests.UI/Services/WebDriverFactory.cs index f65de85e5..0753c142f 100644 --- a/Lombiq.Tests.UI/Services/WebDriverFactory.cs +++ b/Lombiq.Tests.UI/Services/WebDriverFactory.cs @@ -119,7 +119,7 @@ public static Task> CreateFirefoxDriverAsync(BrowserConfigur configuration.BrowserOptionsConfigurator?.Invoke(options); var arguments = typeof(FirefoxOptions) - .GetField("firefoxArguments", BindingFlags.Instance | BindingFlags.NonPublic)? + .GetField("firefoxArguments", BindingFlags.Instance | BindingFlags.NonPublic)? // #spell-check-ignore-line .GetValue(options) as IList ?? []; configuration.Arguments.SetItems(arguments); From 03da57a576af810c51092a7d3e3f783d4a6a2822 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Tue, 19 Nov 2024 02:07:40 +0100 Subject: [PATCH 46/51] Code cleanup. --- .../Extensions/FrontendUITestContextExtensions.cs | 5 +++++ Lombiq.Tests.UI/Services/WebDriverFactory.cs | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs index 77a6e34ca..eb4c29296 100644 --- a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs @@ -5,6 +5,7 @@ using OpenQA.Selenium.Remote; using OrchardCore.Environment.Shell.Scope; using System; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Reflection; using System.Text.Json; @@ -13,6 +14,10 @@ namespace Lombiq.Tests.UI.Extensions; +[SuppressMessage( + "Naming", + "CA1708:Identifiers should differ by more than case", + Justification = "Needed for backwards compatibility, remove after the next major release.")] public static class FrontendUITestContextExtensions { public const string FrontendPseudoTenantName = "!Frontend"; diff --git a/Lombiq.Tests.UI/Services/WebDriverFactory.cs b/Lombiq.Tests.UI/Services/WebDriverFactory.cs index 0753c142f..ce61c68de 100644 --- a/Lombiq.Tests.UI/Services/WebDriverFactory.cs +++ b/Lombiq.Tests.UI/Services/WebDriverFactory.cs @@ -118,8 +118,11 @@ public static Task> CreateFirefoxDriverAsync(BrowserConfigur configuration.BrowserOptionsConfigurator?.Invoke(options); + // For some reason FirefoxOptions does not expose the argument list like the Chromium-based driver + // options classes do. + const string argumentsFieldName = "firefoxArguments"; // #spell-check-ignore-line var arguments = typeof(FirefoxOptions) - .GetField("firefoxArguments", BindingFlags.Instance | BindingFlags.NonPublic)? // #spell-check-ignore-line + .GetField(argumentsFieldName, BindingFlags.Instance | BindingFlags.NonPublic)? .GetValue(options) as IList ?? []; configuration.Arguments.SetItems(arguments); From a3a35c7d14e16504d3ef73903b71038b4a7f21a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Tue, 19 Nov 2024 02:31:02 +0100 Subject: [PATCH 47/51] Looks like "ignore-certificate-errors" is needed explicitly. --- Lombiq.Tests.UI/ui-testing-toolkit.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/ui-testing-toolkit.mjs b/Lombiq.Tests.UI/ui-testing-toolkit.mjs index be096f58c..6de85f563 100644 --- a/Lombiq.Tests.UI/ui-testing-toolkit.mjs +++ b/Lombiq.Tests.UI/ui-testing-toolkit.mjs @@ -86,7 +86,7 @@ async function runTest(test, configureOptions = null) { if (browserName !== 'Chrome') throw new Error("Only Chrome is supported at this time"); - let options = new chrome.Options(); + let options = new chrome.Options().addArguments('ignore-certificate-errors'); const argumentsPath = path.join(tempDirectory, 'BrowserArguments.json'); if (await _exists(argumentsPath)) { From f3fad2af42c4044537d26b843b87d4d997e551f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Tue, 19 Nov 2024 20:28:18 +0100 Subject: [PATCH 48/51] Spelling --- Lombiq.Tests.UI.Samples/Readme.md | 2 +- Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.mjs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Lombiq.Tests.UI.Samples/Readme.md b/Lombiq.Tests.UI.Samples/Readme.md index e23e1bce0..c02001648 100644 --- a/Lombiq.Tests.UI.Samples/Readme.md +++ b/Lombiq.Tests.UI.Samples/Readme.md @@ -32,7 +32,7 @@ For general details about and on using the Toolbox see the [root Readme](../Read - [Testing remote apps](Tests/RemoteTests.cs) - [Testing time-dependent functionality](Tests/ShiftTimeTests.cs) - [Test headless Orchard Core with a frontend subprocess](FrontendUITestBase.cs) -- [Executing tests written in Javascript](Tests/JavascriptTests.cs) +- [Executing tests written in JavaScript](Tests/JavaScriptTests.cs) ## Adding new tutorials diff --git a/Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.mjs b/Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.mjs index b80c76273..8f0cf3b87 100644 --- a/Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.mjs +++ b/Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.mjs @@ -5,7 +5,7 @@ import { runTest, shouldContainText, navigate } from '../ui-testing-toolkit.mjs' // This function automatically handles the command line arguments and sets up a Chrome driver. await runTest(async (driver, startUrl) => { - // Inside you can use all normal Selenium Javascript code, e.g.: + // Inside you can use all normal Selenium JavaScript code, e.g.: // - https://www.selenium.dev/selenium/docs/api/javascript/WebDriver.html // - https://www.selenium.dev/selenium/docs/api/javascript/By.html await driver.findElement(By.xpath("//a[@href = '/blog/post-1']")).click(); @@ -23,4 +23,4 @@ await runTest(async (driver, startUrl) => { await driver.findElement(By.xpath("id('footer')//a[@href='https://lombiq.com/']")); }); -// END OF TRAINING SECTION: Executing tests written in Javascript. +// END OF TRAINING SECTION: Executing tests written in JavaScript. From 4d0fcb17cccd8724018b0e2ec8bfe878e0f8e3e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Tue, 19 Nov 2024 20:28:30 +0100 Subject: [PATCH 49/51] Simplifying name --- Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs index eb4c29296..fa9e68c61 100644 --- a/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/FrontendUITestContextExtensions.cs @@ -42,7 +42,7 @@ public static void SwitchToBackend(this UITestContext context) => /// cref="FrontendOrchardCoreUITestExecutorConfigurationExtensions.GetFrontendAndBackendUris"/> and presents it as /// switching to a tenant named . This is not a real Orchard Core tenant, so /// this name can only be used for information (for example can't be used with ). + /// cref="IWebApplicationInstanceExtensions.UsingScopeAsync(IWebApplicationInstance,Func{ShellScope,Task},string,bool)"/>). /// public static void SwitchToFrontend(this UITestContext context) => context.SwitchCurrentTenant( From cbef0edd663c3d03a20e0de9969fae677c3bcb8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Tue, 19 Nov 2024 20:36:48 +0100 Subject: [PATCH 50/51] Code styling --- Lombiq.Tests.UI/ui-testing-toolkit.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/ui-testing-toolkit.mjs b/Lombiq.Tests.UI/ui-testing-toolkit.mjs index 6de85f563..0f793b550 100644 --- a/Lombiq.Tests.UI/ui-testing-toolkit.mjs +++ b/Lombiq.Tests.UI/ui-testing-toolkit.mjs @@ -84,7 +84,7 @@ async function runTest(test, configureOptions = null) { if (args.length !== 4) throw new Error('Usage: node script.js driverPath startUrl tempDirectory browserName'); const [driverPath, startUrl, tempDirectory, browserName] = args; - if (browserName !== 'Chrome') throw new Error("Only Chrome is supported at this time"); + if (browserName !== 'Chrome') throw new Error('Only Chrome is supported at this time.'); let options = new chrome.Options().addArguments('ignore-certificate-errors'); From 7c9239f6f8fd367b89a30ea264d00f407370fd52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ra=20El-Saig?= Date: Tue, 19 Nov 2024 21:19:39 +0100 Subject: [PATCH 51/51] Rename SandboxAfterSetupAsync to OpenSandboxAfterSetupAsync. --- Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.cs | 2 +- Lombiq.Tests.UI/OrchardCoreUITestBase.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.cs b/Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.cs index 8e1cc2077..36efc2d1d 100644 --- a/Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.cs @@ -40,7 +40,7 @@ public Task ExampleJavaScriptTestShouldWork() => // test developer's workflow. [Fact] public Task Sandbox() => - SandboxAfterSetupAsync(async context => + OpenSandboxAfterSetupAsync(async context => { await context.SetupNodeSeleniumAsync(_testOutputHelper); await context.SwitchToInteractiveWithJavaScriptTestInfoAsync(Path.Join("Tests", "JavaScriptTests.mjs")); diff --git a/Lombiq.Tests.UI/OrchardCoreUITestBase.cs b/Lombiq.Tests.UI/OrchardCoreUITestBase.cs index 14305f26d..0ab0cc5c0 100644 --- a/Lombiq.Tests.UI/OrchardCoreUITestBase.cs +++ b/Lombiq.Tests.UI/OrchardCoreUITestBase.cs @@ -392,7 +392,7 @@ await ExecuteOrchardCoreTestAsync( /// Starts a "test" with configuration adjustments to make it more suitable for interactive exploration of the UI /// testing setup. In a GitHub Actions environment this method does nothing. /// - protected Task SandboxAfterSetupAsync( + protected Task OpenSandboxAfterSetupAsync( Func testAsync = null, Browser browser = default, Func changeConfigurationAsync = null)