Merge pull request #402 from Lombiq/issue/LMBQ-178

LMBQ-178: Enhancing support for remote tests working with apps behind Cloudflare

Lombiq.Tests.UI.Samples/Tests/RemoteTests.cs

@@ -16,7 +16,10 @@ namespace Lombiq.Tests.UI.Samples.Tests;
 // smoke tests on your production app (e.g.: Can people still log in? Are payments still working?). The UI Testing
 // Toolbox also supports this. Check out the example below!
 
-// Note how the test derives from RemoteUITestBase this time, not UITestBase.
+// Note how the test derives from RemoteUITestBase this time, not UITestBase. This is a generic base class for any
+// remote test. However, if you're testing an app behind Cloudflare, you might see requests rejected with an HTTP 403
+// due to Cloudflare's Bot Fight Mode; in that case, derive from CloudflareRemoteUITestBase instead after checking out
+// its documentation on how to set it up.
 public class RemoteTests : RemoteUITestBase
 {
     public RemoteTests(ITestOutputHelper testOutputHelper)

Lombiq.Tests.UI/CloudflareRemoteUITestBase.cs

@@ -0,0 +1,80 @@
+using Lombiq.Tests.UI.Helpers;
+using Lombiq.Tests.UI.Services;
+using System;
+using System.Threading.Tasks;
+using Xunit.Abstractions;
+
+namespace Lombiq.Tests.UI;
+
+/// <summary>
+/// Base class for UI tests that run on a remote (i.e. not locally running) app behind Cloudflare. Gets around
+/// Cloudflare rejecting automated requests.
+/// </summary>
+public abstract class CloudflareRemoteUITestBase : RemoteUITestBase
+{
+    /// <summary>
+    /// Gets the Cloudflare account's ID, necessary for Cloudflare API calls. Note that due to how the IP Access Rule
+    /// management works, you may only have tests for apps behind a single Cloudflare account in a given test project.
+    /// </summary>
+    /// <remarks>
+    /// <para>
+    /// You can look up your Cloudflare account's ID from the Cloudflare dashboard following the <see
+    /// href="https://developers.cloudflare.com/fundamentals/setup/find-account-and-zone-ids/">Cloudflare
+    /// documentation</see>.
+    /// </para>
+    /// </remarks>
+    protected abstract string CloudflareAccountId { get; }
+
+    /// <summary>
+    /// Gets the Cloudflare API token to use for setting up IP Access Rules, so the machine running the tests won't get
+    /// its requests rejected.
+    /// </summary>
+    /// <remarks>
+    /// <para>
+    /// Create an API token following the <see
+    /// ref="https://developers.cloudflare.com/fundamentals/api/get-started/create-token/">Cloudflare
+    /// documentation</see>. It only has to have the account-level Account Firewall Access Rules permission with Edit
+    /// rights.
+    /// </para>
+    /// <para>
+    /// You can configure the API token by overriding this property in your test class and setting it from any custom
+    /// environment value, or by setting the default <c>Lombiq_Tests_UI__CloudflareApiToken</c> environment variable. We
+    /// don't recommend hard-coding the token.
+    /// </para>
+    /// </remarks>
+    protected virtual string CloudflareApiToken => TestConfigurationManager.GetConfiguration("CloudflareApiToken");
+
+    protected CloudflareRemoteUITestBase(ITestOutputHelper testOutputHelper)
+        : base(testOutputHelper)
+    {
+    }
+
+    protected override Task ExecuteTestAsync(
+        Uri baseUri,
+        Func<UITestContext, Task> testAsync,
+        Browser browser,
+        Func<OrchardCoreUITestExecutorConfiguration, Task> changeConfigurationAsync)
+    {
+        async Task ChangeConfigurationForCloudflareAsync(OrchardCoreUITestExecutorConfiguration configuration)
+        {
+            // Cloudflare's e-mail address obfuscating feature creates invalid iframes.
+            configuration.HtmlValidationConfiguration.WithRelativeConfigPath("PermitNoTitleIframes.htmlvalidate.json");
+
+            if (changeConfigurationAsync != null) await changeConfigurationAsync(configuration);
+        }
+
+        if (string.IsNullOrEmpty(CloudflareApiToken))
+        {
+            _testOutputHelper.WriteLineTimestampedAndDebug(
+                "No Cloudflare API token is set, thus skipping the IP Access Rule setup. Note that Cloudflare might " +
+                "reject automated requests with HTTP 403s.");
+
+            return base.ExecuteTestAsync(baseUri, testAsync, browser, ChangeConfigurationForCloudflareAsync);
+        }
+
+        return CloudflareHelper.ExecuteWrappedInIpAccessRuleManagementAsync(
+            () => base.ExecuteTestAsync(baseUri, testAsync, browser, ChangeConfigurationForCloudflareAsync),
+            CloudflareAccountId,
+            CloudflareApiToken);
+    }
+}

Lombiq.Tests.UI/Helpers/CloudflareHelper.cs

@@ -0,0 +1,218 @@
+using Refit;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
+using System.IO;
+using System.Linq;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Lombiq.Tests.UI.Helpers;
+
+internal static class CloudflareHelper
+{
+    private static readonly SemaphoreSlim _semaphore = new(1, 1);
+    private static int _referenceCount;
+
+    private static string _currentIp;
+    private static string _ipAccessRuleId;
+    private static ICloudflareApi _cloudflareApi;
+
+    public static async Task ExecuteWrappedInIpAccessRuleManagementAsync(
+        Func<Task> testAsync,
+        string cloudflareAccountId,
+        string cloudflareApiToken)
+    {
+        await _semaphore.WaitAsync();
+        Interlocked.Increment(ref _referenceCount);
+
+        Debug.WriteLine("Current reference count at the start of the test: {0}.", _referenceCount);
+
+        try
+        {
+            _cloudflareApi ??= RestService.For<ICloudflareApi>("https://api.cloudflare.com/client/v4", new RefitSettings
+            {
+                AuthorizationHeaderValueGetter = (_, _) => Task.FromResult(cloudflareApiToken),
+            });
+
+            _currentIp ??= _cloudflareApi != null ? await GetPublicIpAsync() : string.Empty;
+
+            if (_ipAccessRuleId == null)
+            {
+                Debug.WriteLine("Creating an IP Access Rule for the IP {0}.", (object)_currentIp);
+
+                // Delete any pre-existing rules for the current IP first.
+                string preexistingRuleId = null;
+                await ReliabilityHelper.DoWithRetriesAndCatchesAsync(
+                    async () =>
+                    {
+                        var rulesResponse = await _cloudflareApi.GetIpAccessRulesAsync(cloudflareAccountId, _currentIp);
+                        preexistingRuleId = rulesResponse.Result?.FirstOrDefault()?.Id;
+                        return rulesResponse.Success;
+                    });
+
+                // preexistingRuleId can be set in the delegate above, so it's not always null.
+#pragma warning disable S2583 // Conditionally executed code should be reachable
+                if (preexistingRuleId != null)
+                {
+                    await DeleteIpAccessRuleWithRetriesAsync(cloudflareAccountId, preexistingRuleId);
+                }
+#pragma warning restore S2583 // Conditionally executed code should be reachable
+
+                // Create the IP Access Rule.
+                var createResponseResult = await ReliabilityHelper.DoWithRetriesAndCatchesAsync(
+                    async () =>
+                    {
+                        var createResponse = await _cloudflareApi.CreateIpAccessRuleAsync(cloudflareAccountId, new IpAccessRuleRequest
+                        {
+                            Mode = "whitelist",
+                            Configuration = new IpAccessRuleConfiguration { Target = "ip", Value = _currentIp },
+                            Notes = "Temporarily allow a remote UI test from GitHub Actions.",
+                        });
+
+                        _ipAccessRuleId = createResponse.Result?.Id;
+
+                        return createResponse.Success && _ipAccessRuleId != null;
+                    });
+
+                ThrowIfNotSuccess(createResponseResult, _currentIp, "didn't save properly");
+
+                // Wait for the rule to appear, to make sure that it's active.
+                var ruleCheckRequestResult = await ReliabilityHelper.DoWithRetriesAndCatchesAsync(
+                    async () =>
+                    {
+                        var rulesResponse = await _cloudflareApi.GetIpAccessRulesAsync(cloudflareAccountId);
+                        return rulesResponse.Success && rulesResponse.Result.Exists(rule => rule.Id == _ipAccessRuleId);
+                    });
+
+                ThrowIfNotSuccess(ruleCheckRequestResult, _currentIp, "didn't get activated");
+            }
+        }
+        finally
+        {
+            _semaphore.Release();
+        }
+
+        try
+        {
+            await testAsync();
+        }
+        finally
+        {
+            // Clean up the IP access rule.
+            if (_ipAccessRuleId != null && Interlocked.Decrement(ref _referenceCount) == 0)
+            {
+                Debug.WriteLine("Removing the IP Access Rule. Current reference count: {0}.", _referenceCount);
+
+                var deleteSucceededResult = await DeleteIpAccessRuleWithRetriesAsync(cloudflareAccountId, _ipAccessRuleId);
+
+                if (deleteSucceededResult.IsSuccess) _ipAccessRuleId = null;
+
+                ThrowIfNotSuccess(deleteSucceededResult, _currentIp, "couldn't be deleted");
+            }
+        }
+    }
+
+    private static async Task<string> GetPublicIpAsync()
+    {
+        using var client = new HttpClient();
+        string ip = string.Empty;
+
+        var ipRequestResult = await ReliabilityHelper.DoWithRetriesAndCatchesAsync(
+            async () =>
+            {
+                ip = await client.GetStringAsync("https://api.ipify.org");
+                return true;
+            });
+
+        if (!ipRequestResult.IsSuccess)
+        {
+            throw new IOException("Couldn't get the public IP address of the runner.", ipRequestResult.Exception);
+        }
+
+        return ip;
+    }
+
+    private static void ThrowIfNotSuccess((bool IsSuccess, Exception InnerException) result, string currentIp, string messagePart)
+    {
+        if (result.IsSuccess) return;
+
+        throw new IOException(
+            $"The Cloudflare IP Access Rule for allowing requests from this runner {messagePart}. There might be a " +
+            $"leftover rule for the IP {currentIp} that needs to be deleted manually." +
+            (result.InnerException is ApiException ex ? $" Response: {ex.Content}" : string.Empty),
+            result.InnerException);
+    }
+
+    public static Task<(bool IsSuccess, Exception Exception)> DeleteIpAccessRuleWithRetriesAsync(
+        string cloudflareAccountId,
+        string ipAccessRuleId) =>
+        ReliabilityHelper.DoWithRetriesAndCatchesAsync(
+            async () =>
+            {
+                var deleteResponse = await _cloudflareApi.DeleteIpAccessRuleAsync(cloudflareAccountId, ipAccessRuleId);
+                return deleteResponse.Success;
+            });
+
+    [Headers("Authorization: Bearer")]
+    [SuppressMessage("StyleCop.CSharp.DocumentationRules", "SA1600:Elements should be documented", Justification = "It's an API client.")]
+    public interface ICloudflareApi
+    {
+        [Post("/accounts/{accountId}/firewall/access_rules/rules")]
+        Task<ApiResponse<IpAccessRuleResponse>> CreateIpAccessRuleAsync(
+            string accountId,
+            [Body] IpAccessRuleRequest request
+        );
+
+        [Get("/accounts/{accountId}/firewall/access_rules/rules")]
+        Task<ApiResponse<IpAccessRuleResponse[]>> GetIpAccessRulesAsync(
+            string accountId,
+            [AliasAs("configuration.value")] string configurationValue = null,
+            [AliasAs("per_page")] int pageSize = 200);
+
+        [Delete("/accounts/{accountId}/firewall/access_rules/rules/{ruleId}")]
+        Task<ApiResponse<DeleteResponse>> DeleteIpAccessRuleAsync(string accountId, string ruleId);
+    }
+
+    public sealed class IpAccessRuleRequest
+    {
+        public string Mode { get; set; }
+        public IpAccessRuleConfiguration Configuration { get; set; }
+        public string Notes { get; set; }
+    }
+
+    [DebuggerDisplay("{Target} <- {Value}")]
+    public sealed class IpAccessRuleConfiguration
+    {
+        public string Target { get; set; }
+        public string Value { get; set; }
+    }
+
+    [DebuggerDisplay("{Id}: {Configuration}")]
+    public sealed class IpAccessRuleResponse
+    {
+        public string Id { get; set; }
+        public IpAccessRuleConfiguration Configuration { get; set; }
+    }
+
+    public sealed class DeleteResponse
+    {
+        public bool Success { get; set; }
+    }
+
+    public sealed class ApiResponse<T>
+    {
+        public bool Success { get; set; }
+        public T Result { get; set; }
+        public IEnumerable<ApiError> Errors { get; set; }
+        public IEnumerable<ApiError> Messages { get; set; }
+    }
+
+    public sealed class ApiError
+    {
+        public string Code { get; set; }
+        public string Message { get; set; }
+    }
+}

Lombiq.Tests.UI/Helpers/ReliabilityHelper.cs

@@ -114,6 +114,55 @@ public static async Task<bool> DoWithRetriesAsync(
         TimeSpan? interval = null) =>
             (await DoWithRetriesInternalAsync(processAsync, timeout, interval)).IsSuccess;
 
+    /// <summary>
+    /// Executes the process repeatedly while it's not successful, with the given timeout and retry intervals.
+    /// Exceptions thrown by the process are caught and treated as failures.
+    /// </summary>
+    /// <param name="processAsync">
+    /// The operation that potentially needs to be retried. Should return <see langword="true"/> if it's successful,
+    /// <see langword="false"/> otherwise.
+    /// </param>
+    /// <param name="timeout">
+    /// The maximum time allowed for the process to complete. Defaults to the default of <see
+    /// cref="SafeWaitAsync{T}.Timeout"/>.
+    /// </param>
+    /// <param name="interval">
+    /// The polling interval used by <see cref="SafeWaitAsync{T}"/>. Defaults to the default of <see
+    /// cref="SafeWaitAsync{T}.PollingInterval"/>.
+    /// </param>
+    /// <returns>
+    /// A tuple with a <see langword="bool"/> indicating if <paramref name="processAsync"/> succeeded (regardless of it
+    /// happening on the first try or during retries, <see langword="false"/> otherwise. The second element is the
+    /// <see cref="Exception"/> that was thrown by the process during the last attempt, if any.
+    /// </returns>
+    public static async Task<(bool IsSuccess, Exception Exception)> DoWithRetriesAndCatchesAsync(
+        Func<Task<bool>> processAsync,
+        TimeSpan? timeout = null,
+        TimeSpan? interval = null)
+    {
+        Exception exception = null;
+
+        var isSuccess = (await DoWithRetriesInternalAsync(
+                async () =>
+                {
+                    exception = null;
+
+                    try
+                    {
+                        return await processAsync();
+                    }
+                    catch (Exception ex)
+                    {
+                        exception = ex;
+                        return false;
+                    }
+                },
+                timeout,
+                interval)).IsSuccess;
+
+        return (isSuccess, exception);
+    }
+
     /// <summary>
     /// Executes the process and retries if an element becomes stale ( <see cref="StaleElementReferenceException"/>). If
     /// the operation didn't succeed then throws a <see cref="TimeoutException"/>.

Lombiq.Tests.UI/Helpers/UrlCheckHelper.cs

@@ -6,6 +6,12 @@ namespace Lombiq.Tests.UI.Helpers;
 
 public static class UrlCheckHelper
 {
+    /// <summary>
+    /// Checks if the current page is NOT driven by Orchard, i.e. excludes admin pages or frontend pages that come from
+    /// built-in modules.
+    /// </summary>
+    public static bool IsNotOrchardPage(UITestContext context) => !IsOrchardPage(context);
+
     /// <summary>
     /// Checks if the current page is driven by Orchard, i.e. admin pages or frontend pages that come from built-in
     /// modules.
@@ -53,7 +59,7 @@ public static bool IsValidatablePage(UITestContext context)
             url.ContainsOrdinalIgnoreCase("://localhost:") &&
             !url.StartsWithOrdinalIgnoreCase(context.SmtpServiceRunningContext?.WebUIUri.ToString() ?? "dummy://") &&
             !url.ContainsOrdinalIgnoreCase("Lombiq.Tests.UI.Shortcuts") &&
-            !IsOrchardPage(context) &&
+            IsNotOrchardPage(context) &&
             context.Driver.Title != "Setup";
     }