diff --git a/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj b/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj index 524752739..2f30ec9a8 100644 --- a/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj +++ b/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj @@ -10,6 +10,7 @@ + @@ -17,6 +18,9 @@ Always + + PreserveNewest + PreserveNewest diff --git a/Lombiq.Tests.UI.Samples/Readme.md b/Lombiq.Tests.UI.Samples/Readme.md index 451c528fe..15419deee 100644 --- a/Lombiq.Tests.UI.Samples/Readme.md +++ b/Lombiq.Tests.UI.Samples/Readme.md @@ -28,6 +28,7 @@ For general details about and on using the Toolbox see the [root Readme](../Read - [Basic visual verification tests](Tests/BasicVisualVerificationTests.cs) - [Testing in tenants](Tests/TenantTests.cs) - [Interactive mode](Tests/InteractiveModeTests.cs) +- [Security scanning](Tests/SecurityScanningTests.cs) ## Adding new tutorials diff --git a/Lombiq.Tests.UI.Samples/Tests/CustomZapAutomationFrameworkPlan.yml b/Lombiq.Tests.UI.Samples/Tests/CustomZapAutomationFrameworkPlan.yml new file mode 100644 index 000000000..ba2162096 --- /dev/null +++ b/Lombiq.Tests.UI.Samples/Tests/CustomZapAutomationFrameworkPlan.yml @@ -0,0 +1,114 @@ +--- +env: + contexts: + - name: "Default Context" + urls: + - "" + excludePaths: [] + authentication: + parameters: {} + verification: + method: "response" + pollFrequency: 60 + pollUnits: "requests" + sessionManagement: + method: "cookie" + parameters: {} + technology: + exclude: + - "C" + - "IBM DB2" + - "PHP" + - "CouchDB" + - "Oracle" + - "JSP/Servlet" + - "Firebird" + - "HypersonicSQL" + - "SAP MaxDB" + - "Ruby" + - "Microsoft Access" + - "Java" + - "Tomcat" + - "Sybase" + - "Python" + parameters: + failOnError: true + failOnWarning: false + progressToStdout: true + vars: {} +jobs: +- parameters: + scanOnlyInScope: true + enableTags: false + disableAllRules: false + rules: + - id: 10035 + name: "Strict-Transport-Security Header" + threshold: "off" + - id: 10038 + name: "Content Security Policy (CSP) Header Not Set" + threshold: "off" + - id: 10020 + name: "Anti-clickjacking Header" + threshold: "off" + - id: 10037 + name: "Server Leaks Information via \"X-Powered-By\" HTTP Response Header Field(s)" + threshold: "off" + - id: 10021 + name: "X-Content-Type-Options Header Missing" + threshold: "off" + name: "passiveScan-config" + type: "passiveScan-config" +- parameters: {} + name: "spider" + type: "spider" +- parameters: {} + name: "passiveScan-wait" + type: "passiveScan-wait" +- parameters: + reportDir: "/zap/wrk/reports" + template: "modern" + theme: "corporate" + reportTitle: "ZAP Scanning Report" + reportDescription: "" + risks: + - "low" + - "medium" + - "high" + confidences: + - "low" + - "medium" + - "high" + - "confirmed" + sections: + - "passingrules" + - "instancecount" + - "alertdetails" + - "alertcount" + - "params" + - "chart" + - "statistics" + name: "report" + type: "report" +- parameters: + template: "sarif-json" + reportDir: "/zap/wrk/reports" + reportFile: "" + reportTitle: "ZAP Scanning Report" + reportDescription: "" + displayReport: false + risks: + - "info" + - "low" + - "medium" + - "high" + confidences: + - "falsepositive" + - "low" + - "medium" + - "high" + - "confirmed" + sites: [] + name: "sarifReport" + type: "report" + diff --git a/Lombiq.Tests.UI.Samples/Tests/InteractiveModeTests.cs b/Lombiq.Tests.UI.Samples/Tests/InteractiveModeTests.cs index 4d4bc1ad7..b0438cd18 100644 --- a/Lombiq.Tests.UI.Samples/Tests/InteractiveModeTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/InteractiveModeTests.cs @@ -70,3 +70,4 @@ await Task.WhenAll( } // END OF TRAINING SECTION: Interactive mode. +// NEXT STATION: Head over to Tests/SecurityScanningTests.cs. diff --git a/Lombiq.Tests.UI.Samples/Tests/SecurityScanningTests.cs b/Lombiq.Tests.UI.Samples/Tests/SecurityScanningTests.cs new file mode 100644 index 000000000..14dbbf4e4 --- /dev/null +++ b/Lombiq.Tests.UI.Samples/Tests/SecurityScanningTests.cs @@ -0,0 +1,157 @@ +using Lombiq.Tests.UI.Extensions; +using Lombiq.Tests.UI.Pages; +using Lombiq.Tests.UI.SecurityScanning; +using Lombiq.Tests.UI.Services; +using OpenQA.Selenium; +using Shouldly; +using System; +using System.Threading.Tasks; +using Xunit; +using Xunit.Abstractions; +using YamlDotNet.RepresentationModel; + +namespace Lombiq.Tests.UI.Samples.Tests; + +// Zed Attack Proxy (ZAP, see https://www.zaproxy.org/) is the world's most widely used web app security scanner, and a +// fellow open-source project we can recommend. And you can use it right from UI tests, on the same app that's run for +// the tests! This is useful to find all kinds of security issues with your app. In this sample we'll see how, but be +// sure to also check out the corresponding documentation page: +// https://github.com/Lombiq/UI-Testing-Toolbox/blob/dev/Lombiq.Tests.UI/Docs/SecurityScanning.md. + +// Note that security scanning has cross-platform support, but due to the limitations of virtualization under Windows in +// GitHub Actions, these tests won't work there. They'll work on a Windows desktop though. +public class SecurityScanningTests : UITestBase +{ + public SecurityScanningTests(ITestOutputHelper testOutputHelper) + : base(testOutputHelper) + { + } + + // Let's see simple use case first: Running a built-in ZAP scan. + + // We're running one of ZAP's built-in scans, the Baseline scan. This, as the name suggests, provides some + // rudimentary security checks. While you can start with this, we recommend running the Full Scan, for which there + // similarly is an extension method as well. + + // If you're new to security scanning, starting with exactly this is probably a good idea. Most possibly your app + // will fail the scan, but don't worry! You'll get a nice report about the findings in the failure dump. + [Fact] + public Task BasicSecurityScanShouldPass() => + ExecuteTestAfterSetupAsync(context => context.RunAndAssertBaselineSecurityScanAsync()); + + // Time for some custom configuration! While this scan also runs the Baseline scan, it does this with several + // adjustments: + // - Also runs ZAP's Ajax Spider (https://www.zaproxy.org/docs/desktop/addons/ajax-spider/automation/). This is + // usually not just unnecessary for a website that's not an SPA, but also slows the scan down by a lot. However, + // if you have an SPA, you need to use it. + // - Excludes certain URLs from the scan completely. Use this if you don't want ZAP to process certain URLs at all. + // - Disables the "Server Leaks Information via "X-Powered-By" HTTP Response Header Field(s)" alert of ZAP's passive + // scan for the whole scan. This is because by default, Orchard Core sends an "X-Powered-By: OrchardCore" header. + // If you want airtight security, you might want to turn this off, but for the sake of example we just ignore the + // alert here. + // - Also disables the "Content Security Policy (CSP) Header Not Set" rule but only for the /about page. Use this to + // disable rules more specifically instead of the whole scan. + // - Configures sign in with a user account. This is what the scan will start with. With the Blog recipe it doesn't + // matter too much, since nothing on the frontend will change, but you can use this to scan authenticated features + // too. Note that since ZAP uses its own spider, not the browser accessed by the test, user sessions are not + // shared, so such an explicit sign in is necessary. + // - The assertion on the scan results is custom. Use this if you (conditionally) want to assert on the results + // differently from the global context.Configuration.SecurityScanningConfiguration.AssertSecurityScanResult. The + // default there is "no scanning alert is allowed"; we expect some alerts here. + [Fact] + public Task SecurityScanWithCustomConfigurationShouldPass() => + ExecuteTestAfterSetupAsync( + context => context.RunAndAssertBaselineSecurityScanAsync( + configuration => configuration + ////.UseAjaxSpider() // This is quite slow so just showing you here but not running it. + .ExcludeUrlWithRegex(".*blog.*") + .DisablePassiveScanRule(10037, "Server Leaks Information via \"X-Powered-By\" HTTP Response Header Field(s)") + .DisableScanRuleForUrlWithRegex(".*/about", 10038, "Content Security Policy (CSP) Header Not Set") + .SignIn(), + sarifLog => sarifLog.Runs[0].Results.Count.ShouldBeLessThan(34))); + + // Let's get low-level into ZAP's configuration now. While the .NET configuration API of the Lombiq UI Testing + // Toolbox covers the most important ways to configure ZAP, sometimes you need more. For this, you have complete + // control over ZAP's configuration via its Automation Framework (see + // https://www.zaproxy.org/docs/automate/automation-framework/ and https://www.youtube.com/watch?v=PnCbIAnauD8 for + // an introduction), what all the packaged scans and .NET configuration uses under the hood too. This way, if you + // know what you want to do in ZAP, you can just directly run in as a UI test! + + // We run a completely custom Automation Framework plan here. It's almost the same as the plan used by the Baseline + // scan, but has some rules disabled by default, so we can assert on no alerts. Note that it has the Content build + // action to copy it to the build output folder. + + // You can also create and configure such plans from the ZAP desktop app, following the guides linked above. The + // plan doesn't need anything special, apart from having at least one context defined, as well as having a + // "sarif-json" report job so assertions can work with it. If something is missing in it, you'll get exceptions + // telling you what the problem is anyway. + + // Then, you can see an example of modifying the ZAP plan from code. You can also do this with the built-in plans to + // customize them if something you need is not surfaced as configuration. + [Fact] + public Task SecurityScanWithCustomAutomationFrameworkPlanShouldPass() => + ExecuteTestAfterSetupAsync( + context => context.RunAndAssertSecurityScanAsync( + "Tests/CustomZapAutomationFrameworkPlan.yml", + configuration => configuration + .ModifyZapPlan(plan => + { + // "plan" here is a representation of the YAML document containing the plan. It's a low-level + // representation, but you can do anything with it. + + // We'll change a parameter for ZAP's spider. This of course could be done right in our custom + // plan, but we wanted to demo this too; furthermore, from code, you can change the plan even + // based on the context dynamically, so it's more flexible than trying to configure everything + // in the plan's YAML file. + + var spiderJob = plan.GetSpiderJob(); + var spiderParameters = (YamlMappingNode)spiderJob["parameters"]; + // The default maxDepth is 5. 8 will let the spider run for a bit more, potentially discovering + // more pages to be scanned. + spiderParameters.Add("maxDepth", "8"); + }), + sarifLog => SecurityScanningConfiguration.AssertSecurityScanHasNoAlerts(context, sarifLog))); + + // Overriding the default setup so we can have a simpler site, simplifying the security scan for the purpose of this + // demo. For a real app's security scan you needn't (shouldn't) do this though; always run the scan on the actual + // app with everything set up how you run it in production. + protected override Task ExecuteTestAfterSetupAsync( + Func testAsync, + Browser browser, + Func changeConfigurationAsync) => + ExecuteTestAsync( + testAsync, + browser, + async context => + { + var homepageUri = await context.GoToSetupPageAndSetupOrchardCoreAsync( + new OrchardCoreSetupParameters(context) + { + SiteName = "Lombiq's OSOCE - UI Testing", + RecipeId = "Blog", + TablePrefix = "OSOCE", + SiteTimeZoneValue = "Europe/Budapest", + }); + + context.Exists(By.ClassName("site-heading")); + + return homepageUri; + }, + async configuration => + { + configuration.HtmlValidationConfiguration.RunHtmlValidationAssertionOnAllPageChanges = false; + + // Note how we specify an assertion too. This is because ZAP actually notices a few security issues with + // vanilla Orchard Core. These, however, are more like artifacts of running the app locally and out of + // the box without any real configuration. So, to make the tests pass, we need to override the default + // assertion that would fail the test if any issue is found. + + // Don't do this at home! Fix the issues instead. This is only here to have a smoother demo. + configuration.SecurityScanningConfiguration.AssertSecurityScanResult = (_, _) => { }; + // Check out the rest of SecurityScanningConfiguration too! + + await changeConfigurationAsync(configuration); + }); +} + +// END OF TRAINING SECTION: Security scanning. diff --git a/Lombiq.Tests.UI.Shortcuts/Controllers/CurrentUserController.cs b/Lombiq.Tests.UI.Shortcuts/Controllers/CurrentUserController.cs index 926580e3b..08f46b510 100644 --- a/Lombiq.Tests.UI.Shortcuts/Controllers/CurrentUserController.cs +++ b/Lombiq.Tests.UI.Shortcuts/Controllers/CurrentUserController.cs @@ -9,5 +9,5 @@ public class CurrentUserController : Controller { // Needs to return a string even if there's no user, otherwise it'd return an HTTP 204 without a body, see: // https://weblog.west-wind.com/posts/2020/Feb/24/Null-API-Responses-and-HTTP-204-Results-in-ASPNET-Core. - public string Index() => "UserName: " + User?.Identity?.Name; + public string Index() => User.Identity.IsAuthenticated ? "UserName: " + User.Identity.Name : "Unauthenticated"; } diff --git a/Lombiq.Tests.UI.Shortcuts/Lombiq.Tests.UI.Shortcuts.csproj b/Lombiq.Tests.UI.Shortcuts/Lombiq.Tests.UI.Shortcuts.csproj index bd6f9b01f..cc4504bda 100644 --- a/Lombiq.Tests.UI.Shortcuts/Lombiq.Tests.UI.Shortcuts.csproj +++ b/Lombiq.Tests.UI.Shortcuts/Lombiq.Tests.UI.Shortcuts.csproj @@ -50,7 +50,7 @@ - + diff --git a/Lombiq.Tests.UI/Docs/Attachments/ZapReportScreenshot.png b/Lombiq.Tests.UI/Docs/Attachments/ZapReportScreenshot.png new file mode 100644 index 000000000..39a5438c0 Binary files /dev/null and b/Lombiq.Tests.UI/Docs/Attachments/ZapReportScreenshot.png differ diff --git a/Lombiq.Tests.UI/Docs/Configuration.md b/Lombiq.Tests.UI/Docs/Configuration.md index a5291028a..933dc2d2c 100644 --- a/Lombiq.Tests.UI/Docs/Configuration.md +++ b/Lombiq.Tests.UI/Docs/Configuration.md @@ -20,7 +20,7 @@ Note that since the tests are xUnit tests you can configure general parameters o ``` -Note also that some projects' _xunit.runner.json_ files may include the flag [stopOnFail](https://xunit.net/docs/configuration-files#stopOnFail) set to `true`, which makes further tests stop once a failing test is encountered. +Note also that some projects' _xunit.runner.json_ files may include the flag [`stopOnFail`](https://xunit.net/docs/configuration-files#stopOnFail) set to `true`, which makes further tests stop once a failing test is encountered. Certain test execution parameters can be configured externally too, the ones retrieved via the `TestConfigurationManager` class. All configuration options are basic key-value pairs and can be provided in one of the two ways: diff --git a/Lombiq.Tests.UI/Docs/CreatingTests.md b/Lombiq.Tests.UI/Docs/CreatingTests.md index d9f31a1ac..6ea697c02 100644 --- a/Lombiq.Tests.UI/Docs/CreatingTests.md +++ b/Lombiq.Tests.UI/Docs/CreatingTests.md @@ -14,6 +14,7 @@ We also recommend always running some highly automated tests that need very litt - The suite of tests for checking that all the basic Orchard Core features work, like login, registration, and content management. Use `context.TestBasicOrchardFeatures()` to run all such tests but see the other, more granular tests too. This is also demonstrated in `Lombiq.Tests.UI.Samples` and in [this video](https://www.youtube.com/watch?v=jmhq63sRZrI). - [Monkey tests](https://en.wikipedia.org/wiki/Monkey_testing) can also be useful. Use `context.TestCurrentPageAsMonkeyRecursively()` to run a monkey testing process. This walks through site pages and does random interactions with pages, like clicking, scrolling, form filling, etc. It's recommended to have at least 3 monkey tests that execute with different user states: As an admin, as a regular registered user and as an anonymous user. The admin test can start execution on admin dashboard page, while other tests can start on home page. This is also demonstrated in `Lombiq.Tests.UI.Samples` and in [this video](https://www.youtube.com/watch?v=pZbEsEz3tuE). +- Security scans with [Zed Attack Proxy (ZAP)](https://www.zaproxy.org/), see [Security scanning](SecurityScanning.md). ## Steps for creating a test class diff --git a/Lombiq.Tests.UI/Docs/SecurityScanning.md b/Lombiq.Tests.UI/Docs/SecurityScanning.md new file mode 100644 index 000000000..b26c1acf6 --- /dev/null +++ b/Lombiq.Tests.UI/Docs/SecurityScanning.md @@ -0,0 +1,30 @@ +# Security scanning with ZAP + +## Overview + +You can create detailed security scans of your app with [Zed Attack Proxy (ZAP)](https://www.zaproxy.org/) right from the Lombiq UI Testing Toolbox, with nice reports. ZAP is the world's most widely used web app security scanner, and a fellow open-source project we can recommend. + +![Sample ZAP security scan report](Attachments/ZapReportScreenshot.png) + +- The most important default ZAP scans, Baseline, Full, GraphQL, and OpenAPI scans are included and readily usable. Note that these are modified to be more applicable to Orchard Core apps run on localhost during a UI testing scenario (notably, [`ajaxSpider`](https://www.zaproxy.org/docs/desktop/addons/ajax-spider/automation/) is removed, since most Orchard Core apps don't need it but it takes a lot of time, and rules as well as selected technologies are adjusted). If you want to scan remote (and especially production) apps, then you'll need to create your own scans based on ZAP's default ones. These can then be run from inside UI tests too. +- You can assert on scan results and thus fail the test if there are security warnings. +- Since we use [ZAP's Automation Framework](https://www.zaproxy.org/docs/automate/automation-framework/) for configuration, you have complete and detailed control over how the scans are configured, but you can also start with a simple configuration available in the .NET API. +- [SARIF](https://sarifweb.azurewebsites.net/) reports are available to integrate with other InfoSec tools. + +## Working with ZAP in the Lombiq UI Testing Toolbox + +- We recommend you first check out the [related samples in the `Lombiq.Tests.UI.Samples` project](../../Lombiq.Tests.UI.Samples/Tests/SecurityScanningTests.cs). +- If you're new to ZAP, you can start learning by checking out [ZAP's getting started guide](https://www.zaproxy.org/getting-started/), as well as the [ZAP Chat Videos](https://www.zaproxy.org/zap-chat/). The [documentation on ZAP's Automation Framework](https://www.zaproxy.org/docs/automate/automation-framework/) and the [ZAP Chat 06 Automation Introduction video](https://www.youtube.com/watch?v=PnCbIAnauD8) (as well as the subsequent videos about it in the series) will help you understand what we use under the hood to instruct ZAP, and will allow you to use your completely custom Automation Framework plans too. +- Be aware that ZAP scans run its own spider or with an internally managed browser instance, not in the browser launched by the test. +- While ZAP is fully managed for you, Docker needs to be available and running to host the ZAP instance. On your development machine, you can install [Docker Desktop](https://www.docker.com/products/docker-desktop/). +- The full scan of a website with even just 1-200 pages can take 5-10 minutes. So, be careful to fine-tune the ZAP configuration to make it suitable for your app. + +## Troubleshooting + +- If you're unsure what happens in a scan, run the [ZAP desktop app](https://www.zaproxy.org/download/) and load the Automation Framework plan's YAML file into it. If you use the default scans, then these will be available under the build output directory (like _bin/Debug_) under _SecurityScanning/AutomationFrameworkPlans_. Then, you can open and run them as demonstrated [in this video](https://youtu.be/PnCbIAnauD8?si=u0vi63Uvv9wZINzb&t=1173). +- If an alert is a false positive, follow [the official docs](https://www.zaproxy.org/faq/how-do-i-handle-a-false-positive/). You can use the [`alertFilter` job](https://www.zaproxy.org/docs/desktop/addons/alert-filters/automation/) to ignore alerts in very specific conditions. You can also access this via the .NET configuration API. +- ZAP didn't find everything in your app? By default, ZAP has a crawl depth of 5 for its standard spider and 10 for its AJAX spider. Set `maxDepth` (and `maxChildren`) [for `spider`](https://www.zaproxy.org/docs/desktop/addons/automation-framework/job-spider/) and `maxCrawlDepth` [for `spiderAjax`](https://www.zaproxy.org/docs/desktop/addons/ajax-spider/automation/). +- Do you sometimes get slightly different scan results? This is normal, and ZAP can be inconsistent/appear random within limits, see [the official docs page](https://www.zaproxy.org/faq/why-can-zap-scans-be-inconsistent/). +- Is the active scan too slow? + - You can find out which rules take the most time by adding a script displaying each rules' runtime with `YamlDocumentExtensions.AddDisplayActiveScanRuleRuntimesScriptAfterActiveScan()`. + - The ["Cross Site Scripting (DOM Based)" active scan rule](https://www.zaproxy.org/docs/desktop/addons/dom-xss-active-scan-rule/), unlike other rules, launches browsers and thus will take 1-2 orders of magnitude more time than other scans, usually causing the bulk of an active scan's runtime. Also see [the official docs](https://www.zaproxy.org/docs/desktop/addons/dom-xss-active-scan-rule/). You can tune it so it completes faster but still produces acceptable results for your app. You can do this from the Automation Framework plan's YAML file (see the samples on how you can use a custom one), or with `SecurityScanConfiguration.ConfigureActiveScanRule()`. diff --git a/Lombiq.Tests.UI/Docs/Tools.md b/Lombiq.Tests.UI/Docs/Tools.md index 6fbdd0fc2..e0b87edc6 100644 --- a/Lombiq.Tests.UI/Docs/Tools.md +++ b/Lombiq.Tests.UI/Docs/Tools.md @@ -14,3 +14,4 @@ - Monkey testing is implemented using [Gremlins.js](https://github.com/marmelab/gremlins.js/) library. - Visual verification is implemented using [ImageSharpCompare](https://github.com/Codeuctivity/ImageSharp.Compare). - [Ben.Demystifier](https://github.com/benaadams/Ben.Demystifier) is used to simplify stack traces, mainly around async methods. +- Security scans are done with [Zed Attack Proxy (ZAP)](https://www.zaproxy.org/). diff --git a/Lombiq.Tests.UI/Docs/Troubleshooting.md b/Lombiq.Tests.UI/Docs/Troubleshooting.md index cca05a2e2..ffe94dca0 100644 --- a/Lombiq.Tests.UI/Docs/Troubleshooting.md +++ b/Lombiq.Tests.UI/Docs/Troubleshooting.md @@ -65,3 +65,7 @@ - The last monkey testing interaction before a failure is logged. You can correlate with the coordinates of it with the last page screenshot. - If you want to test the failed page granularly, you can write a test that navigates to that page and executes `context.TestCurrentPageAsMonkey(_monkeyTestingOptions, 12345);`, where `12345` is the random seed number that can be found in a failed test log. - It is also possible to set a larger time value to the `MonkeyTestingOptions.GremlinsAttackDelay` property in order to make gremlin interaction slower, thus allowing you to watch what's happening. + +## Security scanning with ZAP + +Check out the [security scanning docs](SecurityScanning.md). diff --git a/Lombiq.Tests.UI/Extensions/FailureDumpUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/FailureDumpUITestContextExtensions.cs index fdeca8613..19872250f 100644 --- a/Lombiq.Tests.UI/Extensions/FailureDumpUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/FailureDumpUITestContextExtensions.cs @@ -11,10 +11,34 @@ namespace Lombiq.Tests.UI.Extensions; public static class FailureDumpUITestContextExtensions { + /// + /// Appends a local directory's whole content to be collected on failure dump. + /// + /// The full file system path of the directory. + /// A message to display in case the desired file already exists in the dump. + public static void AppendDirectoryToFailureDump( + this UITestContext context, + string directoryPath, + string messageIfExists = null) => + RecursivelyAppendFolderContent(context, directoryPath, string.Empty, messageIfExists); + + /// + /// Appends a local file's content to be collected on failure dump. + /// + /// The full file system path of the file. + /// A message to display in case the desired file already exists in the dump. + public static void AppendFailureDump( + this UITestContext context, + string filePath, + string messageIfExists = null) => + context.AppendFailureDump( + Path.GetFileName(filePath), + context => Task.FromResult((Stream)File.OpenRead(filePath)), + messageIfExists); + /// /// Appends stream as file content to be collected on failure dump. /// - /// instance. /// The name of the file. /// Gets called in failure dump collection. /// A message to display in case the desired file already exists in the dump. @@ -31,7 +55,6 @@ public static void AppendFailureDump( /// /// Appends string as file content to be collected on failure dump. /// - /// instance. /// The name of the file. /// File content. /// A message to display in case the desired file already exists in the dump. @@ -51,7 +74,6 @@ public static void AppendFailureDump( /// /// Appends generic content as file content to be collected on failure dump. /// - /// instance. /// The name of the file. /// File content. /// Function to get a new from content. @@ -72,7 +94,6 @@ public static void AppendFailureDump( /// /// Appends as file content to be collected on failure dump. /// - /// instance. /// The name of the file. /// File content. The will be disposed at the end. /// A message to display in case the desired file already exists in the dump. @@ -100,4 +121,28 @@ private static void AppendFailureDumpInternal( context.FailureDumpContainer.Add(fileName, item); } + + private static void RecursivelyAppendFolderContent( + UITestContext context, + string directoryPath, + string failureDumpDirectoryPath, + string messageIfExists = null) + { + foreach (var filePath in Directory.GetFiles(directoryPath)) + { + context.AppendFailureDump( + Path.Combine(failureDumpDirectoryPath, Path.GetFileName(filePath)), + context => Task.FromResult((Stream)File.OpenRead(filePath)), + messageIfExists); + } + + foreach (var subDirectoryPath in Directory.GetDirectories(directoryPath)) + { + RecursivelyAppendFolderContent( + context, + subDirectoryPath, + Path.Combine(failureDumpDirectoryPath, Path.GetFileName(subDirectoryPath)), + messageIfExists); + } + } } diff --git a/Lombiq.Tests.UI/Extensions/ShortcutsUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/ShortcutsUITestContextExtensions.cs index 3d09bb55b..5a41fd2e8 100644 --- a/Lombiq.Tests.UI/Extensions/ShortcutsUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/ShortcutsUITestContextExtensions.cs @@ -108,6 +108,7 @@ public static async Task GetCurrentUserNameAsync(this UITestContext cont { await context.GoToAsync(controller => controller.Index()); var userNameContainer = context.Get(By.CssSelector("pre")).Text; + if (userNameContainer == "Unauthenticated") return string.Empty; return userNameContainer["UserName: ".Length..]; } @@ -590,12 +591,26 @@ await UsingScopeAsync( return eventUrl; } + /// + /// Switches to an interactive mode where control from the test is handed over and you can use the web app as an + /// ordinary user from the browser or access its web APIs. To switch back to the test, click the button + /// that'll be displayed in the browser, or open . + /// + public static async Task SwitchToInteractiveAsync(this UITestContext context) + { + await context.EnterInteractiveModeAsync(); + await context.WaitInteractiveModeAsync(); + + context.Driver.Close(); + context.SwitchToLastWindow(); + } + /// /// Opens a new tab with the /// page. Visiting this page enables the interactive mode flag so it can be awaited with the extension method. /// - public static Task EnterInteractiveModeAsync(this UITestContext context) + internal static Task EnterInteractiveModeAsync(this UITestContext context) { context.Driver.SwitchTo().NewWindow(WindowType.Tab); context.Driver.SwitchTo().Window(context.Driver.WindowHandles[^1]); @@ -607,7 +622,7 @@ public static Task EnterInteractiveModeAsync(this UITestContext context) /// Periodically polls the and waits half a second if it's /// . /// - public static async Task WaitInteractiveModeAsync(this UITestContext context) + internal static async Task WaitInteractiveModeAsync(this UITestContext context) { var client = context.GetApi(); while (await client.IsInteractiveModeEnabledAsync()) @@ -616,15 +631,6 @@ public static async Task WaitInteractiveModeAsync(this UITestContext context) } } - public static async Task SwitchToInteractiveAsync(this UITestContext context) - { - await context.EnterInteractiveModeAsync(); - await context.WaitInteractiveModeAsync(); - - context.Driver.Close(); - context.SwitchToLastWindow(); - } - private static bool IsAdminTheme(IManifestInfo manifest) => manifest.Tags.Any(tag => tag.EqualsOrdinalIgnoreCase(ManifestConstants.AdminTag)); diff --git a/Lombiq.Tests.UI/Extensions/TypedRouteUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/TypedRouteUITestContextExtensions.cs index 7467663bb..cd2033e44 100644 --- a/Lombiq.Tests.UI/Extensions/TypedRouteUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/TypedRouteUITestContextExtensions.cs @@ -4,6 +4,7 @@ using Microsoft.Extensions.DependencyInjection; using OrchardCore.Admin; using System; +using System.Diagnostics.CodeAnalysis; using System.Linq.Expressions; using System.Threading.Tasks; @@ -20,9 +21,7 @@ public static Task GoToAsync( Expression> actionExpression, params (string Key, object Value)[] additionalArguments) where TController : ControllerBase => - context.GoToRelativeUrlAsync(TypedRoute - .CreateFromExpression(actionExpression, additionalArguments, CreateServiceProvider(context)) - .ToString()); + context.GoToRelativeUrlAsync(context.GetRelativeUrlOfAction(actionExpression, additionalArguments)); /// /// Navigates to the relative URL generated by for the ( Expression> actionExpressionAsync, params (string Key, object Value)[] additionalArguments) where TController : ControllerBase => - context.GoToRelativeUrlAsync(TypedRoute + context.GoToRelativeUrlAsync(context.GetRelativeUrlOfAction(actionExpressionAsync, additionalArguments)); + + /// + /// Gets the relative URL generated by for the in the + /// . + /// + [SuppressMessage("Design", "CA1055:URI-like return values should not be strings", Justification = "Other APIs need it as a string.")] + public static string GetRelativeUrlOfAction( + this UITestContext context, + Expression> actionExpression, + params (string Key, object Value)[] additionalArguments) + where TController : ControllerBase => + TypedRoute + .CreateFromExpression(actionExpression, additionalArguments, CreateServiceProvider(context)) + .ToString(); + + /// + /// Gets the relative URL generated by for the in + /// the . + /// + [SuppressMessage("Design", "CA1055:URI-like return values should not be strings", Justification = "Other APIs need it as a string.")] + public static string GetRelativeUrlOfAction( + this UITestContext context, + Expression> actionExpressionAsync, + params (string Key, object Value)[] additionalArguments) + where TController : ControllerBase => + TypedRoute .CreateFromExpression(actionExpressionAsync.StripResult(), additionalArguments, CreateServiceProvider(context)) - .ToString()); + .ToString(); private static IServiceProvider CreateServiceProvider(UITestContext context) { diff --git a/Lombiq.Tests.UI/Helpers/DirectoryHelper.cs b/Lombiq.Tests.UI/Helpers/DirectoryHelper.cs index fdaaa0166..8af0780d7 100644 --- a/Lombiq.Tests.UI/Helpers/DirectoryHelper.cs +++ b/Lombiq.Tests.UI/Helpers/DirectoryHelper.cs @@ -18,7 +18,7 @@ public static void SafelyDeleteDirectoryIfExists(string path, int maxTryCount = { Directory.Delete(path, recursive: true); // Even after the delete seemingly succeeding the folder can remain there with some empty subfolders. - // Perhaps this happens when one opens it in Explorer and that keeps a handle open. + // Perhaps this happens when one opens it in Windows Explorer and that keeps a handle open. if (!Directory.Exists(path)) return; } catch (DirectoryNotFoundException) diff --git a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj index 1e4e5c038..28dae6c07 100644 --- a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj +++ b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj @@ -18,7 +18,7 @@ Lombiq Technologies Copyright © 2020, Lombiq Technologies Ltd. Lombiq UI Testing Toolbox for Orchard Core: Web UI testing toolbox mostly for Orchard Core applications. Everything you need to do UI testing with Selenium for an Orchard app is here. See the project website for detailed documentation. - OrchardCore;Lombiq;AspNetCore;Selenium;Atata;Shouldly;xUnit;Axe;AccessibilityTesting;UITesting;Testing;Automation + OrchardCore;Lombiq;AspNetCore;Selenium;Atata;Shouldly;xUnit;Axe;AccessibilityTesting;UITesting;Testing;Automation;ZAP;Zed Attack Proxy;Security;Scanning;OWASP NuGetIcon.png https://github.com/Lombiq/UI-Testing-Toolbox https://github.com/Lombiq/UI-Testing-Toolbox @@ -44,6 +44,14 @@ PreserveNewest true + + PreserveNewest + true + + + PreserveNewest + true + PreserveNewest true @@ -69,11 +77,13 @@ + + @@ -93,9 +103,9 @@ - - - + + + @@ -113,5 +123,14 @@ + + + + + + + + + diff --git a/Lombiq.Tests.UI/SecurityScanning/AutomationFrameworkPlanFragments/DisplayActiveScanRuleRuntimesScript.yml b/Lombiq.Tests.UI/SecurityScanning/AutomationFrameworkPlanFragments/DisplayActiveScanRuleRuntimesScript.yml new file mode 100644 index 000000000..464dc14cd --- /dev/null +++ b/Lombiq.Tests.UI/SecurityScanning/AutomationFrameworkPlanFragments/DisplayActiveScanRuleRuntimesScript.yml @@ -0,0 +1,25 @@ +- parameters: + action: "add" + type: "active" + engine: "ECMAScript : Graal.js" + name: "displayRuleRuntimes" + target: "" + inline: "var extAscan = control.getExtensionLoader().getExtension(\n org.zaproxy.zap.extension.ascan.ExtensionActiveScan.NAME);\n\ + \nif (extAscan != null) {\n var lastScan = extAscan.getLastScan();\n if (lastScan\ + \ != null) {\n var hps = lastScan.getHostProcesses().toArray();\n for\ + \ (var i=0; i < hps.length; i++) {\n var hp = hps[i];\n var plugins\ + \ = hp.getCompleted().toArray();\n for (var j=0; j < plugins.length; j++)\ + \ {\n var plugin = plugins[j];\n var timeTaken = plugin.getTimeFinished().getTime()\n\ + \ - plugin.getTimeStarted().getTime();\n print(plugin.getName()\ + \ + \"\\t\" + timeTaken);\n }\n }\n }\n}\n" + name: "script" + type: "script" +- parameters: + action: "run" + type: "standalone" + engine: "" + name: "displayRuleRuntimes" + target: "" + inline: "" + name: "script" + type: "script" diff --git a/Lombiq.Tests.UI/SecurityScanning/AutomationFrameworkPlanFragments/RequestorJob.yml b/Lombiq.Tests.UI/SecurityScanning/AutomationFrameworkPlanFragments/RequestorJob.yml new file mode 100644 index 000000000..b1693febf --- /dev/null +++ b/Lombiq.Tests.UI/SecurityScanning/AutomationFrameworkPlanFragments/RequestorJob.yml @@ -0,0 +1,12 @@ +parameters: + user: "" +requests: +- url: "" + name: "" + method: "" + httpVersion: "" + headers: [] + data: "" + responseCode: 200 +name: "requestor" +type: "requestor" diff --git a/Lombiq.Tests.UI/SecurityScanning/AutomationFrameworkPlanFragments/SpiderAjaxJob.yml b/Lombiq.Tests.UI/SecurityScanning/AutomationFrameworkPlanFragments/SpiderAjaxJob.yml new file mode 100644 index 000000000..aa35251d8 --- /dev/null +++ b/Lombiq.Tests.UI/SecurityScanning/AutomationFrameworkPlanFragments/SpiderAjaxJob.yml @@ -0,0 +1,8 @@ +parameters: + maxDuration: 60 + maxCrawlDepth: 10 + numberOfBrowsers: 64 + inScopeOnly: true + runOnlyIfModern: true +name: "spiderAjax" +type: "spiderAjax" diff --git a/Lombiq.Tests.UI/SecurityScanning/AutomationFrameworkPlanFragmentsPaths.cs b/Lombiq.Tests.UI/SecurityScanning/AutomationFrameworkPlanFragmentsPaths.cs new file mode 100644 index 000000000..b9fe60f43 --- /dev/null +++ b/Lombiq.Tests.UI/SecurityScanning/AutomationFrameworkPlanFragmentsPaths.cs @@ -0,0 +1,14 @@ +using System.IO; + +namespace Lombiq.Tests.UI.SecurityScanning; + +public static class AutomationFrameworkPlanFragmentsPaths +{ + private static readonly string AutomationFrameworkPlanFragmentsPath = + Path.Combine("SecurityScanning", "AutomationFrameworkPlanFragments"); + + public static readonly string DisplayActiveScanRuleRuntimesScriptPath = + Path.Combine(AutomationFrameworkPlanFragmentsPath, "DisplayActiveScanRuleRuntimesScript.yml"); + public static readonly string RequestorJobPath = Path.Combine(AutomationFrameworkPlanFragmentsPath, "RequestorJob.yml"); + public static readonly string SpiderAjaxJobPath = Path.Combine(AutomationFrameworkPlanFragmentsPath, "SpiderAjaxJob.yml"); +} diff --git a/Lombiq.Tests.UI/SecurityScanning/AutomationFrameworkPlanPaths.cs b/Lombiq.Tests.UI/SecurityScanning/AutomationFrameworkPlanPaths.cs new file mode 100644 index 000000000..09a2cc65f --- /dev/null +++ b/Lombiq.Tests.UI/SecurityScanning/AutomationFrameworkPlanPaths.cs @@ -0,0 +1,13 @@ +using System.IO; + +namespace Lombiq.Tests.UI.SecurityScanning; + +public static class AutomationFrameworkPlanPaths +{ + private static readonly string AutomationFrameworkPlansPath = Path.Combine("SecurityScanning", "AutomationFrameworkPlans"); + + public static readonly string BaselinePlanPath = Path.Combine(AutomationFrameworkPlansPath, "Baseline.yml"); + public static readonly string FullScanPlanPath = Path.Combine(AutomationFrameworkPlansPath, "FullScan.yml"); + public static readonly string GraphQLPlanPath = Path.Combine(AutomationFrameworkPlansPath, "GraphQL.yml"); + public static readonly string OpenAPIPlanPath = Path.Combine(AutomationFrameworkPlansPath, "OpenAPI.yml"); +} diff --git a/Lombiq.Tests.UI/SecurityScanning/AutomationFrameworkPlans/Baseline.yml b/Lombiq.Tests.UI/SecurityScanning/AutomationFrameworkPlans/Baseline.yml new file mode 100644 index 000000000..3ebde1b96 --- /dev/null +++ b/Lombiq.Tests.UI/SecurityScanning/AutomationFrameworkPlans/Baseline.yml @@ -0,0 +1,102 @@ +--- +env: + contexts: + - name: "Default Context" + urls: + - "" + excludePaths: [] + authentication: + parameters: {} + verification: + method: "response" + pollFrequency: 60 + pollUnits: "requests" + sessionManagement: + method: "cookie" + parameters: {} + technology: + exclude: + - "C" + - "IBM DB2" + - "PHP" + - "CouchDB" + - "Oracle" + - "JSP/Servlet" + - "Firebird" + - "HypersonicSQL" + - "SAP MaxDB" + - "Ruby" + - "Microsoft Access" + - "Java" + - "Tomcat" + - "Sybase" + - "Python" + parameters: + failOnError: true + failOnWarning: false + progressToStdout: true + vars: {} +jobs: +- parameters: + scanOnlyInScope: true + enableTags: false + disableAllRules: false + rules: + - id: 10035 + name: "Strict-Transport-Security Header" + threshold: "off" + name: "passiveScan-config" + type: "passiveScan-config" +- parameters: {} + name: "spider" + type: "spider" +- parameters: {} + name: "passiveScan-wait" + type: "passiveScan-wait" +- parameters: + reportDir: "/zap/wrk/reports" + template: "modern" + theme: "corporate" + reportTitle: "ZAP Scanning Report" + reportDescription: "" + risks: + - "low" + - "medium" + - "high" + confidences: + - "low" + - "medium" + - "high" + - "confirmed" + sections: + - "passingrules" + - "instancecount" + - "alertdetails" + - "alertcount" + - "params" + - "chart" + - "statistics" + name: "report" + type: "report" +- parameters: + template: "sarif-json" + reportDir: "/zap/wrk/reports" + reportFile: "" + reportTitle: "ZAP Scanning Report" + reportDescription: "" + displayReport: false + risks: + - "info" + - "low" + - "medium" + - "high" + confidences: + - "falsepositive" + - "low" + - "medium" + - "high" + - "confirmed" + sites: [] + name: "sarifReport" + type: "report" + diff --git a/Lombiq.Tests.UI/SecurityScanning/AutomationFrameworkPlans/FullScan.yml b/Lombiq.Tests.UI/SecurityScanning/AutomationFrameworkPlans/FullScan.yml new file mode 100644 index 000000000..19bf336b2 --- /dev/null +++ b/Lombiq.Tests.UI/SecurityScanning/AutomationFrameworkPlans/FullScan.yml @@ -0,0 +1,107 @@ +--- +env: + contexts: + - name: "Default Context" + urls: + - "" + excludePaths: [] + authentication: + parameters: {} + verification: + method: "response" + pollFrequency: 60 + pollUnits: "requests" + sessionManagement: + method: "cookie" + parameters: {} + technology: + exclude: + - "C" + - "IBM DB2" + - "PHP" + - "CouchDB" + - "Oracle" + - "JSP/Servlet" + - "Firebird" + - "HypersonicSQL" + - "SAP MaxDB" + - "Ruby" + - "Microsoft Access" + - "Java" + - "Tomcat" + - "Sybase" + - "Python" + parameters: + failOnError: true + failOnWarning: false + progressToStdout: true + vars: {} +jobs: +- parameters: + scanOnlyInScope: true + enableTags: false + disableAllRules: false + rules: + - id: 10035 + name: "Strict-Transport-Security Header" + threshold: "off" + name: "passiveScan-config" + type: "passiveScan-config" +- parameters: {} + name: "spider" + type: "spider" +- parameters: {} + name: "passiveScan-wait" + type: "passiveScan-wait" +- parameters: {} + policyDefinition: + rules: [] + name: "activeScan" + type: "activeScan" +- parameters: + reportDir: "/zap/wrk/reports" + template: "modern" + theme: "corporate" + reportTitle: "ZAP Scanning Report" + reportDescription: "" + risks: + - "low" + - "medium" + - "high" + confidences: + - "low" + - "medium" + - "high" + - "confirmed" + sections: + - "passingrules" + - "instancecount" + - "alertdetails" + - "alertcount" + - "params" + - "chart" + - "statistics" + name: "report" + type: "report" +- parameters: + template: "sarif-json" + reportDir: "/zap/wrk/reports" + reportFile: "" + reportTitle: "ZAP Scanning Report" + reportDescription: "" + displayReport: false + risks: + - "info" + - "low" + - "medium" + - "high" + confidences: + - "falsepositive" + - "low" + - "medium" + - "high" + - "confirmed" + sites: [] + name: "sarifReport" + type: "report" + diff --git a/Lombiq.Tests.UI/SecurityScanning/AutomationFrameworkPlans/GraphQL.yml b/Lombiq.Tests.UI/SecurityScanning/AutomationFrameworkPlans/GraphQL.yml new file mode 100644 index 000000000..24f14d902 --- /dev/null +++ b/Lombiq.Tests.UI/SecurityScanning/AutomationFrameworkPlans/GraphQL.yml @@ -0,0 +1,116 @@ +--- +env: + contexts: + - name: "Default Context" + urls: + - "" + excludePaths: [] + authentication: + parameters: {} + verification: + method: "response" + pollFrequency: 60 + pollUnits: "requests" + sessionManagement: + method: "cookie" + parameters: {} + technology: + exclude: + - "C" + - "IBM DB2" + - "PHP" + - "CouchDB" + - "Oracle" + - "JSP/Servlet" + - "Firebird" + - "HypersonicSQL" + - "SAP MaxDB" + - "Ruby" + - "Microsoft Access" + - "Java" + - "Tomcat" + - "Sybase" + - "Python" + parameters: + failOnError: true + failOnWarning: false + progressToStdout: true + vars: {} +jobs: +- parameters: + scanOnlyInScope: true + enableTags: false + disableAllRules: false + rules: + - id: 10035 + name: "Strict-Transport-Security Header" + threshold: "off" + name: "passiveScan-config" + type: "passiveScan-config" +- parameters: + queryGenEnabled: true + maxQueryDepth: 5 + lenientMaxQueryDepthEnabled: true + maxAdditionalQueryDepth: 5 + maxArgsDepth: 5 + optionalArgsEnabled: true + argsType: "both" + querySplitType: "leaf" + requestMethod: "post_json" + name: "graphql" + type: "graphql" +- parameters: {} + name: "passiveScan-wait" + type: "passiveScan-wait" +- parameters: {} + policyDefinition: + rules: [] + name: "activeScan" + type: "activeScan" +- parameters: + reportDir: "/zap/wrk/reports" + template: "modern" + theme: "corporate" + reportTitle: "ZAP Scanning Report" + reportDescription: "" + risks: + - "low" + - "medium" + - "high" + confidences: + - "low" + - "medium" + - "high" + - "confirmed" + sections: + - "passingrules" + - "instancecount" + - "alertdetails" + - "alertcount" + - "params" + - "chart" + - "statistics" + name: "report" + type: "report" +- parameters: + template: "sarif-json" + reportDir: "/zap/wrk/reports" + reportFile: "" + reportTitle: "ZAP Scanning Report" + reportDescription: "" + displayReport: false + risks: + - "info" + - "low" + - "medium" + - "high" + confidences: + - "falsepositive" + - "low" + - "medium" + - "high" + - "confirmed" + sites: [] + name: "sarifReport" + type: "report" + diff --git a/Lombiq.Tests.UI/SecurityScanning/AutomationFrameworkPlans/OpenAPI.yml b/Lombiq.Tests.UI/SecurityScanning/AutomationFrameworkPlans/OpenAPI.yml new file mode 100644 index 000000000..9327dc09a --- /dev/null +++ b/Lombiq.Tests.UI/SecurityScanning/AutomationFrameworkPlans/OpenAPI.yml @@ -0,0 +1,107 @@ +--- +env: + contexts: + - name: "Default Context" + urls: + - "" + excludePaths: [] + authentication: + parameters: {} + verification: + method: "response" + pollFrequency: 60 + pollUnits: "requests" + sessionManagement: + method: "cookie" + parameters: {} + technology: + exclude: + - "C" + - "IBM DB2" + - "PHP" + - "CouchDB" + - "Oracle" + - "JSP/Servlet" + - "Firebird" + - "HypersonicSQL" + - "SAP MaxDB" + - "Ruby" + - "Microsoft Access" + - "Java" + - "Tomcat" + - "Sybase" + - "Python" + parameters: + failOnError: true + failOnWarning: false + progressToStdout: true + vars: {} +jobs: +- parameters: + scanOnlyInScope: true + enableTags: false + disableAllRules: false + rules: + - id: 10035 + name: "Strict-Transport-Security Header" + threshold: "off" + name: "passiveScan-config" + type: "passiveScan-config" +- parameters: {} + name: "openapi" + type: "openapi" +- parameters: {} + name: "passiveScan-wait" + type: "passiveScan-wait" +- parameters: {} + policyDefinition: + rules: [] + name: "activeScan" + type: "activeScan" +- parameters: + reportDir: "/zap/wrk/reports" + template: "modern" + theme: "corporate" + reportTitle: "ZAP Scanning Report" + reportDescription: "" + risks: + - "low" + - "medium" + - "high" + confidences: + - "low" + - "medium" + - "high" + - "confirmed" + sections: + - "passingrules" + - "instancecount" + - "alertdetails" + - "alertcount" + - "params" + - "chart" + - "statistics" + name: "report" + type: "report" +- parameters: + template: "sarif-json" + reportDir: "/zap/wrk/reports" + reportFile: "" + reportTitle: "ZAP Scanning Report" + reportDescription: "" + displayReport: false + risks: + - "info" + - "low" + - "medium" + - "high" + confidences: + - "falsepositive" + - "low" + - "medium" + - "high" + - "confirmed" + sites: [] + name: "sarifReport" + type: "report" + diff --git a/Lombiq.Tests.UI/SecurityScanning/SecurityScanConfiguration.cs b/Lombiq.Tests.UI/SecurityScanning/SecurityScanConfiguration.cs new file mode 100644 index 000000000..4aad3938e --- /dev/null +++ b/Lombiq.Tests.UI/SecurityScanning/SecurityScanConfiguration.cs @@ -0,0 +1,265 @@ +using Lombiq.HelpfulLibraries.OrchardCore.Mvc; +using Lombiq.Tests.UI.Constants; +using Lombiq.Tests.UI.Extensions; +using Lombiq.Tests.UI.Services; +using Lombiq.Tests.UI.Shortcuts.Controllers; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using YamlDotNet.RepresentationModel; + +namespace Lombiq.Tests.UI.SecurityScanning; + +/// +/// High-level configuration for a security scan with Zed Attack Proxy (ZAP). +/// +/// +/// +/// This class and intentionally use different terminology, the latter assuming you +/// know ZAP. Here, we provide a simplified configuration for people who just want to use security scans without having +/// to understand ZAP's configuration too much. +/// +/// +public class SecurityScanConfiguration +{ + public Uri StartUri { get; private set; } + public IList AdditionalUris { get; } = new List(); + public bool AjaxSpiderIsUsed { get; private set; } + public string SignInUserName { get; private set; } + public IList ExcludedUrlRegexPatterns { get; } = new List(); + public IList DisabledActiveScanRules { get; } = new List(); + public IDictionary ConfiguredActiveScanRules { get; } = + new Dictionary(); + public IList DisabledPassiveScanRules { get; } = new List(); + public IDictionary DisabledRulesForUrls { get; } = new Dictionary(); + public IList> ZapPlanModifiers { get; } = new List>(); + + internal SecurityScanConfiguration() + { + } + + /// + /// Sets the start URL under the app where to start the scan from. + /// + /// The under the app where to start the scan from. + public SecurityScanConfiguration StartAtUri(Uri startUri) + { + StartUri = startUri; + return this; + } + + /// + /// Adds an additional URL to visit during the scan. This is useful if you want to scan URLs that are otherwise + /// unreachable from . + /// + /// The under the app to also cover during the scan. + public SecurityScanConfiguration AddAdditionalUri(Uri additionalUri) + { + AdditionalUris.Add(additionalUri); + return this; + } + + /// + /// Enables the ZAP Ajax + /// Spider. This is useful if you have an SPA; it unnecessarily slows down the scan otherwise. + /// + public SecurityScanConfiguration UseAjaxSpider() + { + AjaxSpiderIsUsed = true; + return this; + } + + /// + /// Signs in directly (see ) with the given user at the start + /// of the scan. + /// + /// The name of the user to sign in with directly. + public SecurityScanConfiguration SignIn(string userName = DefaultUser.UserName) + { + SignInUserName = userName; + return this; + } + + /// + /// Excludes URLs from the scan that are matched by the supplied regex. + /// + /// + /// The regex pattern to match URLs against. It will be matched against the whole absolute URL, e.g., ".*blog.*" + /// will match https://example.com/blog, https://example.com/blog/my-post, etc. + /// + public SecurityScanConfiguration ExcludeUrlWithRegex(string excludedUrlRegex) + { + ExcludedUrlRegexPatterns.Add(excludedUrlRegex); + return this; + } + + /// + /// Disable a certain active scan rule for the whole scan. If you only want to disable a rule for specific pages + /// matched by a regex, use instead. + /// + /// The ID of the rule. In the scan report, this is usually displayed as "Plugin Id". + /// + /// The human-readable name of the rule. Not required to turn off the rule, and its value doesn't matter. It's just + /// useful for the readability of the method call. + /// + public SecurityScanConfiguration DisableActiveScanRule(int id, string name = "") + { + DisabledActiveScanRules.Add(new ScanRule(id, name)); + return this; + } + + /// + /// Configures a certain active scan rule for the whole scan. + /// + /// The ID of the rule. In the scan report, this is usually displayed as "Plugin Id". + /// + /// Controls how likely ZAP is to report potential vulnerabilities. See the official docs. + /// + /// + /// Controls the number of attacks that ZAP will perform. See the official docs. + /// + /// + /// The human-readable name of the rule. Not required to turn off the rule, and its value doesn't matter. It's just + /// useful for the readability of the method call. + /// + public SecurityScanConfiguration ConfigureActiveScanRule(int id, ScanRuleThreshold threshold, ScanRuleStrength strength, string name = "") + { + ConfiguredActiveScanRules.Add(new ScanRule(id, name), (threshold, strength)); + return this; + } + + /// + /// Disable a certain passive scan rule for the whole scan. If you only want to disable a rule for specific pages + /// matched by a regex, use instead. + /// + /// The ID of the rule. In the scan report, this is usually displayed as "Plugin Id". + /// + /// The human-readable name of the rule. Not required to turn off the rule, and its value doesn't matter. It's just + /// useful for the readability of the method call. + /// + public SecurityScanConfiguration DisablePassiveScanRule(int id, string name = "") + { + DisabledPassiveScanRules.Add(new ScanRule(id, name)); + return this; + } + + /// + /// Disables a rule (can be any rule, including e.g. both active or passive scan rules) for just URLs matching the + /// given regular expression pattern. + /// + /// + /// The regex pattern to match URLs against. It will be matched against the whole absolute URL, e.g., ".*blog.*" + /// will match https://example.com/blog, https://example.com/blog/my-post, etc. + /// + /// The ID of the rule. In the scan report, this is usually displayed as "Plugin Id". + /// + /// The human-readable name of the rule. Not required to turn off the rule, and its value doesn't matter. It's just + /// useful for the readability of the method call. + /// + public SecurityScanConfiguration DisableScanRuleForUrlWithRegex(string urlRegex, int ruleId, string ruleName = "") + { + DisabledRulesForUrls[urlRegex] = new ScanRule(ruleId, ruleName); + return this; + } + + /// + /// Modifies the Automation Framework + /// plan of Zed Attack Proxy (ZAP), the tool used for the security scan. + /// You can use this to do any arbitrary ZAP configuration. + /// + /// + /// A delegate to modify the deserialized representation of the ZAP Automation Framework plan in YAML. + /// + public SecurityScanConfiguration ModifyZapPlan(Func modifyPlan) + { + ZapPlanModifiers.Add(modifyPlan); + return this; + } + + /// + /// Modifies the Automation Framework + /// plan of Zed Attack Proxy (ZAP), the tool used for the security scan. + /// You can use this to do any arbitrary ZAP configuration. + /// + /// + /// A delegate to modify the deserialized representation of the ZAP Automation Framework plan in YAML. + /// + public SecurityScanConfiguration ModifyZapPlan(Action modifyPlan) + { + ZapPlanModifiers.Add(yamlDocument => + { + modifyPlan(yamlDocument); + return Task.CompletedTask; + }); + + return this; + } + + internal async Task ApplyToPlanAsync(YamlDocument yamlDocument, UITestContext context) + { + yamlDocument.SetStartUrl(StartUri); + + foreach (var uri in AdditionalUris) yamlDocument.AddUrl(uri); + + if (AjaxSpiderIsUsed) yamlDocument.AddSpiderAjaxAfterSpider(); + + if (!string.IsNullOrEmpty(SignInUserName)) + { + yamlDocument.AddRequestor( + context.GetAbsoluteUri( + context.GetRelativeUrlOfAction(controller => controller.SignInDirectly(SignInUserName))) + .ToString()); + + // With such direct sign in we don't need to utilize ZAP's authentication and user managements mechanisms + // (see https://www.zaproxy.org/docs/desktop/start/features/authmethods/ and + // https://www.zaproxy.org/docs/desktop/addons/automation-framework/authentication/). If using the standard + // Orchard Core login screen, that would also require using a (headless) browser, bringing all kinds of + // WebDriver compatibility issues we've already solved here. + + // Also, it might be that later such a verification for the login state will need to be needed, but this + // seems unnecessary now. + // verification: + // method: "response" + // method: "poll" + // loggedInRegex: "Unauthenticated" + // loggedOutRegex: "UserName: .*" + // pollFrequency: 60 + // pollUnits: "requests" + // pollUrl: "https://localhost:44335/Lombiq.Tests.UI.Shortcuts/CurrentUser/Index" + // pollPostData: "" + } + + yamlDocument.AddExcludePathsRegex(ExcludedUrlRegexPatterns.ToArray()); + foreach (var rule in DisabledActiveScanRules) yamlDocument.DisableActiveScanRule(rule.Id, rule.Name); + + foreach (var ruleConfiguration in ConfiguredActiveScanRules) + { + yamlDocument.ConfigureActiveScanRule( + ruleConfiguration.Key.Id, + ruleConfiguration.Value.Threshold, + ruleConfiguration.Value.Strength, + ruleConfiguration.Key.Name); + } + + foreach (var rule in DisabledPassiveScanRules) yamlDocument.DisablePassiveScanRule(rule.Id, rule.Name); + foreach (var urlToRule in DisabledRulesForUrls) yamlDocument.AddAlertFilter(urlToRule.Key, urlToRule.Value.Id, urlToRule.Value.Name); + foreach (var modifier in ZapPlanModifiers) await modifier(yamlDocument); + } + + public class ScanRule + { + public int Id { get; } + public string Name { get; } + + public ScanRule(int id, string name) + { + Id = id; + Name = name; + } + } +} diff --git a/Lombiq.Tests.UI/SecurityScanning/SecurityScanResult.cs b/Lombiq.Tests.UI/SecurityScanning/SecurityScanResult.cs new file mode 100644 index 000000000..269e1d044 --- /dev/null +++ b/Lombiq.Tests.UI/SecurityScanning/SecurityScanResult.cs @@ -0,0 +1,15 @@ +using Microsoft.CodeAnalysis.Sarif; + +namespace Lombiq.Tests.UI.SecurityScanning; + +public class SecurityScanResult +{ + public string ReportsDirectoryPath { get; } + public SarifLog SarifLog { get; } + + public SecurityScanResult(string reportsDirectoryPath, SarifLog sarifLog) + { + ReportsDirectoryPath = reportsDirectoryPath; + SarifLog = sarifLog; + } +} diff --git a/Lombiq.Tests.UI/SecurityScanning/SecurityScanningAssertionException.cs b/Lombiq.Tests.UI/SecurityScanning/SecurityScanningAssertionException.cs new file mode 100644 index 000000000..138907dd3 --- /dev/null +++ b/Lombiq.Tests.UI/SecurityScanning/SecurityScanningAssertionException.cs @@ -0,0 +1,27 @@ +using System; + +namespace Lombiq.Tests.UI.Exceptions; + +public class SecurityScanningAssertionException : Exception +{ + public SecurityScanningAssertionException(Exception innerException) + : base( + "Asserting the security scan result failed. Check the security scan report in the failure dump for details.", + innerException) + { + } + + public SecurityScanningAssertionException() + { + } + + public SecurityScanningAssertionException(string message) + : base(message) + { + } + + public SecurityScanningAssertionException(string message, Exception innerException) + : base(message, innerException) + { + } +} diff --git a/Lombiq.Tests.UI/SecurityScanning/SecurityScanningConfiguration.cs b/Lombiq.Tests.UI/SecurityScanning/SecurityScanningConfiguration.cs new file mode 100644 index 000000000..0bf3df4f7 --- /dev/null +++ b/Lombiq.Tests.UI/SecurityScanning/SecurityScanningConfiguration.cs @@ -0,0 +1,36 @@ +using Lombiq.Tests.UI.Services; +using Microsoft.CodeAnalysis.Sarif; +using Shouldly; +using System; +using System.Threading.Tasks; +using YamlDotNet.RepresentationModel; + +namespace Lombiq.Tests.UI.SecurityScanning; + +public class SecurityScanningConfiguration +{ + /// + /// Gets or sets a value indicating whether to save a report to the failure dump for every scan, even passing ones. + /// + /// + /// + /// Won't work until https://github.com/Lombiq/UI-Testing-Toolbox/issues/323 is implemented, hence it's internal. + /// + /// + internal bool CreateReportAlways { get; set; } + + /// + /// Gets or sets a delegate that may modify the deserialized representation of the ZAP Automation Framework plan in + /// YAML. + /// + public Func ZapAutomationFrameworkPlanModifier { get; set; } + + /// + /// Gets or sets a delegate to run assertions on the when security scanning happens. + /// + public Action AssertSecurityScanResult { get; set; } = AssertSecurityScanHasNoAlerts; + + public static readonly Action AssertSecurityScanHasNoAlerts = + (_, sarifLog) => sarifLog.Runs[0].Results.ShouldNotContain(result => + result.Kind == ResultKind.Fail && result.Level != FailureLevel.None && result.Level != FailureLevel.Note); +} diff --git a/Lombiq.Tests.UI/SecurityScanning/SecurityScanningException.cs b/Lombiq.Tests.UI/SecurityScanning/SecurityScanningException.cs new file mode 100644 index 000000000..c78eef82a --- /dev/null +++ b/Lombiq.Tests.UI/SecurityScanning/SecurityScanningException.cs @@ -0,0 +1,20 @@ +using System; + +namespace Lombiq.Tests.UI.SecurityScanning; + +public class SecurityScanningException : Exception +{ + public SecurityScanningException() + { + } + + public SecurityScanningException(string message) + : base(message) + { + } + + public SecurityScanningException(string message, Exception innerException) + : base(message, innerException) + { + } +} diff --git a/Lombiq.Tests.UI/SecurityScanning/SecurityScanningUITestContextExtensions.cs b/Lombiq.Tests.UI/SecurityScanning/SecurityScanningUITestContextExtensions.cs new file mode 100644 index 000000000..95c5c790c --- /dev/null +++ b/Lombiq.Tests.UI/SecurityScanning/SecurityScanningUITestContextExtensions.cs @@ -0,0 +1,163 @@ +using Lombiq.Tests.UI.Exceptions; +using Lombiq.Tests.UI.Extensions; +using Lombiq.Tests.UI.Services; +using Microsoft.CodeAnalysis.Sarif; +using System; +using System.Threading.Tasks; + +namespace Lombiq.Tests.UI.SecurityScanning; + +public static class SecurityScanningUITestContextExtensions +{ + /// + /// Run a Zed Attack Proxy (ZAP) security scan against an app with the + /// Baseline Automation Framework profile except for the spiderAjax job, and runs assertions on the result (see for the official docs on the legacy version of this + /// scan). + /// + /// A delegate to configure the security scan in detail. + /// + /// A delegate to run assertions on the one the scan finishes. + /// + public static Task RunAndAssertBaselineSecurityScanAsync( + this UITestContext context, + Action configure = null, + Action assertSecurityScanResult = null) => + context.RunAndAssertSecurityScanAsync( + AutomationFrameworkPlanPaths.BaselinePlanPath, + configure, + assertSecurityScanResult); + + /// + /// Run a Zed Attack Proxy (ZAP) security scan against an app with the + /// Full Scan Automation Framework profile except for the spiderAjax job, and runs assertions on the result (see + /// for the official docs on the legacy version of this + /// scan). + /// + /// A delegate to configure the security scan in detail. + /// + /// A delegate to run assertions on the one the scan finishes. + /// + public static Task RunAndAssertFullSecurityScanAsync( + this UITestContext context, + Action configure = null, + Action assertSecurityScanResult = null) => + context.RunAndAssertSecurityScanAsync( + AutomationFrameworkPlanPaths.FullScanPlanPath, + configure, + assertSecurityScanResult); + + /// + /// Run a Zed Attack Proxy (ZAP) security scan against an app with the + /// GraphQL Automation Framework profile and runs assertions on the result (see for the official docs on ZAP's GraphQL + /// support). + /// + /// A delegate to configure the security scan in detail. + /// + /// A delegate to run assertions on the one the scan finishes. + /// + public static Task RunAndAssertGraphQLSecurityScanAsync( + this UITestContext context, + Action configure = null, + Action assertSecurityScanResult = null) => + context.RunAndAssertSecurityScanAsync( + AutomationFrameworkPlanPaths.GraphQLPlanPath, + configure, + assertSecurityScanResult); + + /// + /// Run a Zed Attack Proxy (ZAP) security scan against an app with the + /// OpenAPI Automation Framework profile and runs assertions on the result (see for the official docs on ZAP's GraphQL + /// support). + /// + /// A delegate to configure the security scan in detail. + /// + /// A delegate to run assertions on the one the scan finishes. + /// + public static Task RunAndAssertOpenApiSecurityScanAsync( + this UITestContext context, + Action configure = null, + Action assertSecurityScanResult = null) => + context.RunAndAssertSecurityScanAsync( + AutomationFrameworkPlanPaths.OpenAPIPlanPath, + configure, + assertSecurityScanResult); + + /// + /// Run a Zed Attack Proxy (ZAP) security scan against an app and runs + /// assertions on the result. + /// + /// + /// File system path to the YAML configuration file of ZAP's Automation Framework. See for details. + /// + /// A delegate to configure the security scan in detail. + /// + /// A delegate to run assertions on the one the scan finishes. + /// + /// + /// A instance containing the SARIF () report of the scan. + /// + public static async Task RunAndAssertSecurityScanAsync( + this UITestContext context, + string automationFrameworkYamlPath, + Action configure = null, + Action assertSecurityScanResult = null) + { + var configuration = context.Configuration.SecurityScanningConfiguration; + + SecurityScanResult result = null; + try + { + result = await context.RunSecurityScanAsync(automationFrameworkYamlPath, configure); + + if (assertSecurityScanResult != null) assertSecurityScanResult(result.SarifLog); + else configuration?.AssertSecurityScanResult(context, result.SarifLog); + + if (configuration.CreateReportAlways) context.AppendDirectoryToFailureDump(result.ReportsDirectoryPath); + } + catch (Exception ex) + { + if (result != null) context.AppendDirectoryToFailureDump(result.ReportsDirectoryPath); + throw new SecurityScanningAssertionException(ex); + } + } + + /// + /// Run a Zed Attack Proxy (ZAP) security scan against an app. + /// + /// + /// File system path to the YAML configuration file of ZAP's Automation Framework. See + /// for details. + /// + /// A delegate to configure the security scan in detail. + /// + /// A instance containing the SARIF () report of the scan. + /// + public static Task RunSecurityScanAsync( + this UITestContext context, + string automationFrameworkYamlPath, + Action configure = null) + { + var configuration = new SecurityScanConfiguration(); + + configuration.StartAtUri(context.GetCurrentUri()); + + if (context.Configuration.SecurityScanningConfiguration.ZapAutomationFrameworkPlanModifier != null) + { + configuration.ModifyZapPlan(async plan => + await context.Configuration.SecurityScanningConfiguration.ZapAutomationFrameworkPlanModifier(context, plan)); + } + + configure?.Invoke(configuration); + + return context.ZapManager.RunSecurityScanAsync( + context, + automationFrameworkYamlPath, + async plan => await configuration.ApplyToPlanAsync(plan, context)); + } +} diff --git a/Lombiq.Tests.UI/SecurityScanning/YamlDocumentExtensions.cs b/Lombiq.Tests.UI/SecurityScanning/YamlDocumentExtensions.cs new file mode 100644 index 000000000..9966e63ad --- /dev/null +++ b/Lombiq.Tests.UI/SecurityScanning/YamlDocumentExtensions.cs @@ -0,0 +1,422 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using YamlDotNet.RepresentationModel; + +namespace Lombiq.Tests.UI.SecurityScanning; + +public static class YamlDocumentExtensions +{ + /// + /// Sets the start URL under the app where to start the scan from in the current context of the ZAP Automation + /// Framework plan. + /// + /// The absolute to start the scan from. + public static YamlDocument SetStartUrl(this YamlDocument yamlDocument, Uri startUri) + { + // Setting includePaths in the context is not necessary because by default everything under "urls" will be + // scanned. + + var urls = yamlDocument.GetUrls(); + urls.Children.Clear(); + urls.Add(startUri.ToString()); + + return yamlDocument; + } + + /// + /// Adds a URL to the "urls" section of the current context of the ZAP Automation Framework plan. + /// + /// + /// The to add to the "urls" section of the current context of the ZAP Automation Framework + /// plan. + /// + public static YamlDocument AddUrl(this YamlDocument yamlDocument, Uri uri) + { + var urls = yamlDocument.GetUrls(); + urls.Add(uri.ToString()); + return yamlDocument; + } + + /// + /// Adds the ZAP Ajax Spider + /// to the ZAP Automation Framework plan, just after the job named "spider". + /// + /// + /// Thrown if no job named "spider" is found in the ZAP Automation Framework plan. + /// + public static YamlDocument AddSpiderAjaxAfterSpider(this YamlDocument yamlDocument) + { + var jobs = yamlDocument.GetJobs(); + var spiderJob = + yamlDocument.GetSpiderJob() ?? + throw new ArgumentException( + "No job named \"spider\" found in the Automation Framework Plan. We can only add the ajaxSpider job " + + "immediately after it."); + + var spiderIndex = jobs.Children.IndexOf(spiderJob); + var spiderAjaxJob = YamlHelper.LoadDocument(AutomationFrameworkPlanFragmentsPaths.SpiderAjaxJobPath); + jobs.Children.Insert(spiderIndex + 1, spiderAjaxJob.RootNode); + + return yamlDocument; + } + + /// + /// Adds a script to the ZAP Automation Framework plan that displays the runtime of each active scan rule, in milliseconds, + /// just after the first job with the type "activeScan". + /// + /// + /// Thrown if no job with the type "activeScan" is found in the ZAP Automation Framework plan. + /// + /// + /// + /// Script code taken from . + /// + /// + public static YamlDocument AddDisplayActiveScanRuleRuntimesScriptAfterActiveScan(this YamlDocument yamlDocument) + { + var jobs = yamlDocument.GetJobs(); + var activeScanJob = + yamlDocument.GetJobByType("activeScan") ?? + throw new ArgumentException( + "No job with the type \"activeScan\" found in the Automation Framework Plan. We can only add the " + + "active scan rule runtime-displaying script immediately after it."); + + var activeScanIndex = jobs.Children.IndexOf(activeScanJob); + var scriptJobs = ((YamlSequenceNode)YamlHelper + .LoadDocument(AutomationFrameworkPlanFragmentsPaths.DisplayActiveScanRuleRuntimesScriptPath).RootNode) + .Children; + + for (int i = scriptJobs.Count - 1; i >= 0; i--) + { + jobs.Children.Insert(activeScanIndex + 1, scriptJobs[i]); + } + + return yamlDocument; + } + + /// + /// Adds one or more regex patterns to the ZAP Automation Framework plan's excludePaths config under the current + /// context. + /// + /// + /// One or more regex patterns to be added to the ZAP Automation Framework plan's excludePaths config under the + /// current context. These should be regex patterns that match the whole absolute URL, so something like ".*blog.*" + /// to match /blog, /blog/my-post, etc. + /// + public static YamlDocument AddExcludePathsRegex(this YamlDocument yamlDocument, params string[] excludePathsRegexPatterns) + { + var currentContext = yamlDocument.GetCurrentContext(); + + if (!currentContext.Children.ContainsKey("excludePaths")) currentContext.Add("excludePaths", new YamlSequenceNode()); + + var excludePaths = (YamlSequenceNode)currentContext["excludePaths"]; + foreach (var pattern in excludePathsRegexPatterns) + { + excludePaths.Add(pattern); + } + + return yamlDocument; + } + + /// + /// Disable a certain ZAP passive scan rule for the whole scan in the ZAP Automation Framework plan. If you only + /// want to disable a rule for a given page, use + /// instead. + /// + /// The ID of the rule. In the scan report, this is usually displayed as "Plugin Id". + /// + /// The human-readable name of the rule. Not required to turn off the rule, and its value doesn't matter. It's just + /// useful for the readability of the method call. + /// + /// + /// Thrown if no job with the type "passiveScan-config" is found in the Automation Framework Plan. + /// + public static YamlDocument DisablePassiveScanRule(this YamlDocument yamlDocument, int id, string name = "") + { + var passiveScanConfigJob = yamlDocument.GetPassiveScanConfigJobOrThrow(); + + if (!passiveScanConfigJob.Children.ContainsKey("rules")) passiveScanConfigJob.Add("rules", new YamlSequenceNode()); + + var newRule = new YamlMappingNode + { + { "id", id.ToTechnicalString() }, + { "name", name }, + { "threshold", "off" }, + }; + + ((YamlSequenceNode)passiveScanConfigJob["rules"]).Add(newRule); + + return yamlDocument; + } + + /// + /// Disable a certain ZAP active scan rule for the whole scan in the ZAP Automation Framework plan. If you only want + /// to disable a rule for a given page, use + /// instead. + /// + /// The ID of the rule. In the scan report, this is usually displayed as "Plugin Id". + /// + /// The human-readable name of the rule. Not required to turn off the rule, and its value doesn't matter. It's just + /// useful for the readability of the method call. + /// + /// + /// Thrown if no job with the type "activeScan" is found in the Automation Framework Plan, or if it doesn't have a + /// policyDefinition property. + /// + public static YamlDocument DisableActiveScanRule(this YamlDocument yamlDocument, int id, string name = "") => + yamlDocument.ConfigureActiveScanRule(id, ScanRuleThreshold.Off, ScanRuleStrength.Default, name); + + /// + /// Configures a certain ZAP active scan rule for the whole scan in the ZAP Automation Framework plan. + /// + /// The ID of the rule. In the scan report, this is usually displayed as "Plugin Id". + /// + /// Controls how likely ZAP is to report potential vulnerabilities. See the official docs. + /// + /// + /// Controls the number of attacks that ZAP will perform. See the official docs. + /// + /// + /// The human-readable name of the rule. Not required to configure the rule, and its value doesn't matter. It's just + /// useful for the readability of the method call. + /// + /// + /// Thrown if no job with the type "activeScan" is found in the Automation Framework Plan, or if it doesn't have a + /// policyDefinition property. + /// + public static YamlDocument ConfigureActiveScanRule( + this YamlDocument yamlDocument, + int id, + ScanRuleThreshold threshold, + ScanRuleStrength strength, + string name = "") + { + var activeScanConfigJob = + (YamlMappingNode)yamlDocument.GetJobByType("activeScan") ?? + throw new ArgumentException( + "No job with the type \"activeScan\" found in the Automation Framework Plan so the rule can't be added."); + + if (!activeScanConfigJob.Children.ContainsKey("policyDefinition")) + { + throw new ArgumentException("The \"activeScan\" job should contain a policyDefinition."); + } + + var policyDefinition = (YamlMappingNode)activeScanConfigJob["policyDefinition"]; + + if (!policyDefinition.Children.ContainsKey("rules")) policyDefinition.Add("rules", new YamlSequenceNode()); + + var newRule = new YamlMappingNode + { + { "id", id.ToTechnicalString() }, + { "name", name }, + { "threshold", threshold.ToString() }, + { "strength", strength.ToString() }, + }; + + ((YamlSequenceNode)policyDefinition["rules"]).Add(newRule); + + return yamlDocument; + } + + /// + /// Adds an Alert Filter to the ZAP + /// Automation Framework plan. + /// + /// The ID of the rule. In the scan report, this is usually displayed as "Plugin Id". + /// + /// A regular expression pattern to match URLs against. This should be a regex pattern that matches the whole + /// absolute URL, so something like ".*blog.*" to match /blog, /blog/my-post, etc. + /// + /// + /// The human-readable name of the rule. Not required to turn off the rule, and its value doesn't matter. It's just + /// useful for the readability of the method call. + /// + /// + /// If you disable the rule because it's a false positive, then set this to . This helps the + /// development of ZAP by collecting which rules have the highest false positive rate (see ). + /// + public static YamlDocument AddAlertFilter( + this YamlDocument yamlDocument, + string urlMatchingRegexPattern, + int ruleId, + string ruleName = "", + bool isFalsePositive = false) + { + var jobs = yamlDocument.GetJobs(); + + if (yamlDocument.GetJobByType("alertFilter") is not YamlMappingNode alertFilterJob) + { + alertFilterJob = new YamlMappingNode + { + { "type", "alertFilter" }, + { "name", "alertFilter" }, + }; + + var passiveScanConfigJob = yamlDocument.GetPassiveScanConfigJobOrThrow(); + var passiveScanConfigIndex = jobs.Children.IndexOf(passiveScanConfigJob); + jobs.Children.Insert(passiveScanConfigIndex + 1, alertFilterJob); + } + + if (!alertFilterJob.Children.ContainsKey("alertFilters")) alertFilterJob.Add("alertFilters", new YamlSequenceNode()); + + var newRule = new YamlMappingNode + { + { "ruleId", ruleId.ToTechnicalString() }, + { "ruleName", ruleName }, + { "url", urlMatchingRegexPattern }, + { "urlRegex", "true" }, + { "newRisk", isFalsePositive ? "False Positive" : "Info" }, + }; + + ((YamlSequenceNode)alertFilterJob["alertFilters"]).Add(newRule); + + return yamlDocument; + } + + /// + /// Adds a "requestor" job to the ZAP Automation Framework plan just before the job named "spider". + /// + /// The URL the requestor job will access. + /// + /// If no job named "spider" is found in the ZAP Automation Framework plan. + /// + public static YamlDocument AddRequestor(this YamlDocument yamlDocument, string url) + { + var jobs = yamlDocument.GetJobs(); + + var spiderJob = + yamlDocument.GetSpiderJob() ?? + throw new ArgumentException( + "No job named \"spider\" found in the Automation Framework Plan. We can only add the requestor job " + + "immediately before it."); + + var requestorJob = YamlHelper.LoadDocument(AutomationFrameworkPlanFragmentsPaths.RequestorJobPath).GetRootNode(); + + ((YamlSequenceNode)requestorJob["requests"]).Children[0]["url"].SetValue(url); + + var spiderIndex = jobs.Children.IndexOf(spiderJob); + jobs.Children.Insert(spiderIndex, requestorJob); + + return yamlDocument; + } + + /// + /// Gets cast to . + /// + public static YamlMappingNode GetRootNode(this YamlDocument yamlDocument) => (YamlMappingNode)yamlDocument.RootNode; + + /// + /// Gets the "jobs" section of the ZAP Automation Framework plan. + /// + public static YamlSequenceNode GetJobs(this YamlDocument yamlDocument) => + (YamlSequenceNode)yamlDocument.GetRootNode()["jobs"]; + + /// + /// Gets the job from the "jobs" section of the ZAP Automation Framework with the name "spider". + /// + public static YamlNode GetSpiderJob(this YamlDocument yamlDocument) => yamlDocument.GetJobByName("spider"); + + /// + /// Gets a job from the "jobs" section of the ZAP Automation Framework plan by its name. + /// + /// The "name" field of the job to search for. + public static YamlNode GetJobByName(this YamlDocument yamlDocument, string jobName) => + yamlDocument.GetJobs().FirstOrDefault(job => (string)job["name"] == jobName); + + /// + /// Gets a job from the "jobs" section of the ZAP Automation Framework plan by its type. + /// + /// The "type" field of the job to search for. + public static YamlNode GetJobByType(this YamlDocument yamlDocument, string jobType) => + yamlDocument.GetJobs().FirstOrDefault(job => (string)job["type"] == jobType); + + /// + /// Gets the "urls" section of the current context in the ZAP Automation Framework plan. + /// + public static YamlSequenceNode GetUrls(this YamlDocument yamlDocument) + { + var currentContext = yamlDocument.GetCurrentContext(); + + if (!currentContext.Children.ContainsKey("urls")) currentContext.Add("urls", new YamlSequenceNode()); + + return (YamlSequenceNode)currentContext["urls"]; + } + + /// + /// Gets the first context or the one named "Default Context" from the ZAP Automation Framework plan. + /// + /// + /// Thrown if the ZAP Automation Framework plan doesn't contain a context. + /// + public static YamlMappingNode GetCurrentContext(this YamlDocument yamlDocument) + { + var contexts = (YamlSequenceNode)yamlDocument.GetRootNode()["env"]["contexts"]; + + if (!contexts.Any()) + { + throw new ArgumentException( + "The supplied ZAP Automation Framework plan YAML file should contain at least one context."); + } + + var currentContext = (YamlMappingNode)contexts[0]; + + if (contexts.Count() > 1) + { + currentContext = (YamlMappingNode)contexts.FirstOrDefault(context => context["Name"].ToString() == "Default Context") + ?? currentContext; + } + + return currentContext; + } + + /// + /// Gets the job with the type "passiveScan-config" from the ZAP Automation Framework plan. + /// + /// + /// Thrown if the ZAP Automation Framework plan doesn't contain a job with the type "passiveScan-config". + /// + public static YamlMappingNode GetPassiveScanConfigJobOrThrow(this YamlDocument yamlDocument) => + (YamlMappingNode)yamlDocument.GetJobByType("passiveScan-config") ?? + throw new ArgumentException( + "No job with the type \"passiveScan-config\" found in the Automation Framework Plan."); + + /// + /// Shortcuts to to be able to chain extensions in an + /// async method/delegate. + /// + /// . + public static Task CompletedTaskAsync(this YamlDocument yamlDocument) => Task.CompletedTask; + + /// + /// Merge the given into the current one. + /// + /// + /// The to merge from, which overrides the current one. + /// + /// The merged . + public static YamlDocument MergeFrom(this YamlDocument baseDocument, YamlDocument overrideDocument) + { + var baseMapping = baseDocument.GetRootNode(); + var overrideMapping = overrideDocument.GetRootNode(); + + foreach (var entry in overrideMapping.Children) + { + if (baseMapping.Children.ContainsKey(entry.Key)) + { + // Override existing property. + baseMapping.Children[entry.Key] = entry.Value; + } + else + { + // Add new property. + baseMapping.Children.Add(entry.Key, entry.Value); + } + } + + return baseDocument; + } +} diff --git a/Lombiq.Tests.UI/SecurityScanning/YamlHelper.cs b/Lombiq.Tests.UI/SecurityScanning/YamlHelper.cs new file mode 100644 index 000000000..f0fadff09 --- /dev/null +++ b/Lombiq.Tests.UI/SecurityScanning/YamlHelper.cs @@ -0,0 +1,15 @@ +using System.IO; +using YamlDotNet.RepresentationModel; + +namespace Lombiq.Tests.UI.SecurityScanning; + +public static class YamlHelper +{ + public static YamlDocument LoadDocument(string yamlFilePath) + { + using var streamReader = new StreamReader(yamlFilePath); + var yamlStream = new YamlStream(); + yamlStream.Load(streamReader); + return yamlStream.Documents[0]; + } +} diff --git a/Lombiq.Tests.UI/SecurityScanning/YamlNodeExtensions.cs b/Lombiq.Tests.UI/SecurityScanning/YamlNodeExtensions.cs new file mode 100644 index 000000000..df567c6cc --- /dev/null +++ b/Lombiq.Tests.UI/SecurityScanning/YamlNodeExtensions.cs @@ -0,0 +1,25 @@ +using System; +using YamlDotNet.RepresentationModel; + +namespace Lombiq.Tests.UI.SecurityScanning; + +public static class YamlNodeExtensions +{ + /// + /// Sets to the given value. + /// + /// The value to set to. + /// + /// Thrown if the supplied YamlNode can't be cast to YamlScalarNode and thus can't have a value set. + /// + public static void SetValue(this YamlNode yamlNode, string value) + { + if (yamlNode is not YamlScalarNode) + { + throw new ArgumentException( + "The supplied YamlNode can't be cast to YamlScalarNode and thus can't have a value set.", nameof(yamlNode)); + } + + ((YamlScalarNode)yamlNode).Value = value; + } +} diff --git a/Lombiq.Tests.UI/SecurityScanning/ZapEnums.cs b/Lombiq.Tests.UI/SecurityScanning/ZapEnums.cs new file mode 100644 index 000000000..d3f3612c7 --- /dev/null +++ b/Lombiq.Tests.UI/SecurityScanning/ZapEnums.cs @@ -0,0 +1,27 @@ +namespace Lombiq.Tests.UI.SecurityScanning; + +/// +/// Controls how likely ZAP is to report potential vulnerabilities. See the official docs. +/// +public enum ScanRuleThreshold +{ + Off, + Default, + Low, + Medium, + High, +} + +/// +/// Controls the number of attacks that ZAP will perform. See the official docs. +/// +public enum ScanRuleStrength +{ + Default, + Low, + Medium, + High, + Insane, +} diff --git a/Lombiq.Tests.UI/SecurityScanning/ZapManager.cs b/Lombiq.Tests.UI/SecurityScanning/ZapManager.cs new file mode 100644 index 000000000..addea5713 --- /dev/null +++ b/Lombiq.Tests.UI/SecurityScanning/ZapManager.cs @@ -0,0 +1,248 @@ +using CliWrap; +using Lombiq.HelpfulLibraries.Cli; +using Lombiq.Tests.UI.Constants; +using Lombiq.Tests.UI.Services; +using Lombiq.Tests.UI.Services.GitHub; +using Microsoft.CodeAnalysis.Sarif; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Xunit.Abstractions; +using YamlDotNet.RepresentationModel; + +namespace Lombiq.Tests.UI.SecurityScanning; + +/// +/// Service to manage Zed Attack Proxy (ZAP) instances and security scans +/// for a given test. +/// +public sealed class ZapManager : IAsyncDisposable +{ + // Using the then-latest stable release of ZAP. You can check for newer version tags here: + // https://hub.docker.com/r/softwaresecurityproject/zap-stable/tags. + // When updating this version, also regenerate the Automation Framework YAML config files so we don't miss any + // changes to those. + private const string _zapImage = "softwaresecurityproject/zap-stable:2.14.0"; // #spell-check-ignore-line + private const string _zapWorkingDirectoryPath = "/zap/wrk/"; // #spell-check-ignore-line + private const string _zapReportsDirectoryName = "reports"; + + private static readonly SemaphoreSlim _pullSemaphore = new(1, 1); + private static readonly CliProgram _docker = new("docker"); + + private readonly ITestOutputHelper _testOutputHelper; + private readonly CancellationTokenSource _cancellationTokenSource = new(); + + private static bool _wasPulled; + + internal ZapManager(ITestOutputHelper testOutputHelper) => _testOutputHelper = testOutputHelper; + + /// + /// Run a Zed Attack Proxy (ZAP) security scan against an app. + /// + /// The of the currently executing test. + /// + /// File system path to the YAML configuration file of ZAP's Automation Framework. See for details. + /// + /// + /// A delegate to modify the deserialized representation of the ZAP Automation Framework plan in YAML. + /// + /// + /// A instance containing the SARIF () report of the scan. + /// + public async Task RunSecurityScanAsync( + UITestContext context, + string automationFrameworkYamlPath, + Func modifyPlan = null) + { + // Being able to run more than one scan in a test would complicate report generation and processing the SARIF + // report so rather just preventing it here (really nobody should want it anyway). + const string customContextKey = "ZapManager.ScanWasRun"; + + if (context.CustomContext.ContainsKey(customContextKey)) + { + throw new NotSupportedException("You may only run a single ZAP scan in a given test."); + } + + context.CustomContext.Add(customContextKey, string.Empty); + + await EnsureInitializedAsync(); + + if (string.IsNullOrEmpty(automationFrameworkYamlPath)) + { + automationFrameworkYamlPath = AutomationFrameworkPlanPaths.BaselinePlanPath; + } + + var mountedDirectoryPath = DirectoryPaths.GetTempSubDirectoryPath(context.Id, "Zap"); + var reportsDirectoryPath = Path.Combine(mountedDirectoryPath, _zapReportsDirectoryName); + Directory.CreateDirectory(reportsDirectoryPath); + + // Giving write permission to all users to the reports folder. This is to avoid issues under GitHub-hosted + // runners in GitHub Actions (BuildJet ones work without this too) at ZAP not being able to create the report. + // Pre-creating the report's folder would just prompt ZAP to try another folder name suffixed with "2". + if (GitHubHelper.IsGitHubEnvironment) + { + await new CliProgram("chmod").ExecuteAsync(_cancellationTokenSource.Token, "a+w", reportsDirectoryPath); + } + + var yamlFileName = Path.GetFileName(automationFrameworkYamlPath); + var yamlFileCopyPath = Path.Combine(mountedDirectoryPath, yamlFileName); + + File.Copy(automationFrameworkYamlPath, yamlFileCopyPath, overwrite: true); + + await PreparePlanAsync(yamlFileCopyPath, modifyPlan); + + // Explanation on the CLI arguments used below: + // - --add-host and --network host: Lets us connect to the host OS's localhost, where the OC app runs, with + // https://localhost. See https://stackoverflow.com/a/24326540/220230. --network host serves the same, but + // only works under Linux. See https://docs.docker.com/engine/reference/commandline/run/#network and + // https://docs.docker.com/network/drivers/host/. + // - --rm: Removes the container after completion. Otherwise, unused containers would pile up in Docker. See + // https://docs.docker.com/engine/reference/run/#clean-up---rm for the official docs. + // - --volume: Mounts the given host folder as a volume under the given container path. This is to pass files + // back and forth between the host and the container. + // - --tty: Allocates a pseudo-teletypewriter, i.e. redirects the output of ZAP to the CLI's output. + // - zap.sh: The entry point of ZAP. Everything that comes after this is executed in the container. + + // Also see https://www.zaproxy.org/docs/docker/about/#automation-framework. + + // Running a ZAP desktop in the browser with Webswing with the same config under Windows: #spell-check-ignore-line +#pragma warning disable S103 // Lines should not be too long + // docker run --add-host localhost:host-gateway -u zap -p 8080:8080 -p 8090:8090 -i softwaresecurityproject/zap-stable zap-webswing.sh #spell-check-ignore-line +#pragma warning restore S103 // Lines should not be too long + + var cliParameters = new List { "run" }; + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + cliParameters.Add("--network"); + cliParameters.Add("host"); + } + else + { + cliParameters.Add("--add-host"); + cliParameters.Add("localhost:host-gateway"); + } + + cliParameters.AddRange(new object[] + { + "--rm", + "--volume", + $"{mountedDirectoryPath}:{_zapWorkingDirectoryPath}:rw", + "--tty", + _zapImage, + "zap.sh", + "-cmd", + "-autorun", + _zapWorkingDirectoryPath + yamlFileName, + }); + + var stdErrBuffer = new StringBuilder(); + + // Here we use a new container instance for every scan. This is viable and is not an overhead big enough to + // worry about, but an optimization would be to run multiple scans (also possible simultaneously with background + // commands) with the same instance. This needs the -dir option to configure a different home directory per + // scan, see https://www.zaproxy.org/docs/desktop/cmdline/#options. + var result = await _docker + .GetCommand(cliParameters) + .WithStandardOutputPipe(PipeTarget.ToDelegate(line => _testOutputHelper.WriteLineTimestampedAndDebug(line))) + .WithStandardErrorPipe(PipeTarget.ToStringBuilder(stdErrBuffer)) + // This is so no exception is thrown by CliWrap if the exit code is not 0. + .WithValidation(CommandResultValidation.None) + .ExecuteAsync(_cancellationTokenSource.Token); + + _testOutputHelper.WriteLineTimestampedAndDebug("Security scanning completed with the exit code {0}.", result.ExitCode); + + if (result.ExitCode == 1) + { + throw new SecurityScanningException("Security scanning didn't successfully finish. Check the test's output log for details."); + } + + var jsonReports = Directory.EnumerateFiles(reportsDirectoryPath, "*.json").ToList(); + + if (jsonReports.Count > 1) + { + throw new SecurityScanningException( + "There were more than one JSON reports generated for the ZAP scan. The supplied ZAP Automation " + + "Framework YAML file should contain exactly one JSON report job, generating a SARIF report."); + } + + if (jsonReports.Count != 1) + { + throw new SecurityScanningException( + "No SARIF JSON report was generated for the ZAP scan. This indicates that the scan couldn't finish. " + + "Check the test output for details."); + } + + return new SecurityScanResult(reportsDirectoryPath, SarifLog.Load(jsonReports[0])); + } + + public ValueTask DisposeAsync() + { + if (_cancellationTokenSource != null && !_cancellationTokenSource.IsCancellationRequested) + { + _cancellationTokenSource.Cancel(); + _cancellationTokenSource.Dispose(); + } + + return ValueTask.CompletedTask; + } + + private async Task EnsureInitializedAsync() + { + try + { + var token = _cancellationTokenSource.Token; + + await _pullSemaphore.WaitAsync(token); + + if (_wasPulled) return; + + // Without --quiet, "What's Next?" hints will be written to stderr by Docker. See + // https://github.com/docker/for-mac/issues/6904. + await _docker.ExecuteAsync(token, "pull", _zapImage, "--quiet"); + _wasPulled = true; + } + finally + { + _pullSemaphore.Release(); + } + } + + private static async Task PreparePlanAsync(string yamlFilePath, Func modifyPlan) + { + var yamlDocument = YamlHelper.LoadDocument(yamlFilePath); + + // Setting report directories to the conventional one and verifying that there's exactly one SARIF report. + + var sarifReportCount = 0; + + foreach (var job in yamlDocument.GetJobs()) + { + if ((string)job["type"] != "report") continue; + + var parameters = (YamlMappingNode)job["parameters"]; + parameters["reportDir"].SetValue(_zapWorkingDirectoryPath + _zapReportsDirectoryName); + + if ((string)parameters["template"] == "sarif-json") sarifReportCount++; + } + + if (sarifReportCount != 1) + { + throw new ArgumentException( + "The supplied ZAP Automation Framework YAML file should contain exactly one SARIF report job."); + } + + if (modifyPlan != null) await modifyPlan(yamlDocument); + + using var streamWriter = new StreamWriter(yamlFilePath); + var yamlStream = new YamlStream(yamlDocument); + yamlStream.Save(streamWriter, assignAnchors: false); + } +} diff --git a/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs b/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs index 9c1725653..77fb8e271 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs @@ -1,4 +1,5 @@ using Lombiq.Tests.UI.Extensions; +using Lombiq.Tests.UI.SecurityScanning; using Lombiq.Tests.UI.Services.GitHub; using OpenQA.Selenium; using Shouldly; @@ -141,6 +142,8 @@ public class OrchardCoreUITestExecutorConfiguration public HtmlValidationConfiguration HtmlValidationConfiguration { get; set; } = new(); + public SecurityScanningConfiguration SecurityScanningConfiguration { get; set; } = new(); + /// /// Gets or sets a value indicating whether the test should verify the Orchard Core logs and the browser logs for /// errors after every page load. When enabled and there is an error the test is failed immediately which prevents diff --git a/Lombiq.Tests.UI/Services/SmtpService.cs b/Lombiq.Tests.UI/Services/SmtpService.cs index 166176861..d198932c8 100644 --- a/Lombiq.Tests.UI/Services/SmtpService.cs +++ b/Lombiq.Tests.UI/Services/SmtpService.cs @@ -110,7 +110,7 @@ public async Task StartAsync() // An empty db parameter means an in-memory DB. For all possible command line arguments see: // https://github.com/rnwood/smtp4dev/blob/master/Rnwood.Smtp4dev/Program.cs#L132. await CliProgram.DotNet.GetCommand( - new object[] { "tool", "run", "smtp4dev", "--db", string.Empty, "--smtpport", _smtpPort, "--urls", webUIUri }) + "tool", "run", "smtp4dev", "--db", string.Empty, "--smtpport", _smtpPort, "--urls", webUIUri) .ExecuteDotNetApplicationAsync( stdErr => throw new IOException( diff --git a/Lombiq.Tests.UI/Services/SqlServerManager.cs b/Lombiq.Tests.UI/Services/SqlServerManager.cs index 57cb7fd5b..b0df97248 100644 --- a/Lombiq.Tests.UI/Services/SqlServerManager.cs +++ b/Lombiq.Tests.UI/Services/SqlServerManager.cs @@ -263,23 +263,17 @@ public async Task RestoreSnapshotAsync( } private Task DockerExecuteAsync(string containerName, params object[] command) => - _docker.ExecuteAsync( - CreateArguments(containerName, command), - additionalExceptionText: null, - CancellationToken.None); + _docker.ExecuteAsync(CancellationToken.None, CreateArguments(containerName, command)); private Task DockerExecuteAndGetOutputAsync(string containerName, params object[] command) => - _docker.ExecuteAndGetOutputAsync( - CreateArguments(containerName, command), - additionalExceptionText: null, - CancellationToken.None); + _docker.ExecuteAndGetOutputAsync(CancellationToken.None, CreateArguments(containerName, command)); - private static List CreateArguments(string containerName, params object[] command) + private static object[] CreateArguments(string containerName, params object[] command) { var arguments = new List { "exec", "-u", 0, containerName }; arguments.AddRange(command); - return arguments; + return arguments.ToArray(); } public async ValueTask DisposeAsync() diff --git a/Lombiq.Tests.UI/Services/UITestContext.cs b/Lombiq.Tests.UI/Services/UITestContext.cs index e9806ec4b..a1da23400 100644 --- a/Lombiq.Tests.UI/Services/UITestContext.cs +++ b/Lombiq.Tests.UI/Services/UITestContext.cs @@ -1,6 +1,7 @@ using Lombiq.Tests.UI.Exceptions; using Lombiq.Tests.UI.Extensions; using Lombiq.Tests.UI.Models; +using Lombiq.Tests.UI.SecurityScanning; using OpenQA.Selenium; using System; using System.Collections.Generic; @@ -61,6 +62,13 @@ public class UITestContext /// public AzureBlobStorageRunningContext AzureBlobStorageRunningContext { get; } + /// + /// Gets the service to manage Zed Attack Proxy (ZAP) instances for + /// security scanning. Usually, it's recommended to use the higher-level ZAP extension methods instead. + /// + public ZapManager ZapManager { get; } + /// /// Gets a cumulative log of browser console entries. /// @@ -76,7 +84,7 @@ public class UITestContext public Dictionary CustomContext { get; } = new(); /// - /// Gets a dictionary storing some custom data for collecting in failure dump. + /// Gets a dictionary storing some custom data for collecting in the failure dump. /// public IDictionary FailureDumpContainer { get; } = new Dictionary(); @@ -105,7 +113,8 @@ public UITestContext( OrchardCoreUITestExecutorConfiguration configuration, IWebApplicationInstance application, AtataScope scope, - RunningContextContainer runningContextContainer) + RunningContextContainer runningContextContainer, + ZapManager zapManager) { Id = id; TestManifest = testManifest; @@ -115,6 +124,7 @@ public UITestContext( Scope = scope; SmtpServiceRunningContext = runningContextContainer.SmtpServiceRunningContext; AzureBlobStorageRunningContext = runningContextContainer.AzureBlobStorageRunningContext; + ZapManager = zapManager; } /// diff --git a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs index c66926f65..95e1a9aef 100644 --- a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs +++ b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs @@ -6,6 +6,7 @@ using Lombiq.Tests.UI.Extensions; using Lombiq.Tests.UI.Helpers; using Lombiq.Tests.UI.Models; +using Lombiq.Tests.UI.SecurityScanning; using Lombiq.Tests.UI.Services.GitHub; using Microsoft.VisualBasic.FileIO; using Mono.Unix; @@ -39,6 +40,7 @@ internal sealed class UITestExecutionSession : IAsyncDisposable private SqlServerManager _sqlServerManager; private SmtpService _smtpService; private AzureBlobStorageManager _azureBlobStorageManager; + private ZapManager _zapManager; private IWebApplicationInstance _applicationInstance; private UITestContext _context; private DockerConfiguration _dockerConfiguration; @@ -161,12 +163,13 @@ private async ValueTask ShutdownAsync() if (_applicationInstance != null) await _applicationInstance.DisposeAsync(); + string contextId = null; + if (_context != null) { + contextId = _context.Id; _context.Scope?.Dispose(); - DirectoryHelper.SafelyDeleteDirectoryIfExists(DirectoryPaths.GetTempSubDirectoryPath(_context.Id)); - _context.FailureDumpContainer.Values.ForEach(value => value.Dispose()); _context.FailureDumpContainer.Clear(); } @@ -178,6 +181,25 @@ private async ValueTask ShutdownAsync() if (_smtpService != null) await _smtpService.DisposeAsync(); if (_azureBlobStorageManager != null) await _azureBlobStorageManager.DisposeAsync(); + if (_zapManager != null) await _zapManager.DisposeAsync(); + + // First the context needs to be disposed before anything else, and then, once the other services free up any + // handles to the temp folder, that can be cleaned up too. + if (!string.IsNullOrEmpty(contextId)) + { + try + { + DirectoryHelper.SafelyDeleteDirectoryIfExists(DirectoryPaths.GetTempSubDirectoryPath(contextId)); + } + catch (Exception ex) when (GitHubHelper.IsGitHubEnvironment) + { + // This can be caused by running a security scan via ZapManager. + _testOutputHelper.WriteLineTimestampedAndDebug( + "Cleaning up the temporary directory failed with the following exception. Due to using ephemeral " + + "GitHub Actions runners, this is not a fatal error. Exception details: {0}", + ex); + } + } _screenshotCount = 0; @@ -584,6 +606,8 @@ private async Task CreateContextAsync() if (_configuration.UseAzureBlobStorage) azureBlobStorageContext = await SetUpAzureBlobStorageAsync(); if (_configuration.UseSmtpService) smtpContext = await StartSmtpServiceAsync(); + _zapManager = new ZapManager(_testOutputHelper); + Task UITestingBeforeAppStartHandlerAsync(string contentRootPath, InstanceCommandLineArgumentsBuilder arguments) { _configuration.OrchardCoreConfiguration.BeforeAppStart -= UITestingBeforeAppStartHandlerAsync; @@ -640,7 +664,8 @@ Task UITestingBeforeAppStartHandlerAsync(string contentRootPath, InstanceCommand _configuration, _applicationInstance, atataScope, - new RunningContextContainer(sqlServerContext, smtpContext, azureBlobStorageContext)); + new RunningContextContainer(sqlServerContext, smtpContext, azureBlobStorageContext), + _zapManager); } private string GetSetupHashCode() => diff --git a/Readme.md b/Readme.md index 731c24da9..4bc2e6f85 100644 --- a/Readme.md +++ b/Readme.md @@ -26,6 +26,7 @@ Highlights: - Visual verification testing: You can make the test fail if the look of the app changes. Demo video [here](https://www.youtube.com/watch?v=a-1zKjxTKkk). - If your app uses a camera, a fake video capture source in Chrome is supported. [Here's a demo video of the feature](https://www.youtube.com/watch?v=sGcD0eJ2ytc), and check out the docs [here](Lombiq.Tests.UI/Docs/FakeVideoCaptureSource.md). - Interactive mode for debugging the app while the test is paused. [Here's a demo of the feature](Lombiq.Tests.UI.Samples/Tests/InteractiveModeTests.cs), and a [demo video here](https://www.youtube.com/watch?v=ItNltaruWTY). +- Security scanning with [Zed Attack Proxy (ZAP)](https://www.zaproxy.org/), the world's most widely used web app security scanner. See a demo video of the project [here](https://www.youtube.com/watch?v=mEUg6-pad-E), and the Orchard Harvest 2023 conference talk about automated QA in Orchard Core [here](https://youtu.be/CHdhwD2NHBU). Also, see our [Testing Toolbox](https://github.com/Lombiq/Testing-Toolbox) for similar features for lower-level tests. @@ -50,6 +51,7 @@ Do you want to quickly try out this project and see it in action? Check it out i - [Configuration](Lombiq.Tests.UI/Docs/Configuration.md) - [Executing tests](Lombiq.Tests.UI/Docs/ExecutingTests.md) - [Fake video capture source](Lombiq.Tests.UI/Docs/FakeVideoCaptureSource.md) +- [Security scanning](Lombiq.Tests.UI/Docs/SecurityScanning.md) - [Troubleshooting](Lombiq.Tests.UI/Docs/Troubleshooting.md) - [Limits](Lombiq.Tests.UI/Docs/Limits.md) - [Tools we use](Lombiq.Tests.UI/Docs/Tools.md)