From cf18b48f032b9ce2de5755248c752e910d19f734 Mon Sep 17 00:00:00 2001 From: rabelenda Date: Thu, 2 May 2024 15:41:56 -0300 Subject: [PATCH] Add support for response assertions --- .../Assertions/DslResponseAssertionTest.cs | 33 +++ .../Core/Assertions/DslResponseAssertion.cs | 276 ++++++++++++++++++ Abstracta.JmeterDsl/JmeterDsl.cs | 34 +++ devbox.lock | 1 + docs/guide/response-processing/index.md | 1 + .../response-processing/response-assertion.md | 30 ++ 6 files changed, 375 insertions(+) create mode 100644 Abstracta.JmeterDsl.Tests/Core/Assertions/DslResponseAssertionTest.cs create mode 100644 Abstracta.JmeterDsl/Core/Assertions/DslResponseAssertion.cs create mode 100644 docs/guide/response-processing/response-assertion.md diff --git a/Abstracta.JmeterDsl.Tests/Core/Assertions/DslResponseAssertionTest.cs b/Abstracta.JmeterDsl.Tests/Core/Assertions/DslResponseAssertionTest.cs new file mode 100644 index 0000000..44d9db4 --- /dev/null +++ b/Abstracta.JmeterDsl.Tests/Core/Assertions/DslResponseAssertionTest.cs @@ -0,0 +1,33 @@ +namespace Abstracta.JmeterDsl.Core.Assertions +{ + using static JmeterDsl; + + public class DslResponseAssertionTest + { + [Test] + public void ShouldNotFailAssertionWhenResponseAssertionWithMatchingCondition() + { + var stats = TestPlan( + ThreadGroup(1, 1, + DummySampler("OK") + .Children( + ResponseAssertion().ContainsSubstrings("OK") + ) + )).Run(); + Assert.That(stats.Overall.ErrorsCount, Is.EqualTo(0)); + } + + [Test] + public void ShouldFailAssertionWhenResponseAssertionWithNotMatchingCondition() + { + var stats = TestPlan( + ThreadGroup(1, 1, + DummySampler("OK") + .Children( + ResponseAssertion().ContainsSubstrings("FAIL") + ) + )).Run(); + Assert.That(stats.Overall.ErrorsCount, Is.EqualTo(1)); + } + } +} \ No newline at end of file diff --git a/Abstracta.JmeterDsl/Core/Assertions/DslResponseAssertion.cs b/Abstracta.JmeterDsl/Core/Assertions/DslResponseAssertion.cs new file mode 100644 index 0000000..11cea78 --- /dev/null +++ b/Abstracta.JmeterDsl/Core/Assertions/DslResponseAssertion.cs @@ -0,0 +1,276 @@ +using Abstracta.JmeterDsl.Core.TestElements; + +namespace Abstracta.JmeterDsl.Core.Assertions +{ + /// + /// Allows marking a request result as success or failure by a specific result field value. + /// + public class DslResponseAssertion : DslScopedTestElement + { + private TargetField _fieldToTest = TargetField.ResponseBody; + private bool _ignoreStatus; + private string[] _containsSubstrings; + private string[] _equalsToStrings; + private string[] _containsRegexes; + private string[] _matchesRegexes; + private bool _invertCheck; + private bool _anyMatch; + + public DslResponseAssertion(string name) + : base(name) + { + } + + /// + /// Identifies a particular field to apply the assertion to. + /// + public enum TargetField + { + /// + /// Applies the assertion to the response body. + /// + ResponseBody, + + /// + /// Applies the assertion to the text obtained through Apache Tika + /// from the response body (which might be a pdf, excel, etc.). + /// + ResponseBodyAsDocument, + + /// + /// Applies the assertion to the response code (eg: the HTTP response code, like 200). + /// + ResponseCode, + + /// + /// Applies the assertion to the response message (eg: the HTTP response message, like OK). + /// + ResponseMessage, + + /// + /// Applies the assertion to the response headers. Response headers is a string with headers + /// separated by new lines and names and values separated by colons. + /// + ResponseHeaders, + + /// + /// Applies the assertion to the set of request headers. Request headers is a string with headers + /// separated by new lines and names and values separated by colons. + /// + RequestHeaders, + + /// + /// Applies the assertion to the request URL. + /// + RequestUrl, + + /// + /// Applies the assertion to the request body. + /// + RequestBody, + } + + /// + /// Specifies what field to apply the assertion to. + ///
+ /// When not specified it will apply the given assertion to the response body. + /// + /// specifies the field to apply the assertion to. + /// the response assertion for further configuration or usage. + /// + public DslResponseAssertion FieldToTest(TargetField fieldToTest) + { + _fieldToTest = fieldToTest; + return this; + } + + /// + /// Specifies that any previously status set to the request should be ignored, and request should + /// be marked as success by default. + ///
+ /// This allows overriding the default behavior provided by JMeter when marking requests as failed + /// (eg: HTTP status codes like 4xx or 5xx). This is particularly useful when tested application + /// returns an unsuccessful response (eg: 400) but you want to consider some of those cases still + /// as successful using a different criteria to determine when they are actually a failure (an + /// unexpected response). + ///
+ /// Take into consideration that if you specify multiple response assertions to the same sampler, + /// then if this flag is enabled, any previous assertion result in same sampler will be ignored + /// (marked as success). So, consider setting this flag in first response assertion only. + /// + /// the response assertion for further configuration or usage. + public DslResponseAssertion IgnoreStatus() => + IgnoreStatus(true); + + /// + /// Same as but allowing to enable or disable it. + ///
+ /// This is helpful when the resolution is taken at runtime. + /// + /// specifies to enable or disable the setting. By default, it is set to false. + /// the response assertion for further configuration or usage. + /// + public DslResponseAssertion IgnoreStatus(bool enable) + { + _ignoreStatus = enable; + return this; + } + + /// + /// Checks if the specified contains the given substrings. + ///
+ /// By default, the main sample (not sub samples) response body will be checked, and all supplied + /// substrings must be contained. Review other methods in this class if you need to check + /// substrings but in some other ways (eg: in response headers, any match is enough, or none of + /// specified substrings should be contained). + /// + /// list of strings to be searched in the given field to test (by default + /// response body). + /// the response assertion for further configuration or usage. + public DslResponseAssertion ContainsSubstrings(params string[] substrings) + { + _containsSubstrings = substrings; + return this; + } + + /// + /// Compares the configured to the given strings for equality. + ///
+ /// By default, the main sample (not sub samples) response body will be checked, and all supplied + /// strings must be equal to the body (in default setting only makes sense to specify one string). + /// Review other methods in this class if you need to check equality to entire strings but in some + /// other ways (eg: in response headers, any match is enough, or none of specified strings should + /// be equal to the field value). + /// + /// list of strings to be compared against the given field to test (by default + /// response body). + /// the response assertion for further configuration or usage. + public DslResponseAssertion EqualsToStrings(params string[] strings) + { + _equalsToStrings = strings; + return this; + } + + /// + /// Checks if the configured contains matches for given regular + /// expressions. + ///
+ /// By default, the main sample (not sub samples) response body will be checked, and all supplied + /// regular expressions must contain a match in the body. Review other methods in this class if you + /// need to check regular expressions matches are contained but in some other ways (eg: in response + /// headers, any regex match is enough, or none of specified regex should be contained in the field + /// value). + ///
+ /// By default, regular expressions evaluate in multi-line mode, which means that '.' does not + /// match new lines, '^' matches start of lines and '$' matches end of lines. To use single-line + /// mode prefix '(?s)' to the regular expressions. Regular expressions are also by default + /// case-sensitive, which can be changed to insensitive by adding '(?i)' to the regex. + /// + /// list of regular expressions to search for matches in the field to test (by + /// default response body). + /// the response assertion for further configuration or usage. + public DslResponseAssertion ContainsRegexes(params string[] regexes) + { + _containsRegexes = regexes; + return this; + } + + /// + /// Checks if the configured matches (completely, and not just + /// part of it) given regular expressions. + ///
+ /// By default, the main sample (not sub samples) response body will be checked, and all supplied + /// regular expressions must match the entire body. Review other methods in this class if you need + /// to check regular expressions matches but in some other ways (eg: in response headers, any regex + /// match is enough, or none of specified regex should be matched with the field value). + ///
+ /// By default, regular expressions evaluate in multi-line mode, which means that '.' does not + /// match new lines, '^' matches start of lines and '$' matches end of lines. To use single-line + /// mode prefix '(?s)' to the regular expressions. Regular expressions are also by default + /// case-sensitive, which can be changed to insensitive by adding '(?i)' to the regex. + /// + /// list of regular expressions the field to test (by default response body) must + /// match. + /// the response assertion for further configuration or usage. + public DslResponseAssertion MatchesRegexes(params string[] regexes) + { + _matchesRegexes = regexes; + return this; + } + + /// + /// Allows inverting/negating each of the checks applied by the assertion. + ///
+ /// This is the same as the "Not" option in Response Assertion in JMeter GUI. + ///
+ /// It is important to note that the inversion of the check happens at each check and not to the + /// final result. Eg: + /// ResponseAssertion().ContainsSubstrings("error", "failure").InvertCheck() + ///
+ /// Will check that the response does not contain "error" and does not contain "failure". You can + /// think it as !(ContainsSubstring("error")) && !(ContainsSubstring("failure")). + ///
+ /// Similar logic applies when using in combination with anyMatch method. Eg: + /// ResponseAssertion().ContainsSubstrings("error", "failure").InvertCheck().MatchAny() + ///
+ /// Will check that response does not contain both "error" and "failure" at the same time. This is + /// analogous to !(ContainsSubstring("error")) || !(ContainsSubstring("failure)), which is + /// equivalent to !(ContainsSubstring("error") && ContainsSubstring("failure)). + ///
+ /// Keep in mind that order of invocations of methods in response assertion is irrelevant (so + /// InvertCheck().MatchAny() gets the same result as MatchAny().InvertCheck()). + /// + /// the response assertion for further configuration or usage. + public DslResponseAssertion InvertCheck() => + InvertCheck(true); + + /// + /// Same as but allowing to enable or disable it. + ///
+ /// This is helpful when the resolution is taken at runtime. + /// + /// specifies to enable or disable the setting. By default, it is set to false. + /// the response assertion for further configuration or usage. + /// + public DslResponseAssertion InvertCheck(bool enable) + { + _invertCheck = enable; + return this; + } + + /// + /// Specifies that if any check matches then the response assertion is satisfied. + ///
+ /// This is the same as the "Or" option in Response Assertion in JMeter GUI. + ///
+ /// By default, when you use something like this: + /// ResponseAssertion().ContainsSubstrings("success", "OK") + ///
+ /// The response assertion will be success when both "success" and "OK" sub strings appear in + /// response body (if one or both don't appear, then it fails). You can think of it like + /// ContainsSubstring("success") && ContainsSubstring("OK"). + ///
+ /// If you want to check that any of them matches then use anyMatch, like this: + /// ResponseAssertion().ContainsSubstrings("success", "OK").AnyMatch() + ///
+ /// Which you can interpret as ContainsSubstring("success") || ContainsSubstring("OK"). + /// + /// the response assertion for further configuration or usage. + public DslResponseAssertion AnyMatch() => + AnyMatch(true); + + /// + /// Same as but allowing to enable or disable it. + ///
+ /// This is helpful when the resolution is taken at runtime. + /// + /// specifies to enable or disable the setting. By default, it is set to false. + /// the response assertion for further configuration or usage. + /// + public DslResponseAssertion AnyMatch(bool enable) + { + _anyMatch = enable; + return this; + } + } +} \ No newline at end of file diff --git a/Abstracta.JmeterDsl/JmeterDsl.cs b/Abstracta.JmeterDsl/JmeterDsl.cs index 981cb40..72cf091 100644 --- a/Abstracta.JmeterDsl/JmeterDsl.cs +++ b/Abstracta.JmeterDsl/JmeterDsl.cs @@ -1,5 +1,6 @@ using System; using Abstracta.JmeterDsl.Core; +using Abstracta.JmeterDsl.Core.Assertions; using Abstracta.JmeterDsl.Core.Configs; using Abstracta.JmeterDsl.Core.Controllers; using Abstracta.JmeterDsl.Core.Listeners; @@ -295,6 +296,39 @@ public static DslJsr223PreProcessor Jsr223PreProcessor(string name, string scrip public static DslRegexExtractor RegexExtractor(string variableName, string regex) => new DslRegexExtractor(variableName, regex); + /// + /// Builds a Response Assertion to be able to check that obtained sample result is the expected + /// one. + ///
+ /// JMeter by default uses repose codes (eg: 4xx and 5xx HTTP response codes are error codes) to + /// determine if a request was success or not, but in some cases this might not be enough or + /// correct. In some cases applications might not behave in this way, for example, they might + /// return a 200 HTTP status code but with an error message in the body, or the response might be a + /// success one, but the information contained within the response is not the expected one to + /// continue executing the test. In such scenarios you can use response assertions to properly + /// verify your assumptions before continuing with next request in the test plan. + ///
+ /// By default, response assertion will use the response body of the main sample result (not sub + /// samples as redirects, or embedded resources) to check the specified criteria (substring match, + /// entire string equality, contained regex or entire regex match) against. + /// + /// the created Response Assertion which should be modified to apply the proper criteria. + /// Check for all available options. + /// + public static DslResponseAssertion ResponseAssertion() => + new DslResponseAssertion(null); + + /// + /// Same as but allowing to set a name on the assertion, which can be + /// later used to identify assertion results and differentiate it from other assertions. + /// + /// is the name to be assigned to the assertion + /// the created Response Assertion which should be modified to apply the proper criteria. + /// Check for all available options. + /// + public static DslResponseAssertion ResponseAssertion(string name) => + new DslResponseAssertion(name); + /// /// Builds a Simple Data Writer to write all collected results to a JTL file. ///
diff --git a/devbox.lock b/devbox.lock index 5c4e5a0..77a5623 100644 --- a/devbox.lock +++ b/devbox.lock @@ -21,6 +21,7 @@ }, "nodejs@latest": { "last_modified": "2023-09-10T10:53:27Z", + "plugin_version": "0.0.2", "resolved": "github:NixOS/nixpkgs/78058d810644f5ed276804ce7ea9e82d92bee293#nodejs_20", "source": "devbox-search", "version": "20.6.1" diff --git a/docs/guide/response-processing/index.md b/docs/guide/response-processing/index.md index 52ecf23..4d8dadd 100644 --- a/docs/guide/response-processing/index.md +++ b/docs/guide/response-processing/index.md @@ -1,3 +1,4 @@ ## Response processing + diff --git a/docs/guide/response-processing/response-assertion.md b/docs/guide/response-processing/response-assertion.md new file mode 100644 index 0000000..e84fd74 --- /dev/null +++ b/docs/guide/response-processing/response-assertion.md @@ -0,0 +1,30 @@ +### Check for expected response + +By default, JMeter marks any HTTP request with a fail response code (4xx or 5xx) as failed, which allows you to easily identify when some request unexpectedly fails. But in many cases, this is not enough or desirable, and you need to check for the response body (or some other field) to contain (or not) a certain string. + +This is usually accomplished in JMeter with the usage of Response Assertions, which provides an easy and fast way to verify that you get the proper response for each step of the test plan, marking the request as a failure when the specified condition is not met. + +Here is an example of how to specify a response assertion in JMeter DSL: + +```cs +using static Abstracta.JmeterDsl.JmeterDsl; + +public class PerformanceTest +{ + [Test] + public void LoadTest() + { + var stats = TestPlan( + ThreadGroup(2, 10, + HttpSampler("http://my.service") + .Children( + ResponseAssertion().ContainsSubstrings("OK") + ) + ) + ).Run(); + Assert.That(stats.Overall.SampleTimePercentile99, Is.LessThan(TimeSpan.FromSeconds(5))); + } +} +``` + +Check [Response Assertion](/Abstracta.JmeterDsl/Core/Assertions/DslResponseAssertion.cs) for more details and additional options.