From ad4434ad121152af882f04ad7e59a85835d58a1e Mon Sep 17 00:00:00 2001 From: Anna Date: Fri, 4 Oct 2024 15:20:47 +0200 Subject: [PATCH 1/4] Renames and simplify syntax --- .../Rcm/AspNetCore5AsmData.cs | 54 +++++++++---------- 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/tracer/test/Datadog.Trace.Security.IntegrationTests/Rcm/AspNetCore5AsmData.cs b/tracer/test/Datadog.Trace.Security.IntegrationTests/Rcm/AspNetCore5AsmData.cs index 27885d7f2193..70b01e13ac32 100644 --- a/tracer/test/Datadog.Trace.Security.IntegrationTests/Rcm/AspNetCore5AsmData.cs +++ b/tracer/test/Datadog.Trace.Security.IntegrationTests/Rcm/AspNetCore5AsmData.cs @@ -12,8 +12,6 @@ using System.Collections.Immutable; using System.Linq; using System.Threading.Tasks; -using Datadog.Trace.AppSec; -using Datadog.Trace.AppSec.Rcm; using Datadog.Trace.AppSec.Rcm.Models.AsmData; using Datadog.Trace.AppSec.Rcm.Models.AsmFeatures; using Datadog.Trace.Configuration; @@ -75,18 +73,17 @@ public async Task RunTest(string test, string url) var sanitisedUrl = VerifyHelper.SanitisePathsForVerify(url); // we want to see the ip here var scrubbers = VerifyHelper.SpanScrubbers.Where(s => s.RegexPattern.ToString() != @"http.client_ip: (.)*(?=,)"); - var settings = VerifyHelper.GetSpanVerifierSettings(scrubbers: scrubbers, parameters: new object[] { test, sanitisedUrl }); + var settings = VerifyHelper.GetSpanVerifierSettings(scrubbers: scrubbers, parameters: [test, sanitisedUrl]); var spanBeforeAsmData = await SendRequestsAsync(agent, url); await agent.SetupRcmAndWait( Output, - new[] - { - ((object)new Payload { RulesData = new[] { new RuleData { Id = "blocked_ips", Type = "ip_with_expiration", Data = new[] { new Data { Expiration = 5545453532, Value = MainIp } } } } }, + [ + (new Payload { RulesData = [new RuleData { Id = "blocked_ips", Type = "ip_with_expiration", Data = [new Data { Expiration = 5545453532, Value = MainIp }] }] }, RcmProducts.AsmData, nameof(AspNetCore5AsmDataBlockingRequestIp)), - (new Payload { RulesData = new[] { new RuleData { Id = "blocked_ips", Type = "ip_with_expiration", Data = new[] { new Data { Expiration = 1545453532, Value = MainIp } } } } }, + (new Payload { RulesData = [new RuleData { Id = "blocked_ips", Type = "ip_with_expiration", Data = [new Data { Expiration = 1545453532, Value = MainIp }] }] }, RcmProducts.AsmData, nameof(AspNetCore5AsmDataBlockingRequestIp) + "2"), - }); + ]); var spanAfterAsmData = await SendRequestsAsync(agent, url); var spans = new List(); @@ -113,11 +110,11 @@ public async Task RunTest(string test, string url) var sanitisedUrl = VerifyHelper.SanitisePathsForVerify(url); // we want to see the ip here var scrubbers = VerifyHelper.SpanScrubbers.Where(s => s.RegexPattern.ToString() != @"http.client_ip: (.)*(?=,)"); - var settings = VerifyHelper.GetSpanVerifierSettings(scrubbers: scrubbers, parameters: new object[] { test, sanitisedUrl }); + var settings = VerifyHelper.GetSpanVerifierSettings(scrubbers: scrubbers, parameters: [test, sanitisedUrl]); var spanBeforeAsmData = await SendRequestsAsync(agent, url); var asmFeaturesFileId = nameof(AspNetCore5AsmDataSecurityEnabledBlockingRequestIpOneClick); - var request = await agent.SetupRcmAndWait(Output, new[] { ((object)new AsmFeatures { Asm = new AsmFeature { Enabled = true } }, RcmProducts.AsmFeatures, asmFeaturesFileId) }); + var request = await agent.SetupRcmAndWait(Output, [(new AsmFeatures { Asm = new AsmFeature { Enabled = true } }, RcmProducts.AsmFeatures, asmFeaturesFileId)]); request.Should().NotBeNull(); request.CachedTargetFiles.Should().HaveCount(1); var spanAfterAsmActivated = await SendRequestsAsync(agent, url); @@ -127,13 +124,12 @@ public async Task RunTest(string test, string url) request = await agent.SetupRcmAndWait( Output, - new[] - { - ((object)new AsmFeatures { Asm = new AsmFeature { Enabled = true } }, RcmProducts.AsmFeatures, asmFeaturesFileId), - (new Payload { RulesData = new[] { new RuleData { Id = "blocked_ips", Type = "ip_with_expiration", Data = new[] { new Data { Expiration = 5545453532, Value = MainIp }, new Data { Expiration = null, Value = "123.1.1.1" } } } } }, RcmProducts.AsmData, fileId), - ((object)new Payload { RulesData = new[] { new RuleData { Id = "blocked_ips", Type = "ip_with_expiration", Data = new[] { new Data { Expiration = 1545453532, Value = MainIp } } } } }, RcmProducts.AsmData, + [ + (new AsmFeatures { Asm = new AsmFeature { Enabled = true } }, RcmProducts.AsmFeatures, asmFeaturesFileId), + (new Payload { RulesData = [new RuleData { Id = "blocked_ips", Type = "ip_with_expiration", Data = [new Data { Expiration = 5545453532, Value = MainIp }, new Data { Expiration = null, Value = "123.1.1.1" }] }] }, RcmProducts.AsmData, fileId), + (new Payload { RulesData = [new RuleData { Id = "blocked_ips", Type = "ip_with_expiration", Data = [new Data { Expiration = 1545453532, Value = MainIp }] }] }, RcmProducts.AsmData, fileId2) - }); + ]); request.Should().NotBeNull(); request.CachedTargetFiles.Should().HaveCount(3); request.CachedTargetFiles.Any(c => c.Path.Contains(fileId)).Should().BeTrue(); @@ -143,28 +139,26 @@ public async Task RunTest(string test, string url) request = await agent.SetupRcmAndWait( Output, - new[] - { - ((object)new AsmFeatures { Asm = new AsmFeature { Enabled = false } }, RcmProducts.AsmFeatures, + [ + (new AsmFeatures { Asm = new AsmFeature { Enabled = false } }, RcmProducts.AsmFeatures, asmFeaturesFileId), - (new Payload { RulesData = new[] { new RuleData { Id = "blocked_ips", Type = "ip_with_expiration", Data = new[] { new Data { Expiration = 5545453532, Value = MainIp }, new Data { Expiration = null, Value = "123.1.1.1" } } } } }, + (new Payload { RulesData = [new RuleData { Id = "blocked_ips", Type = "ip_with_expiration", Data = [new Data { Expiration = 5545453532, Value = MainIp }, new Data { Expiration = null, Value = "123.1.1.1" }] }] }, RcmProducts.AsmData, fileId), - ((object)new Payload { RulesData = new[] { new RuleData { Id = "blocked_ips", Type = "ip_with_expiration", Data = new[] { new Data { Expiration = 1545453532, Value = MainIp } } } } }, + (new Payload { RulesData = [new RuleData { Id = "blocked_ips", Type = "ip_with_expiration", Data = [new Data { Expiration = 1545453532, Value = MainIp }] }] }, RcmProducts.AsmData, fileId2) - }); + ]); request.Should().NotBeNull(); request.CachedTargetFiles.Should().HaveCount(3); var spanAfterAsmDeactivated = await SendRequestsAsync(agent, url); request = await agent.SetupRcmAndWait( Output, - new[] - { - ((object)new AsmFeatures { Asm = new AsmFeature { Enabled = true } }, RcmProducts.AsmFeatures, asmFeaturesFileId), (new Payload { RulesData = new[] { new RuleData { Id = "blocked_ips", Type = "ip_with_expiration", Data = new[] { new Data { Expiration = 5545453532, Value = MainIp }, new Data { Expiration = null, Value = "123.1.1.1" } } } } }, - RcmProducts.AsmData, fileId), - ((object)new Payload { RulesData = new[] { new RuleData { Id = "blocked_ips", Type = "ip_with_expiration", Data = new[] { new Data { Expiration = 1545453532, Value = MainIp } } } } }, + [ + (new AsmFeatures { Asm = new AsmFeature { Enabled = true } }, RcmProducts.AsmFeatures, asmFeaturesFileId), + (new Payload { RulesData = [new RuleData { Id = "blocked_ips", Type = "ip_with_expiration", Data = [new Data { Expiration = 5545453532, Value = MainIp }, new Data { Expiration = null, Value = "123.1.1.1" }] }] }, RcmProducts.AsmData, fileId), + (new Payload { RulesData = [new RuleData { Id = "blocked_ips", Type = "ip_with_expiration", Data = [new Data { Expiration = 1545453532, Value = MainIp }] }] }, RcmProducts.AsmData, fileId2) - }); + ]); request.Should().NotBeNull(); request.CachedTargetFiles.Should().HaveCount(3); var spanAfterAsmDataReactivated = await SendRequestsAsync(agent, url); @@ -196,7 +190,7 @@ public async Task RunTest(string test, string url) await TryStartApp(); var agent = Fixture.Agent; var sanitisedUrl = VerifyHelper.SanitisePathsForVerify(url); - var settings = VerifyHelper.GetSpanVerifierSettings(parameters: new object[] { test, sanitisedUrl }); + var settings = VerifyHelper.GetSpanVerifierSettings(parameters: [test, sanitisedUrl]); var spanBeforeAsmData = await SendRequestsAsync(agent, url); // make sure this is unique if it s going to be run parallel @@ -204,7 +198,7 @@ public async Task RunTest(string test, string url) await agent.SetupRcmAndWait( Output, - new[] { ((object)new Payload { RulesData = new[] { new RuleData { Id = "blocked_users", Type = "data_with_expiration", Data = new[] { new Data { Expiration = 5545453532, Value = "user3" } } } } }, RcmProducts.AsmData, acknowledgedId: fileId) }); + [(new Payload { RulesData = [new RuleData { Id = "blocked_users", Type = "data_with_expiration", Data = [new Data { Expiration = 5545453532, Value = "user3" }] }] }, RcmProducts.AsmData, fileId)]); var spanAfterAsmData = await SendRequestsAsync(agent, url); var spans = new List(); spans.AddRange(spanBeforeAsmData); From 2a3a05d364f0cfbc363aaa7b50342262996e0bf6 Mon Sep 17 00:00:00 2001 From: Anna Date: Tue, 8 Oct 2024 15:04:30 +0200 Subject: [PATCH 2/4] Fix the flakiness first --- .../Rcm/AspNetCore5AsmData.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tracer/test/Datadog.Trace.Security.IntegrationTests/Rcm/AspNetCore5AsmData.cs b/tracer/test/Datadog.Trace.Security.IntegrationTests/Rcm/AspNetCore5AsmData.cs index 70b01e13ac32..cb1c78ead0d8 100644 --- a/tracer/test/Datadog.Trace.Security.IntegrationTests/Rcm/AspNetCore5AsmData.cs +++ b/tracer/test/Datadog.Trace.Security.IntegrationTests/Rcm/AspNetCore5AsmData.cs @@ -151,6 +151,14 @@ public async Task RunTest(string test, string url) request.CachedTargetFiles.Should().HaveCount(3); var spanAfterAsmDeactivated = await SendRequestsAsync(agent, url); + // we have to send first asm features = true, because asm_data won't be taken into account as rcm subscriptions to asm_data have been removed when turning off the waf. and then, later on, send, separately the asm data. That's the trade off of not subscribing to asm_data and asm when appsec is turned off + request = await agent.SetupRcmAndWait( + Output, + [ + (new AsmFeatures { Asm = new AsmFeature { Enabled = true } }, RcmProducts.AsmFeatures, asmFeaturesFileId)]); + request.Should().NotBeNull(); + request.CachedTargetFiles.Should().HaveCount(1); + request = await agent.SetupRcmAndWait( Output, [ @@ -198,7 +206,7 @@ public async Task RunTest(string test, string url) await agent.SetupRcmAndWait( Output, - [(new Payload { RulesData = [new RuleData { Id = "blocked_users", Type = "data_with_expiration", Data = [new Data { Expiration = 5545453532, Value = "user3" }] }] }, RcmProducts.AsmData, fileId)]); + [(new Payload { RulesData = [new RuleData { Id = "blocked_users", Type = "data_with_expiration", Data = [new Data { Expiration = 5545453532, Value = "user3" }] }] }, RcmProducts.AsmData, fileId)]); var spanAfterAsmData = await SendRequestsAsync(agent, url); var spans = new List(); spans.AddRange(spanBeforeAsmData); From 25166aaad7753afda4e026ca01537adf632743fe Mon Sep 17 00:00:00 2001 From: Anna Date: Tue, 8 Oct 2024 15:04:46 +0200 Subject: [PATCH 3/4] Simplify rcm updates --- tracer/missing-nullability-files.csv | 1 - .../Datadog.Trace/AppSec/Rcm/AsmDdProduct.cs | 11 +--- .../AppSec/Rcm/ConfigurationStatus.cs | 58 +++++++++---------- .../AppSec/Rcm/Models/AsmDd/RuleSet.cs | 37 +++++++++--- tracer/src/Datadog.Trace/AppSec/Security.cs | 11 +--- tracer/src/Datadog.Trace/AppSec/Waf/IWaf.cs | 6 +- .../Waf/Initialization/WafConfigurator.cs | 7 +++ .../Waf/ReturnTypes.Managed/InitResult.cs | 3 - tracer/src/Datadog.Trace/AppSec/Waf/Waf.cs | 39 +++++++------ 9 files changed, 91 insertions(+), 82 deletions(-) diff --git a/tracer/missing-nullability-files.csv b/tracer/missing-nullability-files.csv index ad2464eeee8e..f65cc13d46b1 100644 --- a/tracer/missing-nullability-files.csv +++ b/tracer/missing-nullability-files.csv @@ -270,7 +270,6 @@ src/Datadog.Trace/Agent/Transports/MimeTypes.cs src/Datadog.Trace/Agent/Transports/SocketHandlerRequestFactory.cs src/Datadog.Trace/AppSec/Concurrency/ReaderWriterLock.Core.cs src/Datadog.Trace/AppSec/Concurrency/ReaderWriterLock.Framework.cs -src/Datadog.Trace/AppSec/Waf/IWaf.cs src/Datadog.Trace/AppSec/Waf/WafConstants.cs src/Datadog.Trace/AppSec/Waf/WafReturnCode.cs src/Datadog.Trace/Ci/Agent/ApmAgentWriter.cs diff --git a/tracer/src/Datadog.Trace/AppSec/Rcm/AsmDdProduct.cs b/tracer/src/Datadog.Trace/AppSec/Rcm/AsmDdProduct.cs index 2f89e8f7154a..08e2bc4f826f 100644 --- a/tracer/src/Datadog.Trace/AppSec/Rcm/AsmDdProduct.cs +++ b/tracer/src/Datadog.Trace/AppSec/Rcm/AsmDdProduct.cs @@ -40,18 +40,9 @@ public void ProcessUpdates(ConfigurationStatus configurationStatus, List removedConfigsForThisProduct) { - var oneRemoved = false; foreach (var removedConfig in removedConfigsForThisProduct) { - oneRemoved |= configurationStatus.RulesByFile.Remove(removedConfig.Path); - } - - if (configurationStatus.RulesByFile.Count == 0) - { - configurationStatus.IncomingUpdateState.FallbackToEmbeddedRuleset(); - } - else if (oneRemoved) - { + configurationStatus.RulesByFile.Remove(removedConfig.Path); configurationStatus.IncomingUpdateState.WafKeysToApply.Add(ConfigurationStatus.WafRulesKey); } } diff --git a/tracer/src/Datadog.Trace/AppSec/Rcm/ConfigurationStatus.cs b/tracer/src/Datadog.Trace/AppSec/Rcm/ConfigurationStatus.cs index 2e4ce01d72f2..0ef41a46f94a 100644 --- a/tracer/src/Datadog.Trace/AppSec/Rcm/ConfigurationStatus.cs +++ b/tracer/src/Datadog.Trace/AppSec/Rcm/ConfigurationStatus.cs @@ -13,7 +13,6 @@ using Datadog.Trace.AppSec.Rcm.Models.AsmDd; using Datadog.Trace.AppSec.Rcm.Models.AsmFeatures; using Datadog.Trace.AppSec.Waf.Initialization; -using Datadog.Trace.ExtensionMethods; using Datadog.Trace.Logging; using Datadog.Trace.RemoteConfigurationManagement; using Datadog.Trace.Vendors.Newtonsoft.Json.Linq; @@ -48,8 +47,6 @@ internal record ConfigurationStatus public ConfigurationStatus(string? embeddedRulesPath) => _embeddedRulesPath = embeddedRulesPath; - internal RuleSet? FallbackEmbeddedRuleSet { get; set; } - internal bool? EnableAsm { get; set; } = null; internal string? AutoUserInstrumMode { get; set; } = null; @@ -104,67 +101,73 @@ internal static List MergeRuleData(IEnumerable res) return finalRuleData; } - internal Dictionary BuildDictionaryForWafAccordingToIncomingUpdate() + internal object? BuildDictionaryForWafAccordingToIncomingUpdate(string? embeddedRulesetPath) { - var dictionary = new Dictionary(); + var configuration = new Dictionary(); if (IncomingUpdateState.WafKeysToApply.Contains(WafExclusionsKey)) { var exclusions = ExclusionsByFile.SelectMany(x => x.Value).ToList(); - dictionary.Add(WafExclusionsKey, new JArray(exclusions)); + configuration.Add(WafExclusionsKey, new JArray(exclusions)); } if (IncomingUpdateState.WafKeysToApply.Contains(WafRulesOverridesKey)) { var overrides = RulesOverridesByFile.SelectMany(x => x.Value).ToList(); - dictionary.Add(WafRulesOverridesKey, overrides.Select(r => r.ToKeyValuePair()).ToArray()); + configuration.Add(WafRulesOverridesKey, overrides.Select(r => r.ToKeyValuePair()).ToArray()); } if (IncomingUpdateState.WafKeysToApply.Contains(WafRulesDataKey)) { var rulesData = MergeRuleData(RulesDataByFile.SelectMany(x => x.Value)); - dictionary.Add(WafRulesDataKey, rulesData.Select(r => r.ToKeyValuePair()).ToArray()); + configuration.Add(WafRulesDataKey, rulesData.Select(r => r.ToKeyValuePair()).ToArray()); } if (IncomingUpdateState.WafKeysToApply.Contains(WafExclusionsDataKey)) { var rulesData = MergeRuleData(ExclusionsDataByFile.SelectMany(x => x.Value)); - dictionary.Add(WafExclusionsDataKey, rulesData.Select(r => r.ToKeyValuePair()).ToArray()); + configuration.Add(WafExclusionsDataKey, rulesData.Select(r => r.ToKeyValuePair()).ToArray()); } if (IncomingUpdateState.WafKeysToApply.Contains(WafActionsKey)) { var actions = ActionsByFile.SelectMany(x => x.Value).ToList(); - dictionary.Add(WafActionsKey, actions.Select(r => r.ToKeyValuePair()).ToArray()); + configuration.Add(WafActionsKey, actions.Select(r => r.ToKeyValuePair()).ToArray()); } if (IncomingUpdateState.WafKeysToApply.Contains(WafCustomRulesKey)) { var customRules = CustomRulesByFile.SelectMany(x => x.Value).ToList(); var mergedCustomRules = new JArray(customRules); - dictionary.Add(WafCustomRulesKey, mergedCustomRules); + configuration.Add(WafCustomRulesKey, mergedCustomRules); } - if (IncomingUpdateState.FallbackToEmbeddedRulesetAtNextUpdate) + // if there's incoming rules or empty rules, or if asm is to be activated, we also want the rules key in waf arguments + if (IncomingUpdateState.WafKeysToApply.Contains(WafRulesKey) || (IncomingUpdateState.SecurityStateChange && (EnableAsm ?? false))) { - if (FallbackEmbeddedRuleSet == null) + var rulesetFromRcm = RulesByFile.Values.FirstOrDefault(); + // should deserialize from LocalRuleFile + if (rulesetFromRcm is null) { - var result = WafConfigurator.DeserializeEmbeddedOrStaticRules(_embeddedRulesPath); - if (result != null) + var deserializedFromLocalRules = WafConfigurator.DeserializeEmbeddedOrStaticRules(embeddedRulesetPath); + if (deserializedFromLocalRules is not null) { - FallbackEmbeddedRuleSet = RuleSet.From(result); + if (configuration.Count == 0) + { + return deserializedFromLocalRules; + } + + var ruleSet = RuleSet.From(deserializedFromLocalRules); + ruleSet.AddToDictionaryAtRoot(configuration); } } - - FallbackEmbeddedRuleSet?.AddToDictionaryAtRoot(dictionary); - } - else if (IncomingUpdateState.WafKeysToApply.Contains(WafRulesKey)) - { - var rulesetFromRcm = RulesByFile.Values.FirstOrDefault(); - rulesetFromRcm?.AddToDictionaryAtRoot(dictionary); + else + { + rulesetFromRcm?.AddToDictionaryAtRoot(configuration); + } } - return dictionary; + return configuration.Count > 0 ? configuration : null; } /// @@ -247,7 +250,7 @@ public bool StoreLastConfigState(Dictionary> c } } - // only treat asm_features as it will decide if asm gets toggled on and if we deserialize all the others + // only deserialize and apply asm_features as it will decide if asm gets toggled on and if we deserialize all the others // (the enable of auto user instrumentation as added to asm_features) _asmFeatureProduct.ProcessUpdates(this, asmFeaturesToUpdate); _asmFeatureProduct.ProcessRemovals(this, asmFeaturesToRemove); @@ -282,19 +285,14 @@ internal record IncomingUpdateStatus { internal HashSet WafKeysToApply { get; } = new(); - internal bool FallbackToEmbeddedRulesetAtNextUpdate { get; private set; } - internal bool SecurityStateChange { get; set; } public void Reset() { - FallbackToEmbeddedRulesetAtNextUpdate = false; WafKeysToApply.Clear(); SecurityStateChange = false; } - public void FallbackToEmbeddedRuleset() => FallbackToEmbeddedRulesetAtNextUpdate = true; - public void SignalSecurityStateChange() => SecurityStateChange = true; } } diff --git a/tracer/src/Datadog.Trace/AppSec/Rcm/Models/AsmDd/RuleSet.cs b/tracer/src/Datadog.Trace/AppSec/Rcm/Models/AsmDd/RuleSet.cs index 8c9480e8bd55..890ceacbb31c 100644 --- a/tracer/src/Datadog.Trace/AppSec/Rcm/Models/AsmDd/RuleSet.cs +++ b/tracer/src/Datadog.Trace/AppSec/Rcm/Models/AsmDd/RuleSet.cs @@ -1,4 +1,4 @@ -// +// // Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. // @@ -22,13 +22,22 @@ internal class RuleSet [JsonProperty("processors")] internal JToken? Processors { get; set; } + [JsonProperty("actions")] + internal JToken? Actions { get; set; } + [JsonProperty("scanners")] internal JToken? Scanners { get; set; } - public JToken? All { get; set; } + [JsonProperty("exclusions")] + internal JToken? Exclusions { get; set; } + + [JsonProperty("custom_rules")] + internal JToken? CustomRules { get; set; } public static RuleSet From(JToken result) { + // can rules from rc contains exclusions and custom rules? + var ruleset = new RuleSet { Version = result["version"]?.ToString(), @@ -36,7 +45,9 @@ public static RuleSet From(JToken result) Rules = result["rules"], Processors = result["processors"], Scanners = result["scanners"], - All = result + Actions = result["actions"], + Exclusions = result["exclusions"], + CustomRules = result["custom_rules"] }; return ruleset; } @@ -49,27 +60,37 @@ public void AddToDictionaryAtRoot(Dictionary dictionary) { if (Rules != null) { - dictionary.Add("rules", Rules); + dictionary["rules"] = Rules; } if (Metadata != null) { - dictionary.Add("metadata", Metadata); + dictionary["metadata"] = Metadata; } if (Version != null) { - dictionary.Add("version", Version); + dictionary["version"] = Version; } if (Processors != null) { - dictionary.Add("processors", Processors); + dictionary["processors"] = Processors; } if (Scanners != null) { - dictionary.Add("scanners", Scanners); + dictionary["scanners"] = Scanners; + } + + if (Exclusions is not null) + { + dictionary["exclusions"] = Exclusions; + } + + if (CustomRules is not null) + { + dictionary["custom_rules"] = CustomRules; } } } diff --git a/tracer/src/Datadog.Trace/AppSec/Security.cs b/tracer/src/Datadog.Trace/AppSec/Security.cs index cb9f0420e0a1..b96b7585e1ed 100644 --- a/tracer/src/Datadog.Trace/AppSec/Security.cs +++ b/tracer/src/Datadog.Trace/AppSec/Security.cs @@ -208,7 +208,6 @@ private ApplyDetails[] UpdateFromRcm(Dictionary 0) { var newSubscription = new Subscription(UpdateFromRcm, newKeys); diff --git a/tracer/src/Datadog.Trace/AppSec/Waf/IWaf.cs b/tracer/src/Datadog.Trace/AppSec/Waf/IWaf.cs index 6e88a9479ba4..c5cf36ee2e82 100644 --- a/tracer/src/Datadog.Trace/AppSec/Waf/IWaf.cs +++ b/tracer/src/Datadog.Trace/AppSec/Waf/IWaf.cs @@ -2,7 +2,7 @@ // Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. // - +#nullable enable using System; using Datadog.Trace.AppSec.Rcm; using Datadog.Trace.AppSec.Waf.NativeBindings; @@ -14,11 +14,11 @@ internal interface IWaf : IDisposable { public string Version { get; } - public IContext CreateContext(); + public IContext? CreateContext(); internal unsafe WafReturnCode Run(IntPtr contextHandle, DdwafObjectStruct* rawPersistentData, DdwafObjectStruct* rawEphemeralData, ref DdwafResultStruct retNative, ulong timeoutMicroSeconds); - UpdateResult UpdateWafFromConfigurationStatus(ConfigurationStatus configurationStatus); + UpdateResult UpdateWafFromConfigurationStatus(ConfigurationStatus configurationStatus, string? staticRulesFilePath = null); public string[] GetKnownAddresses(); diff --git a/tracer/src/Datadog.Trace/AppSec/Waf/Initialization/WafConfigurator.cs b/tracer/src/Datadog.Trace/AppSec/Waf/Initialization/WafConfigurator.cs index 72a393024aee..d277e8b43ead 100644 --- a/tracer/src/Datadog.Trace/AppSec/Waf/Initialization/WafConfigurator.cs +++ b/tracer/src/Datadog.Trace/AppSec/Waf/Initialization/WafConfigurator.cs @@ -82,6 +82,13 @@ private static void LogRuleDetailsIfDebugEnabled(JToken root) return File.OpenRead(rulesFile); } + /// + /// Deserialize rules for the waf as Jtoken + /// If null is passed, will deserialize embedded rule file in the app + /// If a path is given but file isn't found, it won't fallback on the embedded rule file + /// + /// if null, will fallback on embedded rules file + /// the rules, might be null if file not found internal static JToken? DeserializeEmbeddedOrStaticRules(string? rulesFilePath) { JToken root; diff --git a/tracer/src/Datadog.Trace/AppSec/Waf/ReturnTypes.Managed/InitResult.cs b/tracer/src/Datadog.Trace/AppSec/Waf/ReturnTypes.Managed/InitResult.cs index e74bcb239e66..b62f4808afec 100644 --- a/tracer/src/Datadog.Trace/AppSec/Waf/ReturnTypes.Managed/InitResult.cs +++ b/tracer/src/Datadog.Trace/AppSec/Waf/ReturnTypes.Managed/InitResult.cs @@ -22,7 +22,6 @@ private InitResult(ushort failedToLoadRules, ushort loadedRules, string ruleFile { HasErrors = errors.Count > 0; Errors = errors; - EmbeddedRules = embeddedRules; FailedToLoadRules = failedToLoadRules; LoadedRules = loadedRules; RuleFileVersion = ruleFileVersion; @@ -55,8 +54,6 @@ private InitResult(ushort failedToLoadRules, ushort loadedRules, string ruleFile internal IReadOnlyDictionary Errors { get; } - public JToken? EmbeddedRules { get; set; } - internal string ErrorMessage { get; } internal bool HasErrors { get; } diff --git a/tracer/src/Datadog.Trace/AppSec/Waf/Waf.cs b/tracer/src/Datadog.Trace/AppSec/Waf/Waf.cs index 1f0be2ffb0f6..0bb02c80e839 100644 --- a/tracer/src/Datadog.Trace/AppSec/Waf/Waf.cs +++ b/tracer/src/Datadog.Trace/AppSec/Waf/Waf.cs @@ -6,11 +6,14 @@ #nullable enable using System; +using System.Collections; using System.Collections.Generic; using System.Linq; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using Datadog.Trace.AppSec.Rcm; using Datadog.Trace.AppSec.Rcm.Models.AsmData; +using Datadog.Trace.AppSec.Rcm.Models.AsmDd; using Datadog.Trace.AppSec.Waf.Initialization; using Datadog.Trace.AppSec.Waf.NativeBindings; using Datadog.Trace.AppSec.Waf.ReturnTypes.Managed; @@ -20,6 +23,7 @@ using Datadog.Trace.Telemetry; using Datadog.Trace.Vendors.Newtonsoft.Json; using Datadog.Trace.Vendors.Serilog.Events; +using static Datadog.Trace.AppSec.Rcm.ConfigurationStatus; namespace Datadog.Trace.AppSec.Waf { @@ -54,7 +58,7 @@ internal Waf(IntPtr wafHandle, WafLibraryInvoker wafLibraryInvoker, IEncoder enc /// the regex that will be used to obfuscate possible sensitive data in values that are highlighted WAF as potentially malicious, /// empty string means use default embedded in the WAF /// can be null, means use rules embedded in the manifest - /// can be null. RemoteConfig rules json. Takes precedence over rulesFile + /// can be null. RemoteConfig rules json. Takes precedence over rulesFile /// use legacy encoder /// if debug level logs should be enabled for the WAF /// the waf wrapper around waf native @@ -63,7 +67,7 @@ internal static InitResult Create( string obfuscationParameterKeyRegex, string obfuscationParameterValueRegex, string? embeddedRulesetPath = null, - ConfigurationStatus? configurationStatus = null, + ConfigurationStatus? remoteConfigStatus = null, bool useUnsafeEncoder = false, bool wafDebugEnabled = false) { @@ -71,18 +75,16 @@ internal static InitResult Create( // set the log level and setup the logger wafLibraryInvoker.SetupLogging(wafDebugEnabled); - object? configurationToEncode = null; - if (configurationStatus is not null) + if (remoteConfigStatus is not null) { - var configFromRcm = configurationStatus.BuildDictionaryForWafAccordingToIncomingUpdate(); - if (configFromRcm.Count > 0) - { - configurationToEncode = configFromRcm; - } + configurationToEncode = remoteConfigStatus.BuildDictionaryForWafAccordingToIncomingUpdate(embeddedRulesetPath); + } + else + { + var deserializedFromLocalRules = WafConfigurator.DeserializeEmbeddedOrStaticRules(embeddedRulesetPath); + configurationToEncode = deserializedFromLocalRules; } - - configurationToEncode ??= WafConfigurator.DeserializeEmbeddedOrStaticRules(embeddedRulesetPath)!; if (configurationToEncode is null) { @@ -102,8 +104,7 @@ internal static InitResult Create( try { - var initResult = wafConfigurator.Configure(ref rulesObj, encoder, configWafStruct, ref diagnostics, configurationStatus == null ? embeddedRulesetPath : "RemoteConfig"); - initResult.EmbeddedRules = initResult.EmbeddedRules; + var initResult = wafConfigurator.Configure(ref rulesObj, encoder, configWafStruct, ref diagnostics, remoteConfigStatus == null ? embeddedRulesetPath : "RemoteConfig"); return initResult; } finally @@ -169,16 +170,16 @@ private unsafe UpdateResult UpdateWafAndDispose(IEncodeResult updateData) return res; } - public UpdateResult UpdateWafFromConfigurationStatus(ConfigurationStatus configurationStatus) + public UpdateResult UpdateWafFromConfigurationStatus(ConfigurationStatus configurationStatus, string? rulesPath = null) { - var dic = configurationStatus.BuildDictionaryForWafAccordingToIncomingUpdate(); - if (dic.IsEmpty()) + var dic = configurationStatus.BuildDictionaryForWafAccordingToIncomingUpdate(rulesPath); + if (dic is null) { Log.Warning("A waf update came from remote configuration but final merged dictionary for waf is empty, no update will be performed."); return UpdateResult.FromNothingToUpdate(); } - return Update(dic); + return Update(dic!); } /// @@ -215,7 +216,7 @@ public UpdateResult UpdateWafFromConfigurationStatus(ConfigurationStatus configu return Context.GetContext(contextHandle, this, _wafLibraryInvoker, _encoder); } - private UpdateResult Update(IDictionary arguments) + private UpdateResult Update(object arguments) { UpdateResult updated; try @@ -229,7 +230,7 @@ private UpdateResult Update(IDictionary arguments) updated = UpdateWafAndDispose(encodedArgs); // only if rules are provided will the waf give metrics - if (arguments.ContainsKey("rules")) + if (arguments is Dictionary dic && dic.ContainsKey("rules")) { TelemetryFactory.Metrics.RecordCountWafUpdates(); } From 6a3967413c36d788ef2c2c5fe57c4860e9012cb1 Mon Sep 17 00:00:00 2001 From: Anna Date: Tue, 8 Oct 2024 17:26:36 +0200 Subject: [PATCH 4/4] answer to comment --- .../Datadog.Trace/AppSec/Waf/ReturnTypes.Managed/InitResult.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tracer/src/Datadog.Trace/AppSec/Waf/ReturnTypes.Managed/InitResult.cs b/tracer/src/Datadog.Trace/AppSec/Waf/ReturnTypes.Managed/InitResult.cs index b62f4808afec..7cdb4e25026b 100644 --- a/tracer/src/Datadog.Trace/AppSec/Waf/ReturnTypes.Managed/InitResult.cs +++ b/tracer/src/Datadog.Trace/AppSec/Waf/ReturnTypes.Managed/InitResult.cs @@ -10,7 +10,6 @@ using Datadog.Trace.AppSec.WafEncoding; using Datadog.Trace.Logging; using Datadog.Trace.Vendors.Newtonsoft.Json; -using Datadog.Trace.Vendors.Newtonsoft.Json.Linq; namespace Datadog.Trace.AppSec.Waf.ReturnTypes.Managed { @@ -18,7 +17,7 @@ internal class InitResult { private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(InitResult)); - private InitResult(ushort failedToLoadRules, ushort loadedRules, string ruleFileVersion, IReadOnlyDictionary errors, JToken? embeddedRules = null, bool unusableRuleFile = false, IntPtr? wafHandle = null, WafLibraryInvoker? wafLibraryInvoker = null, IEncoder? encoder = null, bool shouldEnableWaf = true, bool incompatibleWaf = false) + private InitResult(ushort failedToLoadRules, ushort loadedRules, string ruleFileVersion, IReadOnlyDictionary errors, bool unusableRuleFile = false, IntPtr? wafHandle = null, WafLibraryInvoker? wafLibraryInvoker = null, IEncoder? encoder = null, bool shouldEnableWaf = true, bool incompatibleWaf = false) { HasErrors = errors.Count > 0; Errors = errors;