[WIP] Organise collections into a single module

.gitlab/benchmarks.yml

@@ -11,7 +11,7 @@ benchmarks:
   script:
     - export ARTIFACTS_DIR="$(pwd)/reports" && (mkdir "${ARTIFACTS_DIR}" || :)
     - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/"
-    - git clone --branch libddwaf https://github.com/DataDog/benchmarking-platform /platform && cd /platform
+    - git clone --branch anilm3/reduced-iterations https://github.com/DataDog/benchmarking-platform /platform && cd /platform
     - ./steps/capture-hardware-software-info.sh
     - ./steps/run-benchmarks.sh
     - ./steps/analyze-results.sh

cmake/objects.cmake

@@ -8,7 +8,7 @@ set(LIBDDWAF_SOURCE
     ${libddwaf_SOURCE_DIR}/src/event.cpp
     ${libddwaf_SOURCE_DIR}/src/object.cpp
     ${libddwaf_SOURCE_DIR}/src/object_store.cpp
-    ${libddwaf_SOURCE_DIR}/src/collection.cpp
+    ${libddwaf_SOURCE_DIR}/src/module.cpp
     ${libddwaf_SOURCE_DIR}/src/expression.cpp
     ${libddwaf_SOURCE_DIR}/src/ruleset_info.cpp
     ${libddwaf_SOURCE_DIR}/src/ip_utils.cpp

src/context.cpp

@@ -9,7 +9,6 @@
 #include <vector>
 
 #include "clock.hpp"
-#include "collection.hpp"
 #include "context.hpp"
 #include "ddwaf.h"
 #include "event.hpp"
@@ -235,38 +234,9 @@ std::vector<event> context::eval_rules(
 {
     std::vector<ddwaf::event> events;
 
-    auto eval_collection = [&](const auto &type, const auto &collection) {
-        auto it = collection_cache_.find(type);
-        if (it == collection_cache_.end()) {
-            auto [new_it, res] = collection_cache_.emplace(type, collection_cache{});
-            it = new_it;
-        }
-        collection.match(events, store_, it->second, policy, ruleset_->rule_matchers, deadline);
-    };
-
-    // Evaluate user priority collections first
-    for (auto &[type, collection] : ruleset_->user_priority_collections) {
-        DDWAF_DEBUG("Evaluating user priority collection '{}'", type);
-        eval_collection(type, collection);
-    }
-
-    // Evaluate priority collections first
-    for (auto &[type, collection] : ruleset_->base_priority_collections) {
-        DDWAF_DEBUG("Evaluating priority collection '{}'", type);
-        eval_collection(type, collection);
-    }
-
-    // Evaluate regular collection after
-    for (auto &[type, collection] : ruleset_->user_collections) {
-        DDWAF_DEBUG("Evaluating user collection '{}'", type);
-        eval_collection(type, collection);
-    }
-
-    // Evaluate regular collection after
-    for (auto &[type, collection] : ruleset_->base_collections) {
-        DDWAF_DEBUG("Evaluating base collection '{}'", type);
-        eval_collection(type, collection);
-    }
+    DDWAF_DEBUG("Evaluating rules");
+    ruleset_->rules.eval(
+        events, store_, collection_cache_, policy, ruleset_->rule_matchers, deadline);
 
     return events;
 }

src/context.hpp

@@ -16,6 +16,7 @@
 #include "exclusion/common.hpp"
 #include "exclusion/input_filter.hpp"
 #include "exclusion/rule_filter.hpp"
+#include "module.hpp"
 #include "obfuscator.hpp"
 #include "rule.hpp"
 #include "ruleset.hpp"
@@ -33,7 +34,7 @@ class context {
     {
         rule_filter_cache_.reserve(ruleset_->rule_filters.size());
         input_filter_cache_.reserve(ruleset_->input_filters.size());
-        collection_cache_.reserve(ruleset_->collection_types.size());
+        ruleset_->rules.init_cache(collection_cache_);
     }
 
     context(const context &) = delete;
@@ -83,12 +84,12 @@ class context {
     memory::unordered_map<base_processor *, processor_cache> processor_cache_;
 
     // Caches of filters and conditions
-    memory::unordered_map<rule_filter *, rule_filter::cache_type> rule_filter_cache_{};
+    memory::unordered_map<rule_filter *, rule_filter::cache_type> rule_filter_cache_;
     memory::unordered_map<input_filter *, input_filter::cache_type> input_filter_cache_;
     exclusion::context_policy exclusion_policy_;
 
     // Cache of collections to avoid processing once a result has been obtained
-    memory::unordered_map<std::string_view, collection::cache_type> collection_cache_{};
+    collection_module::cache_type collection_cache_{};
 };
 
 class context_wrapper {

src/collection.cpp → src/module.cpp

@@ -3,6 +3,7 @@
 //
 // This product includes software developed at Datadog (https://www.datadoghq.com/).
 // Copyright 2021 Datadog, Inc.
+#include <cstddef>
 #include <memory>
 #include <optional>
 #include <string>
@@ -12,38 +13,38 @@
 #include <vector>
 
 #include "clock.hpp"
-#include "collection.hpp"
 #include "context_allocator.hpp"
 #include "event.hpp"
 #include "exception.hpp"
 #include "exclusion/common.hpp"
 #include "log.hpp"
 #include "matcher/base.hpp"
+#include "module.hpp"
 #include "object_store.hpp"
 #include "rule.hpp"
 
 namespace ddwaf {
 
-std::optional<event> match_rule(rule *rule, const object_store &store,
-    memory::unordered_map<ddwaf::rule *, rule::cache_type> &cache,
-    const exclusion::context_policy &policy,
+namespace {
+std::optional<event> match_rule(ddwaf::rule &rule, const object_store &store,
+    rule::cache_type &cache, const exclusion::context_policy &policy,
     const std::unordered_map<std::string, std::shared_ptr<matcher::base>> &dynamic_matchers,
     ddwaf::timer &deadline)
 {
-    const auto &id = rule->get_id();
+    const auto &id = rule.get_id();
 
     if (deadline.expired()) {
         DDWAF_INFO("Ran out of time while evaluating rule '{}'", id);
         throw timeout_exception();
     }
 
-    if (!rule->is_enabled()) {
+    if (!rule.is_enabled()) {
         DDWAF_DEBUG("Rule '{}' is disabled", id);
         return std::nullopt;
     }
 
     std::string_view action_override;
-    auto exclusion = policy.find(rule);
+    auto exclusion = policy.find(&rule);
     if (exclusion.mode == exclusion::filter_mode::bypass) {
         DDWAF_DEBUG("Bypassing rule '{}'", id);
         return std::nullopt;
@@ -60,15 +61,8 @@ std::optional<event> match_rule(rule *rule, const object_store &store,
     }
 
     try {
-        auto it = cache.find(rule);
-        if (it == cache.end()) {
-            auto [new_it, res] = cache.emplace(rule, rule::cache_type{});
-            it = new_it;
-        }
-
-        rule::cache_type &rule_cache = it->second;
         std::optional<event> event;
-        event = rule->match(store, rule_cache, exclusion.objects, dynamic_matchers, deadline);
+        event = rule.match(store, cache, exclusion.objects, dynamic_matchers, deadline);
 
         if (event.has_value()) {
             event->action_override = action_override;
@@ -83,39 +77,42 @@ std::optional<event> match_rule(rule *rule, const object_store &store,
     return std::nullopt;
 }
 
-template <typename Derived>
-void base_collection<Derived>::match(std::vector<event> &events, object_store &store,
-    collection_cache &cache, const exclusion::context_policy &exclusion,
+} // namespace
+
+void collection_module::eval(std::vector<event> &events, object_store &store, module_cache &cache,
+    const exclusion::context_policy &exclusion,
     const std::unordered_map<std::string, std::shared_ptr<matcher::base>> &dynamic_matchers,
     ddwaf::timer &deadline) const
 {
-    if (cache.result >= Derived::type()) {
-        // If the result was cached but ephemeral, clear it. Note that this is
-        // just a workaround taking advantage of the order of evaluation of
-        // collections. Collections might be removed in the future altogether.
-        if (cache.result == Derived::type() && cache.ephemeral) {
-            cache.result = collection_type::none;
-            cache.ephemeral = false;
-        } else {
-            return;
+    for (const auto &collection : collections_) {
+        DDWAF_DEBUG("Evaluating collection: {}", collection.name);
+        auto &collection_cache = cache.collections[collection.name];
+        if (collection_cache.type >= collection.type) {
+            // If the result was cached but ephemeral, clear it. Note that this is
+            // just a workaround taking advantage of the order of evaluation of
+            // collections. Collections might be removed in the future altogether.
+            if (collection_cache.type == collection.type && collection_cache.ephemeral) {
+                collection_cache.type = collection_type::none;
+                collection_cache.ephemeral = false;
+            } else {
+                continue;
+            }
         }
-    }
 
-    for (auto *rule : rules_) {
-        auto event =
-            match_rule(rule, store, cache.rule_cache, exclusion, dynamic_matchers, deadline);
-        if (event.has_value()) {
-            cache.result = Derived::type();
-            cache.ephemeral = event->ephemeral;
-
-            events.emplace_back(std::move(*event));
-            DDWAF_DEBUG("Found event on rule {}", rule->get_id());
-            break;
+        for (std::size_t i = collection.start; i < collection.end; ++i) {
+            auto &rule = *rules_[i];
+            auto event =
+                match_rule(rule, store, cache.rules[i], exclusion, dynamic_matchers, deadline);
+            if (event.has_value()) {
+                collection_cache.type = collection.type;
+                collection_cache.ephemeral = event->ephemeral;
+
+                events.emplace_back(std::move(*event));
+                DDWAF_DEBUG("Found event on rule {}", rule.get_id());
+                break;
+            }
         }
     }
 }
 
-template class base_collection<collection>;
-template class base_collection<priority_collection>;
-
 } // namespace ddwaf