WIP: FastADC implementation

src/core/algorithms/dc/FastADC/misc/misc.h

@@ -0,0 +1,55 @@
+#pragma once
+
+#include "model/table/typed_column_data.h"
+
+namespace algos::fastadc {
+
+namespace {
+// Helper to trigger a compile-time error for unsupported types
+template <typename T>
+struct DependentFalse : std::false_type {};
+}  // namespace
+
+template <typename T>
+[[nodiscard]] inline T const& GetValue(model::TypedColumnData const& column, size_t row) {
+    model::Type const& type = column.GetType();
+
+    if (!column.IsNullOrEmpty(row)) {
+        return type.GetValue<T>(column.GetValue(row));
+    }
+
+    /*
+     * Mimicking the Java behavior:
+     * https://github.com/RangerShaw/FastADC/blob/master/src/main/java/de/metanome/algorithms/dcfinder/input/Column.java#L71
+     *
+     * public Long getLong(int line) {
+     *     return values.get(line).isEmpty() ? Long.MIN_VALUE :
+     *             Long.parseLong(values.get(line));
+     * }
+     *
+     * public Double getDouble(int line) {
+     *     return values.get(line).isEmpty() ? Double.MIN_VALUE :
+     *            Double.parseDouble(values.get(line));
+     * }
+     *
+     * public String getString(int line) {
+     *     return values.get(line) == null ? "" : values.get(line);
+     * }
+     */
+    if constexpr (std::is_same_v<T, std::string>) {
+        static std::string const kEmptyStr = "";
+        return kEmptyStr;
+    } else if constexpr (std::is_same_v<T, int64_t>) {
+        static int64_t const kMinInt = std::numeric_limits<int64_t>::min();
+        return kMinInt;
+    } else if constexpr (std::is_same_v<T, double>) {
+        static double const kMinDouble = std::numeric_limits<double>::lowest();
+        return kMinDouble;
+    } else {
+        static_assert(DependentFalse<T>::value,
+                      "FastADC algorithm supports only int64_t, string, or double as column types. "
+                      "This function should not be called with other types.");
+    }
+}
+
+}  // namespace algos::fastadc

src/core/algorithms/dc/FastADC/misc/typed_column_data_value_differences.cpp

@@ -0,0 +1,90 @@
+#include "typed_column_data_value_differences.h"
+
+#include <easylogging++.h>
+
+#include "misc.h"
+
+namespace algos::fastadc {
+
+template <typename T>
+static double GetSharedPercentageTyped(model::TypedColumnData const& c1,
+                                       model::TypedColumnData const& c2) {
+    std::unordered_map<T, size_t> freq_map1;
+    std::unordered_map<T, size_t> freq_map2;
+
+    freq_map1.reserve(c1.GetNumRows());
+    freq_map2.reserve(c2.GetNumRows());
+
+    for (size_t i = 0; i < c1.GetNumRows(); i++) freq_map1[GetValue<T>(c1, i)]++;
+    for (size_t i = 0; i < c2.GetNumRows(); i++) freq_map2[GetValue<T>(c2, i)]++;
+
+    size_t shared_count = 0;
+    size_t total_count = 0;
+
+    for (auto const& [data, frequency1] : freq_map1) {
+        auto it = freq_map2.find(data);
+        size_t frequency2 = (it == freq_map2.end()) ? 0 : it->second;
+
+        shared_count += std::min(frequency1, frequency2);
+        total_count += std::max(frequency1, frequency2);
+    }
+
+    return total_count > 0 ? static_cast<double>(shared_count) / total_count : 0.0;
+}
+
+template <typename T>
+static double CalculateAverageTyped(model::TypedColumnData const& column) {
+    double sum = 0.0;
+
+    if (column.GetNumRows() == 0) {
+        return sum;
+    }
+
+    for (size_t i = 0; i < column.GetNumRows(); ++i) {
+        sum += static_cast<double>(GetValue<T>(column, i));
+    }
+
+    return sum / column.GetNumRows();
+}
+
+double GetSharedPercentage(model::TypedColumnData const& c1, model::TypedColumnData const& c2) {
+    if (c1.GetColumn() == c2.GetColumn()) return 1.;
+
+    switch (c1.GetTypeId()) {
+        case model::TypeId::kInt:
+            return GetSharedPercentageTyped<int64_t>(c1, c2);
+        case model::TypeId::kDouble:
+            return GetSharedPercentageTyped<double>(c1, c2);
+        case model::TypeId::kString:
+            return GetSharedPercentageTyped<std::string>(c1, c2);
+        default:
+            LOG(DEBUG) << "Column " << c1.GetColumn()->ToString() << " with type "
+                       << c1.GetType().ToString()
+                       << " is not supported for shared percentage calculation";
+            return -1;
+    }
+}
+
+double GetAverageRatio(model::TypedColumnData const& c1, model::TypedColumnData const& c2) {
+    if (c1.GetColumn() == c2.GetColumn()) return 1.;
+
+    double avg1 = 0.0, avg2 = 0.0;
+
+    switch (c1.GetTypeId()) {
+        case model::TypeId::kInt:
+            avg1 = CalculateAverageTyped<int64_t>(c1);
+            avg2 = CalculateAverageTyped<int64_t>(c2);
+            break;
+        case model::TypeId::kDouble:
+            avg1 = CalculateAverageTyped<double>(c1);
+            avg2 = CalculateAverageTyped<double>(c2);
+            break;
+        default:
+            LOG(DEBUG) << "Column type  " << c1.GetType().ToString() << " is not numeric";
+            return -1;
+    }
+
+    return std::min(avg1, avg2) / std::max(avg1, avg2);
+}
+
+}  // namespace algos::fastadc

src/core/algorithms/dc/FastADC/misc/typed_column_data_value_differences.h

@@ -0,0 +1,32 @@
+#include "model/table/typed_column_data.h"
+
+namespace algos::fastadc {
+
+/**
+ * Computes the shared percentage of values between two columns of the same type.
+ * The function is specialized for columns containing only int, double, or string
+ * types. It calculates the frequency of each unique value in both columns and
+ * determines the ratio of the shared values to the total values.
+ *
+ * Assumes that types of values in both columns are the same.
+ *
+ * @return A double representing the shared percentage of values between the two
+ *         columns if the types are int, double, or string. Returns -1 (i.e this indicates
+ *         that there are no similaties whatsoever) if the column types are not
+ *         supported for comparison
+ */
+double GetSharedPercentage(model::TypedColumnData const& c1, model::TypedColumnData const& c2);
+
+/**
+ * Calculates the ratio of the smaller average to the larger average of two columns.
+ *
+ * Assumes that types of values in both columns are numeric and the same.
+ *
+ * @return A double representing the ratio of the smaller average to the larger average
+ *         of the two columns if both columns are numeric. Returns -1 (i.e this indicates
+ *         that there are no similaties whatsoever) if the column types are not
+ *         types are not numeric.
+ */
+double GetAverageRatio(model::TypedColumnData const& c1, model::TypedColumnData const& c2);
+
+}  // namespace algos::fastadc

src/core/algorithms/dc/FastADC/model/column_operand.cpp

@@ -0,0 +1,9 @@
+#include "column_operand.h"
+
+namespace algos::fastadc {
+
+size_t hash_value(ColumnOperand const& col_op) noexcept {
+    return std::hash<ColumnOperand>()(col_op);
+}
+
+}  // namespace algos::fastadc

src/core/algorithms/dc/FastADC/model/column_operand.h

@@ -0,0 +1,73 @@
+#pragma once
+
+#include "table/column.h"
+
+namespace algos::fastadc {
+
+/**
+ * @brief Represents a column operand within a predicate for FastADC.
+ *
+ * FastADC processes Denial Constraints (DCs) that involve comparisons between
+ * pairs of rows within a dataset. A typical DC example, derived from a Functional
+ * Dependency such as A -> B, is expressed as: `forall t, s in r, not (t.A = s.A and t.B != s.B)`
+ * This denotes that for any pair of rows in the relation, it should not be the case
+ * that while the values in column "A" are equal, the values in column "B" are unequal.
+ *
+ * A predicate in this context (e.g., t.A == s.A) comprises three elements to be fully
+ * represented: the column operand from the first tuple ("t.A"), the comparison operator
+ * ("="), and the column operand from the second tuple ("s.A"). The `ColumnOperand` class
+ * encapsulates the column operand part of a predicate, such as "t.A" or "s.A".
+ *
+ * The class distinguishes between operands derived from the first tuple (t) and those
+ * from the second tuple (s) using a boolean flag `tuple_`, where `true` indicates an
+ * operand from the first tuple (t), and `false` indicates an operand from the second
+ * tuple (s).
+ */
+class ColumnOperand {
+private:
+    Column const* column_;
+    bool tuple_;
+
+public:
+    ColumnOperand(Column const* column, bool tuple) : column_(column), tuple_(tuple) {}
+
+    bool operator==(ColumnOperand const& rhs) const {
+        return column_ == rhs.column_ && tuple_ == rhs.tuple_;
+    }
+
+    Column const* GetColumn() const {
+        return column_;
+    }
+
+    bool GetTuple() const {
+        return tuple_;
+    }
+
+    // here TS means (t, s)
+    ColumnOperand GetInvTS() const {
+        return ColumnOperand(column_, !tuple_);
+    }
+
+    std::string ToString() const {
+        return (tuple_ ? "t." : "s.") + column_->GetName();
+    }
+};
+
+// NOLINTBEGIN(readability-identifier-naming)
+size_t hash_value(ColumnOperand const& k) noexcept;
+// NOLINTEND(readability-identifier-naming)
+
+}  // namespace algos::fastadc
+
+namespace std {
+template <>
+struct hash<algos::fastadc::ColumnOperand> {
+    size_t operator()(algos::fastadc::ColumnOperand const& k) const noexcept {
+        size_t seed = 0;
+        boost::hash_combine(seed, k.GetColumn()->GetIndex());
+        boost::hash_combine(seed, k.GetTuple());
+        return seed;
+    }
+};
+
+}  // namespace std

src/core/algorithms/dc/FastADC/model/denial_constraint.h

@@ -0,0 +1,64 @@
+#pragma once
+
+#include <sstream>
+#include <string>
+
+#include "predicate_set.h"
+
+namespace algos::fastadc {
+
+class DenialConstraint {
+private:
+    PredicateSet predicate_set_;
+
+public:
+    explicit DenialConstraint(PredicateSet const& predicateSet) : predicate_set_(predicateSet) {}
+
+    DenialConstraint(boost::dynamic_bitset<> const& predicates,
+                     PredicateIndexProvider* predicate_index_provider)
+        : predicate_set_(predicates, predicate_index_provider) {
+        assert(predicate_index_provider);
+    }
+
+    DenialConstraint GetInvT1T2DC(PredicateProvider* predicate_provider) const {
+        return DenialConstraint(predicate_set_.GetInvTS(predicate_provider));
+    }
+
+    PredicateSet const& GetPredicateSet() const {
+        return predicate_set_;
+    }
+
+    size_t GetPredicateCount() const {
+        return predicate_set_.Size();
+    }
+
+    std::string ToString() const {
+        std::string const c_not = "\u00AC";
+        std::string const c_and = " ∧ ";
+        std::ostringstream sb;
+        sb << c_not << "{ ";
+        bool first = true;
+        for (PredicatePtr predicate : predicate_set_) {
+            if (!first) {
+                sb << c_and;
+            }
+            sb << predicate->ToString();
+            first = false;
+        }
+        sb << " }";
+        return sb.str();
+    }
+
+    bool operator==(DenialConstraint const& other) const = default;
+};
+
+}  // namespace algos::fastadc
+
+namespace std {
+template <>
+struct hash<algos::fastadc::DenialConstraint> {
+    size_t operator()(algos::fastadc::DenialConstraint const& k) const noexcept {
+        return k.GetPredicateSet().Hash();
+    }
+};
+}  // namespace std

src/core/algorithms/dc/FastADC/model/evidence.h

@@ -0,0 +1,29 @@
+#include "../util/common_clue_set_builder.h"
+#include "predicate.h"
+
+namespace algos::fastadc {
+
+struct Evidence {
+    int64_t count;
+    PredicateBitset evidence;
+
+    Evidence(Clue satisfied, int64_t count, PredicateBitset const& cardinalityMask,
+             std::vector<PredicateBitset> const& correctionMap)
+        : count(count) {
+        evidence = cardinalityMask;
+
+        Clue tmp = satisfied;
+        size_t pos = 0;
+        while (tmp.any()) {
+            if (tmp.test(0)) {
+                evidence ^= correctionMap[pos];
+            }
+            tmp >>= 1;
+            pos++;
+        }
+    }
+
+    Evidence(Clue bitSet, int64_t count) : count(count), evidence(bitSet) {}
+};
+
+}  // namespace algos::fastadc