From 729841561acdcfc08097be15f4c1ebc73600c872 Mon Sep 17 00:00:00 2001 From: Phi Date: Fri, 4 Aug 2023 20:24:59 +0000 Subject: [PATCH] Implementation of the initialization, setup, and execution phase of the reach-only protocol. (#1129) Implement the phases of the new sparse reach only protocol: 1. Initialization phase: Duchies sample local El Gamal keypair. 2. Setup phase: a) Non-aggregators add noise registers to the crv and shuffle the modified crv. They encrypt their excessive noise using the composite El Gamal public key. They send the crv and the excessive noise ciphertext to the aggregator. b) Aggregator: Waits for the crv and the excessive noise ciphertexts from non-aggregators. It adds noise to the crv, shuffles it, and encrypts its excessive noise with the composite El Gamal key. It then combines the excessive noise ciphertexts by adding them together. It sends the modified crv and the excessive noise ciphertext to the next worker. 3. Execution phase: Duchies collaborate to decrypt, randomize the register indices, and decrypt the excessive noise ciphertext. The aggregator can count the number of distinct registers, obtain the total amount of excessive noise, then estimate the reach based on the available information. --- .../crypto/encryption_utility_helper.cc | 59 +- .../common/crypto/encryption_utility_helper.h | 27 +- .../common/crypto/protocol_cryptor.cc | 56 +- .../common/crypto/protocol_cryptor.h | 4 + .../protocol/liquid_legions_v2/BUILD.bazel | 28 + .../liquid_legions_v2_encryption_utility.cc | 65 +- ...ly_liquid_legions_v2_encryption_utility.cc | 661 +++++++++++++++++ ...nly_liquid_legions_v2_encryption_utility.h | 89 +++ .../liquid_legions_v2/testing/BUILD.bazel | 28 + ...id_legions_v2_encryption_utility_helper.cc | 63 ++ ...uid_legions_v2_encryption_utility_helper.h | 44 ++ .../daemon/herald/LiquidLegionsV2Starter.kt | 4 +- ...iquidLegionsSketchAggregationV2Protocol.kt | 4 +- .../computationcontrol/ProtocolStages.kt | 4 +- .../measurement/internal/duchy/crypto.proto | 22 - ...liquid_legions_sketch_aggregation_v2.proto | 2 +- ...liquid_legions_sketch_aggregation_v2.proto | 10 - ...liquid_legions_v2_encryption_methods.proto | 82 +-- .../protocol/liquid_legions_v2/BUILD.bazel | 21 + ...quid_legions_v2_encryption_utility_test.cc | 48 +- ...quid_legions_v2_encryption_utility_test.cc | 664 ++++++++++++++++++ ...etchAggregationV2ProtocolEnumStagesTest.kt | 2 +- 22 files changed, 1796 insertions(+), 191 deletions(-) create mode 100644 src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/reach_only_liquid_legions_v2_encryption_utility.cc create mode 100644 src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/reach_only_liquid_legions_v2_encryption_utility.h create mode 100644 src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/testing/BUILD.bazel create mode 100644 src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/testing/liquid_legions_v2_encryption_utility_helper.cc create mode 100644 src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/testing/liquid_legions_v2_encryption_utility_helper.h create mode 100644 src/test/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/reach_only_liquid_legions_v2_encryption_utility_test.cc diff --git a/src/main/cc/wfa/measurement/common/crypto/encryption_utility_helper.cc b/src/main/cc/wfa/measurement/common/crypto/encryption_utility_helper.cc index 0ddf2cd6ecd..70e7f1541c5 100644 --- a/src/main/cc/wfa/measurement/common/crypto/encryption_utility_helper.cc +++ b/src/main/cc/wfa/measurement/common/crypto/encryption_utility_helper.cc @@ -14,6 +14,7 @@ #include "wfa/measurement/common/crypto/encryption_utility_helper.h" +#include #include #include "absl/status/status.h" @@ -47,26 +48,6 @@ absl::StatusOr ExtractElGamalCiphertextFromString( std::string(str.substr(kBytesPerEcPoint, kBytesPerEcPoint))); } -absl::StatusOr> GetBlindedRegisterIndexes( - absl::string_view data, ProtocolCryptor& protocol_cryptor) { - ASSIGN_OR_RETURN(size_t register_count, - GetNumberOfBlocks(data, kBytesPerCipherRegister)); - std::vector blinded_register_indexes; - blinded_register_indexes.reserve(register_count); - for (size_t index = 0; index < register_count; ++index) { - // The size of data_block is guaranteed to be equal to - // kBytesPerCipherText - absl::string_view data_block = - data.substr(index * kBytesPerCipherRegister, kBytesPerCipherText); - ASSIGN_OR_RETURN(ElGamalCiphertext ciphertext, - ExtractElGamalCiphertextFromString(data_block)); - ASSIGN_OR_RETURN(std::string decrypted_el_gamal, - protocol_cryptor.DecryptLocalElGamal(ciphertext)); - blinded_register_indexes.push_back(std::move(decrypted_el_gamal)); - } - return blinded_register_indexes; -} - absl::StatusOr ExtractKeyCountPairFromSubstring( absl::string_view str) { if (str.size() != kBytesPerCipherText * 2) { @@ -121,6 +102,17 @@ absl::Status WriteEcPointPairToString(const ElGamalEcPointPair& ec_point_pair, return absl::OkStatus(); } +absl::StatusOr GetEcPointPairFromString( + absl::string_view str, int curve_id) { + Context ctx; + ASSIGN_OR_RETURN(ECGroup ec_group, ECGroup::Create(curve_id, &ctx)); + ASSIGN_OR_RETURN(ElGamalCiphertext ciphertext, + ExtractElGamalCiphertextFromString(str)); + ASSIGN_OR_RETURN(ElGamalEcPointPair ec_point, + GetElGamalEcPoints(ciphertext, ec_group)); + return ec_point; +} + absl::StatusOr> GetCountValuesPlaintext( int maximum_value, int curve_id) { if (maximum_value < 1) { @@ -142,4 +134,31 @@ absl::StatusOr> GetCountValuesPlaintext( return result; } +absl::Status EncryptCompositeElGamalAndAppendToString( + ProtocolCryptor& protocol_cryptor, CompositeType composite_type, + absl::string_view plaintext_ec, std::string& data) { + ASSIGN_OR_RETURN( + ElGamalCiphertext key, + protocol_cryptor.EncryptCompositeElGamal(plaintext_ec, composite_type)); + data.append(key.first); + data.append(key.second); + return absl::OkStatus(); +} + +absl::Status EncryptCompositeElGamalAndWriteToString( + ProtocolCryptor& protocol_cryptor, CompositeType composite_type, + absl::string_view plaintext_ec, size_t pos, std::string& result) { + if (pos + kBytesPerCipherText > result.size()) { + return absl::InvalidArgumentError("result is not long enough to write."); + } + ASSIGN_OR_RETURN( + ElGamalCiphertext key, + protocol_cryptor.EncryptCompositeElGamal(plaintext_ec, composite_type)); + + result.replace(pos, kBytesPerEcPoint, key.first); + result.replace(pos + kBytesPerEcPoint, kBytesPerEcPoint, key.second); + + return absl::OkStatus(); +} + } // namespace wfa::measurement::common::crypto diff --git a/src/main/cc/wfa/measurement/common/crypto/encryption_utility_helper.h b/src/main/cc/wfa/measurement/common/crypto/encryption_utility_helper.h index 90ae46fe235..65da3709569 100644 --- a/src/main/cc/wfa/measurement/common/crypto/encryption_utility_helper.h +++ b/src/main/cc/wfa/measurement/common/crypto/encryption_utility_helper.h @@ -26,6 +26,8 @@ namespace wfa::measurement::common::crypto { +using ::wfa::measurement::common::crypto::CompositeType; + // A pair of ciphertexts which store the key and count values of a liquidlegions // register. struct KeyCountPairCipherText { @@ -41,11 +43,6 @@ absl::StatusOr GetNumberOfBlocks(absl::string_view data, absl::StatusOr ExtractElGamalCiphertextFromString( absl::string_view str); -// Blinds the last layer of ElGamal Encryption of register indexes, and return -// the deterministically encrypted results. -absl::StatusOr> GetBlindedRegisterIndexes( - absl::string_view data, ProtocolCryptor& protocol_cryptor); - // Extracts a KeyCountPairCipherText from a string_view. absl::StatusOr ExtractKeyCountPairFromSubstring( absl::string_view str); @@ -66,10 +63,30 @@ absl::Status AppendEcPointPairToString(const ElGamalEcPointPair& ec_point_pair, absl::Status WriteEcPointPairToString(const ElGamalEcPointPair& ec_point_pair, size_t pos, std::string& result); +// Extract a ElGamalEcPointPair from a string_view. +absl::StatusOr GetEcPointPairFromString( + absl::string_view str, int curve_id); + // Returns the vector of ECPoints for count values from 1 to maximum_value. absl::StatusOr> GetCountValuesPlaintext( int maximum_value, int curve_id); +// Encrypts plaintext and appends bytes of the cipher text to a target string. +// The length of bytes appened is kBytesPerCipherText = kBytesPerEcPoint * 2. +absl::Status EncryptCompositeElGamalAndAppendToString( + ProtocolCryptor& protocol_cryptor, CompositeType composite_type, + absl::string_view plaintext_ec, std::string& data); + +// Encrypts plaintext and writes bytes of the cipher text to a target string at +// a certain position. +// Bytes are written by replacing content of the string starting at pos. The +// length of bytes written is kBytesPerCipherText = kBytesPerEcPoint * 2. +// Returns a Status with code `INVALID_ARGUMENT` when the result string is not +// long enough. +absl::Status EncryptCompositeElGamalAndWriteToString( + ProtocolCryptor& protocol_cryptor, CompositeType composite_type, + absl::string_view plaintext_ec, size_t pos, std::string& result); + } // namespace wfa::measurement::common::crypto #endif // SRC_MAIN_CC_WFA_MEASUREMENT_COMMON_CRYPTO_ENCRYPTION_UTILITY_HELPER_H_ diff --git a/src/main/cc/wfa/measurement/common/crypto/protocol_cryptor.cc b/src/main/cc/wfa/measurement/common/crypto/protocol_cryptor.cc index 9a4489453b5..ea3cb0b6493 100644 --- a/src/main/cc/wfa/measurement/common/crypto/protocol_cryptor.cc +++ b/src/main/cc/wfa/measurement/common/crypto/protocol_cryptor.cc @@ -66,6 +66,8 @@ class ProtocolCryptorImpl : public ProtocolCryptor { CompositeType composite_type) override; absl::StatusOr EncryptCompositeElGamal( absl::string_view plain_ec_point, CompositeType composite_type) override; + absl::StatusOr EncryptIntegerToStringCompositeElGamal( + int64_t value) override; absl::StatusOr ReRandomize( const ElGamalCiphertext& ciphertext, CompositeType composite_type) override; @@ -173,6 +175,58 @@ absl::StatusOr ProtocolCryptorImpl::EncryptCompositeElGamal( : partial_composite_el_gamal_cipher_->Encrypt(plain_ec_point); } +absl::StatusOr +ProtocolCryptorImpl::EncryptIntegerToStringCompositeElGamal(int64_t value) { + Context ctx; + std::string ciphertext; + ciphertext.resize(kBytesPerCipherText); + if (value < 0) { + return absl::InvalidArgumentError( + absl::StrCat("The value should be non-negative, but is ", value)); + } + if (value == 0) { + ASSIGN_OR_RETURN( + ElGamalEcPointPair zero_ec, + EncryptIdentityElementToEcPointsCompositeElGamal(CompositeType::kFull)); + + if (absl::StatusOr result = zero_ec.u.ToBytesCompressed(); + result.ok()) { + ciphertext.replace(0, kBytesPerEcPoint, *result); + } else { + return result.status(); + } + + if (absl::StatusOr result = zero_ec.e.ToBytesCompressed(); + result.ok()) { + ciphertext.replace(kBytesPerEcPoint, kBytesPerEcPoint, *result); + } else { + return result.status(); + } + } else { + ASSIGN_OR_RETURN(ElGamalEcPointPair one_ec, + EncryptPlaintextToEcPointsCompositeElGamal( + kUnitECPointSeed, CompositeType::kFull)); + ASSIGN_OR_RETURN( + ElGamalEcPointPair point_ec, + MultiplyEcPointPairByScalar(one_ec, ctx.CreateBigNum(value))); + + if (absl::StatusOr result = point_ec.u.ToBytesCompressed(); + result.ok()) { + ciphertext.replace(0, kBytesPerEcPoint, *result); + } else { + return result.status(); + } + + if (absl::StatusOr result = point_ec.e.ToBytesCompressed(); + result.ok()) { + ciphertext.replace(kBytesPerEcPoint, kBytesPerEcPoint, *result); + } else { + return result.status(); + } + } + return ciphertext; +} + absl::StatusOr ProtocolCryptorImpl::ReRandomize( const ElGamalCiphertext& ciphertext, CompositeType composite_type) { ASSIGN_OR_RETURN( @@ -251,7 +305,7 @@ absl::Status ProtocolCryptorImpl::BatchProcess(absl::string_view data, ASSIGN_OR_RETURN(std::string temp, DecryptLocalElGamal(ciphertext)); // The first part of the ciphertext is the random number which is still // required to decrypt the other layers of ElGamal encryptions (at the - // subsequent duchies. So we keep it. + // subsequent duchies). So we keep it. result.replace(pos, kBytesPerEcPoint, ciphertext.first); pos += kBytesPerEcPoint; result.replace(pos, kBytesPerEcPoint, temp); diff --git a/src/main/cc/wfa/measurement/common/crypto/protocol_cryptor.h b/src/main/cc/wfa/measurement/common/crypto/protocol_cryptor.h index 1809619e1be..f7a7ecc68e0 100644 --- a/src/main/cc/wfa/measurement/common/crypto/protocol_cryptor.h +++ b/src/main/cc/wfa/measurement/common/crypto/protocol_cryptor.h @@ -70,6 +70,10 @@ class ProtocolCryptor { // Encrypts the plain EcPoint using the full or partial composite ElGamal Key. virtual absl::StatusOr EncryptCompositeElGamal( absl::string_view plain_ec_point, CompositeType composite_type) = 0; + // Maps the integer onto the curve and then encrypts the EcPoint with the full + // composite ElGamal Key, returns the string representation of the ciphertext. + virtual absl::StatusOr EncryptIntegerToStringCompositeElGamal( + int64_t value) = 0; // Encrypts the Identity Element using the full or partial composite ElGamal // Key, returns the result as an ElGamalEcPointPair. virtual absl::StatusOr diff --git a/src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/BUILD.bazel b/src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/BUILD.bazel index 1932eb30b7a..79de5b8dfca 100644 --- a/src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/BUILD.bazel +++ b/src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/BUILD.bazel @@ -4,6 +4,7 @@ load("@wfa_common_jvm//build:defs.bzl", "test_target") package(default_visibility = [ ":__pkg__", test_target(":__pkg__"), + "//src/main/cc/wfa/measurement:__subpackages__", "//src/main/swig/protocol:__subpackages__", ]) @@ -36,6 +37,33 @@ cc_library( ], ) +cc_library( + name = "reach_only_liquid_legions_v2_encryption_utility", + srcs = [ + "reach_only_liquid_legions_v2_encryption_utility.cc", + ], + hdrs = [ + "reach_only_liquid_legions_v2_encryption_utility.h", + ], + strip_include_prefix = _INCLUDE_PREFIX, + deps = [ + ":multithreading_helper", + ":noise_parameters_computation", + "//src/main/cc/wfa/measurement/common/crypto:constants", + "//src/main/cc/wfa/measurement/common/crypto:encryption_utility_helper", + "//src/main/cc/wfa/measurement/common/crypto:protocol_cryptor", + "//src/main/proto/wfa/measurement/internal/duchy/protocol:reach_only_liquid_legions_v2_encryption_methods_cc_proto", + "@any_sketch//src/main/cc/estimation:estimators", + "@any_sketch//src/main/cc/math:distributed_discrete_gaussian_noiser", + "@any_sketch//src/main/cc/math:distributed_geometric_noiser", + "@com_google_absl//absl/algorithm:container", + "@com_google_private_join_and_compute//private_join_and_compute/crypto:commutative_elgamal", + "@wfa_common_cpp//src/main/cc/common_cpp/jni:jni_wrap", + "@wfa_common_cpp//src/main/cc/common_cpp/macros", + "@wfa_common_cpp//src/main/cc/common_cpp/time:started_thread_cpu_timer", + ], +) + cc_library( name = "liquid_legions_v2_encryption_utility_wrapper", srcs = [ diff --git a/src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/liquid_legions_v2_encryption_utility.cc b/src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/liquid_legions_v2_encryption_utility.cc index 00b4bbf6603..62b7efc8c77 100644 --- a/src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/liquid_legions_v2_encryption_utility.cc +++ b/src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/liquid_legions_v2_encryption_utility.cc @@ -75,6 +75,31 @@ using ::wfa::measurement::common::crypto::ProtocolCryptorOptions; using ::wfa::measurement::internal::duchy::ElGamalPublicKey; using ::wfa::measurement::internal::duchy::protocol::LiquidLegionsV2NoiseConfig; +// Blinds the last layer of ElGamal Encryption of register indexes, and return +// the deterministically encrypted results. +absl::StatusOr> GetBlindedRegisterIndexes( + absl::string_view data, MultithreadingHelper& helper) { + ASSIGN_OR_RETURN(size_t register_count, + GetNumberOfBlocks(data, kBytesPerCipherRegister)); + std::vector blinded_register_indexes; + blinded_register_indexes.resize(register_count); + + absl::AnyInvocable f = + [&](ProtocolCryptor& cryptor, size_t index) -> absl::Status { + absl::string_view data_block = + data.substr(index * kBytesPerCipherRegister, kBytesPerCipherText); + ASSIGN_OR_RETURN(ElGamalCiphertext ciphertext, + ExtractElGamalCiphertextFromString(data_block)); + ASSIGN_OR_RETURN(std::string decrypted_el_gamal, + cryptor.DecryptLocalElGamal(ciphertext)); + blinded_register_indexes[index] = std::move(decrypted_el_gamal); + return absl::OkStatus(); + }; + RETURN_IF_ERROR(helper.Execute(register_count, f)); + + return blinded_register_indexes; +} + // Merge all the counts in each group using the SameKeyAggregation algorithm. // The calculated (flag_1, flag_2, flag_3, count) tuple is appended to the // response. 'sub_permutation' contains the locations of the registers belonging @@ -240,39 +265,6 @@ absl::StatusOr> GetSameKeyAggregatorMatrixBase( return std::move(result); } -absl::Status EncryptCompositeElGamalAndAppendToString( - ProtocolCryptor& protocol_cryptor, CompositeType composite_type, - absl::string_view plaintext_ec, std::string& data) { - ASSIGN_OR_RETURN( - ElGamalCiphertext key, - protocol_cryptor.EncryptCompositeElGamal(plaintext_ec, composite_type)); - data.append(key.first); - data.append(key.second); - return absl::OkStatus(); -} - -// Encrypts plaintext and writes bytes of the cipher text to a target string at -// a certain position. -// Bytes are written by replacing content of the string starting at pos. The -// length of bytes written is kBytesPerCipherText = kBytesPerEcPoint * 2. -// Returns a Status with code `INVALID_ARGUMENT` when the result string is not -// long enough. -absl::Status EncryptCompositeElGamalAndWriteToString( - ProtocolCryptor& protocol_cryptor, CompositeType composite_type, - absl::string_view plaintext_ec, size_t pos, std::string& result) { - if (pos + kBytesPerCipherText > result.size()) { - return absl::InvalidArgumentError("result is not long enough to write."); - } - ASSIGN_OR_RETURN( - ElGamalCiphertext key, - protocol_cryptor.EncryptCompositeElGamal(plaintext_ec, composite_type)); - - result.replace(pos, kBytesPerEcPoint, key.first); - result.replace(pos + kBytesPerEcPoint, kBytesPerEcPoint, key.second); - - return absl::OkStatus(); -} - // Adds encrypted blinded-histogram-noise registers to the end of data. // returns the number of such noise registers added. absl::StatusOr AddBlindedHistogramNoise( @@ -939,10 +931,9 @@ CompleteExecutionPhaseOneAtAggregator( MultithreadingHelper::CreateMultithreadingHelper( request.parallelism(), protocol_cryptor_options)); - ASSIGN_OR_RETURN( - std::vector blinded_register_indexes, - GetBlindedRegisterIndexes(request.combined_register_vector(), - multithreading_helper->GetProtocolCryptor())); + ASSIGN_OR_RETURN(std::vector blinded_register_indexes, + GetBlindedRegisterIndexes(request.combined_register_vector(), + *multithreading_helper)); // Create a sorting permutation of the blinded register indexes, such that we // don't need to modify the sketch data, whose size could be huge. We only diff --git a/src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/reach_only_liquid_legions_v2_encryption_utility.cc b/src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/reach_only_liquid_legions_v2_encryption_utility.cc new file mode 100644 index 00000000000..bb52b3b6be0 --- /dev/null +++ b/src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/reach_only_liquid_legions_v2_encryption_utility.cc @@ -0,0 +1,661 @@ +// Copyright 2023 The Cross-Media Measurement Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "wfa/measurement/internal/duchy/protocol/liquid_legions_v2/reach_only_liquid_legions_v2_encryption_utility.h" + +#include +#include +#include +#include +#include + +#include "absl/algorithm/container.h" +#include "absl/memory/memory.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/types/span.h" +#include "common_cpp/macros/macros.h" +#include "common_cpp/time/started_thread_cpu_timer.h" +#include "estimation/estimators.h" +#include "math/distributed_noiser.h" +#include "private_join_and_compute/crypto/commutative_elgamal.h" +#include "wfa/measurement/common/crypto/constants.h" +#include "wfa/measurement/common/crypto/encryption_utility_helper.h" +#include "wfa/measurement/common/crypto/protocol_cryptor.h" +#include "wfa/measurement/common/string_block_sorter.h" +#include "wfa/measurement/internal/duchy/protocol/liquid_legions_v2/multithreading_helper.h" +#include "wfa/measurement/internal/duchy/protocol/liquid_legions_v2/noise_parameters_computation.h" + +namespace wfa::measurement::internal::duchy::protocol::liquid_legions_v2 { + +namespace { + +using ::private_join_and_compute::BigNum; +using ::private_join_and_compute::CommutativeElGamal; +using ::private_join_and_compute::Context; +using ::private_join_and_compute::ECGroup; +using ::wfa::measurement::common::SortStringByBlock; +using ::wfa::measurement::common::crypto::Action; +using ::wfa::measurement::common::crypto::CompositeType; +using ::wfa::measurement::common::crypto::CreateProtocolCryptor; +using ::wfa::measurement::common::crypto::ElGamalCiphertext; +using ::wfa::measurement::common::crypto::ElGamalEcPointPair; +using ::wfa::measurement::common::crypto::ExtractElGamalCiphertextFromString; +using ::wfa::measurement::common::crypto::ExtractKeyCountPairFromRegisters; +using ::wfa::measurement::common::crypto::GetCountValuesPlaintext; +using ::wfa::measurement::common::crypto::GetEcPointPairFromString; +using ::wfa::measurement::common::crypto::GetElGamalEcPoints; +using ::wfa::measurement::common::crypto::GetNumberOfBlocks; +using ::wfa::measurement::common::crypto::kBlindedHistogramNoiseRegisterKey; +using ::wfa::measurement::common::crypto::kBytesPerCipherText; +using ::wfa::measurement::common::crypto::kBytesPerEcPoint; +using ::wfa::measurement::common::crypto::kBytesPerFlagsCountTuple; +using ::wfa::measurement::common::crypto::kDefaultEllipticCurveId; +using ::wfa::measurement::common::crypto::kDestroyedRegisterKey; +using ::wfa::measurement::common::crypto::KeyCountPairCipherText; +using ::wfa::measurement::common::crypto::kFlagZeroBase; +using ::wfa::measurement::common::crypto::kGenerateNewCompositeCipher; +using ::wfa::measurement::common::crypto::kGenerateNewParitialCompositeCipher; +using ::wfa::measurement::common::crypto::kGenerateWithNewElGamalPrivateKey; +using ::wfa::measurement::common::crypto::kGenerateWithNewElGamalPublicKey; +using ::wfa::measurement::common::crypto::kGenerateWithNewPohligHellmanKey; +using ::wfa::measurement::common::crypto::kPaddingNoiseRegisterId; +using ::wfa::measurement::common::crypto::kPublisherNoiseRegisterId; +using ::wfa::measurement::common::crypto::kUnitECPointSeed; +using ::wfa::measurement::common::crypto::MultiplyEcPointPairByScalar; +using ::wfa::measurement::common::crypto::ProtocolCryptor; +using ::wfa::measurement::common::crypto::ProtocolCryptorOptions; +using ::wfa::measurement::internal::duchy::ElGamalPublicKey; +using ::wfa::measurement::internal::duchy::protocol::LiquidLegionsV2NoiseConfig; + +// Blinds the last layer of ElGamal Encryption of register indexes, and return +// the deterministically encrypted results. +absl::StatusOr> GetRollv2BlindedRegisterIndexes( + absl::string_view data, MultithreadingHelper& helper) { + ASSIGN_OR_RETURN(size_t register_count, + GetNumberOfBlocks(data, kBytesPerCipherText)); + std::vector blinded_register_indexes; + blinded_register_indexes.resize(register_count); + + absl::AnyInvocable f = + [&](ProtocolCryptor& cryptor, size_t index) -> absl::Status { + absl::string_view data_block = + data.substr(index * kBytesPerCipherText, kBytesPerCipherText); + ASSIGN_OR_RETURN(ElGamalCiphertext ciphertext, + ExtractElGamalCiphertextFromString(data_block)); + ASSIGN_OR_RETURN(std::string decrypted_el_gamal, + cryptor.DecryptLocalElGamal(ciphertext)); + blinded_register_indexes[index] = std::move(decrypted_el_gamal); + return absl::OkStatus(); + }; + RETURN_IF_ERROR(helper.Execute(register_count, f)); + + return blinded_register_indexes; +} + +absl::StatusOr EstimateReach(double liquid_legions_decay_rate, + int64_t liquid_legions_size, + size_t non_empty_register_count, + float sampling_rate = 1.0) { + if (liquid_legions_decay_rate <= 1.0) { + return absl::InvalidArgumentError(absl::StrCat( + "The decay rate should be > 1, but is ", liquid_legions_decay_rate)); + } + if (liquid_legions_size <= non_empty_register_count) { + return absl::InvalidArgumentError(absl::StrCat( + "liquid legions size (", liquid_legions_size, + ") should be greater then the number of non empty registers (", + non_empty_register_count, ").")); + } + return wfa::estimation::EstimateCardinalityLiquidLegions( + liquid_legions_decay_rate, liquid_legions_size, non_empty_register_count, + sampling_rate); +} + +int64_t CountUniqueElements(const std::vector& arr) { + if (arr.empty()) { + return 0; + } + // Create a sorting permutation of the array, such that we don't need to + // modify the data, whose size could be huge. + std::vector permutation(arr.size()); + absl::c_iota(permutation, 0); + absl::c_sort(permutation, + [&](size_t a, size_t b) { return arr[a] < arr[b]; }); + + // Counting the number of unique elements by iterating through the indices. + int64_t count = 1; + int start = 0; + for (size_t i = 0; i < arr.size(); ++i) { + if (arr[permutation[i]] == arr[permutation[start]]) { + // This register has the same index, it belongs to the same group; + continue; + } else { + // This register belongs to a new group. Increase the unique register + // count by 1. + count++; + // Reset the starting point. + start = i; + } + } + return count; +} + +// Adds encrypted blinded-histogram-noise registers to the end of data. +// returns the number of such noise registers added. +absl::StatusOr AddReachOnlyBlindedHistogramNoise( + ProtocolCryptor& protocol_cryptor, int total_sketches_count, + const math::DistributedNoiser& distributed_noiser, size_t pos, + std::string& data, int64_t& num_unique_noise_id) { + int64_t noise_register_added = 0; + num_unique_noise_id = 0; + + for (int k = 1; k <= total_sketches_count; ++k) { + // The random number of distinct register_ids that should appear k times. + ASSIGN_OR_RETURN(int64_t noise_register_count_for_bucket_k, + distributed_noiser.GenerateNoiseComponent()); + num_unique_noise_id += noise_register_count_for_bucket_k; + + // Add noise_register_count_for_bucket_k such distinct register ids. + for (int i = 0; i < noise_register_count_for_bucket_k; ++i) { + // The prefix is to ensure the value is not in the regular id space. + std::string register_id = + absl::StrCat("blinded_histogram_noise", + protocol_cryptor.NextRandomBigNumAsString()); + ASSIGN_OR_RETURN(std::string register_id_ec, + protocol_cryptor.MapToCurve(register_id)); + // Add k registers with the same register_id. + for (int j = 0; j < k; ++j) { + // Add register_id + RETURN_IF_ERROR(EncryptCompositeElGamalAndWriteToString( + protocol_cryptor, CompositeType::kFull, register_id_ec, pos, data)); + pos += kBytesPerCipherText; + + ++noise_register_added; + } + } + } + + return noise_register_added; +} + +// Adds encrypted noise-for-publisher-noise registers to the end of data. +// returns the number of such noise registers added. +absl::StatusOr AddReachOnlyNoiseForPublisherNoise( + MultithreadingHelper& helper, + const math::DistributedNoiser& distributed_noiser, size_t pos, + std::string& data) { + ASSIGN_OR_RETURN( + std::string publisher_noise_register_id_ec, + helper.GetProtocolCryptor().MapToCurve(kPublisherNoiseRegisterId)); + + ASSIGN_OR_RETURN(int64_t noise_registers_count, + distributed_noiser.GenerateNoiseComponent()); + // Make sure that there is at least one publisher noise added so that we can + // always subtract 1 for the publisher noise later. This is to avoid the + // corner case where the noise_registers_count is zero for all workers. + noise_registers_count++; + + absl::AnyInvocable f = + [&](ProtocolCryptor& cryptor, size_t index) -> absl::Status { + size_t current_pos = pos + kBytesPerCipherText * index; + RETURN_IF_ERROR(EncryptCompositeElGamalAndWriteToString( + cryptor, CompositeType::kFull, publisher_noise_register_id_ec, + current_pos, data)); + + return absl::OkStatus(); + }; + RETURN_IF_ERROR(helper.Execute(noise_registers_count, f)); + + return noise_registers_count; +} + +// Adds encrypted global-reach-DP-noise registers to the end of data. +// returns the number of such noise registers added. +absl::StatusOr AddReachOnlyGlobalReachDpNoise( + MultithreadingHelper& helper, + const math::DistributedNoiser& distributed_noiser, size_t pos, + std::string& data) { + ASSIGN_OR_RETURN(int64_t noise_registers_count, + distributed_noiser.GenerateNoiseComponent()); + absl::AnyInvocable f = + [&](ProtocolCryptor& cryptor, size_t index) -> absl::Status { + size_t current_pos = pos + kBytesPerCipherText * index; + // Add register id, a random number. + // The prefix is to ensure the value is not in the regular id space. + std::string register_id = + absl::StrCat("reach_dp_noise", cryptor.NextRandomBigNumAsString()); + ASSIGN_OR_RETURN(std::string register_id_ec, + cryptor.MapToCurve(register_id)); + RETURN_IF_ERROR(EncryptCompositeElGamalAndWriteToString( + cryptor, CompositeType::kFull, register_id_ec, current_pos, data)); + + return absl::OkStatus(); + }; + RETURN_IF_ERROR(helper.Execute(noise_registers_count, f)); + + return noise_registers_count; +} + +// Adds encrypted padding-noise registers to the end of data. +absl::Status AddReachOnlyPaddingReachNoise(MultithreadingHelper& helper, + int64_t count, size_t pos, + std::string& data) { + if (count < 0) { + return absl::InvalidArgumentError("Count should >= 0."); + } + + ASSIGN_OR_RETURN( + std::string padding_noise_register_id_ec, + helper.GetProtocolCryptor().MapToCurve(kPaddingNoiseRegisterId)); + + absl::AnyInvocable f = + [&](ProtocolCryptor& cryptor, size_t index) -> absl::Status { + size_t current_pos = pos + kBytesPerCipherText * index; + // Add register_id, a predefined constant + RETURN_IF_ERROR(EncryptCompositeElGamalAndWriteToString( + cryptor, CompositeType::kFull, padding_noise_register_id_ec, + current_pos, data)); + + return absl::OkStatus(); + }; + RETURN_IF_ERROR(helper.Execute(count, f)); + pos += kBytesPerCipherText * count; + + return absl::OkStatus(); +} + +absl::Status ValidateReachOnlySetupNoiseParameters( + const RegisterNoiseGenerationParameters& parameters) { + if (parameters.contributors_count() < 1) { + return absl::InvalidArgumentError("contributors_count should be positive."); + } + if (parameters.total_sketches_count() < 1) { + return absl::InvalidArgumentError( + "total_sketches_count should be positive."); + } + if (parameters.dp_params().blind_histogram().epsilon() <= 0 || + parameters.dp_params().blind_histogram().delta() <= 0) { + return absl::InvalidArgumentError( + "Invalid blind_histogram dp parameter. epsilon/delta should be " + "positive."); + } + if (parameters.dp_params().noise_for_publisher_noise().epsilon() <= 0 || + parameters.dp_params().noise_for_publisher_noise().delta() <= 0) { + return absl::InvalidArgumentError( + "Invalid noise_for_publisher_noise dp parameter. epsilon/delta should " + "be positive."); + } + if (parameters.dp_params().global_reach_dp_noise().epsilon() <= 0 || + parameters.dp_params().global_reach_dp_noise().delta() <= 0) { + return absl::InvalidArgumentError( + "Invalid global_reach_dp_noise dp parameter. epsilon/delta should be " + "positive."); + } + return absl::OkStatus(); +} + +} // namespace + +absl::StatusOr +CompleteReachOnlyInitializationPhase( + const CompleteReachOnlyInitializationPhaseRequest& request) { + StartedThreadCpuTimer timer; + + ASSIGN_OR_RETURN( + std::unique_ptr cipher, + CommutativeElGamal::CreateWithNewKeyPair(request.curve_id())); + ASSIGN_OR_RETURN(ElGamalCiphertext public_key, cipher->GetPublicKeyBytes()); + ASSIGN_OR_RETURN(std::string private_key, cipher->GetPrivateKeyBytes()); + + CompleteReachOnlyInitializationPhaseResponse response; + response.mutable_el_gamal_key_pair()->mutable_public_key()->set_generator( + public_key.first); + response.mutable_el_gamal_key_pair()->mutable_public_key()->set_element( + public_key.second); + response.mutable_el_gamal_key_pair()->set_secret_key(private_key); + + response.set_elapsed_cpu_time_millis(timer.ElapsedMillis()); + return response; +} + +absl::StatusOr CompleteReachOnlySetupPhase( + const CompleteReachOnlySetupPhaseRequest& request) { + StartedThreadCpuTimer timer; + + CompleteReachOnlySetupPhaseResponse response; + std::string* response_crv = response.mutable_combined_register_vector(); + + *response_crv = request.combined_register_vector(); + + ProtocolCryptorOptions protocol_cryptor_options{ + .curve_id = static_cast(request.curve_id()), + .local_el_gamal_public_key = kGenerateWithNewElGamalPublicKey, + .local_el_gamal_private_key = + std::string(kGenerateWithNewElGamalPrivateKey), + .local_pohlig_hellman_private_key = + std::string(kGenerateWithNewPohligHellmanKey), + .composite_el_gamal_public_key = + std::make_pair(request.composite_el_gamal_public_key().generator(), + request.composite_el_gamal_public_key().element()), + .partial_composite_el_gamal_public_key = + kGenerateNewParitialCompositeCipher}; + + int64_t excessive_noise_count = 0; + + if (request.has_noise_parameters()) { + const RegisterNoiseGenerationParameters& noise_parameters = + request.noise_parameters(); + + auto blind_histogram_noiser = GetBlindHistogramNoiser( + noise_parameters.dp_params().blind_histogram(), + noise_parameters.contributors_count(), request.noise_mechanism()); + + auto publisher_noiser = GetPublisherNoiser( + noise_parameters.dp_params().noise_for_publisher_noise(), + noise_parameters.total_sketches_count(), + noise_parameters.contributors_count(), request.noise_mechanism()); + + auto global_reach_dp_noiser = GetGlobalReachDpNoiser( + noise_parameters.dp_params().global_reach_dp_noise(), + noise_parameters.contributors_count(), request.noise_mechanism()); + + // The total noise registers added. There are additional 2 noise count here + // to make sure that at least 1 publisher noise and 1 padding noise will be + // added. + int64_t total_noise_registers_count = + publisher_noiser->options().shift_offset * 2 + + global_reach_dp_noiser->options().shift_offset * 2 + + blind_histogram_noiser->options().shift_offset * + noise_parameters.total_sketches_count() * + (noise_parameters.total_sketches_count() + 1) + + 2; + + // Resize the space to hold all output data. + size_t pos = response_crv->size(); + response_crv->resize(request.combined_register_vector().size() + + total_noise_registers_count * kBytesPerCipherText); + + RETURN_IF_ERROR(ValidateReachOnlySetupNoiseParameters(noise_parameters)); + ASSIGN_OR_RETURN(auto multithreading_helper, + MultithreadingHelper::CreateMultithreadingHelper( + request.parallelism(), protocol_cryptor_options)); + + // 1. Add blinded histogram noise. + ASSIGN_OR_RETURN( + int64_t blinded_histogram_noise_count, + AddReachOnlyBlindedHistogramNoise( + multithreading_helper->GetProtocolCryptor(), + noise_parameters.total_sketches_count(), *blind_histogram_noiser, + pos, *response_crv, excessive_noise_count)); + pos += kBytesPerCipherText * blinded_histogram_noise_count; + // 2. Add noise for publisher noise. Publisher noise count is at least 1. + ASSIGN_OR_RETURN( + int64_t publisher_noise_count, + AddReachOnlyNoiseForPublisherNoise( + *multithreading_helper, *publisher_noiser, pos, *response_crv)); + pos += kBytesPerCipherText * publisher_noise_count; + // 3. Add reach DP noise. + ASSIGN_OR_RETURN(int64_t reach_dp_noise_count, + AddReachOnlyGlobalReachDpNoise(*multithreading_helper, + *global_reach_dp_noiser, + pos, *response_crv)); + pos += kBytesPerCipherText * reach_dp_noise_count; + // 4. Add padding noise. Padding noise count will be at least 1. + int64_t padding_noise_count = total_noise_registers_count - + blinded_histogram_noise_count - + publisher_noise_count - reach_dp_noise_count; + RETURN_IF_ERROR(AddReachOnlyPaddingReachNoise( + *multithreading_helper, padding_noise_count, pos, *response_crv)); + } + + // Encrypt the excessive noise. + ASSIGN_OR_RETURN(std::unique_ptr protocol_cryptor, + CreateProtocolCryptor(protocol_cryptor_options)); + ASSIGN_OR_RETURN(std::string serialized_excessive_noise_ciphertext, + protocol_cryptor->EncryptIntegerToStringCompositeElGamal( + excessive_noise_count)); + + response.set_serialized_excessive_noise_ciphertext( + serialized_excessive_noise_ciphertext); + + RETURN_IF_ERROR(SortStringByBlock( + *response.mutable_combined_register_vector())); + + response.set_elapsed_cpu_time_millis(timer.ElapsedMillis()); + return response; +} + +absl::StatusOr +CompleteReachOnlySetupPhaseAtAggregator( + const CompleteReachOnlySetupPhaseRequest& request) { + StartedThreadCpuTimer timer; + ASSIGN_OR_RETURN(CompleteReachOnlySetupPhaseResponse response, + CompleteReachOnlySetupPhase(request)); + + // Get the ElGamal encryption of the excessive noise on the aggregator. + ASSIGN_OR_RETURN( + ElGamalEcPointPair ec_point, + GetEcPointPairFromString(response.serialized_excessive_noise_ciphertext(), + request.curve_id())); + + // Combined the excessive_noise ciphertexts. + int num_ciphertexts = request.serialized_excessive_noise_ciphertext().size() / + kBytesPerCipherText; + for (int i = 0; i < num_ciphertexts; i++) { + ASSIGN_OR_RETURN(ElGamalEcPointPair temp, + GetEcPointPairFromString( + request.serialized_excessive_noise_ciphertext().substr( + i * kBytesPerCipherText, kBytesPerCipherText), + request.curve_id())); + ASSIGN_OR_RETURN(ec_point, AddEcPointPairs(ec_point, temp)); + } + + std::string excessive_noise_string; + excessive_noise_string.resize(kBytesPerCipherText); + RETURN_IF_ERROR( + WriteEcPointPairToString(ec_point, 0, excessive_noise_string)); + + response.set_serialized_excessive_noise_ciphertext(excessive_noise_string); + + response.set_elapsed_cpu_time_millis(timer.ElapsedMillis()); + return response; +} + +absl::StatusOr +CompleteReachOnlyExecutionPhase( + const CompleteReachOnlyExecutionPhaseRequest& request) { + StartedThreadCpuTimer timer; + + ASSIGN_OR_RETURN(size_t register_count, + GetNumberOfBlocks(request.combined_register_vector(), + kBytesPerCipherText)); + + ProtocolCryptorOptions protocol_cryptor_options{ + .curve_id = static_cast(request.curve_id()), + .local_el_gamal_public_key = std::make_pair( + request.local_el_gamal_key_pair().public_key().generator(), + request.local_el_gamal_key_pair().public_key().element()), + .local_el_gamal_private_key = + request.local_el_gamal_key_pair().secret_key(), + .local_pohlig_hellman_private_key = + std::string(kGenerateWithNewPohligHellmanKey), + .composite_el_gamal_public_key = kGenerateNewCompositeCipher, + .partial_composite_el_gamal_public_key = + kGenerateNewParitialCompositeCipher}; + ASSIGN_OR_RETURN(auto multithreading_helper, + MultithreadingHelper::CreateMultithreadingHelper( + request.parallelism(), protocol_cryptor_options)); + + CompleteReachOnlyExecutionPhaseResponse response; + // Partially decrypt the aggregated excessive noise ciphertext. + ASSIGN_OR_RETURN(std::unique_ptr protocol_cryptor, + CreateProtocolCryptor(protocol_cryptor_options)); + + std::string updated_noise_ciphertext; + updated_noise_ciphertext.resize(kBytesPerCipherText); + RETURN_IF_ERROR(protocol_cryptor->BatchProcess( + request.serialized_excessive_noise_ciphertext(), + {Action::kPartialDecrypt}, 0, updated_noise_ciphertext)); + response.set_serialized_excessive_noise_ciphertext(updated_noise_ciphertext); + + std::string* response_crv = response.mutable_combined_register_vector(); + // The output crv is the same size with the input crv. + size_t start_pos = 0; + response_crv->resize(request.combined_register_vector().size()); + + absl::AnyInvocable f = + [&](ProtocolCryptor& cryptor, size_t index) -> absl::Status { + absl::string_view current_block = + absl::string_view(request.combined_register_vector()) + .substr(index * kBytesPerCipherText, kBytesPerCipherText); + size_t pos = start_pos + kBytesPerCipherText * index; + + RETURN_IF_ERROR(cryptor.BatchProcess(current_block, {Action::kBlind}, pos, + *response_crv)); + + return absl::OkStatus(); + }; + + RETURN_IF_ERROR(multithreading_helper->Execute(register_count, f)); + RETURN_IF_ERROR(SortStringByBlock(*response_crv)); + + response.set_elapsed_cpu_time_millis(timer.ElapsedMillis()); + return response; +} + +absl::StatusOr +CompleteReachOnlyExecutionPhaseAtAggregator( + const CompleteReachOnlyExecutionPhaseAtAggregatorRequest& request) { + StartedThreadCpuTimer timer; + + if (request.combined_register_vector().size() % kBytesPerCipherText != 0) { + return absl::InvalidArgumentError(absl::StrCat( + "The size of byte array is not divisible by the block_size: ", + request.combined_register_vector().size())); + } + + ProtocolCryptorOptions protocol_cryptor_options{ + .curve_id = static_cast(request.curve_id()), + .local_el_gamal_public_key = std::make_pair( + request.local_el_gamal_key_pair().public_key().generator(), + request.local_el_gamal_key_pair().public_key().element()), + .local_el_gamal_private_key = + request.local_el_gamal_key_pair().secret_key(), + .local_pohlig_hellman_private_key = + std::string(kGenerateWithNewPohligHellmanKey), + .composite_el_gamal_public_key = kGenerateNewCompositeCipher, + .partial_composite_el_gamal_public_key = + kGenerateNewParitialCompositeCipher}; + + // Decrypt the aggregated excessive noise ciphertext to get the excessive + // noise count. + int64_t excessive_noise_count = 0; + ASSIGN_OR_RETURN(std::unique_ptr protocol_cryptor, + CreateProtocolCryptor(protocol_cryptor_options)); + ASSIGN_OR_RETURN(ElGamalCiphertext ciphertext, + ExtractElGamalCiphertextFromString( + request.serialized_excessive_noise_ciphertext())); + ASSIGN_OR_RETURN( + bool isZero, + protocol_cryptor->IsDecryptLocalElGamalResultZero(ciphertext)); + if (!isZero) { + ASSIGN_OR_RETURN(std::string plaintext, + protocol_cryptor->DecryptLocalElGamal(ciphertext)); + + auto blind_histogram_noiser = GetBlindHistogramNoiser( + request.noise_parameters().dp_params().blind_histogram(), + request.noise_parameters().contributors_count(), + request.noise_mechanism()); + // For each a in [1; number_of_EDPs], each worker samples at most + // blind_histogram_noiser->options().shift_offset * 2 noise registers. So + // all workers sample at most #EDPs*#max_per_worker noise registers. + int max_excessive_noise = + blind_histogram_noiser->options().shift_offset * 2 * + request.noise_parameters().total_sketches_count() * + request.noise_parameters().contributors_count(); + // The lookup table stores the max_excessive_noise EC points where + // ec_lookup_table[i] = (i+1)*ec_generator. + ASSIGN_OR_RETURN( + std::vector ec_lookup_table, + GetCountValuesPlaintext(max_excessive_noise, request.curve_id())); + // Decrypt the excessive noise using the lookup table. + int i = 0; + for (i = 0; i < ec_lookup_table.size(); i++) { + if (ec_lookup_table[i] == plaintext) { + excessive_noise_count = i + 1; + break; + } + } + // Returns an error if the decryption fails. + if (i == ec_lookup_table.size()) { + return absl::InternalError( + "Failed to decrypt the excessive noise ciphertext."); + } + } + + ASSIGN_OR_RETURN(auto multithreading_helper, + MultithreadingHelper::CreateMultithreadingHelper( + request.parallelism(), protocol_cryptor_options)); + + ASSIGN_OR_RETURN( + std::vector blinded_register_indexes, + GetRollv2BlindedRegisterIndexes(request.combined_register_vector(), + *multithreading_helper)); + CompleteReachOnlyExecutionPhaseAtAggregatorResponse response; + + // Counting the number of unique registers. + int64_t non_empty_register_count = + CountUniqueElements(blinded_register_indexes); + + // Excluding the blind histogram, padding noise, and the excessive noise from + // the unique register count. It is guaranteed that if noise is added, then + // there exist publisher noise and padding noise. + if (request.has_noise_parameters()) { + non_empty_register_count -= 2; + } + // Remove the total excessive blind histogram noise. + non_empty_register_count = non_empty_register_count - excessive_noise_count; + // Remove the reach dp noise baseline. + if (request.has_reach_dp_noise_baseline()) { + auto noiser = GetGlobalReachDpNoiser( + request.reach_dp_noise_baseline().global_reach_dp_noise(), + request.reach_dp_noise_baseline().contributors_count(), + request.noise_mechanism()); + const auto& noise_options = noiser->options(); + int64_t global_reach_dp_noise_baseline = + noise_options.shift_offset * noise_options.contributor_count; + non_empty_register_count -= global_reach_dp_noise_baseline; + } + + // If the noise added is less than the baseline, the non empty register count + // could be negative. Make sure that it is non-negative. + if (non_empty_register_count < 0) { + non_empty_register_count = 0; + } + + // Estimate the reach + ASSIGN_OR_RETURN( + int64_t reach, + EstimateReach(request.liquid_legions_parameters().decay_rate(), + request.liquid_legions_parameters().size(), + non_empty_register_count, + request.vid_sampling_interval_width())); + + response.set_reach(reach); + response.set_elapsed_cpu_time_millis(timer.ElapsedMillis()); + return response; +} + +} // namespace wfa::measurement::internal::duchy::protocol::liquid_legions_v2 diff --git a/src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/reach_only_liquid_legions_v2_encryption_utility.h b/src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/reach_only_liquid_legions_v2_encryption_utility.h new file mode 100644 index 00000000000..311ab5e9f29 --- /dev/null +++ b/src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/reach_only_liquid_legions_v2_encryption_utility.h @@ -0,0 +1,89 @@ +// Copyright 2023 The Cross-Media Measurement Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef SRC_MAIN_CC_WFA_MEASUREMENT_INTERNAL_DUCHY_PROTOCOL_LIQUID_LEGIONS_V2_REACH_ONLY_LIQUID_LEGIONS_V2_ENCRYPTION_UTILITY_H_ +#define SRC_MAIN_CC_WFA_MEASUREMENT_INTERNAL_DUCHY_PROTOCOL_LIQUID_LEGIONS_V2_REACH_ONLY_LIQUID_LEGIONS_V2_ENCRYPTION_UTILITY_H_ + +#include "absl/status/statusor.h" +#include "wfa/measurement/internal/duchy/protocol/reach_only_liquid_legions_v2_encryption_methods.pb.h" + +namespace wfa::measurement::internal::duchy::protocol::liquid_legions_v2 { + +using ::wfa::measurement::internal::duchy::protocol:: + CompleteReachOnlyExecutionPhaseAtAggregatorRequest; +using ::wfa::measurement::internal::duchy::protocol:: + CompleteReachOnlyExecutionPhaseAtAggregatorResponse; +using ::wfa::measurement::internal::duchy::protocol:: + CompleteReachOnlyExecutionPhaseRequest; +using ::wfa::measurement::internal::duchy::protocol:: + CompleteReachOnlyExecutionPhaseResponse; +using ::wfa::measurement::internal::duchy::protocol:: + CompleteReachOnlyInitializationPhaseRequest; +using ::wfa::measurement::internal::duchy::protocol:: + CompleteReachOnlyInitializationPhaseResponse; +using ::wfa::measurement::internal::duchy::protocol:: + CompleteReachOnlySetupPhaseRequest; +using ::wfa::measurement::internal::duchy::protocol:: + CompleteReachOnlySetupPhaseResponse; + +// Complete work in the initialization phase at both the aggregator and +// non-aggregator workers. More specifically, the worker would generate a random +// set of ElGamal Key pair. +absl::StatusOr +CompleteReachOnlyInitializationPhase( + const CompleteReachOnlyInitializationPhaseRequest& request); + +// Complete work in the setup phase at the non-aggregator workers. More +// specifically, the worker would +// 1. add local noise registers (if configured to). +// 2. shuffle all registers. +// 3. encrypt the amount of excessive noise with the composit ElGamal public +// key. +absl::StatusOr CompleteReachOnlySetupPhase( + const CompleteReachOnlySetupPhaseRequest& request); + +// Complete work in the setup phase at the aggregator. More specifically, the +// aggregator would +// 1. add local noise registers (if configured to). +// 2. shuffle all registers. +// 3. encrypt its excessive noise using the composite ElGamal public key. +// 4. combine its noise ciphertext with those from the workers. +absl::StatusOr +CompleteReachOnlySetupPhaseAtAggregator( + const CompleteReachOnlySetupPhaseRequest& request); + +// Complete work in the execution phase one at a non-aggregator worker. +// More specifically, the worker would +// 1. blind the positions (decrypt local ElGamal layer and then add another +// layer of deterministic pohlig_hellman encryption. +// 2. partially decrypt the noise ciphertext using its partial ElGamal +// private key. +// 3. shuffle all registers. +absl::StatusOr +CompleteReachOnlyExecutionPhase( + const CompleteReachOnlyExecutionPhaseRequest& request); + +// Complete work in the execution phase one at the aggregator worker. +// More specifically, the worker would +// 1. decrypt the local ElGamal encryption on the positions. +// 2. decrypt the total excessive noise. +// 3. count the number of unique registers, excluding the blinded histogram +// noise, the publisher noise, and the excessive noise. +absl::StatusOr +CompleteReachOnlyExecutionPhaseAtAggregator( + const CompleteReachOnlyExecutionPhaseAtAggregatorRequest& request); + +} // namespace wfa::measurement::internal::duchy::protocol::liquid_legions_v2 + +#endif // SRC_MAIN_CC_WFA_MEASUREMENT_INTERNAL_DUCHY_PROTOCOL_LIQUID_LEGIONS_V2_REACH_ONLY_LIQUID_LEGIONS_V2_ENCRYPTION_UTILITY_H_ diff --git a/src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/testing/BUILD.bazel b/src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/testing/BUILD.bazel new file mode 100644 index 00000000000..0fceac27edc --- /dev/null +++ b/src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/testing/BUILD.bazel @@ -0,0 +1,28 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") + +package( + default_testonly = True, + default_visibility = [ + "//src/test/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2:__subpackages__", + ], +) + +_INCLUDE_PREFIX = "/src/main/cc" + +cc_library( + name = "liquid_legions_v2_encryption_utility_helper", + srcs = [ + "liquid_legions_v2_encryption_utility_helper.cc", + ], + hdrs = [ + "liquid_legions_v2_encryption_utility_helper.h", + ], + strip_include_prefix = _INCLUDE_PREFIX, + deps = [ + "//src/main/proto/wfa/measurement/internal/duchy:crypto_cc_proto", + "//src/main/proto/wfa/measurement/internal/duchy:differential_privacy_cc_proto", + "@any_sketch//src/main/cc/any_sketch/crypto:sketch_encrypter", + "@any_sketch//src/main/cc/estimation:estimators", + "@com_google_absl//absl/status:statusor", + ], +) diff --git a/src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/testing/liquid_legions_v2_encryption_utility_helper.cc b/src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/testing/liquid_legions_v2_encryption_utility_helper.cc new file mode 100644 index 00000000000..601e6326225 --- /dev/null +++ b/src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/testing/liquid_legions_v2_encryption_utility_helper.cc @@ -0,0 +1,63 @@ +// Copyright 2023 The Cross-Media Measurement Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "wfa/measurement/internal/duchy/protocol/liquid_legions_v2/testing/liquid_legions_v2_encryption_utility_helper.h" + +#include "estimation/estimators.h" + +namespace wfa::measurement::internal::duchy::protocol::liquid_legions_v2 { + +using ::wfa::any_sketch::Sketch; +using ::wfa::any_sketch::SketchConfig; +using ::wfa::measurement::internal::duchy::ElGamalPublicKey; + +::wfa::any_sketch::crypto::ElGamalPublicKey ToAnySketchElGamalKey( + ElGamalPublicKey key) { + ::wfa::any_sketch::crypto::ElGamalPublicKey result; + result.set_generator(key.generator()); + result.set_element(key.element()); + return result; +} + +ElGamalPublicKey ToDuchyInternalElGamalKey( + ::wfa::any_sketch::crypto::ElGamalPublicKey key) { + ElGamalPublicKey result; + result.set_generator(key.generator()); + result.set_element(key.element()); + return result; +} + +Sketch CreateEmptyLiquidLegionsSketch() { + Sketch plain_sketch; + plain_sketch.mutable_config()->add_values()->set_aggregator( + SketchConfig::ValueSpec::UNIQUE); + plain_sketch.mutable_config()->add_values()->set_aggregator( + SketchConfig::ValueSpec::SUM); + return plain_sketch; +} + +Sketch CreateReachOnlyEmptyLiquidLegionsSketch() { + Sketch plain_sketch; + return plain_sketch; +} + +DifferentialPrivacyParams MakeDifferentialPrivacyParams(double epsilon, + double delta) { + DifferentialPrivacyParams params; + params.set_epsilon(epsilon); + params.set_delta(delta); + return params; +} + +} // namespace wfa::measurement::internal::duchy::protocol::liquid_legions_v2 diff --git a/src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/testing/liquid_legions_v2_encryption_utility_helper.h b/src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/testing/liquid_legions_v2_encryption_utility_helper.h new file mode 100644 index 00000000000..e7775c8f67e --- /dev/null +++ b/src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/testing/liquid_legions_v2_encryption_utility_helper.h @@ -0,0 +1,44 @@ +// Copyright 2023 The Cross-Media Measurement Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef SRC_MAIN_CC_WFA_MEASUREMENT_INTERNAL_DUCHY_PROTOCOL_LIQUID_LEGIONS_V2_TESTING_LIQUID_LEGIONS_V2_ENCRYPTION_UTILITY_HELPER_H_ +#define SRC_MAIN_CC_WFA_MEASUREMENT_INTERNAL_DUCHY_PROTOCOL_LIQUID_LEGIONS_V2_TESTING_LIQUID_LEGIONS_V2_ENCRYPTION_UTILITY_HELPER_H_ + +#include "absl/status/statusor.h" +#include "any_sketch/crypto/sketch_encrypter.h" +#include "wfa/measurement/internal/duchy/crypto.pb.h" +#include "wfa/measurement/internal/duchy/differential_privacy.pb.h" + +namespace wfa::measurement::internal::duchy::protocol::liquid_legions_v2 { + +using ::wfa::any_sketch::Sketch; +using ::wfa::measurement::internal::duchy::DifferentialPrivacyParams; +using ::wfa::measurement::internal::duchy::ElGamalPublicKey; + +::wfa::any_sketch::crypto::ElGamalPublicKey ToAnySketchElGamalKey( + ElGamalPublicKey key); + +ElGamalPublicKey ToDuchyInternalElGamalKey( + ::wfa::any_sketch::crypto::ElGamalPublicKey key); + +Sketch CreateEmptyLiquidLegionsSketch(); + +Sketch CreateReachOnlyEmptyLiquidLegionsSketch(); + +DifferentialPrivacyParams MakeDifferentialPrivacyParams(double epsilon, + double delta); + +} // namespace wfa::measurement::internal::duchy::protocol::liquid_legions_v2 + +#endif // SRC_MAIN_CC_WFA_MEASUREMENT_INTERNAL_DUCHY_PROTOCOL_LIQUID_LEGIONS_V2_TESTING_LIQUID_LEGIONS_V2_ENCRYPTION_UTILITY_HELPER_H_ diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/daemon/herald/LiquidLegionsV2Starter.kt b/src/main/kotlin/org/wfanet/measurement/duchy/daemon/herald/LiquidLegionsV2Starter.kt index da840b2e677..00ee53c174b 100644 --- a/src/main/kotlin/org/wfanet/measurement/duchy/daemon/herald/LiquidLegionsV2Starter.kt +++ b/src/main/kotlin/org/wfanet/measurement/duchy/daemon/herald/LiquidLegionsV2Starter.kt @@ -197,7 +197,7 @@ object LiquidLegionsV2Starter { // For weird stages, we throw. Stage.UNRECOGNIZED, - Stage.STAGE_UNKNOWN -> { + Stage.STAGE_UNSPECIFIED -> { error("[id=${token.globalComputationId}]: Unrecognized stage '$stage'") } } @@ -254,7 +254,7 @@ object LiquidLegionsV2Starter { // For weird stages, we throw. Stage.UNRECOGNIZED, - Stage.STAGE_UNKNOWN -> { + Stage.STAGE_UNSPECIFIED -> { error("[id=${token.globalComputationId}]: Unrecognized stage '$stage'") } } diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/db/computation/LiquidLegionsSketchAggregationV2Protocol.kt b/src/main/kotlin/org/wfanet/measurement/duchy/db/computation/LiquidLegionsSketchAggregationV2Protocol.kt index cb5cdac58f8..d8de7f5dea7 100644 --- a/src/main/kotlin/org/wfanet/measurement/duchy/db/computation/LiquidLegionsSketchAggregationV2Protocol.kt +++ b/src/main/kotlin/org/wfanet/measurement/duchy/db/computation/LiquidLegionsSketchAggregationV2Protocol.kt @@ -121,7 +121,7 @@ object LiquidLegionsSketchAggregationV2Protocol { COMPLETE -> error("Computation should be ended with call to endComputation(...)") // Stages that we can't transition to ever. UNRECOGNIZED, - LiquidLegionsSketchAggregationV2.Stage.STAGE_UNKNOWN, + LiquidLegionsSketchAggregationV2.Stage.STAGE_UNSPECIFIED, INITIALIZATION_PHASE -> error("Cannot make transition function to stage $stage") } } @@ -151,7 +151,7 @@ object LiquidLegionsSketchAggregationV2Protocol { COMPLETE -> error("Computation should be ended with call to endComputation(...)") // Stages that we can't transition to ever. UNRECOGNIZED, - LiquidLegionsSketchAggregationV2.Stage.STAGE_UNKNOWN, + LiquidLegionsSketchAggregationV2.Stage.STAGE_UNSPECIFIED, INITIALIZATION_PHASE -> error("Cannot make transition function to stage $stage") } } diff --git a/src/main/kotlin/org/wfanet/measurement/duchy/service/internal/computationcontrol/ProtocolStages.kt b/src/main/kotlin/org/wfanet/measurement/duchy/service/internal/computationcontrol/ProtocolStages.kt index f900ec4c776..4510c236c4e 100644 --- a/src/main/kotlin/org/wfanet/measurement/duchy/service/internal/computationcontrol/ProtocolStages.kt +++ b/src/main/kotlin/org/wfanet/measurement/duchy/service/internal/computationcontrol/ProtocolStages.kt @@ -83,7 +83,7 @@ class LiquidLegionsV2Stages() : LiquidLegionsSketchAggregationV2.Stage.EXECUTION_PHASE_TWO, LiquidLegionsSketchAggregationV2.Stage.EXECUTION_PHASE_THREE, LiquidLegionsSketchAggregationV2.Stage.COMPLETE, - LiquidLegionsSketchAggregationV2.Stage.STAGE_UNKNOWN, + LiquidLegionsSketchAggregationV2.Stage.STAGE_UNSPECIFIED, LiquidLegionsSketchAggregationV2.Stage.UNRECOGNIZED -> throw IllegalStageException(token.computationStage) { "Unexpected $stageType stage: $protocolStage" @@ -112,7 +112,7 @@ class LiquidLegionsV2Stages() : LiquidLegionsSketchAggregationV2.Stage.EXECUTION_PHASE_TWO, LiquidLegionsSketchAggregationV2.Stage.EXECUTION_PHASE_THREE, LiquidLegionsSketchAggregationV2.Stage.COMPLETE, - LiquidLegionsSketchAggregationV2.Stage.STAGE_UNKNOWN, + LiquidLegionsSketchAggregationV2.Stage.STAGE_UNSPECIFIED, LiquidLegionsSketchAggregationV2.Stage.UNRECOGNIZED -> throw IllegalStageException(stage) { "Next $stageType stage unknown for $protocolStage" } }.toProtocolStage() diff --git a/src/main/proto/wfa/measurement/internal/duchy/crypto.proto b/src/main/proto/wfa/measurement/internal/duchy/crypto.proto index 8992321dc27..3b4c3061b62 100644 --- a/src/main/proto/wfa/measurement/internal/duchy/crypto.proto +++ b/src/main/proto/wfa/measurement/internal/duchy/crypto.proto @@ -59,25 +59,3 @@ message EncryptionPublicKey { // decrypt messages given a private key. bytes data = 2; } - -// Holds a Paillier Public Key. -message PaillierPublicKey { - // The Paillier modulus n. - optional bytes n = 1; - // Contains the Damgard-Jurik exponent corresponding to this key. The Paillier - // modulus will be n^(s+1), and the message space will be n^s. - optional int32 s = 2; -} - -// Holds a Paillier Private Key. -message PaillierPrivateKey { - // One of the two large prime factors of the Paillier modulus n. - optional bytes p = 1; - - // One of the two large prime factors of the Paillier modulus n. - optional bytes q = 2; - - // Contains the Damgard-Jurik exponent corresponding to this key. The Paillier - // modulus will be n^(s+1), and the message space will be n^s. - optional int32 s = 3; -} diff --git a/src/main/proto/wfa/measurement/internal/duchy/protocol/liquid_legions_sketch_aggregation_v2.proto b/src/main/proto/wfa/measurement/internal/duchy/protocol/liquid_legions_sketch_aggregation_v2.proto index a7d99d3a154..bc49b00bd17 100644 --- a/src/main/proto/wfa/measurement/internal/duchy/protocol/liquid_legions_sketch_aggregation_v2.proto +++ b/src/main/proto/wfa/measurement/internal/duchy/protocol/liquid_legions_sketch_aggregation_v2.proto @@ -28,7 +28,7 @@ option java_multiple_files = true; message LiquidLegionsSketchAggregationV2 { enum Stage { // The computation stage is unknown. This is never set intentionally. - STAGE_UNKNOWN = 0; + STAGE_UNSPECIFIED = 0; // The worker is in the initialization phase. // More specifically, each worker will create a new ElGamal key pair solely diff --git a/src/main/proto/wfa/measurement/internal/duchy/protocol/reach_only_liquid_legions_sketch_aggregation_v2.proto b/src/main/proto/wfa/measurement/internal/duchy/protocol/reach_only_liquid_legions_sketch_aggregation_v2.proto index 490b60d0516..7fcc042a28c 100644 --- a/src/main/proto/wfa/measurement/internal/duchy/protocol/reach_only_liquid_legions_sketch_aggregation_v2.proto +++ b/src/main/proto/wfa/measurement/internal/duchy/protocol/reach_only_liquid_legions_sketch_aggregation_v2.proto @@ -152,16 +152,6 @@ message ReachOnlyLiquidLegionsSketchAggregationV2 { // TODO(@ple13): delete this field when we switch to use a secure key // store for duchy private keys. ElGamalKeyPair local_elgamal_key = 7; - - // Paillier Private Key used to decrypt the total excessive noise to be - // removed from the register count in the execution phase. - // TODO(@ple13): delete this field when we switch to use a secure key - // store for duchy private keys. Only the aggregator samples and stores this - // key. - PaillierPrivateKey local_paillier_key = 8; - - // Noise to be removed. - int64 excess_noise = 9; } // Details about a particular attempt of running a stage of the LiquidLegionV2 diff --git a/src/main/proto/wfa/measurement/internal/duchy/protocol/reach_only_liquid_legions_v2_encryption_methods.proto b/src/main/proto/wfa/measurement/internal/duchy/protocol/reach_only_liquid_legions_v2_encryption_methods.proto index 3974294c487..93a5523ca32 100644 --- a/src/main/proto/wfa/measurement/internal/duchy/protocol/reach_only_liquid_legions_v2_encryption_methods.proto +++ b/src/main/proto/wfa/measurement/internal/duchy/protocol/reach_only_liquid_legions_v2_encryption_methods.proto @@ -53,13 +53,21 @@ message CompleteReachOnlySetupPhaseRequest { // The CRV is only needed so the noise can be interleaved and hidden in the // CRV. The registers in the CRV are unchanged, except for their orders. bytes combined_register_vector = 1; + // The elliptical curve to work on. + int64 curve_id = 2; // The parameters required for generating noise registers. // if unset, the worker only shuffles the register without adding any noise. - RegisterNoiseGenerationParameters noise_parameters = 2; + RegisterNoiseGenerationParameters noise_parameters = 3; // The mechanism used to generate noise. - LiquidLegionsV2NoiseConfig.NoiseMechanism noise_mechanism = 3; + LiquidLegionsV2NoiseConfig.NoiseMechanism noise_mechanism = 4; + // Public Key of the composite ElGamal cipher. Used to encrypt the excessive + // noise (which is zero) when noise_parameters is not available. + ElGamalPublicKey composite_el_gamal_public_key = 5; + // This field is only set for the aggregator. There will be one encrypted + // noise element for each non-aggregator worker. + bytes serialized_excessive_noise_ciphertext = 6; // The maximum number of threads used by crypto actions. - int32 parallelism = 4; + int32 parallelism = 7; } // Response of the CompleteReachOnlySetupPhase method. @@ -68,29 +76,11 @@ message CompleteReachOnlySetupPhaseResponse { // and noise registers. bytes combined_register_vector = 1; // The excessive noise that can be removed in the execution phase. - int64 excessive_noise = 2; + bytes serialized_excessive_noise_ciphertext = 2; // The CPU time of processing the request. int64 elapsed_cpu_time_millis = 3; } -// Response of the CompleteReachOnlySetupPhase method at the aggregate worker. -// Different from the non-aggregator, the aggregator samples the Paillier key -// pair and encrypts its excessive noise with the public key. -message CompleteReachOnlySetupPhaseAtAggregatorResponse { - // The output combined register vector (CRV), which contains shuffled input - // and noise registers. - bytes combined_register_vector = 1; - // The Paillier private key. - PaillierPrivateKey paillier_private_key = 2; - // The Paillier public key. - PaillierPublicKey paillier_public_key = 3; - // The serialized Paillier ciphertext that encrypts the aggregated excessive - // noise of the aggregator. - bytes serialized_aggregated_noise_ciphertext = 4; - // The CPU time of processing the request. - int64 elapsed_cpu_time_millis = 5; -} - // The request to complete work in the execution phase at a non-aggregator // worker. message CompleteReachOnlyExecutionPhaseRequest { @@ -100,21 +90,13 @@ message CompleteReachOnlyExecutionPhaseRequest { bytes combined_register_vector = 1; // Key pair of the local ElGamal cipher. Required. ElGamalKeyPair local_el_gamal_key_pair = 2; - // Public Key of the composite ElGamal cipher. Used to re-randomize the keys - // and counts. - ElGamalPublicKey composite_el_gamal_public_key = 3; // The elliptical curve to work on. - int64 curve_id = 4; - // The excessive noise that will be removed. The noise was computed during the - // Setup phase, and stored in the database. - int64 excessive_noise = 5; - // The Paillier public key. - PaillierPublicKey paillier_public_key = 6; - // The serialized Paillier ciphertext that encrypts the aggregated excessive + int64 curve_id = 3; + // The serialized El Gamal ciphertext that encrypts the aggregated excessive // noise. - bytes serialized_aggregated_noise_ciphertext = 7; + bytes serialized_excessive_noise_ciphertext = 4; // The maximum number of threads used by crypto actions. - int32 parallelism = 8; + int32 parallelism = 5; } // Response of the CompleteReachOnlyExecution method. @@ -124,9 +106,9 @@ message CompleteReachOnlyExecutionPhaseResponse { // bytes ElGamal ciphertext. In other words, the CRV size should be divisible // by 66. bytes combined_register_vector = 1; - // The serialized Paillier ciphertext that encrypts the aggregated excessive + // The serialized El Gamal ciphertext that encrypts the aggregated excessive // noise. - bytes serialized_aggregated_noise_ciphertext = 2; + bytes serialized_excessive_noise_ciphertext = 2; // The CPU time of processing the request. int64 elapsed_cpu_time_millis = 3; } @@ -142,29 +124,33 @@ message CompleteReachOnlyExecutionPhaseAtAggregatorRequest { bytes combined_register_vector = 1; // Key pair of the local ElGamal cipher. Required. ElGamalKeyPair local_el_gamal_key_pair = 2; - // Public Key of the composite ElGamal cipher. Used to encrypt the random - // numbers in SameKeyAggregation. - ElGamalPublicKey composite_el_gamal_public_key = 3; // The elliptical curve to work on. - int64 curve_id = 4; - // The Paillier private key to decrypt the aggregated noise ciphertext. - PaillierPrivateKey paillier_private_key = 5; - // The serialized Paillier ciphertext that encrypts the aggregated excessive + int64 curve_id = 3; + // The serialized El Gamal ciphertext that encrypts the aggregated excessive // noise. - bytes serialized_aggregated_noise_ciphertext = 6; + bytes serialized_excessive_noise_ciphertext = 4; + // Parameters for computing the noise baseline of the global reach DP noise + // registers added in the setup phase. + // The baseline is subtracted before reach is estimated. + GlobalReachDpNoiseBaseline reach_dp_noise_baseline = 5; // LiquidLegions parameters used for reach estimation. - LiquidLegionsSketchParameters liquid_legions_parameters = 7; + LiquidLegionsSketchParameters liquid_legions_parameters = 6; // The sampling rate to be used by the LiquidLegionsV2 protocol. // This is taken from the VidSamplingInterval.width parameter in the // MeasurementSpec. - float vid_sampling_interval_width = 8; + float vid_sampling_interval_width = 7; + // The parameters required for generating noise registers. + // if unset, the worker only shuffles the register without adding any noise. + RegisterNoiseGenerationParameters noise_parameters = 8; + // The mechanism used to generate noise in previous phases. + LiquidLegionsV2NoiseConfig.NoiseMechanism noise_mechanism = 9; // The maximum number of threads used by crypto actions. - int32 parallelism = 9; + int32 parallelism = 10; } // The response of the CompleteReachOnlyExecutionAtAggregator method. message CompleteReachOnlyExecutionPhaseAtAggregatorResponse { - // The number of register count. + // The estimated reach. int64 reach = 1; // The CPU time of processing the request. int64 elapsed_cpu_time_millis = 2; diff --git a/src/test/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/BUILD.bazel b/src/test/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/BUILD.bazel index c1b7631349a..27c843d8678 100644 --- a/src/test/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/BUILD.bazel +++ b/src/test/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/BUILD.bazel @@ -9,6 +9,27 @@ cc_test( ], deps = [ "//src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2:liquid_legions_v2_encryption_utility", + "//src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/testing:liquid_legions_v2_encryption_utility_helper", + "//src/main/proto/wfa/measurement/internal/duchy/protocol:liquid_legions_v2_noise_config_cc_proto", + "@any_sketch//src/main/cc/any_sketch/crypto:sketch_encrypter", + "@any_sketch//src/main/cc/estimation:estimators", + "@any_sketch//src/main/proto/wfa/any_sketch:sketch_cc_proto", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + "@wfa_common_cpp//src/main/cc/common_cpp/testing:status", + ], +) + +cc_test( + name = "reach_only_liquid_legions_v2_encryption_utility_test", + size = "small", + timeout = "moderate", + srcs = [ + "reach_only_liquid_legions_v2_encryption_utility_test.cc", + ], + deps = [ + "//src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2:reach_only_liquid_legions_v2_encryption_utility", + "//src/main/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/testing:liquid_legions_v2_encryption_utility_helper", "//src/main/proto/wfa/measurement/internal/duchy/protocol:liquid_legions_v2_noise_config_cc_proto", "@any_sketch//src/main/cc/any_sketch/crypto:sketch_encrypter", "@any_sketch//src/main/cc/estimation:estimators", diff --git a/src/test/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/liquid_legions_v2_encryption_utility_test.cc b/src/test/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/liquid_legions_v2_encryption_utility_test.cc index d3c9696f8ef..909c98ef13f 100644 --- a/src/test/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/liquid_legions_v2_encryption_utility_test.cc +++ b/src/test/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/liquid_legions_v2_encryption_utility_test.cc @@ -30,6 +30,7 @@ #include "wfa/measurement/common/crypto/constants.h" #include "wfa/measurement/common/crypto/ec_point_util.h" #include "wfa/measurement/common/crypto/encryption_utility_helper.h" +#include "wfa/measurement/internal/duchy/protocol/liquid_legions_v2/testing/liquid_legions_v2_encryption_utility_helper.h" #include "wfa/measurement/internal/duchy/protocol/liquid_legions_v2_encryption_methods.pb.h" namespace wfa::measurement::internal::duchy::protocol::liquid_legions_v2 { @@ -86,39 +87,6 @@ void AddRegister(Sketch* sketch, const int index, const int key, register_ptr->add_values(count); } -::wfa::any_sketch::crypto::ElGamalPublicKey ToAnysketchElGamalKey( - ElGamalPublicKey key) { - ::wfa::any_sketch::crypto::ElGamalPublicKey result; - result.set_generator(key.generator()); - result.set_element(key.element()); - return result; -} - -ElGamalPublicKey ToCmmsElGamalKey( - ::wfa::any_sketch::crypto::ElGamalPublicKey key) { - ElGamalPublicKey result; - result.set_generator(key.generator()); - result.set_element(key.element()); - return result; -} - -Sketch CreateEmptyLiquidLegionsSketch() { - Sketch plain_sketch; - plain_sketch.mutable_config()->add_values()->set_aggregator( - SketchConfig::ValueSpec::UNIQUE); - plain_sketch.mutable_config()->add_values()->set_aggregator( - SketchConfig::ValueSpec::SUM); - return plain_sketch; -} - -DifferentialPrivacyParams MakeDifferentialPrivacyParams(double epsilon, - double delta) { - DifferentialPrivacyParams params; - params.set_epsilon(epsilon); - params.set_delta(delta); - return params; -} - // Partition the char vector 33 by 33, and convert the results to strings std::vector GetCipherStrings(absl::string_view bytes) { ABSL_ASSERT(bytes.size() % 66 == 0); @@ -290,18 +258,18 @@ class TestData { // Combine the el_gamal keys from all duchies to generate the data provider // el_gamal key. - client_el_gamal_public_key_ = ToCmmsElGamalKey( + client_el_gamal_public_key_ = ToDuchyInternalElGamalKey( any_sketch::crypto::CombineElGamalPublicKeys( kTestCurveId, - {ToAnysketchElGamalKey(duchy_1_el_gamal_key_pair_.public_key()), - ToAnysketchElGamalKey(duchy_2_el_gamal_key_pair_.public_key()), - ToAnysketchElGamalKey(duchy_3_el_gamal_key_pair_.public_key())}) + {ToAnySketchElGamalKey(duchy_1_el_gamal_key_pair_.public_key()), + ToAnySketchElGamalKey(duchy_2_el_gamal_key_pair_.public_key()), + ToAnySketchElGamalKey(duchy_3_el_gamal_key_pair_.public_key())}) .value()); - duchy_2_3_composite_public_key_ = ToCmmsElGamalKey( + duchy_2_3_composite_public_key_ = ToDuchyInternalElGamalKey( any_sketch::crypto::CombineElGamalPublicKeys( kTestCurveId, - {ToAnysketchElGamalKey(duchy_2_el_gamal_key_pair_.public_key()), - ToAnysketchElGamalKey(duchy_3_el_gamal_key_pair_.public_key())}) + {ToAnySketchElGamalKey(duchy_2_el_gamal_key_pair_.public_key()), + ToAnySketchElGamalKey(duchy_3_el_gamal_key_pair_.public_key())}) .value()); any_sketch::crypto::CiphertextString client_public_key = { diff --git a/src/test/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/reach_only_liquid_legions_v2_encryption_utility_test.cc b/src/test/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/reach_only_liquid_legions_v2_encryption_utility_test.cc new file mode 100644 index 00000000000..575e5285cae --- /dev/null +++ b/src/test/cc/wfa/measurement/internal/duchy/protocol/liquid_legions_v2/reach_only_liquid_legions_v2_encryption_utility_test.cc @@ -0,0 +1,664 @@ +// Copyright 2023 The Cross-Media Measurement Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "wfa/measurement/internal/duchy/protocol/liquid_legions_v2/reach_only_liquid_legions_v2_encryption_utility.h" + +#include "absl/container/flat_hash_map.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "any_sketch/crypto/sketch_encrypter.h" +#include "common_cpp/testing/status_macros.h" +#include "common_cpp/testing/status_matchers.h" +#include "estimation/estimators.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "openssl/obj_mac.h" +#include "private_join_and_compute/crypto/commutative_elgamal.h" +#include "private_join_and_compute/crypto/ec_commutative_cipher.h" +#include "wfa/any_sketch/sketch.pb.h" +#include "wfa/measurement/common/crypto/constants.h" +#include "wfa/measurement/common/crypto/ec_point_util.h" +#include "wfa/measurement/common/crypto/encryption_utility_helper.h" +#include "wfa/measurement/internal/duchy/protocol/liquid_legions_v2/testing/liquid_legions_v2_encryption_utility_helper.h" +#include "wfa/measurement/internal/duchy/protocol/liquid_legions_v2_encryption_methods.pb.h" + +namespace wfa::measurement::internal::duchy::protocol::liquid_legions_v2 { +namespace { + +using ::private_join_and_compute::BigNum; +using ::private_join_and_compute::CommutativeElGamal; +using ::private_join_and_compute::Context; +using ::private_join_and_compute::ECCommutativeCipher; +using ::private_join_and_compute::ECGroup; +using ::private_join_and_compute::ECPoint; +using ::testing::DoubleNear; +using ::testing::Pair; +using ::testing::SizeIs; +using ::testing::UnorderedElementsAre; +using ::wfa::any_sketch::Sketch; +using ::wfa::any_sketch::SketchConfig; +using ::wfa::measurement::common::crypto::ElGamalCiphertext; +using ::wfa::measurement::common::crypto::ExtractElGamalCiphertextFromString; +using ::wfa::measurement::common::crypto::GetCountValuesPlaintext; +using ::wfa::measurement::common::crypto::kBlindedHistogramNoiseRegisterKey; +using ::wfa::measurement::common::crypto::kBytesPerCipherText; +using ::wfa::measurement::common::crypto::kDestroyedRegisterKey; +using ::wfa::measurement::common::crypto::kPaddingNoiseRegisterId; +using ::wfa::measurement::common::crypto::kPublisherNoiseRegisterId; +using ::wfa::measurement::internal::duchy::DifferentialPrivacyParams; +using ::wfa::measurement::internal::duchy::ElGamalKeyPair; +using ::wfa::measurement::internal::duchy::ElGamalPublicKey; +using ::wfa::measurement::internal::duchy::protocol::LiquidLegionsV2NoiseConfig; + +constexpr int kWorkerCount = 3; +constexpr int kPublisherCount = 3; +constexpr int kMaxFrequency = 10; +constexpr int kTestCurveId = NID_X9_62_prime256v1; +constexpr int kParallelism = 3; +constexpr int kBytesPerEcPoint = 33; +constexpr int kBytesCipherText = kBytesPerEcPoint * 2; +constexpr int kDecayRate = 12; +constexpr int kLiquidLegionsSize = 100 * 1000; +constexpr float kVidSamplingIntervalWidth = 0.5; + +struct ReachOnlyMpcResult { + int64_t reach; +}; + +void AddRegister(Sketch* sketch, const int index) { + auto register_ptr = sketch->add_registers(); + register_ptr->set_index(index); +} + +MATCHER_P(IsBlockSorted, block_size, "") { + if (arg.length() % block_size != 0) { + return false; + } + for (size_t i = block_size; i < arg.length(); i += block_size) { + if (arg.substr(i, block_size) < arg.substr(i - block_size, block_size)) { + return false; + } + } + return true; +} + +// The ReachOnlyTest generates cipher keys for 3 duchies, and the combined +// public key for the data providers. The duchy 1 and 2 are non-aggregator +// workers, while the duchy 3 is the aggregator. +class ReachOnlyTest { + public: + ElGamalKeyPair duchy_1_el_gamal_key_pair_; + std::string duchy_1_p_h_key_; + ElGamalKeyPair duchy_2_el_gamal_key_pair_; + std::string duchy_2_p_h_key_; + ElGamalKeyPair duchy_3_el_gamal_key_pair_; + std::string duchy_3_p_h_key_; + ElGamalPublicKey client_el_gamal_public_key_; // combined from 3 duchy keys; + ElGamalPublicKey duchy_2_3_composite_public_key_; // combined from duchy 2 + // and duchy_3 public keys; + std::unique_ptr sketch_encrypter_; + + ReachOnlyTest() { + CompleteReachOnlyInitializationPhaseRequest + complete_reach_only_initialization_phase_request; + complete_reach_only_initialization_phase_request.set_curve_id(kTestCurveId); + + duchy_1_el_gamal_key_pair_ = + CompleteReachOnlyInitializationPhase( + complete_reach_only_initialization_phase_request) + ->el_gamal_key_pair(); + duchy_2_el_gamal_key_pair_ = + CompleteReachOnlyInitializationPhase( + complete_reach_only_initialization_phase_request) + ->el_gamal_key_pair(); + duchy_3_el_gamal_key_pair_ = + CompleteReachOnlyInitializationPhase( + complete_reach_only_initialization_phase_request) + ->el_gamal_key_pair(); + + // Combine the el_gamal keys from all duchies to generate the data provider + // el_gamal key. + client_el_gamal_public_key_ = ToDuchyInternalElGamalKey( + any_sketch::crypto::CombineElGamalPublicKeys( + kTestCurveId, + {ToAnySketchElGamalKey(duchy_1_el_gamal_key_pair_.public_key()), + ToAnySketchElGamalKey(duchy_2_el_gamal_key_pair_.public_key()), + ToAnySketchElGamalKey(duchy_3_el_gamal_key_pair_.public_key())}) + .value()); + duchy_2_3_composite_public_key_ = ToDuchyInternalElGamalKey( + any_sketch::crypto::CombineElGamalPublicKeys( + kTestCurveId, + {ToAnySketchElGamalKey(duchy_2_el_gamal_key_pair_.public_key()), + ToAnySketchElGamalKey(duchy_3_el_gamal_key_pair_.public_key())}) + .value()); + + any_sketch::crypto::CiphertextString client_public_key = { + .u = client_el_gamal_public_key_.generator(), + .e = client_el_gamal_public_key_.element(), + }; + + // Create a sketch_encrypter for encrypting plaintext any_sketch data. + sketch_encrypter_ = any_sketch::crypto::CreateWithPublicKey( + kTestCurveId, kMaxFrequency, client_public_key) + .value(); + } + + absl::StatusOr EncryptWithFlaggedKey(const Sketch& sketch) { + return sketch_encrypter_->Encrypt( + sketch, any_sketch::crypto::EncryptSketchRequest::FLAGGED_KEY); + } + + // Helper function to go through the entire MPC protocol using the input data. + // The final ReachOnlyMpcResult are returned. + absl::StatusOr GoThroughEntireMpcProtocol( + const std::string& encrypted_sketch, + RegisterNoiseGenerationParameters* reach_noise_parameters, + LiquidLegionsV2NoiseConfig::NoiseMechanism noise_mechanism) { + // Setup phase at Duchy 1. + // We assume all test data comes from duchy 1 in the test. + CompleteReachOnlySetupPhaseRequest + complete_reach_only_setup_phase_request_1; + complete_reach_only_setup_phase_request_1.set_combined_register_vector( + encrypted_sketch); + + if (reach_noise_parameters != nullptr) { + *complete_reach_only_setup_phase_request_1.mutable_noise_parameters() = + *reach_noise_parameters; + } + complete_reach_only_setup_phase_request_1.set_noise_mechanism( + noise_mechanism); + complete_reach_only_setup_phase_request_1.set_curve_id(kTestCurveId); + *complete_reach_only_setup_phase_request_1 + .mutable_composite_el_gamal_public_key() = client_el_gamal_public_key_; + complete_reach_only_setup_phase_request_1.set_parallelism(kParallelism); + + ASSIGN_OR_RETURN( + CompleteReachOnlySetupPhaseResponse + complete_reach_only_setup_phase_response_1, + CompleteReachOnlySetupPhase(complete_reach_only_setup_phase_request_1)); + EXPECT_THAT( + complete_reach_only_setup_phase_response_1.combined_register_vector(), + IsBlockSorted(kBytesCipherText)); + EXPECT_THAT(complete_reach_only_setup_phase_response_1 + .serialized_excessive_noise_ciphertext(), + SizeIs(kBytesCipherText)); + + // Setup phase at Duchy 2. + // We assume all test data comes from duchy 1 in the test, so there is only + // noise from duchy 2 (if configured) + CompleteReachOnlySetupPhaseRequest + complete_reach_only_setup_phase_request_2; + if (reach_noise_parameters != nullptr) { + *complete_reach_only_setup_phase_request_2.mutable_noise_parameters() = + *reach_noise_parameters; + } + complete_reach_only_setup_phase_request_2.set_noise_mechanism( + noise_mechanism); + complete_reach_only_setup_phase_request_2.set_curve_id(kTestCurveId); + *complete_reach_only_setup_phase_request_2 + .mutable_composite_el_gamal_public_key() = client_el_gamal_public_key_; + complete_reach_only_setup_phase_request_2.set_parallelism(kParallelism); + + ASSIGN_OR_RETURN( + CompleteReachOnlySetupPhaseResponse + complete_reach_only_setup_phase_response_2, + CompleteReachOnlySetupPhase(complete_reach_only_setup_phase_request_2)); + EXPECT_THAT( + complete_reach_only_setup_phase_response_2.combined_register_vector(), + IsBlockSorted(kBytesCipherText)); + EXPECT_THAT(complete_reach_only_setup_phase_response_2 + .serialized_excessive_noise_ciphertext(), + SizeIs(kBytesCipherText)); + + // Setup phase at Duchy 3. + // We assume all test data comes from duchy 1 in the test, so there is only + // noise from duchy 3 (if configured) + CompleteReachOnlySetupPhaseRequest + complete_reach_only_setup_phase_request_3; + if (reach_noise_parameters != nullptr) { + *complete_reach_only_setup_phase_request_3.mutable_noise_parameters() = + *reach_noise_parameters; + } + complete_reach_only_setup_phase_request_3.set_curve_id(kTestCurveId); + complete_reach_only_setup_phase_request_3.set_noise_mechanism( + noise_mechanism); + *complete_reach_only_setup_phase_request_3 + .mutable_composite_el_gamal_public_key() = client_el_gamal_public_key_; + complete_reach_only_setup_phase_request_3.set_parallelism(kParallelism); + + std::string serialized_excessive_noise_ciphertext = + absl::StrCat(complete_reach_only_setup_phase_response_1 + .serialized_excessive_noise_ciphertext(), + complete_reach_only_setup_phase_response_2 + .serialized_excessive_noise_ciphertext()); + complete_reach_only_setup_phase_request_3 + .set_serialized_excessive_noise_ciphertext( + serialized_excessive_noise_ciphertext); + + // Combine all CRVs from the workers. + std::string combine_data = absl::StrCat( + complete_reach_only_setup_phase_response_1.combined_register_vector(), + complete_reach_only_setup_phase_response_2.combined_register_vector()); + complete_reach_only_setup_phase_request_3.set_combined_register_vector( + combine_data); + + ASSIGN_OR_RETURN(CompleteReachOnlySetupPhaseResponse + complete_reach_only_setup_phase_response_3, + CompleteReachOnlySetupPhaseAtAggregator( + complete_reach_only_setup_phase_request_3)); + EXPECT_THAT( + complete_reach_only_setup_phase_response_3.combined_register_vector(), + IsBlockSorted(kBytesCipherText)); + EXPECT_THAT(complete_reach_only_setup_phase_response_3 + .serialized_excessive_noise_ciphertext(), + SizeIs(kBytesCipherText)); + + // Execution phase at duchy 1 (non-aggregator). + CompleteReachOnlyExecutionPhaseRequest + complete_reach_only_execution_phase_request_1; + *complete_reach_only_execution_phase_request_1 + .mutable_local_el_gamal_key_pair() = duchy_1_el_gamal_key_pair_; + complete_reach_only_execution_phase_request_1.set_curve_id(kTestCurveId); + *complete_reach_only_execution_phase_request_1 + .mutable_serialized_excessive_noise_ciphertext() = + complete_reach_only_setup_phase_response_3 + .serialized_excessive_noise_ciphertext(); + complete_reach_only_execution_phase_request_1.set_parallelism(kParallelism); + complete_reach_only_execution_phase_request_1.set_combined_register_vector( + complete_reach_only_setup_phase_response_3.combined_register_vector()); + complete_reach_only_execution_phase_request_1 + .set_serialized_excessive_noise_ciphertext( + complete_reach_only_setup_phase_response_3 + .serialized_excessive_noise_ciphertext()); + ASSIGN_OR_RETURN(CompleteReachOnlyExecutionPhaseResponse + complete_reach_only_execution_phase_response_1, + CompleteReachOnlyExecutionPhase( + complete_reach_only_execution_phase_request_1)); + EXPECT_THAT(complete_reach_only_execution_phase_response_1 + .combined_register_vector(), + IsBlockSorted(kBytesCipherText)); + + // Execution phase at duchy 2 (non-aggregator). + CompleteReachOnlyExecutionPhaseRequest + complete_reach_only_execution_phase_request_2; + *complete_reach_only_execution_phase_request_2 + .mutable_local_el_gamal_key_pair() = duchy_2_el_gamal_key_pair_; + complete_reach_only_execution_phase_request_2.set_curve_id(kTestCurveId); + *complete_reach_only_execution_phase_request_2 + .mutable_serialized_excessive_noise_ciphertext() = + complete_reach_only_execution_phase_response_1 + .serialized_excessive_noise_ciphertext(); + complete_reach_only_execution_phase_request_2.set_parallelism(kParallelism); + complete_reach_only_execution_phase_request_2.set_combined_register_vector( + complete_reach_only_execution_phase_response_1 + .combined_register_vector()); + complete_reach_only_execution_phase_request_2 + .set_serialized_excessive_noise_ciphertext( + complete_reach_only_execution_phase_response_1 + .serialized_excessive_noise_ciphertext()); + ASSIGN_OR_RETURN(CompleteReachOnlyExecutionPhaseResponse + complete_execution_phase_one_response_2, + CompleteReachOnlyExecutionPhase( + complete_reach_only_execution_phase_request_2)); + EXPECT_THAT( + complete_execution_phase_one_response_2.combined_register_vector(), + IsBlockSorted(kBytesCipherText)); + + // Execution phase at duchy 3 (aggregator). + CompleteReachOnlyExecutionPhaseAtAggregatorRequest + complete_reach_only_execution_phase_at_aggregator_request; + complete_reach_only_execution_phase_at_aggregator_request + .set_combined_register_vector( + complete_execution_phase_one_response_2.combined_register_vector()); + *complete_reach_only_execution_phase_at_aggregator_request + .mutable_local_el_gamal_key_pair() = duchy_3_el_gamal_key_pair_; + complete_reach_only_execution_phase_at_aggregator_request.set_curve_id( + kTestCurveId); + complete_reach_only_execution_phase_at_aggregator_request.set_parallelism( + kParallelism); + *complete_reach_only_execution_phase_at_aggregator_request + .mutable_serialized_excessive_noise_ciphertext() = + complete_execution_phase_one_response_2 + .serialized_excessive_noise_ciphertext(); + if (reach_noise_parameters != nullptr) { + complete_reach_only_execution_phase_at_aggregator_request + .mutable_reach_dp_noise_baseline() + ->set_contributors_count(3); + *complete_reach_only_execution_phase_at_aggregator_request + .mutable_reach_dp_noise_baseline() + ->mutable_global_reach_dp_noise() = + reach_noise_parameters->dp_params().global_reach_dp_noise(); + *complete_reach_only_execution_phase_at_aggregator_request + .mutable_noise_parameters() = *reach_noise_parameters; + } + complete_reach_only_execution_phase_at_aggregator_request + .mutable_liquid_legions_parameters() + ->set_decay_rate(kDecayRate); + complete_reach_only_execution_phase_at_aggregator_request + .mutable_liquid_legions_parameters() + ->set_size(kLiquidLegionsSize); + complete_reach_only_execution_phase_at_aggregator_request + .set_vid_sampling_interval_width(kVidSamplingIntervalWidth); + complete_reach_only_execution_phase_at_aggregator_request + .set_noise_mechanism(noise_mechanism); + complete_reach_only_execution_phase_at_aggregator_request + .set_serialized_excessive_noise_ciphertext( + complete_execution_phase_one_response_2 + .serialized_excessive_noise_ciphertext()); + ASSIGN_OR_RETURN( + CompleteReachOnlyExecutionPhaseAtAggregatorResponse + complete_reach_only_execution_phase_at_aggregator_response, + CompleteReachOnlyExecutionPhaseAtAggregator( + complete_reach_only_execution_phase_at_aggregator_request)); + + ReachOnlyMpcResult result; + result.reach = + complete_reach_only_execution_phase_at_aggregator_response.reach(); + return result; + } +}; + +TEST(CompleteReachOnlySetupPhase, WrongInputSketchSizeShouldThrow) { + ReachOnlyTest test_data; + CompleteReachOnlySetupPhaseRequest request; + request.set_curve_id(kTestCurveId); + *request.mutable_composite_el_gamal_public_key() = + test_data.client_el_gamal_public_key_; + request.set_combined_register_vector("1234"); + request.set_parallelism(kParallelism); + + auto result = CompleteReachOnlySetupPhase(request); + ASSERT_FALSE(result.ok()); + EXPECT_THAT(result.status(), + StatusIs(absl::StatusCode::kInvalidArgument, "not divisible")); +} + +TEST(CompleteReachOnlySetupPhase, SetupPhaseWorksAsExpectedWithoutNoise) { + ReachOnlyTest test_data; + CompleteReachOnlySetupPhaseRequest request; + request.set_curve_id(kTestCurveId); + *request.mutable_composite_el_gamal_public_key() = + test_data.client_el_gamal_public_key_; + request.set_parallelism(kParallelism); + + std::string register1 = "abc"; + std::string register2 = "def"; + for (int i = 3; i < kBytesCipherText; i++) { + register1 = register1 + " "; + register2 = register2 + " "; + } + std::string registers = register1 + register2; + + request.set_combined_register_vector(registers); + + auto result = CompleteReachOnlySetupPhase(request); + ASSERT_TRUE(result.ok()); + + std::string response_crv = result->combined_register_vector(); + EXPECT_EQ(registers, response_crv); + EXPECT_EQ(registers.length(), response_crv.length()); + EXPECT_EQ("abc", response_crv.substr(0, 3)); + EXPECT_EQ("def", response_crv.substr(kBytesCipherText, 3)); +} + +TEST(CompleteReachOnlySetupPhase, SetupPhaseWorksAsExpectedWithGeometricNoise) { + Context ctx; + ASSERT_OK_AND_ASSIGN(ECGroup ec_group, ECGroup::Create(kTestCurveId, &ctx)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr el_gamal_cipher, + CommutativeElGamal::CreateWithNewKeyPair(kTestCurveId)); + ASSERT_OK_AND_ASSIGN(auto public_key_pair, + el_gamal_cipher->GetPublicKeyBytes()); + ElGamalPublicKey public_key; + public_key.set_generator(public_key_pair.first); + public_key.set_element(public_key_pair.second); + + int64_t computed_blinded_histogram_noise_offset = 7; + int64_t computed_publisher_noise_offset = 7; + int64_t computed_reach_dp_noise_offset = 4; + int64_t expected_total_register_count = + computed_publisher_noise_offset * 2 + computed_reach_dp_noise_offset * 2 + + computed_blinded_histogram_noise_offset * kPublisherCount * + (kPublisherCount + 1) + + 2; + + CompleteReachOnlySetupPhaseRequest request; + request.set_curve_id(kTestCurveId); + RegisterNoiseGenerationParameters* noise_parameters = + request.mutable_noise_parameters(); + noise_parameters->set_curve_id(kTestCurveId); + noise_parameters->set_total_sketches_count(kPublisherCount); + noise_parameters->set_contributors_count(kWorkerCount); + *noise_parameters->mutable_composite_el_gamal_public_key() = public_key; + // resulted p ~= 0 , offset = 7 + *noise_parameters->mutable_dp_params()->mutable_blind_histogram() = + MakeDifferentialPrivacyParams(40, std::exp(-80)); + // resulted p ~= 0 , offset = 7 + *noise_parameters->mutable_dp_params()->mutable_noise_for_publisher_noise() = + MakeDifferentialPrivacyParams(40, std::exp(-40)); + // resulted p ~= 0 , offset = 4 + *noise_parameters->mutable_dp_params()->mutable_global_reach_dp_noise() = + MakeDifferentialPrivacyParams(40, std::exp(-80)); + request.set_noise_mechanism(LiquidLegionsV2NoiseConfig::GEOMETRIC); + request.set_parallelism(kParallelism); + + ASSERT_OK_AND_ASSIGN(CompleteReachOnlySetupPhaseResponse response, + CompleteReachOnlySetupPhase(request)); + + // There was no data in the request, so all registers in the response are + // noise. + std::string noises = response.combined_register_vector(); + ASSERT_THAT(noises, + SizeIs(expected_total_register_count * kBytesPerCipherText)); +} + +TEST(CompleteReachOnlySetupPhase, SetupPhaseWorksAsExpectedWithGaussianNoise) { + Context ctx; + ASSERT_OK_AND_ASSIGN(ECGroup ec_group, ECGroup::Create(kTestCurveId, &ctx)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr el_gamal_cipher, + CommutativeElGamal::CreateWithNewKeyPair(kTestCurveId)); + ASSERT_OK_AND_ASSIGN(auto public_key_pair, + el_gamal_cipher->GetPublicKeyBytes()); + ElGamalPublicKey public_key; + public_key.set_generator(public_key_pair.first); + public_key.set_element(public_key_pair.second); + + int64_t computed_blinded_histogram_noise_offset = 3; + int64_t computed_publisher_noise_offset = 2; + int64_t computed_reach_dp_noise_offset = 3; + int64_t expected_total_register_count = + computed_publisher_noise_offset * 2 + computed_reach_dp_noise_offset * 2 + + computed_blinded_histogram_noise_offset * kPublisherCount * + (kPublisherCount + 1) + + 2; + + CompleteReachOnlySetupPhaseRequest request; + request.set_curve_id(kTestCurveId); + RegisterNoiseGenerationParameters* noise_parameters = + request.mutable_noise_parameters(); + noise_parameters->set_curve_id(kTestCurveId); + noise_parameters->set_total_sketches_count(kPublisherCount); + noise_parameters->set_contributors_count(kWorkerCount); + *noise_parameters->mutable_composite_el_gamal_public_key() = public_key; + // resulted sigma_distributed ~= 0.18, offset = 3 + *noise_parameters->mutable_dp_params()->mutable_blind_histogram() = + MakeDifferentialPrivacyParams(40, std::exp(-80)); + // resulted sigma_distributed ~= 0.13, offset = 2 + *noise_parameters->mutable_dp_params()->mutable_noise_for_publisher_noise() = + MakeDifferentialPrivacyParams(40, std::exp(-40)); + // resulted sigma_distributed ~= 0.18, offset = 3 + *noise_parameters->mutable_dp_params()->mutable_global_reach_dp_noise() = + MakeDifferentialPrivacyParams(40, std::exp(-80)); + request.set_noise_mechanism(LiquidLegionsV2NoiseConfig::DISCRETE_GAUSSIAN); + request.set_parallelism(kParallelism); + + ASSERT_OK_AND_ASSIGN(CompleteReachOnlySetupPhaseResponse response, + CompleteReachOnlySetupPhase(request)); + // There was no data in the request, so all registers in the response are + // noise. + std::string noises = response.combined_register_vector(); + ASSERT_THAT(noises, + SizeIs(expected_total_register_count * kBytesPerCipherText)); +} + +TEST(CompleteReachOnlyExecutionPhase, WrongInputSketchSizeShouldThrow) { + CompleteReachOnlyExecutionPhaseRequest request; + request.set_combined_register_vector("1234"); + request.set_parallelism(kParallelism); + + auto result = CompleteReachOnlyExecutionPhase(request); + ASSERT_FALSE(result.ok()); + EXPECT_THAT(result.status(), + StatusIs(absl::StatusCode::kInvalidArgument, "not divisible")); +} + +TEST(CompleteReachOnlyExecutionPhaseAtAggregator, + WrongInputSketchSizeShouldThrow) { + CompleteReachOnlyExecutionPhaseAtAggregatorRequest request; + request.set_curve_id(kTestCurveId); + request.set_combined_register_vector("1234"); + request.set_parallelism(kParallelism); + + auto result = CompleteReachOnlyExecutionPhaseAtAggregator(request); + ASSERT_FALSE(result.ok()); + EXPECT_THAT(result.status(), + StatusIs(absl::StatusCode::kInvalidArgument, "not divisible")); +} + +TEST(EndToEnd, SumOfCountsShouldBeCorrectWithoutNoise) { + ReachOnlyTest test_data; + Sketch plain_sketch = CreateReachOnlyEmptyLiquidLegionsSketch(); + int num_registers = 100; + for (int i = 1; i <= num_registers; i++) { + AddRegister(&plain_sketch, /*index=*/i); + } + + std::string encrypted_sketch = + test_data.EncryptWithFlaggedKey(plain_sketch).value(); + int64_t expected_reach = wfa::estimation::EstimateCardinalityLiquidLegions( + kDecayRate, kLiquidLegionsSize, num_registers, kVidSamplingIntervalWidth); + + ASSERT_OK_AND_ASSIGN(ReachOnlyMpcResult result_with_geometric_noise, + test_data.GoThroughEntireMpcProtocol( + encrypted_sketch, /*reach_noise=*/nullptr, + LiquidLegionsV2NoiseConfig::GEOMETRIC)); + + EXPECT_EQ(result_with_geometric_noise.reach, expected_reach); + + ASSERT_OK_AND_ASSIGN(ReachOnlyMpcResult result_with_gaussian_noise, + test_data.GoThroughEntireMpcProtocol( + encrypted_sketch, /*reach_noise=*/nullptr, + LiquidLegionsV2NoiseConfig::DISCRETE_GAUSSIAN)); + EXPECT_EQ(result_with_gaussian_noise.reach, expected_reach); +} + +TEST(EndToEnd, CombinedCasesWithDeterministicReachDpNoises) { + ReachOnlyTest test_data; + Sketch plain_sketch = CreateReachOnlyEmptyLiquidLegionsSketch(); + int valid_register_count = 30; + for (int i = 1; i <= valid_register_count; i++) { + AddRegister(&plain_sketch, /*index=*/i); + } + + std::string encrypted_sketch = + test_data.EncryptWithFlaggedKey(plain_sketch).value(); + + RegisterNoiseGenerationParameters reach_noise_parameters; + reach_noise_parameters.set_curve_id(kTestCurveId); + reach_noise_parameters.set_total_sketches_count(3); + reach_noise_parameters.set_contributors_count(kWorkerCount); + // For geometric noise, resulted p = 0.716531, offset = 15. + // Random blind histogram noise. + *reach_noise_parameters.mutable_dp_params()->mutable_blind_histogram() = + MakeDifferentialPrivacyParams(0.11, 0.11); + // For geometric noise, resulted p = 0.716531, offset = 10. + // Random noise for publisher noise. + *reach_noise_parameters.mutable_dp_params() + ->mutable_noise_for_publisher_noise() = + MakeDifferentialPrivacyParams(1, 1); + // For geometric noise, resulted p ~= 0 , offset = 3. + // Deterministic reach dp noise. + *reach_noise_parameters.mutable_dp_params()->mutable_global_reach_dp_noise() = + MakeDifferentialPrivacyParams(40, std::exp(-80)); + *reach_noise_parameters.mutable_composite_el_gamal_public_key() = + test_data.client_el_gamal_public_key_; + + int64_t expected_reach = wfa::estimation::EstimateCardinalityLiquidLegions( + kDecayRate, kLiquidLegionsSize, valid_register_count, + kVidSamplingIntervalWidth); + + ASSERT_OK_AND_ASSIGN(ReachOnlyMpcResult result_with_geometric_noise, + test_data.GoThroughEntireMpcProtocol( + encrypted_sketch, &reach_noise_parameters, + LiquidLegionsV2NoiseConfig::GEOMETRIC)); + + EXPECT_EQ(result_with_geometric_noise.reach, expected_reach); + + ASSERT_OK_AND_ASSIGN(ReachOnlyMpcResult result_with_gaussian_noise, + test_data.GoThroughEntireMpcProtocol( + encrypted_sketch, &reach_noise_parameters, + LiquidLegionsV2NoiseConfig::DISCRETE_GAUSSIAN)); + + EXPECT_EQ(result_with_gaussian_noise.reach, expected_reach); +} + +TEST(ReachEstimation, NonDpNoiseShouldNotImpactTheResult) { + ReachOnlyTest test_data; + Sketch plain_sketch = CreateReachOnlyEmptyLiquidLegionsSketch(); + int valid_register_count = 30; + for (int i = 1; i <= valid_register_count; ++i) { + AddRegister(&plain_sketch, /*index =*/i); + } + + RegisterNoiseGenerationParameters reach_noise_parameters; + reach_noise_parameters.set_curve_id(kTestCurveId); + reach_noise_parameters.set_total_sketches_count(kPublisherCount); + reach_noise_parameters.set_contributors_count(kWorkerCount); + // For geometric noise, resulted p = 0.716531, offset = 15. + // Random blind histogram noise. + *reach_noise_parameters.mutable_dp_params()->mutable_blind_histogram() = + MakeDifferentialPrivacyParams(0.1, 0.1); + // For geometric noise, resulted p = 0.716531, offset = 10. + // Random noise for publisher noise. + *reach_noise_parameters.mutable_dp_params() + ->mutable_noise_for_publisher_noise() = + MakeDifferentialPrivacyParams(1, 1); + // For geometric noise, resulted p ~= 0 , offset = 3. + // Deterministic reach dp noise. + *reach_noise_parameters.mutable_dp_params()->mutable_global_reach_dp_noise() = + MakeDifferentialPrivacyParams(40, std::exp(-80)); + *reach_noise_parameters.mutable_composite_el_gamal_public_key() = + test_data.client_el_gamal_public_key_; + + std::string encrypted_sketch = + test_data.EncryptWithFlaggedKey(plain_sketch).value(); + + int64_t expected_reach = wfa::estimation::EstimateCardinalityLiquidLegions( + kDecayRate, kLiquidLegionsSize, valid_register_count, + kVidSamplingIntervalWidth); + + ASSERT_OK_AND_ASSIGN(ReachOnlyMpcResult result_with_geometric_noise, + test_data.GoThroughEntireMpcProtocol( + encrypted_sketch, &reach_noise_parameters, + LiquidLegionsV2NoiseConfig::GEOMETRIC)); + EXPECT_EQ(result_with_geometric_noise.reach, expected_reach); + + ASSERT_OK_AND_ASSIGN(ReachOnlyMpcResult result_with_gaussian_noise, + test_data.GoThroughEntireMpcProtocol( + encrypted_sketch, &reach_noise_parameters, + LiquidLegionsV2NoiseConfig::DISCRETE_GAUSSIAN)); + EXPECT_EQ(result_with_gaussian_noise.reach, expected_reach); +} + +} // namespace +} // namespace wfa::measurement::internal::duchy::protocol::liquid_legions_v2 diff --git a/src/test/kotlin/org/wfanet/measurement/duchy/db/computation/LiquidLegionsSketchAggregationV2ProtocolEnumStagesTest.kt b/src/test/kotlin/org/wfanet/measurement/duchy/db/computation/LiquidLegionsSketchAggregationV2ProtocolEnumStagesTest.kt index 4e761a83b78..341f4606ab3 100644 --- a/src/test/kotlin/org/wfanet/measurement/duchy/db/computation/LiquidLegionsSketchAggregationV2ProtocolEnumStagesTest.kt +++ b/src/test/kotlin/org/wfanet/measurement/duchy/db/computation/LiquidLegionsSketchAggregationV2ProtocolEnumStagesTest.kt @@ -86,7 +86,7 @@ class LiquidLegionsSketchAggregationV2ProtocolEnumStagesTest { assertFalse { LiquidLegionsSketchAggregationV2Protocol.EnumStages.validTransition( - LiquidLegionsSketchAggregationV2.Stage.STAGE_UNKNOWN, + LiquidLegionsSketchAggregationV2.Stage.STAGE_UNSPECIFIED, LiquidLegionsSketchAggregationV2.Stage.CONFIRMATION_PHASE ) }