From d96b015639c6e7e1f9396d6e22918d2ed8e39f7e Mon Sep 17 00:00:00 2001 From: Ian Joiner <14581281+iajoiner@users.noreply.github.com> Date: Tue, 26 Nov 2024 10:19:52 -0500 Subject: [PATCH 1/3] feat: add `SliceExec` --- .../src/sql/proof_plans/dyn_proof_plan.rs | 7 +- .../src/sql/proof_plans/filter_exec.rs | 4 +- .../proof-of-sql/src/sql/proof_plans/mod.rs | 8 +- .../src/sql/proof_plans/slice_exec.rs | 206 +++++++++ .../src/sql/proof_plans/slice_exec_test.rs | 436 ++++++++++++++++++ .../src/sql/proof_plans/test_utility.rs | 6 +- 6 files changed, 662 insertions(+), 5 deletions(-) create mode 100644 crates/proof-of-sql/src/sql/proof_plans/slice_exec.rs create mode 100644 crates/proof-of-sql/src/sql/proof_plans/slice_exec_test.rs diff --git a/crates/proof-of-sql/src/sql/proof_plans/dyn_proof_plan.rs b/crates/proof-of-sql/src/sql/proof_plans/dyn_proof_plan.rs index e117ed8d3..dc988b9da 100644 --- a/crates/proof-of-sql/src/sql/proof_plans/dyn_proof_plan.rs +++ b/crates/proof-of-sql/src/sql/proof_plans/dyn_proof_plan.rs @@ -1,4 +1,4 @@ -use super::{EmptyExec, FilterExec, GroupByExec, ProjectionExec, TableExec}; +use super::{EmptyExec, FilterExec, GroupByExec, ProjectionExec, SliceExec, TableExec}; use crate::{ base::{ database::{ColumnField, ColumnRef, OwnedTable, Table, TableEvaluation, TableRef}, @@ -43,4 +43,9 @@ pub enum DynProofPlan { /// SELECT , ..., FROM WHERE /// ``` Filter(FilterExec), + /// `ProofPlan` for queries of the form + /// ```ignore + /// LIMIT [OFFSET ] + /// ``` + Slice(SliceExec), } diff --git a/crates/proof-of-sql/src/sql/proof_plans/filter_exec.rs b/crates/proof-of-sql/src/sql/proof_plans/filter_exec.rs index d0b51c578..b5654e42b 100644 --- a/crates/proof-of-sql/src/sql/proof_plans/filter_exec.rs +++ b/crates/proof-of-sql/src/sql/proof_plans/filter_exec.rs @@ -249,7 +249,7 @@ impl ProverEvaluate for FilterExec { } #[allow(clippy::unnecessary_wraps, clippy::too_many_arguments)] -fn verify_filter( +pub(crate) fn verify_filter( builder: &mut VerificationBuilder, alpha: S, beta: S, @@ -286,7 +286,7 @@ fn verify_filter( } #[allow(clippy::too_many_arguments, clippy::many_single_char_names)] -pub(super) fn prove_filter<'a, S: Scalar + 'a>( +pub(crate) fn prove_filter<'a, S: Scalar + 'a>( builder: &mut FinalRoundBuilder<'a, S>, alloc: &'a Bump, alpha: S, diff --git a/crates/proof-of-sql/src/sql/proof_plans/mod.rs b/crates/proof-of-sql/src/sql/proof_plans/mod.rs index 98ddea04e..9f23adb47 100644 --- a/crates/proof-of-sql/src/sql/proof_plans/mod.rs +++ b/crates/proof-of-sql/src/sql/proof_plans/mod.rs @@ -16,9 +16,9 @@ mod projection_exec_test; pub(crate) mod test_utility; mod filter_exec; -pub(crate) use filter_exec::FilterExec; #[cfg(test)] pub(crate) use filter_exec::OstensibleFilterExec; +pub(crate) use filter_exec::{prove_filter, verify_filter, FilterExec}; #[cfg(all(test, feature = "blitzar"))] mod filter_exec_test; #[cfg(all(test, feature = "blitzar"))] @@ -35,5 +35,11 @@ pub(crate) use group_by_exec::GroupByExec; #[cfg(all(test, feature = "blitzar"))] mod group_by_exec_test; +mod slice_exec; +#[allow(unused_imports)] +pub(crate) use slice_exec::SliceExec; +#[cfg(all(test, feature = "blitzar"))] +mod slice_exec_test; + mod dyn_proof_plan; pub use dyn_proof_plan::DynProofPlan; diff --git a/crates/proof-of-sql/src/sql/proof_plans/slice_exec.rs b/crates/proof-of-sql/src/sql/proof_plans/slice_exec.rs new file mode 100644 index 000000000..ada1a22d7 --- /dev/null +++ b/crates/proof-of-sql/src/sql/proof_plans/slice_exec.rs @@ -0,0 +1,206 @@ +use super::{prove_filter, verify_filter, DynProofPlan}; +use crate::{ + base::{ + database::{ + filter_util::filter_columns, ColumnField, ColumnRef, OwnedTable, Table, + TableEvaluation, TableOptions, TableRef, + }, + map::{IndexMap, IndexSet}, + proof::ProofError, + scalar::Scalar, + }, + sql::proof::{ + CountBuilder, FinalRoundBuilder, FirstRoundBuilder, ProofPlan, ProverEvaluate, + VerificationBuilder, + }, +}; +use alloc::{boxed::Box, vec, vec::Vec}; +use bumpalo::Bump; +use core::iter::repeat_with; +use serde::{Deserialize, Serialize}; + +/// `ProofPlan` for queries of the form +/// ```ignore +/// LIMIT [OFFSET ] +/// ``` +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub struct SliceExec { + pub(super) input: Box, + pub(super) skip: usize, + pub(super) fetch: Option, +} + +/// Get the boolean slice selection from the number of rows, skip and fetch +fn get_slice_select(num_rows: usize, skip: usize, fetch: Option) -> Vec { + if let Some(fetch) = fetch { + let end = skip + fetch; + (0..num_rows).map(|i| i >= skip && i < end).collect() + } else { + (0..num_rows).map(|i| i >= skip).collect() + } +} + +impl SliceExec { + /// Creates a new slice execution plan. + #[allow(dead_code)] + pub fn new(input: Box, skip: usize, fetch: Option) -> Self { + Self { input, skip, fetch } + } +} + +impl ProofPlan for SliceExec +where + SliceExec: ProverEvaluate, +{ + fn count(&self, builder: &mut CountBuilder) -> Result<(), ProofError> { + self.input.count(builder)?; + builder.count_intermediate_mles(self.input.get_column_result_fields().len()); + builder.count_intermediate_mles(2); + builder.count_subpolynomials(3); + builder.count_degree(3); + builder.count_post_result_challenges(2); + Ok(()) + } + + #[allow(unused_variables)] + fn verifier_evaluate( + &self, + builder: &mut VerificationBuilder, + accessor: &IndexMap, + _result: Option<&OwnedTable>, + one_eval_map: &IndexMap, + ) -> Result, ProofError> { + // 1. columns + // TODO: Make sure `GroupByExec` as self.input is supported + let input_table_eval = + self.input + .verifier_evaluate(builder, accessor, None, one_eval_map)?; + let output_one_eval = builder.consume_one_evaluation(); + let columns_evals = input_table_eval.column_evals(); + // 2. selection + // The selected range is (offset_index, max_index] + let offset_one_eval = builder.consume_one_evaluation(); + let max_one_eval = builder.consume_one_evaluation(); + let selection_eval = max_one_eval - offset_one_eval; + // 3. filtered_columns + let filtered_columns_evals: Vec<_> = repeat_with(|| builder.consume_intermediate_mle()) + .take(columns_evals.len()) + .collect(); + let alpha = builder.consume_post_result_challenge(); + let beta = builder.consume_post_result_challenge(); + + verify_filter( + builder, + alpha, + beta, + *input_table_eval.one_eval(), + output_one_eval, + columns_evals, + selection_eval, + &filtered_columns_evals, + )?; + Ok(TableEvaluation::new( + filtered_columns_evals, + output_one_eval, + )) + } + + fn get_column_result_fields(&self) -> Vec { + self.input.get_column_result_fields() + } + + fn get_column_references(&self) -> IndexSet { + self.input.get_column_references() + } + + fn get_table_references(&self) -> IndexSet { + self.input.get_table_references() + } +} + +impl ProverEvaluate for SliceExec { + #[tracing::instrument(name = "SliceExec::result_evaluate", level = "debug", skip_all)] + fn result_evaluate<'a, S: Scalar>( + &self, + alloc: &'a Bump, + table_map: &IndexMap>, + ) -> (Table<'a, S>, Vec) { + // 1. columns + let (input, input_one_eval_lengths) = self.input.result_evaluate(alloc, table_map); + let input_length = input.num_rows(); + let columns = input.columns().copied().collect::>(); + // 2. select + let select = get_slice_select(input_length, self.skip, self.fetch); + // The selected range is (offset_index, max_index] + let offset_index = self.skip.min(input_length); + let max_index = if let Some(fetch) = self.fetch { + (self.skip + fetch).min(input_length) + } else { + input_length + }; + let output_length = max_index - offset_index; + // Compute filtered_columns + let (filtered_columns, _) = filter_columns(alloc, &columns, &select); + let res = Table::<'a, S>::try_from_iter_with_options( + self.get_column_result_fields() + .into_iter() + .map(|expr| expr.name()) + .zip(filtered_columns), + TableOptions::new(Some(output_length)), + ) + .expect("Failed to create table from iterator"); + let mut one_eval_lengths = input_one_eval_lengths; + one_eval_lengths.extend(vec![output_length, offset_index, max_index]); + (res, one_eval_lengths) + } + + fn first_round_evaluate(&self, builder: &mut FirstRoundBuilder) { + self.input.first_round_evaluate(builder); + builder.request_post_result_challenges(2); + } + + #[tracing::instrument(name = "SliceExec::prover_evaluate", level = "debug", skip_all)] + #[allow(unused_variables)] + fn final_round_evaluate<'a, S: Scalar>( + &self, + builder: &mut FinalRoundBuilder<'a, S>, + alloc: &'a Bump, + table_map: &IndexMap>, + ) -> Table<'a, S> { + // 1. columns + let input = self.input.final_round_evaluate(builder, alloc, table_map); + let columns = input.columns().copied().collect::>(); + // 2. select + let select = get_slice_select(input.num_rows(), self.skip, self.fetch); + let select_ref: &'a [_] = alloc.alloc_slice_copy(&select); + let output_length = select.iter().filter(|b| **b).count(); + // Compute filtered_columns and indexes + let (filtered_columns, result_len) = filter_columns(alloc, &columns, &select); + // 3. Produce MLEs + filtered_columns.iter().copied().for_each(|column| { + builder.produce_intermediate_mle(column); + }); + let alpha = builder.consume_post_result_challenge(); + let beta = builder.consume_post_result_challenge(); + + prove_filter::( + builder, + alloc, + alpha, + beta, + &columns, + select_ref, + &filtered_columns, + input.num_rows(), + result_len, + ); + Table::<'a, S>::try_from_iter_with_options( + self.get_column_result_fields() + .into_iter() + .map(|expr| expr.name()) + .zip(filtered_columns), + TableOptions::new(Some(output_length)), + ) + .expect("Failed to create table from iterator") + } +} diff --git a/crates/proof-of-sql/src/sql/proof_plans/slice_exec_test.rs b/crates/proof-of-sql/src/sql/proof_plans/slice_exec_test.rs new file mode 100644 index 000000000..9f05f6571 --- /dev/null +++ b/crates/proof-of-sql/src/sql/proof_plans/slice_exec_test.rs @@ -0,0 +1,436 @@ +use super::test_utility::*; +use crate::{ + base::{ + database::{ + owned_table_utility::*, table_utility::*, ColumnField, ColumnType, OwnedTable, + OwnedTableTestAccessor, TableTestAccessor, TestAccessor, + }, + map::{indexmap, IndexMap}, + math::decimal::Precision, + scalar::Curve25519Scalar, + }, + sql::{ + proof::{ + exercise_verification, ProvableQueryResult, ProverEvaluate, VerifiableQueryResult, + }, + proof_exprs::{test_utility::*, DynProofExpr}, + }, +}; +use blitzar::proof::InnerProductProof; +use bumpalo::Bump; + +#[test] +fn we_can_prove_and_get_the_correct_result_from_a_slice_exec() { + let data = owned_table([ + bigint("a", [1_i64, 2, 3, 4, 5]), + varchar("b", ["1", "2", "3", "4", "5"]), + ]); + let t = "sxt.t".parse().unwrap(); + let accessor = OwnedTableTestAccessor::::new_from_table(t, data, 0, ()); + let ast = slice_exec( + projection(cols_expr_plan(t, &["a", "b"], &accessor), tab(t)), + 1, + Some(2), + ); + let verifiable_res = VerifiableQueryResult::new(&ast, &accessor, &()); + exercise_verification(&verifiable_res, &ast, &accessor, t); + let res = verifiable_res.verify(&ast, &accessor, &()).unwrap().table; + let expected_res = owned_table([bigint("a", [2_i64, 3]), varchar("b", ["2", "3"])]); + assert_eq!(res, expected_res); +} + +#[test] +fn we_can_prove_and_get_the_correct_empty_result_from_a_slice_exec() { + let data = owned_table([ + bigint("a", [1_i64, 2, 3, 4, 5]), + varchar("b", ["1", "2", "3", "4", "5"]), + ]); + let t = "sxt.t".parse().unwrap(); + let accessor = OwnedTableTestAccessor::::new_from_table(t, data, 0, ()); + let where_clause: DynProofExpr = equal(column(t, "a", &accessor), const_int128(2)); + let ast = slice_exec( + filter( + cols_expr_plan(t, &["a", "b"], &accessor), + tab(t), + where_clause, + ), + 1, + Some(2), + ); + let verifiable_res = VerifiableQueryResult::new(&ast, &accessor, &()); + exercise_verification(&verifiable_res, &ast, &accessor, t); + let res = verifiable_res.verify(&ast, &accessor, &()).unwrap().table; + let expected_res = owned_table([bigint("a", [0_i64; 0]), varchar("b", [""; 0])]); + assert_eq!(res, expected_res); +} + +#[test] +fn we_can_get_an_empty_result_from_a_slice_on_an_empty_table_using_result_evaluate() { + let alloc = Bump::new(); + let data = table([ + borrowed_bigint("a", [0; 0], &alloc), + borrowed_bigint("b", [0; 0], &alloc), + borrowed_int128("c", [0; 0], &alloc), + borrowed_varchar("d", [""; 0], &alloc), + borrowed_scalar("e", [0; 0], &alloc), + ]); + let t = "sxt.t".parse().unwrap(); + let table_map = indexmap! { + t => data.clone() + }; + let mut accessor = TableTestAccessor::::new_empty_with_setup(()); + accessor.add_table(t, data, 0); + let where_clause: DynProofExpr = equal(column(t, "a", &accessor), const_int128(999)); + let expr = slice_exec( + filter( + cols_expr_plan(t, &["b", "c", "d", "e"], &accessor), + tab(t), + where_clause, + ), + 1, + Some(2), + ); + + let fields = &[ + ColumnField::new("b".parse().unwrap(), ColumnType::BigInt), + ColumnField::new("c".parse().unwrap(), ColumnType::Int128), + ColumnField::new("d".parse().unwrap(), ColumnType::VarChar), + ColumnField::new( + "e".parse().unwrap(), + ColumnType::Decimal75(Precision::new(75).unwrap(), 0), + ), + ]; + let res: OwnedTable = + ProvableQueryResult::from(expr.result_evaluate(&alloc, &table_map).0) + .to_owned_table(fields) + .unwrap(); + let expected: OwnedTable = owned_table([ + bigint("b", [0; 0]), + int128("c", [0; 0]), + varchar("d", [""; 0]), + decimal75("e", 75, 0, [0; 0]), + ]); + + assert_eq!(res, expected); +} + +#[test] +fn we_can_get_an_empty_result_from_a_slice_using_result_evaluate() { + let alloc = Bump::new(); + let data = table([ + borrowed_bigint("a", [1, 4, 5, 2, 5], &alloc), + borrowed_bigint("b", [1, 2, 3, 4, 5], &alloc), + borrowed_int128("c", [1, 2, 3, 4, 5], &alloc), + borrowed_varchar("d", ["1", "2", "3", "4", "5"], &alloc), + borrowed_scalar("e", [1, 2, 3, 4, 5], &alloc), + ]); + let t = "sxt.t".parse().unwrap(); + let table_map = indexmap! { + t => data.clone() + }; + let mut accessor = TableTestAccessor::::new_empty_with_setup(()); + accessor.add_table(t, data, 0); + let where_clause: DynProofExpr = equal(column(t, "a", &accessor), const_int128(999)); + let expr = slice_exec( + filter( + cols_expr_plan(t, &["b", "c", "d", "e"], &accessor), + tab(t), + where_clause, + ), + 1, + Some(2), + ); + + let fields = &[ + ColumnField::new("b".parse().unwrap(), ColumnType::BigInt), + ColumnField::new("c".parse().unwrap(), ColumnType::Int128), + ColumnField::new("d".parse().unwrap(), ColumnType::VarChar), + ColumnField::new( + "e".parse().unwrap(), + ColumnType::Decimal75(Precision::new(1).unwrap(), 0), + ), + ]; + let res: OwnedTable = + ProvableQueryResult::from(expr.result_evaluate(&alloc, &table_map).0) + .to_owned_table(fields) + .unwrap(); + let expected: OwnedTable = owned_table([ + bigint("b", [0; 0]), + int128("c", [0; 0]), + varchar("d", [""; 0]), + decimal75("e", 1, 0, [0; 0]), + ]); + + assert_eq!(res, expected); +} + +#[test] +fn we_can_get_no_columns_from_a_slice_with_empty_input_using_result_evaluate() { + let alloc = Bump::new(); + let data = table([ + borrowed_bigint("a", [1, 4, 5, 2, 5], &alloc), + borrowed_bigint("b", [1, 2, 3, 4, 5], &alloc), + borrowed_int128("c", [1, 2, 3, 4, 5], &alloc), + borrowed_varchar("d", ["1", "2", "3", "4", "5"], &alloc), + borrowed_scalar("e", [1, 2, 3, 4, 5], &alloc), + ]); + let t = "sxt.t".parse().unwrap(); + let table_map = indexmap! { + t => data.clone() + }; + let mut accessor = TableTestAccessor::::new_empty_with_setup(()); + accessor.add_table(t, data, 0); + let where_clause: DynProofExpr = equal(column(t, "a", &accessor), const_int128(5)); + let expr = slice_exec( + filter(cols_expr_plan(t, &[], &accessor), tab(t), where_clause), + 2, + None, + ); + let fields = &[]; + let res: OwnedTable = + ProvableQueryResult::from(expr.result_evaluate(&alloc, &table_map).0) + .to_owned_table(fields) + .unwrap(); + let expected = OwnedTable::try_new(IndexMap::default()).unwrap(); + assert_eq!(res, expected); +} + +#[test] +fn we_can_get_the_correct_result_from_a_slice_using_result_evaluate() { + let alloc = Bump::new(); + let data = table([ + borrowed_bigint("a", [1, 4, 5, 2, 5], &alloc), + borrowed_bigint("b", [1, 2, 3, 4, 5], &alloc), + borrowed_int128("c", [1, 2, 3, 4, 5], &alloc), + borrowed_varchar("d", ["1", "2", "3", "4", "5"], &alloc), + borrowed_scalar("e", [1, 2, 3, 4, 5], &alloc), + ]); + let t = "sxt.t".parse().unwrap(); + let table_map = indexmap! { + t => data.clone() + }; + let mut accessor = TableTestAccessor::::new_empty_with_setup(()); + accessor.add_table(t, data, 0); + let where_clause: DynProofExpr = equal(column(t, "a", &accessor), const_int128(5)); + let expr = slice_exec( + filter( + cols_expr_plan(t, &["b", "c", "d", "e"], &accessor), + tab(t), + where_clause, + ), + 1, + None, + ); + let fields = &[ + ColumnField::new("b".parse().unwrap(), ColumnType::BigInt), + ColumnField::new("c".parse().unwrap(), ColumnType::Int128), + ColumnField::new("d".parse().unwrap(), ColumnType::VarChar), + ColumnField::new( + "e".parse().unwrap(), + ColumnType::Decimal75(Precision::new(1).unwrap(), 0), + ), + ]; + let res: OwnedTable = + ProvableQueryResult::from(expr.result_evaluate(&alloc, &table_map).0) + .to_owned_table(fields) + .unwrap(); + let expected: OwnedTable = owned_table([ + bigint("b", [5]), + int128("c", [5]), + varchar("d", ["5"]), + decimal75("e", 1, 0, [5]), + ]); + assert_eq!(res, expected); +} + +#[test] +fn we_can_prove_a_slice_exec() { + let data = owned_table([ + bigint("a", [101, 105, 105, 105, 105]), + bigint("b", [1, 2, 3, 4, 7]), + int128("c", [1, 3, 3, 4, 5]), + varchar("d", ["1", "2", "3", "4", "5"]), + scalar("e", [1, 2, 3, 4, 5]), + ]); + let t = "sxt.t".parse().unwrap(); + let mut accessor = OwnedTableTestAccessor::::new_empty_with_setup(()); + accessor.add_table(t, data, 0); + let expr = slice_exec( + filter( + vec![ + col_expr_plan(t, "b", &accessor), + col_expr_plan(t, "c", &accessor), + col_expr_plan(t, "d", &accessor), + col_expr_plan(t, "e", &accessor), + aliased_plan(const_int128(105), "const"), + aliased_plan( + equal(column(t, "b", &accessor), column(t, "c", &accessor)), + "bool", + ), + ], + tab(t), + equal(column(t, "a", &accessor), const_int128(105)), + ), + 2, + Some(1), + ); + let res = VerifiableQueryResult::new(&expr, &accessor, &()); + exercise_verification(&res, &expr, &accessor, t); + let res = res.verify(&expr, &accessor, &()).unwrap().table; + let expected = owned_table([ + bigint("b", [4]), + int128("c", [4]), + varchar("d", ["4"]), + scalar("e", [4]), + int128("const", [105]), + boolean("bool", [true]), + ]); + assert_eq!(res, expected); +} + +#[test] +fn we_can_prove_a_nested_slice_exec() { + let data = owned_table([ + bigint("a", [101, 105, 105, 105, 105]), + bigint("b", [1, 2, 3, 4, 7]), + int128("c", [1, 3, 3, 4, 5]), + varchar("d", ["1", "2", "3", "4", "5"]), + scalar("e", [1, 2, 3, 4, 5]), + ]); + let t = "sxt.t".parse().unwrap(); + let mut accessor = OwnedTableTestAccessor::::new_empty_with_setup(()); + accessor.add_table(t, data, 0); + let expr = slice_exec( + slice_exec( + filter( + vec![ + col_expr_plan(t, "b", &accessor), + col_expr_plan(t, "c", &accessor), + col_expr_plan(t, "d", &accessor), + col_expr_plan(t, "e", &accessor), + aliased_plan(const_int128(105), "const"), + aliased_plan( + equal(column(t, "b", &accessor), column(t, "c", &accessor)), + "bool", + ), + ], + tab(t), + equal(column(t, "a", &accessor), const_int128(105)), + ), + 1, + Some(3), + ), + 1, + Some(1), + ); + let res = VerifiableQueryResult::new(&expr, &accessor, &()); + exercise_verification(&res, &expr, &accessor, t); + let res = res.verify(&expr, &accessor, &()).unwrap().table; + let expected = owned_table([ + bigint("b", [4]), + int128("c", [4]), + varchar("d", ["4"]), + scalar("e", [4]), + int128("const", [105]), + boolean("bool", [true]), + ]); + assert_eq!(res, expected); +} + +#[test] +fn we_can_prove_a_nested_slice_exec_with_no_rows() { + let data = owned_table([ + bigint("a", [101, 105, 105, 105, 105]), + bigint("b", [1, 2, 3, 4, 7]), + int128("c", [1, 3, 3, 4, 5]), + varchar("d", ["1", "2", "3", "4", "5"]), + scalar("e", [1, 2, 3, 4, 5]), + ]); + let t = "sxt.t".parse().unwrap(); + let mut accessor = OwnedTableTestAccessor::::new_empty_with_setup(()); + accessor.add_table(t, data, 0); + let expr = slice_exec( + slice_exec( + filter( + vec![ + col_expr_plan(t, "b", &accessor), + col_expr_plan(t, "c", &accessor), + col_expr_plan(t, "d", &accessor), + col_expr_plan(t, "e", &accessor), + aliased_plan(const_int128(105), "const"), + aliased_plan( + equal(column(t, "b", &accessor), column(t, "c", &accessor)), + "bool", + ), + ], + tab(t), + equal(column(t, "a", &accessor), const_int128(105)), + ), + 1, + Some(3), + ), + 3, + None, + ); + let res = VerifiableQueryResult::new(&expr, &accessor, &()); + exercise_verification(&res, &expr, &accessor, t); + let res = res.verify(&expr, &accessor, &()).unwrap().table; + let expected = owned_table([ + bigint("b", [0; 0]), + int128("c", [0; 0]), + varchar("d", [""; 0]), + scalar("e", [0; 0]), + int128("const", [0; 0]), + boolean("bool", [true; 0]), + ]); + assert_eq!(res, expected); +} + +#[test] +fn we_can_prove_another_nested_slice_exec_with_no_rows() { + let data = owned_table([ + bigint("a", [101, 105, 105, 105, 105]), + bigint("b", [1, 2, 3, 4, 7]), + int128("c", [1, 3, 3, 4, 5]), + varchar("d", ["1", "2", "3", "4", "5"]), + scalar("e", [1, 2, 3, 4, 5]), + ]); + let t = "sxt.t".parse().unwrap(); + let mut accessor = OwnedTableTestAccessor::::new_empty_with_setup(()); + accessor.add_table(t, data, 0); + let expr = slice_exec( + slice_exec( + filter( + vec![ + col_expr_plan(t, "b", &accessor), + col_expr_plan(t, "c", &accessor), + col_expr_plan(t, "d", &accessor), + col_expr_plan(t, "e", &accessor), + aliased_plan(const_int128(105), "const"), + aliased_plan( + equal(column(t, "b", &accessor), column(t, "c", &accessor)), + "bool", + ), + ], + tab(t), + equal(column(t, "a", &accessor), const_int128(105)), + ), + 6, + Some(3), + ), + 3, + None, + ); + let res = VerifiableQueryResult::new(&expr, &accessor, &()); + exercise_verification(&res, &expr, &accessor, t); + let res = res.verify(&expr, &accessor, &()).unwrap().table; + let expected = owned_table([ + bigint("b", [0; 0]), + int128("c", [0; 0]), + varchar("d", [""; 0]), + scalar("e", [0; 0]), + int128("const", [0; 0]), + boolean("bool", [true; 0]), + ]); + assert_eq!(res, expected); +} diff --git a/crates/proof-of-sql/src/sql/proof_plans/test_utility.rs b/crates/proof-of-sql/src/sql/proof_plans/test_utility.rs index b22a61846..93f27db53 100644 --- a/crates/proof-of-sql/src/sql/proof_plans/test_utility.rs +++ b/crates/proof-of-sql/src/sql/proof_plans/test_utility.rs @@ -1,4 +1,4 @@ -use super::{DynProofPlan, FilterExec, GroupByExec, ProjectionExec, TableExec}; +use super::{DynProofPlan, FilterExec, GroupByExec, ProjectionExec, SliceExec, TableExec}; use crate::{ base::database::{ColumnField, TableRef}, sql::proof_exprs::{AliasedDynProofExpr, ColumnExpr, DynProofExpr, TableExpr}, @@ -38,3 +38,7 @@ pub fn group_by( where_clause, )) } + +pub fn slice_exec(input: DynProofPlan, skip: usize, fetch: Option) -> DynProofPlan { + DynProofPlan::Slice(SliceExec::new(Box::new(input), skip, fetch)) +} From 3bb2268002a258afaaca3e2f0c984214027bce2d Mon Sep 17 00:00:00 2001 From: Ian Joiner <14581281+iajoiner@users.noreply.github.com> Date: Tue, 3 Dec 2024 17:58:58 -0500 Subject: [PATCH 2/3] fix: address review comments --- crates/proof-of-sql/src/base/proof/error.rs | 3 + .../src/sql/proof_plans/filter_exec.rs | 2 +- .../proof-of-sql/src/sql/proof_plans/mod.rs | 2 +- .../src/sql/proof_plans/slice_exec.rs | 40 +++-- .../src/sql/proof_plans/slice_exec_test.rs | 154 +++++++++++++++--- .../src/sql/proof_plans/test_utility.rs | 8 +- 6 files changed, 167 insertions(+), 42 deletions(-) diff --git a/crates/proof-of-sql/src/base/proof/error.rs b/crates/proof-of-sql/src/base/proof/error.rs index 5d675060a..d918f4e77 100644 --- a/crates/proof-of-sql/src/base/proof/error.rs +++ b/crates/proof-of-sql/src/base/proof/error.rs @@ -6,4 +6,7 @@ pub enum ProofError { #[snafu(display("Verification error: {error}"))] /// This error occurs when a proof failed to verify. VerificationError { error: &'static str }, + /// Unsupported error + #[snafu(display("Unsupported error: {error}"))] + UnsupportedError { error: &'static str }, } diff --git a/crates/proof-of-sql/src/sql/proof_plans/filter_exec.rs b/crates/proof-of-sql/src/sql/proof_plans/filter_exec.rs index b5654e42b..37b74a0f2 100644 --- a/crates/proof-of-sql/src/sql/proof_plans/filter_exec.rs +++ b/crates/proof-of-sql/src/sql/proof_plans/filter_exec.rs @@ -249,7 +249,7 @@ impl ProverEvaluate for FilterExec { } #[allow(clippy::unnecessary_wraps, clippy::too_many_arguments)] -pub(crate) fn verify_filter( +pub(super) fn verify_filter( builder: &mut VerificationBuilder, alpha: S, beta: S, diff --git a/crates/proof-of-sql/src/sql/proof_plans/mod.rs b/crates/proof-of-sql/src/sql/proof_plans/mod.rs index 9f23adb47..be0202f49 100644 --- a/crates/proof-of-sql/src/sql/proof_plans/mod.rs +++ b/crates/proof-of-sql/src/sql/proof_plans/mod.rs @@ -16,9 +16,9 @@ mod projection_exec_test; pub(crate) mod test_utility; mod filter_exec; +pub(super) use filter_exec::FilterExec; #[cfg(test)] pub(crate) use filter_exec::OstensibleFilterExec; -pub(crate) use filter_exec::{prove_filter, verify_filter, FilterExec}; #[cfg(all(test, feature = "blitzar"))] mod filter_exec_test; #[cfg(all(test, feature = "blitzar"))] diff --git a/crates/proof-of-sql/src/sql/proof_plans/slice_exec.rs b/crates/proof-of-sql/src/sql/proof_plans/slice_exec.rs index ada1a22d7..8ec6a255c 100644 --- a/crates/proof-of-sql/src/sql/proof_plans/slice_exec.rs +++ b/crates/proof-of-sql/src/sql/proof_plans/slice_exec.rs @@ -1,4 +1,7 @@ -use super::{prove_filter, verify_filter, DynProofPlan}; +use super::{ + filter_exec::{prove_filter, verify_filter}, + DynProofPlan, +}; use crate::{ base::{ database::{ @@ -16,7 +19,8 @@ use crate::{ }; use alloc::{boxed::Box, vec, vec::Vec}; use bumpalo::Bump; -use core::iter::repeat_with; +use core::iter::{repeat, repeat_with}; +use itertools::repeat_n; use serde::{Deserialize, Serialize}; /// `ProofPlan` for queries of the form @@ -32,12 +36,11 @@ pub struct SliceExec { /// Get the boolean slice selection from the number of rows, skip and fetch fn get_slice_select(num_rows: usize, skip: usize, fetch: Option) -> Vec { - if let Some(fetch) = fetch { - let end = skip + fetch; - (0..num_rows).map(|i| i >= skip && i < end).collect() - } else { - (0..num_rows).map(|i| i >= skip).collect() - } + repeat_n(false, skip) + .chain(repeat_n(true, fetch.unwrap_or(num_rows))) + .chain(repeat(false)) + .take(num_rows) + .collect() } impl SliceExec { @@ -71,7 +74,12 @@ where one_eval_map: &IndexMap, ) -> Result, ProofError> { // 1. columns - // TODO: Make sure `GroupByExec` as self.input is supported + // We do not support `GroupByExec` as input for now + if let DynProofPlan::GroupBy(_) = *self.input { + return Err(ProofError::UnsupportedError { + error: "GroupByExec as input for another plan is not supported", + }); + } let input_table_eval = self.input .verifier_evaluate(builder, accessor, None, one_eval_map)?; @@ -119,14 +127,16 @@ where } impl ProverEvaluate for SliceExec { - #[tracing::instrument(name = "SliceExec::result_evaluate", level = "debug", skip_all)] - fn result_evaluate<'a, S: Scalar>( + #[tracing::instrument(name = "SliceExec::first_round_evaluate", level = "debug", skip_all)] + fn first_round_evaluate<'a, S: Scalar>( &self, + builder: &mut FirstRoundBuilder, alloc: &'a Bump, table_map: &IndexMap>, ) -> (Table<'a, S>, Vec) { // 1. columns - let (input, input_one_eval_lengths) = self.input.result_evaluate(alloc, table_map); + let (input, input_one_eval_lengths) = + self.input.first_round_evaluate(builder, alloc, table_map); let input_length = input.num_rows(); let columns = input.columns().copied().collect::>(); // 2. select @@ -151,12 +161,8 @@ impl ProverEvaluate for SliceExec { .expect("Failed to create table from iterator"); let mut one_eval_lengths = input_one_eval_lengths; one_eval_lengths.extend(vec![output_length, offset_index, max_index]); - (res, one_eval_lengths) - } - - fn first_round_evaluate(&self, builder: &mut FirstRoundBuilder) { - self.input.first_round_evaluate(builder); builder.request_post_result_challenges(2); + (res, one_eval_lengths) } #[tracing::instrument(name = "SliceExec::prover_evaluate", level = "debug", skip_all)] diff --git a/crates/proof-of-sql/src/sql/proof_plans/slice_exec_test.rs b/crates/proof-of-sql/src/sql/proof_plans/slice_exec_test.rs index 9f05f6571..a1080ce55 100644 --- a/crates/proof-of-sql/src/sql/proof_plans/slice_exec_test.rs +++ b/crates/proof-of-sql/src/sql/proof_plans/slice_exec_test.rs @@ -3,15 +3,17 @@ use crate::{ base::{ database::{ owned_table_utility::*, table_utility::*, ColumnField, ColumnType, OwnedTable, - OwnedTableTestAccessor, TableTestAccessor, TestAccessor, + OwnedTableTestAccessor, TableRef, TableTestAccessor, TestAccessor, }, map::{indexmap, IndexMap}, math::decimal::Precision, + proof::ProofError, scalar::Curve25519Scalar, }, sql::{ proof::{ - exercise_verification, ProvableQueryResult, ProverEvaluate, VerifiableQueryResult, + exercise_verification, FirstRoundBuilder, ProvableQueryResult, ProverEvaluate, + QueryError, VerifiableQueryResult, }, proof_exprs::{test_utility::*, DynProofExpr}, }, @@ -65,7 +67,7 @@ fn we_can_prove_and_get_the_correct_empty_result_from_a_slice_exec() { } #[test] -fn we_can_get_an_empty_result_from_a_slice_on_an_empty_table_using_result_evaluate() { +fn we_can_get_an_empty_result_from_a_slice_on_an_empty_table_using_first_round_evaluate() { let alloc = Bump::new(); let data = table([ borrowed_bigint("a", [0; 0], &alloc), @@ -100,10 +102,13 @@ fn we_can_get_an_empty_result_from_a_slice_on_an_empty_table_using_result_evalua ColumnType::Decimal75(Precision::new(75).unwrap(), 0), ), ]; - let res: OwnedTable = - ProvableQueryResult::from(expr.result_evaluate(&alloc, &table_map).0) - .to_owned_table(fields) - .unwrap(); + let first_round_builder = &mut FirstRoundBuilder::new(); + let res: OwnedTable = ProvableQueryResult::from( + expr.first_round_evaluate(first_round_builder, &alloc, &table_map) + .0, + ) + .to_owned_table(fields) + .unwrap(); let expected: OwnedTable = owned_table([ bigint("b", [0; 0]), int128("c", [0; 0]), @@ -115,7 +120,7 @@ fn we_can_get_an_empty_result_from_a_slice_on_an_empty_table_using_result_evalua } #[test] -fn we_can_get_an_empty_result_from_a_slice_using_result_evaluate() { +fn we_can_get_an_empty_result_from_a_slice_using_first_round_evaluate() { let alloc = Bump::new(); let data = table([ borrowed_bigint("a", [1, 4, 5, 2, 5], &alloc), @@ -150,10 +155,13 @@ fn we_can_get_an_empty_result_from_a_slice_using_result_evaluate() { ColumnType::Decimal75(Precision::new(1).unwrap(), 0), ), ]; - let res: OwnedTable = - ProvableQueryResult::from(expr.result_evaluate(&alloc, &table_map).0) - .to_owned_table(fields) - .unwrap(); + let first_round_builder = &mut FirstRoundBuilder::new(); + let res: OwnedTable = ProvableQueryResult::from( + expr.first_round_evaluate(first_round_builder, &alloc, &table_map) + .0, + ) + .to_owned_table(fields) + .unwrap(); let expected: OwnedTable = owned_table([ bigint("b", [0; 0]), int128("c", [0; 0]), @@ -165,7 +173,7 @@ fn we_can_get_an_empty_result_from_a_slice_using_result_evaluate() { } #[test] -fn we_can_get_no_columns_from_a_slice_with_empty_input_using_result_evaluate() { +fn we_can_get_no_columns_from_a_slice_with_empty_input_using_first_round_evaluate() { let alloc = Bump::new(); let data = table([ borrowed_bigint("a", [1, 4, 5, 2, 5], &alloc), @@ -187,16 +195,19 @@ fn we_can_get_no_columns_from_a_slice_with_empty_input_using_result_evaluate() { None, ); let fields = &[]; - let res: OwnedTable = - ProvableQueryResult::from(expr.result_evaluate(&alloc, &table_map).0) - .to_owned_table(fields) - .unwrap(); + let first_round_builder = &mut FirstRoundBuilder::new(); + let res: OwnedTable = ProvableQueryResult::from( + expr.first_round_evaluate(first_round_builder, &alloc, &table_map) + .0, + ) + .to_owned_table(fields) + .unwrap(); let expected = OwnedTable::try_new(IndexMap::default()).unwrap(); assert_eq!(res, expected); } #[test] -fn we_can_get_the_correct_result_from_a_slice_using_result_evaluate() { +fn we_can_get_the_correct_result_from_a_slice_using_first_round_evaluate() { let alloc = Bump::new(); let data = table([ borrowed_bigint("a", [1, 4, 5, 2, 5], &alloc), @@ -230,10 +241,13 @@ fn we_can_get_the_correct_result_from_a_slice_using_result_evaluate() { ColumnType::Decimal75(Precision::new(1).unwrap(), 0), ), ]; - let res: OwnedTable = - ProvableQueryResult::from(expr.result_evaluate(&alloc, &table_map).0) - .to_owned_table(fields) - .unwrap(); + let first_round_builder = &mut FirstRoundBuilder::new(); + let res: OwnedTable = ProvableQueryResult::from( + expr.first_round_evaluate(first_round_builder, &alloc, &table_map) + .0, + ) + .to_owned_table(fields) + .unwrap(); let expected: OwnedTable = owned_table([ bigint("b", [5]), int128("c", [5]), @@ -434,3 +448,99 @@ fn we_can_prove_another_nested_slice_exec_with_no_rows() { ]); assert_eq!(res, expected); } + +#[test] +fn we_can_create_and_prove_a_slice_exec_on_top_of_a_table_exec() { + let alloc = Bump::new(); + let table_ref = TableRef::new("namespace.table_name".parse().unwrap()); + let plan = slice_exec( + table_exec( + table_ref, + vec![ + ColumnField::new("language_rank".parse().unwrap(), ColumnType::BigInt), + ColumnField::new("language_name".parse().unwrap(), ColumnType::VarChar), + ColumnField::new("space_and_time".parse().unwrap(), ColumnType::VarChar), + ], + ), + 1, + Some(4), + ); + let accessor = TableTestAccessor::::new_from_table( + table_ref, + table([ + borrowed_bigint("language_rank", [0_i64, 1, 2, 3], &alloc), + borrowed_varchar( + "language_name", + ["English", "Español", "Português", "Français"], + &alloc, + ), + borrowed_varchar( + "space_and_time", + [ + "space and time", + "espacio y tiempo", + "espaço e tempo", + "espace et temps", + ], + &alloc, + ), + ]), + 0_usize, + (), + ); + let verifiable_res = VerifiableQueryResult::new(&plan, &accessor, &()); + exercise_verification(&verifiable_res, &plan, &accessor, table_ref); + let res = verifiable_res.verify(&plan, &accessor, &()).unwrap().table; + let expected = owned_table([ + bigint("language_rank", [1_i64, 2, 3]), + varchar("language_name", ["Español", "Português", "Français"]), + varchar( + "space_and_time", + ["espacio y tiempo", "espaço e tempo", "espace et temps"], + ), + ]); + assert_eq!(res, expected); +} + +#[test] +fn we_can_create_and_prove_a_slice_exec_on_top_of_an_empty_exec() { + let empty_table = owned_table([]); + let t = "sxt.t".parse().unwrap(); + let accessor = OwnedTableTestAccessor::::new_empty_with_setup(()); + let expr = slice_exec(empty_exec(), 3, Some(2)); + let res = VerifiableQueryResult::new(&expr, &accessor, &()); + exercise_verification(&res, &expr, &accessor, t); + let res = res.verify(&expr, &accessor, &()).unwrap().table; + assert_eq!(res, empty_table); +} + +#[test] +fn we_cannot_prove_a_slice_exec_if_it_has_groupby_as_input_for_now() { + let data = owned_table([ + bigint("a", [1, 2, 2, 1, 2]), + bigint("b", [99, 99, 99, 99, 0]), + bigint("c", [101, 102, 103, 104, 105]), + ]); + let t = "sxt.t".parse().unwrap(); + let mut accessor = OwnedTableTestAccessor::::new_empty_with_setup(()); + accessor.add_table(t, data, 0); + let expr = slice_exec( + group_by( + cols_expr(t, &["a"], &accessor), + vec![sum_expr(column(t, "c", &accessor), "sum_c")], + "__count__", + tab(t), + equal(column(t, "b", &accessor), const_int128(99)), + ), + 2, + None, + ); + let res: VerifiableQueryResult = + VerifiableQueryResult::new(&expr, &accessor, &()); + assert!(matches!( + res.verify(&expr, &accessor, &()), + Err(QueryError::ProofError { + source: ProofError::UnsupportedError { .. } + }) + )); +} diff --git a/crates/proof-of-sql/src/sql/proof_plans/test_utility.rs b/crates/proof-of-sql/src/sql/proof_plans/test_utility.rs index 93f27db53..e9eb09624 100644 --- a/crates/proof-of-sql/src/sql/proof_plans/test_utility.rs +++ b/crates/proof-of-sql/src/sql/proof_plans/test_utility.rs @@ -1,9 +1,15 @@ -use super::{DynProofPlan, FilterExec, GroupByExec, ProjectionExec, SliceExec, TableExec}; +use super::{ + DynProofPlan, EmptyExec, FilterExec, GroupByExec, ProjectionExec, SliceExec, TableExec, +}; use crate::{ base::database::{ColumnField, TableRef}, sql::proof_exprs::{AliasedDynProofExpr, ColumnExpr, DynProofExpr, TableExpr}, }; +pub fn empty_exec() -> DynProofPlan { + DynProofPlan::Empty(EmptyExec::new()) +} + pub fn table_exec(table_ref: TableRef, schema: Vec) -> DynProofPlan { DynProofPlan::Table(TableExec::new(table_ref, schema)) } From 383320c13db4676f2d52bb399e65dd33e9c4264f Mon Sep 17 00:00:00 2001 From: Ian Joiner <14581281+iajoiner@users.noreply.github.com> Date: Wed, 4 Dec 2024 08:37:02 -0500 Subject: [PATCH 3/3] fix: address NITs --- crates/proof-of-sql/src/base/proof/error.rs | 6 +++--- crates/proof-of-sql/src/sql/proof_plans/filter_exec.rs | 2 +- crates/proof-of-sql/src/sql/proof_plans/mod.rs | 2 +- crates/proof-of-sql/src/sql/proof_plans/slice_exec.rs | 4 ++-- crates/proof-of-sql/src/sql/proof_plans/slice_exec_test.rs | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/proof-of-sql/src/base/proof/error.rs b/crates/proof-of-sql/src/base/proof/error.rs index d918f4e77..489b0144b 100644 --- a/crates/proof-of-sql/src/base/proof/error.rs +++ b/crates/proof-of-sql/src/base/proof/error.rs @@ -6,7 +6,7 @@ pub enum ProofError { #[snafu(display("Verification error: {error}"))] /// This error occurs when a proof failed to verify. VerificationError { error: &'static str }, - /// Unsupported error - #[snafu(display("Unsupported error: {error}"))] - UnsupportedError { error: &'static str }, + /// This error occurs when a query plan is not supported. + #[snafu(display("Unsupported query plan: {error}"))] + UnsupportedQueryPlan { error: &'static str }, } diff --git a/crates/proof-of-sql/src/sql/proof_plans/filter_exec.rs b/crates/proof-of-sql/src/sql/proof_plans/filter_exec.rs index 37b74a0f2..d4fb992f8 100644 --- a/crates/proof-of-sql/src/sql/proof_plans/filter_exec.rs +++ b/crates/proof-of-sql/src/sql/proof_plans/filter_exec.rs @@ -286,7 +286,7 @@ pub(super) fn verify_filter( } #[allow(clippy::too_many_arguments, clippy::many_single_char_names)] -pub(crate) fn prove_filter<'a, S: Scalar + 'a>( +pub(super) fn prove_filter<'a, S: Scalar + 'a>( builder: &mut FinalRoundBuilder<'a, S>, alloc: &'a Bump, alpha: S, diff --git a/crates/proof-of-sql/src/sql/proof_plans/mod.rs b/crates/proof-of-sql/src/sql/proof_plans/mod.rs index be0202f49..71097cb28 100644 --- a/crates/proof-of-sql/src/sql/proof_plans/mod.rs +++ b/crates/proof-of-sql/src/sql/proof_plans/mod.rs @@ -16,7 +16,7 @@ mod projection_exec_test; pub(crate) mod test_utility; mod filter_exec; -pub(super) use filter_exec::FilterExec; +pub(crate) use filter_exec::FilterExec; #[cfg(test)] pub(crate) use filter_exec::OstensibleFilterExec; #[cfg(all(test, feature = "blitzar"))] diff --git a/crates/proof-of-sql/src/sql/proof_plans/slice_exec.rs b/crates/proof-of-sql/src/sql/proof_plans/slice_exec.rs index 8ec6a255c..b06e3b956 100644 --- a/crates/proof-of-sql/src/sql/proof_plans/slice_exec.rs +++ b/crates/proof-of-sql/src/sql/proof_plans/slice_exec.rs @@ -75,8 +75,8 @@ where ) -> Result, ProofError> { // 1. columns // We do not support `GroupByExec` as input for now - if let DynProofPlan::GroupBy(_) = *self.input { - return Err(ProofError::UnsupportedError { + if matches!(*self.input, DynProofPlan::GroupBy(_)) { + return Err(ProofError::UnsupportedQueryPlan { error: "GroupByExec as input for another plan is not supported", }); } diff --git a/crates/proof-of-sql/src/sql/proof_plans/slice_exec_test.rs b/crates/proof-of-sql/src/sql/proof_plans/slice_exec_test.rs index a1080ce55..bd237f4f4 100644 --- a/crates/proof-of-sql/src/sql/proof_plans/slice_exec_test.rs +++ b/crates/proof-of-sql/src/sql/proof_plans/slice_exec_test.rs @@ -540,7 +540,7 @@ fn we_cannot_prove_a_slice_exec_if_it_has_groupby_as_input_for_now() { assert!(matches!( res.verify(&expr, &accessor, &()), Err(QueryError::ProofError { - source: ProofError::UnsupportedError { .. } + source: ProofError::UnsupportedQueryPlan { .. } }) )); }