Merge branch 'perf/dynamic-dory-gpu' of https://github.com/spaceandti…

…melabs/sxt-proof-of-sql into perf/dynamic-dory-gpu

.github/workflows/lint-and-test.yml

@@ -103,11 +103,19 @@ jobs:
       - name: Dry run cargo test (proof-of-sql) (std feature only)
         run: cargo test -p proof-of-sql --no-run --no-default-features --features="std"
       - name: Run cargo test (proof primitives - Dory) (std feature only - i.e. not using blitzar)
-        run: cargo test proof_primitive::dory::dory_compute_commitments_test --no-default-features --features="std"
-      - name: Run hello_world example
-        run: cargo run --example hello_world --features="blitzar test"
-      - name: Run posql_db example
+        run: |
+            cargo test proof_primitive::dory::dory_compute_commitments_test --no-default-features --features="std" && \
+            cargo test proof_primitive::dory::dynamic_dory_compute_commitments_test --no-default-features --features="std"
+      - name: Run hello_world example (With Blitzar)
+        run: cargo run --example hello_world --features="test"
+      - name: Run hello_world example (Without Blitzar and With Rayon)
+        run: cargo run --example hello_world --no-default-features --features="rayon test"
+      - name: Run hello_world example (Without Blitzar and Without Rayon)
+        run: cargo run --example hello_world --no-default-features --features="test"
+      - name: Run posql_db example (With Blitzar)
         run: bash crates/proof-of-sql/examples/posql_db/run_example.sh
+      - name: Run posql_db example (Without Blitzar)
+        run: bash crates/proof-of-sql/examples/posql_db/run_example.sh --no-default-features --features="rayon"
 
   clippy:
     name: Clippy
@@ -136,6 +144,41 @@ jobs:
       - name: Run clippy::pedantic for proof-of-sql-parser
         run: cargo clippy --lib -p proof-of-sql-parser -- -D clippy::pedantic
 
+  coverage:
+    name: Code Coverage
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v3
+      - name: Install Dependencies
+        run: sudo apt-get update && sudo apt-get install -y clang lld
+      - uses: taiki-e/install-action@cargo-llvm-cov
+      - name: Clean Previous Coverage Artifacts
+        run: cargo llvm-cov clean --workspace
+      - name: Run Tests to Generate Coverage Data (All Features)
+        run: cargo llvm-cov --no-report --all-features
+      #- name: Run Tests to Generate Coverage Data (Rayon Only)
+      #  run: cargo llvm-cov --no-report --no-default-features --features="rayon"
+      #- name: Run Tests to Generate Coverage Data (Blitzar Only)
+      #  run: cargo llvm-cov --no-report --no-default-features --features="blitzar"
+      #- name: Run Tests to Generate Coverage Data (std only)
+      #  run: cargo llvm-cov --no-report --no-default-features --features="std"
+      - name: Generate Final LCOV Report (Merged Coverage)
+        run: cargo llvm-cov report --summary-only --fail-under-lines 90
+      # Future CodeCov Integration
+      # To integrate with CodeCov in the future,, follow these steps: 
+      # 1. Add the CodeCov token to the repository secrets.
+      # 2. Use the CodeCov Action to upload the coverage report. For more detailed info refer to [CodeCov Documentation](https://docs.codecov.com/docs).
+      #
+      # - name: Generate Final LCOV Report (Merged Coverage)
+      #   run: cargo llvm-cov report --lcov --output-path lcov.info --fail-under-lines 95
+      # - name: Upload Coverage to Codecov
+      # uses: codecov/codecov-action@v2
+      # with:
+      #   token: ${{ secrets.CODECOV_TOKEN }}
+      #   files: lcov.info
+      #   fail_ci_if_error: true
+
   # Run cargo fmt --all -- --config imports_granularity=Crate,group_imports=One --check
   format:
     name: Format

CONTRIBUTING.md

@@ -14,6 +14,7 @@ The following guideline is heavily based on the [Angular Project Guideline](http
    - [Submitting a Pull Request (PR)](#submit-pr)
    - [Addressing review feedback](#address-review)
    - [Updating the commit message](#updating-commit-message)
+   - [Running Code Coverage Locally](#coverage-locally)
    - [After your pull request is merged](#after-pr-merged)
  - [Coding Rules](#rules)
  - [Commit Message Guidelines](#commit-guidelines)
@@ -246,6 +247,12 @@ In order to update the commit message of the last commit on your branch:
 
 NOTE: If you need to update the commit message of an earlier commit, you can use `git rebase` in interactive mode. See the [git docs](https://git-scm.com/docs/git-rebase#_interactive_mode) for more details.
 
+### <a name="coverage-locally"></a> Running Code Coverage Locally 
+To run code coverage locally, install `cargo-llvm-cov` by following the instructions here: [cargo-llvm-cov Installation](https://github.com/taiki-e/cargo-llvm-cov).
+
+For users of VSCode, you can display coverage reports directly in the IDE by following these instructions: [Display Coverage in VSCode](https://github.com/taiki-e/cargo-llvm-cov?tab=readme-ov-file#display-coverage-in-vs-code).
+
+You can exclude specific functions from coverage by adding an attribute to your code: [Exclude Functions from Coverage](https://github.com/taiki-e/cargo-llvm-cov?tab=readme-ov-file#exclude-function-from-coverage).
 
 ### <a name="after-pr-merged"></a> After your pull request is merged
 
@@ -275,7 +282,6 @@ After your pull request is merged, you can safely delete your branch and pull th
     git pull --ff upstream main
     ```
 
-
 ## <a name="rules"></a> Coding Rules
 To ensure consistency throughout the source code, keep these rules in mind as you are working:
 

Cargo.toml

@@ -83,3 +83,21 @@ cast_lossless = "deny"
 redundant_closure_for_method_calls = "deny"
 inconsistent_struct_constructor = "deny"
 default_trait_access = "deny"
+module_name_repetitions = "deny" 
+wildcard_imports = "deny"
+unused_self = "deny"
+manual_let_else = "deny"
+struct_field_names = "deny"
+unicode_not_nfc = "deny"               
+manual_string_new = "deny"  
+large_types_passed_by_value = "deny" 
+map_unwrap_or = "deny"
+if_not_else = "deny" 
+explicit_deref_methods = "deny" 
+items_after_statements = "deny"
+bool_to_int_with_if = "deny"           
+ptr_as_ptr = "deny"        
+match_wildcard_for_single_variants = "deny"   
+match_bool = "deny"  
+manual_assert = "deny"    
+trivially_copy_pass_by_ref = "deny"

crates/proof-of-sql-parser/src/intermediate_ast_tests.rs

@@ -5,7 +5,11 @@ use crate::{
     utility::*,
     SelectStatement,
 };
-use alloc::{borrow::ToOwned, string::ToString, vec};
+use alloc::{
+    borrow::ToOwned,
+    string::{String, ToString},
+    vec,
+};
 
 // Sting parser tests
 #[test]
@@ -26,7 +30,7 @@ fn we_can_correctly_escape_the_single_quote_character() {
 
 #[test]
 fn we_can_parse_empty_strings() {
-    assert_eq!(StringLiteralParser::new().parse("''"), Ok("".to_string()));
+    assert_eq!(StringLiteralParser::new().parse("''"), Ok(String::new()));
 }
 
 #[test]
@@ -108,6 +112,7 @@ fn we_can_parse_strings_having_control_characters() {
     );
 }
 
+#[allow(clippy::unicode_not_nfc)]
 #[test]
 fn unnormalized_strings_should_differ() {
     let lhs = StringLiteralParser::new().parse("'á'").unwrap();

crates/proof-of-sql/Cargo.toml

@@ -83,11 +83,11 @@ workspace = true
 
 [[example]]
 name = "hello_world"
-required-features = [ "blitzar", "test" ]
+required-features = ["test"]
 
 [[example]]
 name = "posql_db"
-required-features = [ "arrow", "blitzar" ]
+required-features = [ "arrow" ]
 
 [[bench]]
 name = "posql_benches"

crates/proof-of-sql/README.md

@@ -73,8 +73,13 @@ The "Hello World" example demonstrates generating and verifying a proof of the q
 #### Run
 
 ```bash
-cargo run --example hello_world
+cargo run --example hello_world 
 ```
+> [!NOTE]
+> To run this example without the `blitzar` (i.e CPU only) feature:
+> ```bash
+> cargo run --example hello_world --no-default-features --features="rayon test"
+> ```
 
 #### Output
 

crates/proof-of-sql/benches/bench_append_rows.rs

@@ -26,7 +26,6 @@ use proof_of_sql::{
 };
 use proof_of_sql_parser::posql_time::{PoSQLTimeUnit, PoSQLTimeZone};
 use rand::Rng;
-use std::ops::Deref;
 
 /// Bench dory performance when appending rows to a table. This includes the computation of
 /// commitments. Chose the number of columns to randomly generate across supported `PoSQL`
@@ -99,34 +98,28 @@ pub fn generate_random_owned_table<S: Scalar>(
         let identifier = format!("column_{}", rng.gen::<u32>());
 
         match column_type {
-            "bigint" => columns.push(bigint(identifier.deref(), vec![rng.gen::<i64>(); num_rows])),
+            "bigint" => columns.push(bigint(&*identifier, vec![rng.gen::<i64>(); num_rows])),
             "boolean" => columns.push(boolean(
-                identifier.deref(),
+                &*identifier,
                 generate_random_boolean_vector(num_rows),
             )),
-            "int128" => columns.push(int128(
-                identifier.deref(),
-                vec![rng.gen::<i128>(); num_rows],
-            )),
+            "int128" => columns.push(int128(&*identifier, vec![rng.gen::<i128>(); num_rows])),
             "scalar" => columns.push(scalar(
-                identifier.deref(),
+                &*identifier,
                 vec![generate_random_u64_array(); num_rows],
             )),
-            "varchar" => columns.push(varchar(identifier.deref(), gen_rnd_str(num_rows))),
+            "varchar" => columns.push(varchar(&*identifier, gen_rnd_str(num_rows))),
             "decimal75" => columns.push(decimal75(
-                identifier.deref(),
+                &*identifier,
                 12,
                 2,
                 vec![generate_random_u64_array(); num_rows],
             )),
-            "tinyint" => columns.push(tinyint(identifier.deref(), vec![rng.gen::<i8>(); num_rows])),
-            "smallint" => columns.push(smallint(
-                identifier.deref(),
-                vec![rng.gen::<i16>(); num_rows],
-            )),
-            "int" => columns.push(int(identifier.deref(), vec![rng.gen::<i32>(); num_rows])),
+            "tinyint" => columns.push(tinyint(&*identifier, vec![rng.gen::<i8>(); num_rows])),
+            "smallint" => columns.push(smallint(&*identifier, vec![rng.gen::<i16>(); num_rows])),
+            "int" => columns.push(int(&*identifier, vec![rng.gen::<i32>(); num_rows])),
             "timestamptz" => columns.push(timestamptz(
-                identifier.deref(),
+                &*identifier,
                 PoSQLTimeUnit::Second,
                 PoSQLTimeZone::Utc,
                 vec![rng.gen::<i64>(); num_rows],

crates/proof-of-sql/benches/jaeger_benches.rs

@@ -22,6 +22,7 @@ use std::env;
 
 const SIZE: usize = 1_000_000;
 
+#[allow(clippy::items_after_statements)]
 fn main() {
     init_backend();
     use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};

crates/proof-of-sql/benches/scaffold/mod.rs

@@ -73,7 +73,7 @@ pub fn jaeger_scaffold<CP: CommitmentEvaluationProof>(
         .unwrap();
 }
 
-#[allow(dead_code)]
+#[allow(dead_code, clippy::module_name_repetitions)]
 pub fn criterion_scaffold<CP: CommitmentEvaluationProof>(
     c: &mut Criterion,
     title: &str,

crates/proof-of-sql/examples/hello_world/README.md

@@ -12,9 +12,15 @@ This example demonstrates generating and verifying a proof of the query `SELECT
 #### Run
 
 ```bash
-cargo run --example hello_world
+cargo run --example hello_world 
 ```
 
+> [!NOTE]
+> To run this example without the `blitzar` (i.e CPU only) feature:
+> ```bash
+> cargo run --example hello_world --no-default-features --features="test rayon"
+> ```
+
 #### Output
 
 ```
@@ -25,4 +31,4 @@ Generating Proof... 467.45371ms
 Verifying Proof... 7.106864ms
 Valid proof!
 Query result: OwnedTable { table: {Identifier { name: "b" }: VarChar(["hello", "world"])} }
-```
+```

crates/proof-of-sql/examples/hello_world/main.rs

@@ -1,8 +1,14 @@
 #![doc = include_str!("README.md")]
-
-use blitzar::{compute::init_backend, proof::InnerProductProof};
+use ark_std::test_rng;
+use blitzar::compute::init_backend;
 use proof_of_sql::{
-    base::database::{owned_table_utility::*, OwnedTableTestAccessor, TestAccessor},
+    base::database::{
+        owned_table_utility::{bigint, owned_table, varchar},
+        OwnedTableTestAccessor, TestAccessor,
+    },
+    proof_primitive::dory::{
+        DynamicDoryEvaluationProof, ProverSetup, PublicParameters, VerifierSetup,
+    },
     sql::{parse::QueryExpr, proof::QueryProof},
 };
 use std::{
@@ -39,7 +45,11 @@ fn main() {
     init_backend();
     end_timer(timer);
     let timer = start_timer("Loading data");
-    let mut accessor = OwnedTableTestAccessor::<InnerProductProof>::new_empty_with_setup(());
+    let public_parameters = PublicParameters::test_rand(5, &mut test_rng());
+    let prover_setup = ProverSetup::from(&public_parameters);
+    let verifier_setup = VerifierSetup::from(&public_parameters);
+    let mut accessor =
+        OwnedTableTestAccessor::<DynamicDoryEvaluationProof>::new_empty_with_setup(&prover_setup);
     accessor.add_table(
         "sxt.table".parse().unwrap(),
         owned_table([
@@ -58,11 +68,19 @@ fn main() {
     .unwrap();
     end_timer(timer);
     let timer = start_timer("Generating Proof");
-    let (proof, serialized_result) =
-        QueryProof::<InnerProductProof>::new(query.proof_expr(), &accessor, &());
+    let (proof, serialized_result) = QueryProof::<DynamicDoryEvaluationProof>::new(
+        query.proof_expr(),
+        &accessor,
+        &&prover_setup,
+    );
     end_timer(timer);
     let timer = start_timer("Verifying Proof");
-    let result = proof.verify(query.proof_expr(), &accessor, &serialized_result, &());
+    let result = proof.verify(
+        query.proof_expr(),
+        &accessor,
+        &serialized_result,
+        &&verifier_setup,
+    );
     end_timer(timer);
     match result {
         Ok(result) => {

crates/proof-of-sql/examples/posql_db/README.md

@@ -5,6 +5,12 @@ Example demonstrating an implementation of a simple csv-backed database with Pro
 ## Install
 Run `cargo install --example posql_db --path crates/proof-of-sql` to install the example.
 
+> [!NOTE]
+> To run this example without the `blitzar` (i.e CPU only )feature 
+> ```bash
+> cargo install --example posql_db --path crates/proof-of-sql --no-default-features --features="rayon"
+> ```
+
 ## Quick Start Exmaple
 Run the following
 ```bash