From fab704a24aba38fa1acb034a2ffa8dd897843048 Mon Sep 17 00:00:00 2001 From: Rigidity Date: Wed, 13 Mar 2024 17:20:32 -0400 Subject: [PATCH 01/29] Add initial Rust binding --- .gitignore | 6 +- rust-bindings/.gitignore | 1 + rust-bindings/Cargo.lock | 365 ++++++++++++++++++++++++++++++++++++++ rust-bindings/Cargo.toml | 17 ++ rust-bindings/build.rs | 41 +++++ rust-bindings/src/lib.rs | 29 +++ rust-bindings/wrapper.cpp | 30 ++++ rust-bindings/wrapper.h | 12 ++ 8 files changed, 500 insertions(+), 1 deletion(-) create mode 100644 rust-bindings/.gitignore create mode 100644 rust-bindings/Cargo.lock create mode 100644 rust-bindings/Cargo.toml create mode 100644 rust-bindings/build.rs create mode 100644 rust-bindings/src/lib.rs create mode 100644 rust-bindings/wrapper.cpp create mode 100644 rust-bindings/wrapper.h diff --git a/.gitignore b/.gitignore index db1074b..7559c77 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,6 @@ .mypy_cache -*.whl +*.egg-info +/dist +/venv +/build +/var \ No newline at end of file diff --git a/rust-bindings/.gitignore b/rust-bindings/.gitignore new file mode 100644 index 0000000..c41cc9e --- /dev/null +++ b/rust-bindings/.gitignore @@ -0,0 +1 @@ +/target \ No newline at end of file diff --git a/rust-bindings/Cargo.lock b/rust-bindings/Cargo.lock new file mode 100644 index 0000000..beada29 --- /dev/null +++ b/rust-bindings/Cargo.lock @@ -0,0 +1,365 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +dependencies = [ + "memchr", +] + +[[package]] +name = "bindgen" +version = "0.69.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn", + "which", +] + +[[package]] +name = "bitflags" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" + +[[package]] +name = "cc" +version = "1.0.90" +source = "git+https://github.com/0123456789-jpg/cc-rs?rev=41a8731#41a87317593083e0e41ca33fe2209f99c3fef80b" + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chiabip158" +version = "0.1.0" +dependencies = [ + "bindgen", + "cc", +] + +[[package]] +name = "clang-sys" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "either" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" + +[[package]] +name = "errno" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] +name = "libc" +version = "0.2.153" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" + +[[package]] +name = "libloading" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" +dependencies = [ + "cfg-if", + "windows-targets", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" + +[[package]] +name = "log" +version = "0.4.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" + +[[package]] +name = "memchr" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "prettyplease" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustix" +version = "0.38.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "syn" +version = "2.0.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b699d15b36d1f02c3e7c69f8ffef53de37aefae075d8488d4ba1a7788d574a07" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" diff --git a/rust-bindings/Cargo.toml b/rust-bindings/Cargo.toml new file mode 100644 index 0000000..c6afa5a --- /dev/null +++ b/rust-bindings/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "chiabip158" +version = "0.1.0" +edition = "2021" +links = "chiabip158" + +[dependencies] + +[build-dependencies] +bindgen = "0.69.4" + +# This uses a modified version of cc-rs for now https://github.com/rust-lang/cc-rs +# Because of this issue https://github.com/rust-lang/cc-rs/issues/1001 +# Which is fixed in this pull request https://github.com/rust-lang/cc-rs/pull/1007 +# TLDR: --show-sdk-platform-version needs to be replaced with --show-sdk-version +# Once this is released, this can be reverted back to the main crate +cc = { git = "https://github.com/0123456789-jpg/cc-rs", rev = "41a8731" } diff --git a/rust-bindings/build.rs b/rust-bindings/build.rs new file mode 100644 index 0000000..e1475ac --- /dev/null +++ b/rust-bindings/build.rs @@ -0,0 +1,41 @@ +use std::env; +use std::path::PathBuf; + +fn main() { + cc::Build::new() + .cpp(true) + .flag("-std=c++11") + .files([ + "../src/blockfilter.cpp", + "../src/crypto/sha256.cpp", + "../src/crypto/siphash.cpp", + "../src/primitives/block.cpp", + "../src/primitives/transaction.cpp", + "../src/script/script.cpp", + "../src/util/strencodings.cpp", + "../src/util/bytevectorhash.cpp", + "../src/uint256.cpp", + "./wrapper.cpp", + ]) + .warnings(false) + .include("../src") + .compile("chiabip158"); + + let bindings = bindgen::Builder::default() + .header("./wrapper.h") + .clang_arg("-x") + .clang_arg("c++") + .clang_arg("-I../src") + .clang_arg("-std=c++11") + .allowlist_type("EncodedFilter") + .allowlist_function("create_element") + .allowlist_function("get_encoded_filter") + .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) + .generate() + .expect("Unable to generate bindings"); + + let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); + bindings + .write_to_file(out_path.join("bindings.rs")) + .expect("Couldn't write bindings!"); +} diff --git a/rust-bindings/src/lib.rs b/rust-bindings/src/lib.rs new file mode 100644 index 0000000..c11c942 --- /dev/null +++ b/rust-bindings/src/lib.rs @@ -0,0 +1,29 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(clippy::missing_safety_doc)] +#![allow(unused)] + +mod bindings { + include!(concat!(env!("OUT_DIR"), "/bindings.rs")); +} + +#[repr(transparent)] +pub struct Element(bindings::std_vector); + +impl Element { + pub fn new(bytes: &[u8]) -> Self { + Self(unsafe { bindings::create_element(bytes.as_ptr(), bytes.len()) }) + } +} + +pub fn encode_filter(elements: &[Element]) -> Vec { + unsafe { + let mut encoded_filter = + bindings::get_encoded_filter(std::mem::transmute(elements.as_ptr()), elements.len()); + Vec::from_raw_parts( + encoded_filter.bytes(), + encoded_filter.size(), + encoded_filter.size(), + ) + } +} diff --git a/rust-bindings/wrapper.cpp b/rust-bindings/wrapper.cpp new file mode 100644 index 0000000..e60fc03 --- /dev/null +++ b/rust-bindings/wrapper.cpp @@ -0,0 +1,30 @@ +#include "wrapper.h" + +GCSFilter::Element create_element(const unsigned char* hash, size_t length) { + GCSFilter::Element element(length); + for (size_t i = 0; i < length; i++) { + element[i] = hash[i]; + } + return element; +} + +unsigned char* EncodedFilter::bytes() { + return vec.data(); +} + +size_t EncodedFilter::size() { + return vec.size(); +} + +EncodedFilter get_encoded_filter(const GCSFilter::Element hashes[], size_t length) { + GCSFilter::ElementSet elements; + for (size_t i = 0; i < length; i++) { + elements.insert(hashes[i]); + } + GCSFilter* filter = new GCSFilter({0, 0, 20, 1 << 20}, elements); + std::vector encoded = filter->GetEncoded(); + + EncodedFilter encoded_filter; + encoded_filter.vec = std::move(encoded); + return encoded_filter; +} \ No newline at end of file diff --git a/rust-bindings/wrapper.h b/rust-bindings/wrapper.h new file mode 100644 index 0000000..68a0c7b --- /dev/null +++ b/rust-bindings/wrapper.h @@ -0,0 +1,12 @@ +#include + +class EncodedFilter { + public: + std::vector vec; + unsigned char* bytes(); + size_t size(); +}; + +GCSFilter::Element create_element(const unsigned char* hash, size_t length); + +EncodedFilter get_encoded_filter(const GCSFilter::Element hashes[], size_t length); \ No newline at end of file From 1ab75110eff0c5e9e4f624557bc22af60eafa3f1 Mon Sep 17 00:00:00 2001 From: Rigidity Date: Wed, 13 Mar 2024 19:14:35 -0400 Subject: [PATCH 02/29] Tests passing, not sure if leaking --- rust-bindings/build.rs | 7 ++++--- rust-bindings/src/lib.rs | 35 +++++++++++++++++--------------- rust-bindings/wrapper.cpp | 42 ++++++++++++++++++--------------------- rust-bindings/wrapper.h | 14 +++++-------- 4 files changed, 47 insertions(+), 51 deletions(-) diff --git a/rust-bindings/build.rs b/rust-bindings/build.rs index e1475ac..71c2cdc 100644 --- a/rust-bindings/build.rs +++ b/rust-bindings/build.rs @@ -2,6 +2,8 @@ use std::env; use std::path::PathBuf; fn main() { + println!("cargo:rerun-if-changed=wrapper.cpp"); + cc::Build::new() .cpp(true) .flag("-std=c++11") @@ -27,9 +29,8 @@ fn main() { .clang_arg("c++") .clang_arg("-I../src") .clang_arg("-std=c++11") - .allowlist_type("EncodedFilter") - .allowlist_function("create_element") - .allowlist_function("get_encoded_filter") + .allowlist_type("Slice") + .allowlist_function("encode_filter") .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) .generate() .expect("Unable to generate bindings"); diff --git a/rust-bindings/src/lib.rs b/rust-bindings/src/lib.rs index c11c942..2d83cce 100644 --- a/rust-bindings/src/lib.rs +++ b/rust-bindings/src/lib.rs @@ -7,23 +7,26 @@ mod bindings { include!(concat!(env!("OUT_DIR"), "/bindings.rs")); } -#[repr(transparent)] -pub struct Element(bindings::std_vector); - -impl Element { - pub fn new(bytes: &[u8]) -> Self { - Self(unsafe { bindings::create_element(bytes.as_ptr(), bytes.len()) }) +pub fn encode_filter(slices: &[&[u8]]) -> Box<[u8]> { + let mut slices: Vec = slices + .iter() + .map(|slice| bindings::Slice { + bytes: slice.as_ptr(), + length: slice.len(), + }) + .collect(); + unsafe { + let filter = bindings::encode_filter(slices.as_ptr(), slices.len()); + let slice = std::slice::from_raw_parts(filter.bytes, filter.length); + Box::from_raw(slice as *const [u8] as *mut [u8]) } } -pub fn encode_filter(elements: &[Element]) -> Vec { - unsafe { - let mut encoded_filter = - bindings::get_encoded_filter(std::mem::transmute(elements.as_ptr()), elements.len()); - Vec::from_raw_parts( - encoded_filter.bytes(), - encoded_filter.size(), - encoded_filter.size(), - ) - } +#[test] +fn test_filter() { + let elem1 = b"abc"; + let elem2 = b"xyz"; + let elem3 = b"123"; + let filter = encode_filter(&[elem1, elem2, elem3]); + assert_eq!(filter.as_ref(), [3, 95, 172, 194, 74, 190, 73, 221, 182]); } diff --git a/rust-bindings/wrapper.cpp b/rust-bindings/wrapper.cpp index e60fc03..fd6caa3 100644 --- a/rust-bindings/wrapper.cpp +++ b/rust-bindings/wrapper.cpp @@ -1,30 +1,26 @@ #include "wrapper.h" -GCSFilter::Element create_element(const unsigned char* hash, size_t length) { - GCSFilter::Element element(length); - for (size_t i = 0; i < length; i++) { - element[i] = hash[i]; - } - return element; -} - -unsigned char* EncodedFilter::bytes() { - return vec.data(); -} - -size_t EncodedFilter::size() { - return vec.size(); -} - -EncodedFilter get_encoded_filter(const GCSFilter::Element hashes[], size_t length) { +Slice encode_filter(const Slice hashes[], size_t length) { GCSFilter::ElementSet elements; for (size_t i = 0; i < length; i++) { - elements.insert(hashes[i]); + Slice hash = hashes[i]; + GCSFilter::Element element(hash.length); + for (size_t j = 0; j < hash.length; j++) { + element[j] = hash.bytes[j]; + } + elements.insert(element); } - GCSFilter* filter = new GCSFilter({0, 0, 20, 1 << 20}, elements); - std::vector encoded = filter->GetEncoded(); + + GCSFilter filter({0, 0, 20, 1 << 20}, elements); + std::vector encoded = filter.GetEncoded(); + + size_t len = encoded.size(); + unsigned char* ptr = new unsigned char[len]; + std::copy(encoded.begin(), encoded.end(), ptr); - EncodedFilter encoded_filter; - encoded_filter.vec = std::move(encoded); - return encoded_filter; + Slice slice { + .bytes = ptr, + .length = len + }; + return slice; } \ No newline at end of file diff --git a/rust-bindings/wrapper.h b/rust-bindings/wrapper.h index 68a0c7b..e0d993a 100644 --- a/rust-bindings/wrapper.h +++ b/rust-bindings/wrapper.h @@ -1,12 +1,8 @@ #include -class EncodedFilter { - public: - std::vector vec; - unsigned char* bytes(); - size_t size(); -}; +typedef struct { + const unsigned char* bytes; + const size_t length; +} Slice; -GCSFilter::Element create_element(const unsigned char* hash, size_t length); - -EncodedFilter get_encoded_filter(const GCSFilter::Element hashes[], size_t length); \ No newline at end of file +Slice encode_filter(const Slice hashes[], size_t length); \ No newline at end of file From 707d5c4ffa56a1423a4f61a9fe82d0e43b94674c Mon Sep 17 00:00:00 2001 From: Rigidity Date: Wed, 13 Mar 2024 20:37:44 -0400 Subject: [PATCH 03/29] Add basic fuzzer --- rust-bindings/Cargo.lock | 46 ++++++++++++++++++- rust-bindings/Cargo.toml | 4 ++ rust-bindings/fuzz/.gitignore | 4 ++ rust-bindings/fuzz/Cargo.toml | 21 +++++++++ .../fuzz/fuzz_targets/fuzz_filter.rs | 7 +++ rust-bindings/src/lib.rs | 15 +++++- 6 files changed, 94 insertions(+), 3 deletions(-) create mode 100644 rust-bindings/fuzz/.gitignore create mode 100644 rust-bindings/fuzz/Cargo.toml create mode 100644 rust-bindings/fuzz/fuzz_targets/fuzz_filter.rs diff --git a/rust-bindings/Cargo.lock b/rust-bindings/Cargo.lock index beada29..74845df 100644 --- a/rust-bindings/Cargo.lock +++ b/rust-bindings/Cargo.lock @@ -11,6 +11,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" + [[package]] name = "bindgen" version = "0.69.4" @@ -40,6 +46,16 @@ version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" +[[package]] +name = "cc" +version = "1.0.90" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" +dependencies = [ + "jobserver", + "libc", +] + [[package]] name = "cc" version = "1.0.90" @@ -65,7 +81,15 @@ name = "chiabip158" version = "0.1.0" dependencies = [ "bindgen", - "cc", + "cc 1.0.90 (git+https://github.com/0123456789-jpg/cc-rs?rev=41a8731)", +] + +[[package]] +name = "chiabip158-fuzz" +version = "0.0.0" +dependencies = [ + "chiabip158", + "libfuzzer-sys", ] [[package]] @@ -119,6 +143,15 @@ dependencies = [ "either", ] +[[package]] +name = "jobserver" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" +dependencies = [ + "libc", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -137,6 +170,17 @@ version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +[[package]] +name = "libfuzzer-sys" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a96cfd5557eb82f2b83fed4955246c988d331975a002961b07c81584d107e7f7" +dependencies = [ + "arbitrary", + "cc 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)", + "once_cell", +] + [[package]] name = "libloading" version = "0.8.3" diff --git a/rust-bindings/Cargo.toml b/rust-bindings/Cargo.toml index c6afa5a..ede5242 100644 --- a/rust-bindings/Cargo.toml +++ b/rust-bindings/Cargo.toml @@ -4,6 +4,10 @@ version = "0.1.0" edition = "2021" links = "chiabip158" +[workspace] +resolver = "2" +members = ["fuzz"] + [dependencies] [build-dependencies] diff --git a/rust-bindings/fuzz/.gitignore b/rust-bindings/fuzz/.gitignore new file mode 100644 index 0000000..1a45eee --- /dev/null +++ b/rust-bindings/fuzz/.gitignore @@ -0,0 +1,4 @@ +target +corpus +artifacts +coverage diff --git a/rust-bindings/fuzz/Cargo.toml b/rust-bindings/fuzz/Cargo.toml new file mode 100644 index 0000000..d78487b --- /dev/null +++ b/rust-bindings/fuzz/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "chiabip158-fuzz" +version = "0.0.0" +publish = false +edition = "2021" + +[package.metadata] +cargo-fuzz = true + +[dependencies] +libfuzzer-sys = "0.4" + +[dependencies.chiabip158] +path = ".." + +[[bin]] +name = "fuzz_filter" +path = "fuzz_targets/fuzz_filter.rs" +test = false +doc = false +bench = false diff --git a/rust-bindings/fuzz/fuzz_targets/fuzz_filter.rs b/rust-bindings/fuzz/fuzz_targets/fuzz_filter.rs new file mode 100644 index 0000000..1af683a --- /dev/null +++ b/rust-bindings/fuzz/fuzz_targets/fuzz_filter.rs @@ -0,0 +1,7 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: Vec<&[u8]>| { + let _ = chiabip158::encode_filter(&data); +}); diff --git a/rust-bindings/src/lib.rs b/rust-bindings/src/lib.rs index 2d83cce..5309b2b 100644 --- a/rust-bindings/src/lib.rs +++ b/rust-bindings/src/lib.rs @@ -7,7 +7,7 @@ mod bindings { include!(concat!(env!("OUT_DIR"), "/bindings.rs")); } -pub fn encode_filter(slices: &[&[u8]]) -> Box<[u8]> { +pub fn encode_filter(slices: &[&[u8]]) -> &'static [u8] { let mut slices: Vec = slices .iter() .map(|slice| bindings::Slice { @@ -18,7 +18,8 @@ pub fn encode_filter(slices: &[&[u8]]) -> Box<[u8]> { unsafe { let filter = bindings::encode_filter(slices.as_ptr(), slices.len()); let slice = std::slice::from_raw_parts(filter.bytes, filter.length); - Box::from_raw(slice as *const [u8] as *mut [u8]) + // Box::from_raw(slice as *const [u8] as *mut [u8]) + slice } } @@ -30,3 +31,13 @@ fn test_filter() { let filter = encode_filter(&[elem1, elem2, elem3]); assert_eq!(filter.as_ref(), [3, 95, 172, 194, 74, 190, 73, 221, 182]); } + +#[test] +fn test_indefinitely() { + let elem1 = &[1; 10000]; + let elem2 = &[2; 10000]; + let elem3 = &[3; 10000]; + loop { + let _filter = encode_filter(&[elem1, elem2, elem3]); + } +} From 63186c9454b6d9e58ba01bac5540e2d7af353a75 Mon Sep 17 00:00:00 2001 From: Rigidity Date: Wed, 13 Mar 2024 20:38:14 -0400 Subject: [PATCH 04/29] oops --- rust-bindings/src/lib.rs | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/rust-bindings/src/lib.rs b/rust-bindings/src/lib.rs index 5309b2b..2d83cce 100644 --- a/rust-bindings/src/lib.rs +++ b/rust-bindings/src/lib.rs @@ -7,7 +7,7 @@ mod bindings { include!(concat!(env!("OUT_DIR"), "/bindings.rs")); } -pub fn encode_filter(slices: &[&[u8]]) -> &'static [u8] { +pub fn encode_filter(slices: &[&[u8]]) -> Box<[u8]> { let mut slices: Vec = slices .iter() .map(|slice| bindings::Slice { @@ -18,8 +18,7 @@ pub fn encode_filter(slices: &[&[u8]]) -> &'static [u8] { unsafe { let filter = bindings::encode_filter(slices.as_ptr(), slices.len()); let slice = std::slice::from_raw_parts(filter.bytes, filter.length); - // Box::from_raw(slice as *const [u8] as *mut [u8]) - slice + Box::from_raw(slice as *const [u8] as *mut [u8]) } } @@ -31,13 +30,3 @@ fn test_filter() { let filter = encode_filter(&[elem1, elem2, elem3]); assert_eq!(filter.as_ref(), [3, 95, 172, 194, 74, 190, 73, 221, 182]); } - -#[test] -fn test_indefinitely() { - let elem1 = &[1; 10000]; - let elem2 = &[2; 10000]; - let elem3 = &[3; 10000]; - loop { - let _filter = encode_filter(&[elem1, elem2, elem3]); - } -} From 582f752a1502dd28cdfbda09d57f6c817a956b43 Mon Sep 17 00:00:00 2001 From: Rigidity Date: Thu, 14 Mar 2024 08:48:21 -0400 Subject: [PATCH 05/29] Add comments --- rust-bindings/src/lib.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/rust-bindings/src/lib.rs b/rust-bindings/src/lib.rs index 2d83cce..a751e12 100644 --- a/rust-bindings/src/lib.rs +++ b/rust-bindings/src/lib.rs @@ -7,7 +7,9 @@ mod bindings { include!(concat!(env!("OUT_DIR"), "/bindings.rs")); } +/// Encode a BIP158 filter from a list of slices. pub fn encode_filter(slices: &[&[u8]]) -> Box<[u8]> { + // Convert the slices to a type that C/C++ can understand. let mut slices: Vec = slices .iter() .map(|slice| bindings::Slice { @@ -15,9 +17,18 @@ pub fn encode_filter(slices: &[&[u8]]) -> Box<[u8]> { length: slice.len(), }) .collect(); + + // SAFETY: The length provided matches the length of the slice, so this should be safe. unsafe { + // Call the binding to generate the BIP158 filter. let filter = bindings::encode_filter(slices.as_ptr(), slices.len()); + + // Convert the C `Slice` struct to a standard Rust slice. let slice = std::slice::from_raw_parts(filter.bytes, filter.length); + + // We cast the slice to a `Box<[u8]>` to ensure that it is properly deallocated. + // The slice is declared as `const` in the C struct, but nothing else uses it so it should be safe to make mutable. + // And then take ownership with `Box` since it's on the heap and must be freed later. Box::from_raw(slice as *const [u8] as *mut [u8]) } } From 03e57164a73af681dec6e2af9ccb3233b23dff7d Mon Sep 17 00:00:00 2001 From: Rigidity Date: Thu, 14 Mar 2024 08:49:40 -0400 Subject: [PATCH 06/29] Remove warning suppressions --- rust-bindings/src/lib.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/rust-bindings/src/lib.rs b/rust-bindings/src/lib.rs index a751e12..a120764 100644 --- a/rust-bindings/src/lib.rs +++ b/rust-bindings/src/lib.rs @@ -1,7 +1,5 @@ #![allow(non_camel_case_types)] #![allow(non_snake_case)] -#![allow(clippy::missing_safety_doc)] -#![allow(unused)] mod bindings { include!(concat!(env!("OUT_DIR"), "/bindings.rs")); @@ -10,7 +8,7 @@ mod bindings { /// Encode a BIP158 filter from a list of slices. pub fn encode_filter(slices: &[&[u8]]) -> Box<[u8]> { // Convert the slices to a type that C/C++ can understand. - let mut slices: Vec = slices + let slices: Vec = slices .iter() .map(|slice| bindings::Slice { bytes: slice.as_ptr(), From 548e995d01eacf7b7fbf767862ea366f23b0bd3e Mon Sep 17 00:00:00 2001 From: Rigidity Date: Thu, 14 Mar 2024 10:31:28 -0400 Subject: [PATCH 07/29] Use delete and manually free on drop --- rust-bindings/build.rs | 1 + rust-bindings/src/lib.rs | 44 +++++++++++++++++++++++++++++++-------- rust-bindings/wrapper.cpp | 4 ++++ rust-bindings/wrapper.h | 4 +++- 4 files changed, 43 insertions(+), 10 deletions(-) diff --git a/rust-bindings/build.rs b/rust-bindings/build.rs index 71c2cdc..7daf1f8 100644 --- a/rust-bindings/build.rs +++ b/rust-bindings/build.rs @@ -31,6 +31,7 @@ fn main() { .clang_arg("-std=c++11") .allowlist_type("Slice") .allowlist_function("encode_filter") + .allowlist_function("free_slice") .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) .generate() .expect("Unable to generate bindings"); diff --git a/rust-bindings/src/lib.rs b/rust-bindings/src/lib.rs index a120764..03b780e 100644 --- a/rust-bindings/src/lib.rs +++ b/rust-bindings/src/lib.rs @@ -1,12 +1,44 @@ #![allow(non_camel_case_types)] #![allow(non_snake_case)] +use std::fmt; + mod bindings { include!(concat!(env!("OUT_DIR"), "/bindings.rs")); } +pub struct EncodedFilter(bindings::Slice); + +impl fmt::Debug for EncodedFilter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.as_ref().fmt(f) + } +} + +impl PartialEq for EncodedFilter { + fn eq(&self, other: &Self) -> bool { + self.as_ref() == other.as_ref() + } +} + +impl Eq for EncodedFilter {} + +impl AsRef<[u8]> for EncodedFilter { + fn as_ref(&self) -> &[u8] { + // SAFETY: The `Slice` struct is guaranteed to be valid for the lifetime of the `EncodedFilter` struct. + unsafe { std::slice::from_raw_parts(self.0.bytes, self.0.length) } + } +} + +impl Drop for EncodedFilter { + fn drop(&mut self) { + // SAFETY: The `Slice` struct is guaranteed to be valid for the lifetime of the `EncodedFilter` struct. + unsafe { bindings::free_slice(self.0) } + } +} + /// Encode a BIP158 filter from a list of slices. -pub fn encode_filter(slices: &[&[u8]]) -> Box<[u8]> { +pub fn encode_filter(slices: &[&[u8]]) -> EncodedFilter { // Convert the slices to a type that C/C++ can understand. let slices: Vec = slices .iter() @@ -20,14 +52,7 @@ pub fn encode_filter(slices: &[&[u8]]) -> Box<[u8]> { unsafe { // Call the binding to generate the BIP158 filter. let filter = bindings::encode_filter(slices.as_ptr(), slices.len()); - - // Convert the C `Slice` struct to a standard Rust slice. - let slice = std::slice::from_raw_parts(filter.bytes, filter.length); - - // We cast the slice to a `Box<[u8]>` to ensure that it is properly deallocated. - // The slice is declared as `const` in the C struct, but nothing else uses it so it should be safe to make mutable. - // And then take ownership with `Box` since it's on the heap and must be freed later. - Box::from_raw(slice as *const [u8] as *mut [u8]) + EncodedFilter(filter) } } @@ -38,4 +63,5 @@ fn test_filter() { let elem3 = b"123"; let filter = encode_filter(&[elem1, elem2, elem3]); assert_eq!(filter.as_ref(), [3, 95, 172, 194, 74, 190, 73, 221, 182]); + assert_eq!(&filter, &filter); } diff --git a/rust-bindings/wrapper.cpp b/rust-bindings/wrapper.cpp index fd6caa3..eeb2196 100644 --- a/rust-bindings/wrapper.cpp +++ b/rust-bindings/wrapper.cpp @@ -23,4 +23,8 @@ Slice encode_filter(const Slice hashes[], size_t length) { .length = len }; return slice; +} + +void free_slice(Slice slice) { + delete[] slice.bytes; } \ No newline at end of file diff --git a/rust-bindings/wrapper.h b/rust-bindings/wrapper.h index e0d993a..c83a350 100644 --- a/rust-bindings/wrapper.h +++ b/rust-bindings/wrapper.h @@ -5,4 +5,6 @@ typedef struct { const size_t length; } Slice; -Slice encode_filter(const Slice hashes[], size_t length); \ No newline at end of file +Slice encode_filter(const Slice hashes[], size_t length); + +void free_slice(Slice slice); \ No newline at end of file From c19f8a61e1219b187e41f0fb7e12ae9849954d06 Mon Sep 17 00:00:00 2001 From: Rigidity Date: Thu, 14 Mar 2024 11:23:05 -0400 Subject: [PATCH 08/29] Use C++ style struct --- .gitignore | 1 + rust-bindings/wrapper.cpp | 15 ++++----------- rust-bindings/wrapper.h | 8 ++++---- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index 7559c77..5c0af96 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .mypy_cache *.egg-info +.vscode /dist /venv /build diff --git a/rust-bindings/wrapper.cpp b/rust-bindings/wrapper.cpp index eeb2196..91001e5 100644 --- a/rust-bindings/wrapper.cpp +++ b/rust-bindings/wrapper.cpp @@ -1,14 +1,11 @@ #include "wrapper.h" -Slice encode_filter(const Slice hashes[], size_t length) { +Slice encode_filter(Slice const* hashes, size_t length) { GCSFilter::ElementSet elements; for (size_t i = 0; i < length; i++) { Slice hash = hashes[i]; - GCSFilter::Element element(hash.length); - for (size_t j = 0; j < hash.length; j++) { - element[j] = hash.bytes[j]; - } - elements.insert(element); + auto element = GCSFilter::Element(hash.bytes, hash.bytes + length); + elements.insert(std::move(element)); } GCSFilter filter({0, 0, 20, 1 << 20}, elements); @@ -18,11 +15,7 @@ Slice encode_filter(const Slice hashes[], size_t length) { unsigned char* ptr = new unsigned char[len]; std::copy(encoded.begin(), encoded.end(), ptr); - Slice slice { - .bytes = ptr, - .length = len - }; - return slice; + return Slice { ptr, len }; } void free_slice(Slice slice) { diff --git a/rust-bindings/wrapper.h b/rust-bindings/wrapper.h index c83a350..3a51632 100644 --- a/rust-bindings/wrapper.h +++ b/rust-bindings/wrapper.h @@ -1,10 +1,10 @@ #include -typedef struct { - const unsigned char* bytes; +struct Slice { + unsigned char const* const bytes; const size_t length; -} Slice; +}; -Slice encode_filter(const Slice hashes[], size_t length); +Slice encode_filter(Slice const* hashes, size_t length); void free_slice(Slice slice); \ No newline at end of file From 096a81b0544937f94b50fc62f297fabb2a8f5d41 Mon Sep 17 00:00:00 2001 From: Rigidity Date: Thu, 14 Mar 2024 12:07:20 -0400 Subject: [PATCH 09/29] Add Rust workflow --- .github/workflows/rust.yml | 79 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 .github/workflows/rust.yml diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml new file mode 100644 index 0000000..c215b32 --- /dev/null +++ b/.github/workflows/rust.yml @@ -0,0 +1,79 @@ +name: Rust bindings + +on: + push: + branches: + - main + release: + types: [published] + pull_request: + branches: + - "**" + +jobs: + build: + name: Test - ${{ matrix.os.name }} ${{ matrix.arch.name }} + runs-on: ${{ matrix.os.runs-on[matrix.arch.matrix] }} + strategy: + fail-fast: false + matrix: + os: + - name: macOS + matrix: macos + runs-on: + arm: [macOS, ARM64] + intel: [macos-latest] + - name: Ubuntu + matrix: ubuntu + runs-on: + arm: [Linux, ARM64] + intel: [ubuntu-latest] + - name: Windows + matrix: windows + runs-on: + intel: [windows-latest] + steps: + - uses: actions/checkout@v3 + + - name: Set up Rust + uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy + + - name: Build + run: cargo build --release + + - name: Test + run: cargo test + + - name: Test (Release) + run: cargo test --release + + - name: Check formatting + run: cargo fmt -- --check + + - name: Check clippy + run: cargo clippy --all-targets --all-features -- -D warnings + + - name: Create release + continue-on-error: true + if: startsWith(github.event.ref, 'refs/tags') + env: + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} + run: cargo publish + + fuzz_targets: + name: Run Fuzzer + runs-on: ubuntu-latest + env: + CARGO_PROFILE_RELEASE_LTO: false + steps: + - uses: actions/checkout@v3 + - uses: dtolnay/rust-toolchain@nightly + + - name: cargo-fuzz + run: cargo +nightly install cargo-fuzz + + - name: cargo fuzz + run: | + cargo fuzz list | xargs -I "%" sh -c "cargo +nightly fuzz run % -- -max_total_time=120 || exit 255" From 7ef9fedf2dc5014ad88aac9a4259fbb48e8102cd Mon Sep 17 00:00:00 2001 From: Rigidity Date: Thu, 14 Mar 2024 12:08:37 -0400 Subject: [PATCH 10/29] Make concurrent? --- .github/workflows/rust.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index c215b32..767b4df 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -10,6 +10,11 @@ on: branches: - "**" +concurrency: + # SHA is added to the end if on `main` to let all main workflows run + group: ${{ github.ref }}-${{ github.workflow }}-${{ github.event_name }}-${{ (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release/') || startsWith(github.ref, 'refs/heads/long_lived/')) && github.sha || '' }} + cancel-in-progress: true + jobs: build: name: Test - ${{ matrix.os.name }} ${{ matrix.arch.name }} From 90d4aa191a194b4db10fc4acb089758747f1384f Mon Sep 17 00:00:00 2001 From: Rigidity Date: Thu, 14 Mar 2024 12:33:24 -0400 Subject: [PATCH 11/29] Fix arch --- .github/workflows/rust.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 767b4df..0d4294e 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -37,6 +37,20 @@ jobs: matrix: windows runs-on: intel: [windows-latest] + arch: + - name: ARM + matrix: arm + - name: Intel + matrix: intel + exclude: + - os: + name: Windows + matrix: windows + runs-on: + intel: [windows-latest] + arch: + name: ARM + matrix: arm steps: - uses: actions/checkout@v3 From c3f27b9870356bc286dd36a7915b5cd1654de574 Mon Sep 17 00:00:00 2001 From: Rigidity Date: Thu, 14 Mar 2024 12:35:47 -0400 Subject: [PATCH 12/29] Working dir --- .github/workflows/rust.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 0d4294e..2feace2 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -51,6 +51,9 @@ jobs: arch: name: ARM matrix: arm + defaults: + run: + working-directory: ./rust-bindings steps: - uses: actions/checkout@v3 @@ -86,6 +89,9 @@ jobs: runs-on: ubuntu-latest env: CARGO_PROFILE_RELEASE_LTO: false + defaults: + run: + working-directory: ./rust-bindings steps: - uses: actions/checkout@v3 - uses: dtolnay/rust-toolchain@nightly From f5947d1c97ec5d239350bc96fa3dec3073d15751 Mon Sep 17 00:00:00 2001 From: Rigidity Date: Thu, 14 Mar 2024 12:48:19 -0400 Subject: [PATCH 13/29] Use hash length not number of hashes (thanks, fuzzer) --- rust-bindings/wrapper.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-bindings/wrapper.cpp b/rust-bindings/wrapper.cpp index 91001e5..35b5034 100644 --- a/rust-bindings/wrapper.cpp +++ b/rust-bindings/wrapper.cpp @@ -4,7 +4,7 @@ Slice encode_filter(Slice const* hashes, size_t length) { GCSFilter::ElementSet elements; for (size_t i = 0; i < length; i++) { Slice hash = hashes[i]; - auto element = GCSFilter::Element(hash.bytes, hash.bytes + length); + auto element = GCSFilter::Element(hash.bytes, hash.bytes + hash.length); elements.insert(std::move(element)); } From fdef302968f2825aff109774ec11d98909011eaf Mon Sep 17 00:00:00 2001 From: Rigidity Date: Thu, 14 Mar 2024 12:59:49 -0400 Subject: [PATCH 14/29] Install clang on Ubuntu ARM --- .github/workflows/rust.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 2feace2..be837b7 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -57,6 +57,10 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Install Clang on Ubuntu ARM + if: matrix.os.name == 'Ubuntu' && matrix.arch.name == 'ARM' + run: sudo apt-get install -y clang + - name: Set up Rust uses: dtolnay/rust-toolchain@stable with: From d43a0a4dbc7534558873686977adc8fc717fd56b Mon Sep 17 00:00:00 2001 From: Rigidity Date: Thu, 14 Mar 2024 13:18:07 -0400 Subject: [PATCH 15/29] Update to C++14 and fix std --- rust-bindings/build.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust-bindings/build.rs b/rust-bindings/build.rs index 7daf1f8..95b4b4f 100644 --- a/rust-bindings/build.rs +++ b/rust-bindings/build.rs @@ -6,7 +6,7 @@ fn main() { cc::Build::new() .cpp(true) - .flag("-std=c++11") + .std("c++14") .files([ "../src/blockfilter.cpp", "../src/crypto/sha256.cpp", @@ -28,7 +28,7 @@ fn main() { .clang_arg("-x") .clang_arg("c++") .clang_arg("-I../src") - .clang_arg("-std=c++11") + .clang_arg("-std=c++14") .allowlist_type("Slice") .allowlist_function("encode_filter") .allowlist_function("free_slice") From 317b5a3935337d4ad1cf8c975b8155e55aea9b24 Mon Sep 17 00:00:00 2001 From: Rigidity Date: Thu, 14 Mar 2024 13:44:44 -0400 Subject: [PATCH 16/29] Add whl back to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 5c0af96..fb32648 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .mypy_cache *.egg-info +*.whl .vscode /dist /venv From e57ed9077c526adc75d2fe95cb5911114c90f82f Mon Sep 17 00:00:00 2001 From: Rigidity Date: Thu, 14 Mar 2024 17:00:34 -0400 Subject: [PATCH 17/29] Update workflow --- .github/workflows/rust.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index be837b7..d7214d2 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -17,7 +17,7 @@ concurrency: jobs: build: - name: Test - ${{ matrix.os.name }} ${{ matrix.arch.name }} + name: Build - ${{ matrix.os.name }} ${{ matrix.arch.name }} runs-on: ${{ matrix.os.runs-on[matrix.arch.matrix] }} strategy: fail-fast: false @@ -55,7 +55,12 @@ jobs: run: working-directory: ./rust-bindings steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + + - name: Set Env + uses: Chia-Network/actions/setjobenv@main + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Install Clang on Ubuntu ARM if: matrix.os.name == 'Ubuntu' && matrix.arch.name == 'ARM' @@ -83,7 +88,7 @@ jobs: - name: Create release continue-on-error: true - if: startsWith(github.event.ref, 'refs/tags') + if: env.RELEASE == 'true' env: CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} run: cargo publish @@ -97,7 +102,7 @@ jobs: run: working-directory: ./rust-bindings steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@nightly - name: cargo-fuzz From 1fea9f145e99a097f3c94665225b34d44b68c776 Mon Sep 17 00:00:00 2001 From: Rigidity Date: Thu, 14 Mar 2024 19:07:03 -0400 Subject: [PATCH 18/29] Improve API and add testing and matching --- rust-bindings/build.rs | 8 ++ .../fuzz/fuzz_targets/fuzz_filter.rs | 19 +++- rust-bindings/src/lib.rs | 106 ++++++++++++++---- rust-bindings/wrapper.cpp | 27 ++++- rust-bindings/wrapper.h | 6 +- 5 files changed, 140 insertions(+), 26 deletions(-) diff --git a/rust-bindings/build.rs b/rust-bindings/build.rs index 95b4b4f..d487ba4 100644 --- a/rust-bindings/build.rs +++ b/rust-bindings/build.rs @@ -29,8 +29,16 @@ fn main() { .clang_arg("c++") .clang_arg("-I../src") .clang_arg("-std=c++14") + .blocklist_item("std.*") + .blocklist_item("GCSFilter.+") + .blocklist_item("ByteVector.*") + .opaque_type("GCSFilter") + .allowlist_type("GCSFilter") .allowlist_type("Slice") + .allowlist_function("create_filter") .allowlist_function("encode_filter") + .allowlist_function("filter_match") + .allowlist_function("filter_match_any") .allowlist_function("free_slice") .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) .generate() diff --git a/rust-bindings/fuzz/fuzz_targets/fuzz_filter.rs b/rust-bindings/fuzz/fuzz_targets/fuzz_filter.rs index 1af683a..b3e0f76 100644 --- a/rust-bindings/fuzz/fuzz_targets/fuzz_filter.rs +++ b/rust-bindings/fuzz/fuzz_targets/fuzz_filter.rs @@ -1,7 +1,22 @@ #![no_main] +use chiabip158::Bip158Filter; use libfuzzer_sys::fuzz_target; -fuzz_target!(|data: Vec<&[u8]>| { - let _ = chiabip158::encode_filter(&data); +fuzz_target!(|data: (Vec<&[u8]>, Vec<&[u8]>)| { + let filter = Bip158Filter::new(&data.0); + + let encoded = filter.encode(); + let bytes = encoded.as_ref(); + for byte in bytes { + let _ = *byte; + } + + for elem in data.0 { + assert!(filter.matches(elem)); + } + + for elem in data.1 { + assert!(!filter.matches(elem)); + } }); diff --git a/rust-bindings/src/lib.rs b/rust-bindings/src/lib.rs index 03b780e..d4f1a10 100644 --- a/rust-bindings/src/lib.rs +++ b/rust-bindings/src/lib.rs @@ -7,6 +7,7 @@ mod bindings { include!(concat!(env!("OUT_DIR"), "/bindings.rs")); } +/// The encoded form of a BIP 158 filter. pub struct EncodedFilter(bindings::Slice); impl fmt::Debug for EncodedFilter { @@ -37,22 +38,70 @@ impl Drop for EncodedFilter { } } -/// Encode a BIP158 filter from a list of slices. -pub fn encode_filter(slices: &[&[u8]]) -> EncodedFilter { - // Convert the slices to a type that C/C++ can understand. - let slices: Vec = slices - .iter() - .map(|slice| bindings::Slice { - bytes: slice.as_ptr(), - length: slice.len(), - }) - .collect(); - - // SAFETY: The length provided matches the length of the slice, so this should be safe. - unsafe { - // Call the binding to generate the BIP158 filter. - let filter = bindings::encode_filter(slices.as_ptr(), slices.len()); - EncodedFilter(filter) +/// A BIP 158 filter. +pub struct Bip158Filter(bindings::GCSFilter); + +impl Bip158Filter { + /// Encode a BIP 158 filter from a list of slices. + pub fn new(slices: &[&[u8]]) -> Self { + // Convert the slices to a type that C/C++ can understand. + let slices: Vec = slices + .iter() + .map(|slice| bindings::Slice { + bytes: slice.as_ptr(), + length: slice.len(), + }) + .collect(); + + // SAFETY: The length provided matches the number of slices, so this should be safe. + unsafe { Self(bindings::create_filter(slices.as_ptr(), slices.len())) } + } + + /// Encodes the filter. + pub fn encode(&self) -> EncodedFilter { + // SAFETY: The `GCSFilter` struct is guaranteed to be valid for the lifetime of the `Bip158Filter` struct. + unsafe { + EncodedFilter(bindings::encode_filter( + &self.0 as *const bindings::GCSFilter, + )) + } + } + + /// Matches a single slice against the filter. + pub fn matches(&self, slice: &[u8]) -> bool { + // SAFETY: The length provided matches the length of the slice, and the `GCSFilter` + // struct is guaranteed to be valid for the lifetime of the `Bip158Filter` struct, so this should be safe. + unsafe { + bindings::filter_match( + &self.0 as *const bindings::GCSFilter, + bindings::Slice { + bytes: slice.as_ptr(), + length: slice.len(), + }, + ) + } + } + + /// Matches any of a list of slices against the filter. + pub fn matches_any(&self, slices: &[&[u8]]) -> bool { + // Convert the slices to a type that C/C++ can understand. + let slices: Vec = slices + .iter() + .map(|slice| bindings::Slice { + bytes: slice.as_ptr(), + length: slice.len(), + }) + .collect(); + + // SAFETY: The length provided matches the number of slices, and the `GCSFilter` + // struct is guaranteed to be valid for the lifetime of the `Bip158Filter` struct, so this should be safe. + unsafe { + bindings::filter_match_any( + &self.0 as *const bindings::GCSFilter, + slices.as_ptr(), + slices.len(), + ) + } } } @@ -61,7 +110,26 @@ fn test_filter() { let elem1 = b"abc"; let elem2 = b"xyz"; let elem3 = b"123"; - let filter = encode_filter(&[elem1, elem2, elem3]); - assert_eq!(filter.as_ref(), [3, 95, 172, 194, 74, 190, 73, 221, 182]); - assert_eq!(&filter, &filter); + let not_elem1 = b"hello"; + let not_elem2 = b"bye"; + + let filter = Bip158Filter::new(&[elem1, elem2, elem3]); + + let encoded = filter.encode(); + assert_eq!(encoded.as_ref(), [3, 95, 172, 194, 74, 190, 73, 221, 182]); + assert_eq!(&encoded, &encoded); + + assert!(filter.matches(elem1)); + assert!(filter.matches(elem2)); + assert!(filter.matches(elem3)); + + assert!(!filter.matches(not_elem1)); + assert!(!filter.matches(not_elem2)); + + assert!(filter.matches_any(&[elem1, elem2, elem3])); + assert!(filter.matches_any(&[elem1, elem2])); + assert!(filter.matches_any(&[elem3])); + + assert!(!filter.matches_any(&[])); + assert!(!filter.matches_any(&[not_elem1, not_elem2])); } diff --git a/rust-bindings/wrapper.cpp b/rust-bindings/wrapper.cpp index 35b5034..a45d563 100644 --- a/rust-bindings/wrapper.cpp +++ b/rust-bindings/wrapper.cpp @@ -1,15 +1,19 @@ #include "wrapper.h" -Slice encode_filter(Slice const* hashes, size_t length) { +GCSFilter create_filter(Slice const* hashes, size_t length) { GCSFilter::ElementSet elements; + for (size_t i = 0; i < length; i++) { Slice hash = hashes[i]; auto element = GCSFilter::Element(hash.bytes, hash.bytes + hash.length); elements.insert(std::move(element)); } - GCSFilter filter({0, 0, 20, 1 << 20}, elements); - std::vector encoded = filter.GetEncoded(); + return GCSFilter({0, 0, 20, 1 << 20}, elements); +} + +Slice encode_filter(GCSFilter const* filter) { + std::vector encoded = filter->GetEncoded(); size_t len = encoded.size(); unsigned char* ptr = new unsigned char[len]; @@ -18,6 +22,23 @@ Slice encode_filter(Slice const* hashes, size_t length) { return Slice { ptr, len }; } +bool filter_match(GCSFilter const* filter, Slice hash) { + auto element = GCSFilter::Element(hash.bytes, hash.bytes + hash.length); + return filter->Match(element); +} + +bool filter_match_any(GCSFilter const* filter, Slice const* hashes, size_t length) { + GCSFilter::ElementSet elements; + + for (int i = 0; i < length; i++) { + Slice hash = hashes[i]; + auto element = GCSFilter::Element(hash.bytes, hash.bytes + hash.length); + elements.insert(std::move(element)); + } + + return filter->MatchAny(elements); +} + void free_slice(Slice slice) { delete[] slice.bytes; } \ No newline at end of file diff --git a/rust-bindings/wrapper.h b/rust-bindings/wrapper.h index 3a51632..b411ea5 100644 --- a/rust-bindings/wrapper.h +++ b/rust-bindings/wrapper.h @@ -5,6 +5,8 @@ struct Slice { const size_t length; }; -Slice encode_filter(Slice const* hashes, size_t length); - +GCSFilter create_filter(Slice const* hashes, size_t length); +Slice encode_filter(GCSFilter const* filter); +bool filter_match(GCSFilter const* filter, Slice hash); +bool filter_match_any(GCSFilter const* filter, Slice const* hashes, size_t length); void free_slice(Slice slice); \ No newline at end of file From 425a17039fa665e2d4ea81b15f86888ed845d496 Mon Sep 17 00:00:00 2001 From: Rigidity Date: Thu, 14 Mar 2024 19:16:28 -0400 Subject: [PATCH 19/29] Opaque std? --- rust-bindings/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-bindings/build.rs b/rust-bindings/build.rs index d487ba4..236b26b 100644 --- a/rust-bindings/build.rs +++ b/rust-bindings/build.rs @@ -29,10 +29,10 @@ fn main() { .clang_arg("c++") .clang_arg("-I../src") .clang_arg("-std=c++14") - .blocklist_item("std.*") .blocklist_item("GCSFilter.+") .blocklist_item("ByteVector.*") .opaque_type("GCSFilter") + .opaque_type("std.*") .allowlist_type("GCSFilter") .allowlist_type("Slice") .allowlist_function("create_filter") From 0b6f4e9b962f0ccef562caf6793081135e8d461d Mon Sep 17 00:00:00 2001 From: Rigidity Date: Thu, 14 Mar 2024 19:18:17 -0400 Subject: [PATCH 20/29] dead code lint --- rust-bindings/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rust-bindings/src/lib.rs b/rust-bindings/src/lib.rs index d4f1a10..bbaf327 100644 --- a/rust-bindings/src/lib.rs +++ b/rust-bindings/src/lib.rs @@ -1,5 +1,6 @@ #![allow(non_camel_case_types)] #![allow(non_snake_case)] +#![allow(dead_code)] use std::fmt; From 9deb885cd0c3a9759a6ff37a8de3ea7aebb6e04f Mon Sep 17 00:00:00 2001 From: Rigidity Date: Thu, 14 Mar 2024 19:53:13 -0400 Subject: [PATCH 21/29] non upper case globals lint --- rust-bindings/fuzz/fuzz_targets/fuzz_filter.rs | 3 +++ rust-bindings/src/lib.rs | 1 + 2 files changed, 4 insertions(+) diff --git a/rust-bindings/fuzz/fuzz_targets/fuzz_filter.rs b/rust-bindings/fuzz/fuzz_targets/fuzz_filter.rs index b3e0f76..87af594 100644 --- a/rust-bindings/fuzz/fuzz_targets/fuzz_filter.rs +++ b/rust-bindings/fuzz/fuzz_targets/fuzz_filter.rs @@ -12,6 +12,9 @@ fuzz_target!(|data: (Vec<&[u8]>, Vec<&[u8]>)| { let _ = *byte; } + assert!(filter.matches_any(&data.0)); + assert!(!filter.matches_any(&data.1)); + for elem in data.0 { assert!(filter.matches(elem)); } diff --git a/rust-bindings/src/lib.rs b/rust-bindings/src/lib.rs index bbaf327..f9c33ba 100644 --- a/rust-bindings/src/lib.rs +++ b/rust-bindings/src/lib.rs @@ -1,6 +1,7 @@ #![allow(non_camel_case_types)] #![allow(non_snake_case)] #![allow(dead_code)] +#![allow(non_upper_case_globals)] use std::fmt; From b4df311fc35f2a263d2e63158bd93026f8210ff5 Mon Sep 17 00:00:00 2001 From: Rigidity Date: Thu, 14 Mar 2024 20:25:04 -0400 Subject: [PATCH 22/29] Refactor a bit --- rust-bindings/fuzz/fuzz_targets/fuzz_filter.rs | 4 ++-- rust-bindings/wrapper.cpp | 18 ++++++++---------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/rust-bindings/fuzz/fuzz_targets/fuzz_filter.rs b/rust-bindings/fuzz/fuzz_targets/fuzz_filter.rs index 87af594..6bc33bd 100644 --- a/rust-bindings/fuzz/fuzz_targets/fuzz_filter.rs +++ b/rust-bindings/fuzz/fuzz_targets/fuzz_filter.rs @@ -13,13 +13,13 @@ fuzz_target!(|data: (Vec<&[u8]>, Vec<&[u8]>)| { } assert!(filter.matches_any(&data.0)); - assert!(!filter.matches_any(&data.1)); + //assert!(!filter.matches_any(&data.1)); for elem in data.0 { assert!(filter.matches(elem)); } for elem in data.1 { - assert!(!filter.matches(elem)); + //assert!(!filter.matches(elem)); } }); diff --git a/rust-bindings/wrapper.cpp b/rust-bindings/wrapper.cpp index a45d563..c2b2c41 100644 --- a/rust-bindings/wrapper.cpp +++ b/rust-bindings/wrapper.cpp @@ -1,6 +1,6 @@ #include "wrapper.h" -GCSFilter create_filter(Slice const* hashes, size_t length) { +GCSFilter::ElementSet create_element_set(Slice const* hashes, size_t length) { GCSFilter::ElementSet elements; for (size_t i = 0; i < length; i++) { @@ -8,7 +8,12 @@ GCSFilter create_filter(Slice const* hashes, size_t length) { auto element = GCSFilter::Element(hash.bytes, hash.bytes + hash.length); elements.insert(std::move(element)); } - + + return elements; +} + +GCSFilter create_filter(Slice const* hashes, size_t length) { + auto elements = create_element_set(hashes, length); return GCSFilter({0, 0, 20, 1 << 20}, elements); } @@ -28,14 +33,7 @@ bool filter_match(GCSFilter const* filter, Slice hash) { } bool filter_match_any(GCSFilter const* filter, Slice const* hashes, size_t length) { - GCSFilter::ElementSet elements; - - for (int i = 0; i < length; i++) { - Slice hash = hashes[i]; - auto element = GCSFilter::Element(hash.bytes, hash.bytes + hash.length); - elements.insert(std::move(element)); - } - + auto elements = create_element_set(hashes, length); return filter->MatchAny(elements); } From 070b427a5c97a4d049665ba86235aeeeb954689d Mon Sep 17 00:00:00 2001 From: Rigidity Date: Thu, 14 Mar 2024 20:25:38 -0400 Subject: [PATCH 23/29] Fix fuzzer? Maybe need to re-add --- rust-bindings/fuzz/fuzz_targets/fuzz_filter.rs | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/rust-bindings/fuzz/fuzz_targets/fuzz_filter.rs b/rust-bindings/fuzz/fuzz_targets/fuzz_filter.rs index 6bc33bd..dcbf87b 100644 --- a/rust-bindings/fuzz/fuzz_targets/fuzz_filter.rs +++ b/rust-bindings/fuzz/fuzz_targets/fuzz_filter.rs @@ -3,8 +3,8 @@ use chiabip158::Bip158Filter; use libfuzzer_sys::fuzz_target; -fuzz_target!(|data: (Vec<&[u8]>, Vec<&[u8]>)| { - let filter = Bip158Filter::new(&data.0); +fuzz_target!(|data: Vec<&[u8]>| { + let filter = Bip158Filter::new(&data); let encoded = filter.encode(); let bytes = encoded.as_ref(); @@ -12,14 +12,7 @@ fuzz_target!(|data: (Vec<&[u8]>, Vec<&[u8]>)| { let _ = *byte; } - assert!(filter.matches_any(&data.0)); - //assert!(!filter.matches_any(&data.1)); - - for elem in data.0 { + for elem in data { assert!(filter.matches(elem)); } - - for elem in data.1 { - //assert!(!filter.matches(elem)); - } }); From 39f35d9f49c3c7e99431f97109be9b09e8d74c5a Mon Sep 17 00:00:00 2001 From: Rigidity Date: Thu, 14 Mar 2024 20:43:15 -0400 Subject: [PATCH 24/29] Fix fuzz --- rust-bindings/fuzz/fuzz_targets/fuzz_filter.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rust-bindings/fuzz/fuzz_targets/fuzz_filter.rs b/rust-bindings/fuzz/fuzz_targets/fuzz_filter.rs index dcbf87b..de37123 100644 --- a/rust-bindings/fuzz/fuzz_targets/fuzz_filter.rs +++ b/rust-bindings/fuzz/fuzz_targets/fuzz_filter.rs @@ -4,6 +4,10 @@ use chiabip158::Bip158Filter; use libfuzzer_sys::fuzz_target; fuzz_target!(|data: Vec<&[u8]>| { + if data.is_empty() { + return; + } + let filter = Bip158Filter::new(&data); let encoded = filter.encode(); @@ -12,6 +16,8 @@ fuzz_target!(|data: Vec<&[u8]>| { let _ = *byte; } + assert!(filter.matches_any(&data)); + for elem in data { assert!(filter.matches(elem)); } From 284c6bbefa4d42e46f490fbb7699f4f8e41b6663 Mon Sep 17 00:00:00 2001 From: Rigidity Date: Fri, 15 Mar 2024 09:21:26 -0400 Subject: [PATCH 25/29] Fix memory leak --- rust-bindings/build.rs | 1 + rust-bindings/src/lib.rs | 25 +++++++++++-------------- rust-bindings/wrapper.cpp | 8 ++++++-- rust-bindings/wrapper.h | 5 +++-- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/rust-bindings/build.rs b/rust-bindings/build.rs index 236b26b..c18348b 100644 --- a/rust-bindings/build.rs +++ b/rust-bindings/build.rs @@ -40,6 +40,7 @@ fn main() { .allowlist_function("filter_match") .allowlist_function("filter_match_any") .allowlist_function("free_slice") + .allowlist_function("free_filter") .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) .generate() .expect("Unable to generate bindings"); diff --git a/rust-bindings/src/lib.rs b/rust-bindings/src/lib.rs index f9c33ba..c522b07 100644 --- a/rust-bindings/src/lib.rs +++ b/rust-bindings/src/lib.rs @@ -41,7 +41,7 @@ impl Drop for EncodedFilter { } /// A BIP 158 filter. -pub struct Bip158Filter(bindings::GCSFilter); +pub struct Bip158Filter(*const bindings::GCSFilter); impl Bip158Filter { /// Encode a BIP 158 filter from a list of slices. @@ -62,11 +62,7 @@ impl Bip158Filter { /// Encodes the filter. pub fn encode(&self) -> EncodedFilter { // SAFETY: The `GCSFilter` struct is guaranteed to be valid for the lifetime of the `Bip158Filter` struct. - unsafe { - EncodedFilter(bindings::encode_filter( - &self.0 as *const bindings::GCSFilter, - )) - } + unsafe { EncodedFilter(bindings::encode_filter(self.0)) } } /// Matches a single slice against the filter. @@ -75,7 +71,7 @@ impl Bip158Filter { // struct is guaranteed to be valid for the lifetime of the `Bip158Filter` struct, so this should be safe. unsafe { bindings::filter_match( - &self.0 as *const bindings::GCSFilter, + self.0, bindings::Slice { bytes: slice.as_ptr(), length: slice.len(), @@ -97,13 +93,14 @@ impl Bip158Filter { // SAFETY: The length provided matches the number of slices, and the `GCSFilter` // struct is guaranteed to be valid for the lifetime of the `Bip158Filter` struct, so this should be safe. - unsafe { - bindings::filter_match_any( - &self.0 as *const bindings::GCSFilter, - slices.as_ptr(), - slices.len(), - ) - } + unsafe { bindings::filter_match_any(self.0, slices.as_ptr(), slices.len()) } + } +} + +impl Drop for Bip158Filter { + fn drop(&mut self) { + // SAFETY: The `Slice` struct is guaranteed to be valid for the lifetime of the `EncodedFilter` struct. + unsafe { bindings::free_filter(self.0) } } } diff --git a/rust-bindings/wrapper.cpp b/rust-bindings/wrapper.cpp index c2b2c41..f0c427d 100644 --- a/rust-bindings/wrapper.cpp +++ b/rust-bindings/wrapper.cpp @@ -12,9 +12,9 @@ GCSFilter::ElementSet create_element_set(Slice const* hashes, size_t length) { return elements; } -GCSFilter create_filter(Slice const* hashes, size_t length) { +GCSFilter const* create_filter(Slice const* hashes, size_t length) { auto elements = create_element_set(hashes, length); - return GCSFilter({0, 0, 20, 1 << 20}, elements); + return new GCSFilter({0, 0, 20, 1 << 20}, elements); } Slice encode_filter(GCSFilter const* filter) { @@ -39,4 +39,8 @@ bool filter_match_any(GCSFilter const* filter, Slice const* hashes, size_t lengt void free_slice(Slice slice) { delete[] slice.bytes; +} + +void free_filter(GCSFilter const* filter) { + delete filter; } \ No newline at end of file diff --git a/rust-bindings/wrapper.h b/rust-bindings/wrapper.h index b411ea5..633f395 100644 --- a/rust-bindings/wrapper.h +++ b/rust-bindings/wrapper.h @@ -5,8 +5,9 @@ struct Slice { const size_t length; }; -GCSFilter create_filter(Slice const* hashes, size_t length); +GCSFilter const* create_filter(Slice const* hashes, size_t length); Slice encode_filter(GCSFilter const* filter); bool filter_match(GCSFilter const* filter, Slice hash); bool filter_match_any(GCSFilter const* filter, Slice const* hashes, size_t length); -void free_slice(Slice slice); \ No newline at end of file +void free_slice(Slice slice); +void free_filter(GCSFilter const* filter); \ No newline at end of file From 53962e307570ee40eeb88117b6c2a17e494f4741 Mon Sep 17 00:00:00 2001 From: Rigidity Date: Fri, 15 Mar 2024 09:23:07 -0400 Subject: [PATCH 26/29] Fix typo --- rust-bindings/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-bindings/src/lib.rs b/rust-bindings/src/lib.rs index c522b07..42cc0c7 100644 --- a/rust-bindings/src/lib.rs +++ b/rust-bindings/src/lib.rs @@ -99,7 +99,7 @@ impl Bip158Filter { impl Drop for Bip158Filter { fn drop(&mut self) { - // SAFETY: The `Slice` struct is guaranteed to be valid for the lifetime of the `EncodedFilter` struct. + // SAFETY: The `Slice` struct is guaranteed to be valid for the lifetime of the `Bip158Filter` struct. unsafe { bindings::free_filter(self.0) } } } From 477993696fc4fe3e22eef8e5d1aff00ba83a40ff Mon Sep 17 00:00:00 2001 From: Rigidity Date: Fri, 15 Mar 2024 09:45:37 -0400 Subject: [PATCH 27/29] More test --- rust-bindings/Cargo.lock | 55 ++++++++++++++++++++++++ rust-bindings/Cargo.toml | 4 ++ rust-bindings/src/lib.rs | 93 +++++++++++++++++++++++++++++++--------- 3 files changed, 131 insertions(+), 21 deletions(-) diff --git a/rust-bindings/Cargo.lock b/rust-bindings/Cargo.lock index 74845df..d82a64d 100644 --- a/rust-bindings/Cargo.lock +++ b/rust-bindings/Cargo.lock @@ -82,6 +82,8 @@ version = "0.1.0" dependencies = [ "bindgen", "cc 1.0.90 (git+https://github.com/0123456789-jpg/cc-rs?rev=41a8731)", + "rand", + "rand_chacha", ] [[package]] @@ -119,6 +121,17 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "getrandom" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "glob" version = "0.3.1" @@ -231,6 +244,12 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + [[package]] name = "prettyplease" version = "0.2.16" @@ -259,6 +278,36 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + [[package]] name = "regex" version = "1.10.3" @@ -330,6 +379,12 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + [[package]] name = "which" version = "4.4.2" diff --git a/rust-bindings/Cargo.toml b/rust-bindings/Cargo.toml index ede5242..60e602c 100644 --- a/rust-bindings/Cargo.toml +++ b/rust-bindings/Cargo.toml @@ -19,3 +19,7 @@ bindgen = "0.69.4" # TLDR: --show-sdk-platform-version needs to be replaced with --show-sdk-version # Once this is released, this can be reverted back to the main crate cc = { git = "https://github.com/0123456789-jpg/cc-rs", rev = "41a8731" } + +[dev-dependencies] +rand = "0.8.5" +rand_chacha = "0.3.1" diff --git a/rust-bindings/src/lib.rs b/rust-bindings/src/lib.rs index f9c33ba..1189865 100644 --- a/rust-bindings/src/lib.rs +++ b/rust-bindings/src/lib.rs @@ -107,31 +107,82 @@ impl Bip158Filter { } } -#[test] -fn test_filter() { - let elem1 = b"abc"; - let elem2 = b"xyz"; - let elem3 = b"123"; - let not_elem1 = b"hello"; - let not_elem2 = b"bye"; +#[cfg(test)] +mod tests { + use std::collections::HashSet; - let filter = Bip158Filter::new(&[elem1, elem2, elem3]); + use rand::{RngCore, SeedableRng}; + use rand_chacha::ChaCha8Rng; - let encoded = filter.encode(); - assert_eq!(encoded.as_ref(), [3, 95, 172, 194, 74, 190, 73, 221, 182]); - assert_eq!(&encoded, &encoded); + use super::*; - assert!(filter.matches(elem1)); - assert!(filter.matches(elem2)); - assert!(filter.matches(elem3)); + #[test] + fn test_filter() { + let elem1 = b"abc"; + let elem2 = b"xyz"; + let elem3 = b"123"; + let not_elem1 = b"hello"; + let not_elem2 = b"bye"; - assert!(!filter.matches(not_elem1)); - assert!(!filter.matches(not_elem2)); + let filter = Bip158Filter::new(&[elem1, elem2, elem3]); - assert!(filter.matches_any(&[elem1, elem2, elem3])); - assert!(filter.matches_any(&[elem1, elem2])); - assert!(filter.matches_any(&[elem3])); + let encoded = filter.encode(); + assert_eq!(encoded.as_ref(), [3, 95, 172, 194, 74, 190, 73, 221, 182]); + assert_eq!(&encoded, &encoded); - assert!(!filter.matches_any(&[])); - assert!(!filter.matches_any(&[not_elem1, not_elem2])); + assert!(filter.matches(elem1)); + assert!(filter.matches(elem2)); + assert!(filter.matches(elem3)); + + assert!(!filter.matches(not_elem1)); + assert!(!filter.matches(not_elem2)); + + assert!(filter.matches_any(&[elem1, elem2, elem3])); + assert!(filter.matches_any(&[elem1, elem2])); + assert!(filter.matches_any(&[elem3])); + + assert!(!filter.matches_any(&[])); + assert!(!filter.matches_any(&[not_elem1, not_elem2])); + } + + #[test] + fn test_false_positive() { + let mut rng = ChaCha8Rng::seed_from_u64(0); + + let hashes: Vec<[u8; 4]> = (0..100) + .map(|_| { + let mut hash = [0; 4]; + rng.fill_bytes(&mut hash); + hash + }) + .collect(); + + let mut hash_set = HashSet::new(); + for hash in hashes.iter() { + hash_set.insert(hash); + } + + let refs: Vec<&[u8]> = hashes.iter().map(|hash| hash.as_ref()).collect(); + let filter = Bip158Filter::new(&refs); + + let count = 5000000; + let hashes: Vec<[u8; 4]> = (0..count) + .map(|_| { + let mut hash = [0; 4]; + rng.fill_bytes(&mut hash); + hash + }) + .filter(|hash| !hash_set.contains(hash)) + .collect(); + + let mut matches = 0; + + for hash in hashes { + if filter.matches(&hash) { + matches += 1; + } + } + + assert_eq!(matches, 10); + } } From f5291bc9a032a467cd7de302accea350008ab52f Mon Sep 17 00:00:00 2001 From: Rigidity Date: Fri, 15 Mar 2024 10:11:50 -0400 Subject: [PATCH 28/29] Add another Python test to match Rust --- .gitignore | 3 +- rust-bindings/Cargo.lock | 72 ++++++++++++++++++++++++++++++++++++++++ rust-bindings/Cargo.toml | 1 + rust-bindings/src/lib.rs | 21 ++++++++---- tests/simple_test.py | 34 +++++++++++++++++-- 5 files changed, 122 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index fb32648..4590053 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ /dist /venv /build -/var \ No newline at end of file +/var +__pycache__ \ No newline at end of file diff --git a/rust-bindings/Cargo.lock b/rust-bindings/Cargo.lock index d82a64d..60f43df 100644 --- a/rust-bindings/Cargo.lock +++ b/rust-bindings/Cargo.lock @@ -46,6 +46,15 @@ version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "cc" version = "1.0.90" @@ -84,6 +93,7 @@ dependencies = [ "cc 1.0.90 (git+https://github.com/0123456789-jpg/cc-rs?rev=41a8731)", "rand", "rand_chacha", + "sha2", ] [[package]] @@ -105,6 +115,35 @@ dependencies = [ "libloading", ] +[[package]] +name = "cpufeatures" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "either" version = "1.10.0" @@ -121,6 +160,16 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.12" @@ -356,6 +405,17 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "shlex" version = "1.3.0" @@ -373,12 +433,24 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + [[package]] name = "unicode-ident" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" diff --git a/rust-bindings/Cargo.toml b/rust-bindings/Cargo.toml index 60e602c..b1bcee0 100644 --- a/rust-bindings/Cargo.toml +++ b/rust-bindings/Cargo.toml @@ -23,3 +23,4 @@ cc = { git = "https://github.com/0123456789-jpg/cc-rs", rev = "41a8731" } [dev-dependencies] rand = "0.8.5" rand_chacha = "0.3.1" +sha2 = "0.10.8" diff --git a/rust-bindings/src/lib.rs b/rust-bindings/src/lib.rs index 83c64be..d438a21 100644 --- a/rust-bindings/src/lib.rs +++ b/rust-bindings/src/lib.rs @@ -110,21 +110,28 @@ mod tests { use rand::{RngCore, SeedableRng}; use rand_chacha::ChaCha8Rng; + use sha2::{digest::FixedOutput, Digest, Sha256}; use super::*; + fn hash(bytes: &[u8]) -> [u8; 32] { + let mut hasher = Sha256::new(); + hasher.update(bytes); + hasher.finalize_fixed().into() + } + #[test] fn test_filter() { - let elem1 = b"abc"; - let elem2 = b"xyz"; - let elem3 = b"123"; - let not_elem1 = b"hello"; - let not_elem2 = b"bye"; + let elem1 = &hash(b"abc"); + let elem2 = &hash(b"xyz"); + let elem3 = &hash(b"123"); + let not_elem1 = &hash(b"hello"); + let not_elem2 = &hash(b"bye"); let filter = Bip158Filter::new(&[elem1, elem2, elem3]); let encoded = filter.encode(); - assert_eq!(encoded.as_ref(), [3, 95, 172, 194, 74, 190, 73, 221, 182]); + assert_eq!(encoded.as_ref(), [3, 174, 90, 204, 224, 219, 7, 253, 91]); assert_eq!(&encoded, &encoded); assert!(filter.matches(elem1)); @@ -137,6 +144,8 @@ mod tests { assert!(filter.matches_any(&[elem1, elem2, elem3])); assert!(filter.matches_any(&[elem1, elem2])); assert!(filter.matches_any(&[elem3])); + assert!(filter + .matches_any(&[not_elem1, not_elem1, elem1, not_elem2, not_elem1, not_elem2, elem2])); assert!(!filter.matches_any(&[])); assert!(!filter.matches_any(&[not_elem1, not_elem2])); diff --git a/tests/simple_test.py b/tests/simple_test.py index 6898261..bc723c3 100644 --- a/tests/simple_test.py +++ b/tests/simple_test.py @@ -5,6 +5,36 @@ random.seed(158) +def test_rust_equivalency(): + elem1 = bytearray(sha256(b"abc").digest()) + elem2 = bytearray(sha256(b"xyz").digest()) + elem3 = bytearray(sha256(b"123").digest()) + not_elem1 = bytearray(sha256(b"hello").digest()) + not_elem2 = bytearray(sha256(b"bye").digest()) + + filter = PyBIP158([elem1, elem2, elem3]) + + encoded = filter.GetEncoded() + assert encoded == [3, 174, 90, 204, 224, 219, 7, 253, 91] + + assert filter.Match(elem1) + assert filter.Match(elem2) + assert filter.Match(elem3) + + assert not filter.Match(not_elem1) + assert not filter.Match(not_elem2) + + assert filter.MatchAny([elem1, elem2, elem3]) + assert filter.MatchAny([elem1, elem2]) + assert filter.MatchAny([elem3]) + assert filter.MatchAny( + [not_elem1, not_elem1, elem1, not_elem2, not_elem1, not_elem2, elem2] + ) + + assert not filter.MatchAny([]) + assert not filter.MatchAny([not_elem1, not_elem2]) + + def test_simple(): print("BIP 158 test") @@ -23,9 +53,9 @@ def test_simple(): rando = random.randint(0, 6000) matchhash = bytearray(sha256(rando.to_bytes(4, "big")).digest()) if pl.Match(matchhash): - print(str(rando)+" OK") + print(str(rando) + " OK") else: - print(str(rando)+" not found") + print(str(rando) + " not found") matcharray.append(matchhash) if pl.MatchAny(matcharray): print("OK") From 7293737553d15b0e7552752fcffab1750663ea02 Mon Sep 17 00:00:00 2001 From: Rigidity Date: Fri, 15 Mar 2024 11:03:24 -0400 Subject: [PATCH 29/29] Make slices generic --- rust-bindings/src/lib.rs | 42 ++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/rust-bindings/src/lib.rs b/rust-bindings/src/lib.rs index d438a21..9287734 100644 --- a/rust-bindings/src/lib.rs +++ b/rust-bindings/src/lib.rs @@ -45,15 +45,12 @@ pub struct Bip158Filter(*const bindings::GCSFilter); impl Bip158Filter { /// Encode a BIP 158 filter from a list of slices. - pub fn new(slices: &[&[u8]]) -> Self { + pub fn new(slices: &[T]) -> Self + where + T: AsRef<[u8]>, + { // Convert the slices to a type that C/C++ can understand. - let slices: Vec = slices - .iter() - .map(|slice| bindings::Slice { - bytes: slice.as_ptr(), - length: slice.len(), - }) - .collect(); + let slices: Vec = slices.iter().map(construct_slice).collect(); // SAFETY: The length provided matches the number of slices, so this should be safe. unsafe { Self(bindings::create_filter(slices.as_ptr(), slices.len())) } @@ -81,15 +78,12 @@ impl Bip158Filter { } /// Matches any of a list of slices against the filter. - pub fn matches_any(&self, slices: &[&[u8]]) -> bool { + pub fn matches_any(&self, slices: &[T]) -> bool + where + T: AsRef<[u8]>, + { // Convert the slices to a type that C/C++ can understand. - let slices: Vec = slices - .iter() - .map(|slice| bindings::Slice { - bytes: slice.as_ptr(), - length: slice.len(), - }) - .collect(); + let slices: Vec = slices.iter().map(construct_slice).collect(); // SAFETY: The length provided matches the number of slices, and the `GCSFilter` // struct is guaranteed to be valid for the lifetime of the `Bip158Filter` struct, so this should be safe. @@ -104,6 +98,17 @@ impl Drop for Bip158Filter { } } +fn construct_slice(slice: &T) -> bindings::Slice +where + T: AsRef<[u8]>, +{ + let slice_ref = slice.as_ref(); + bindings::Slice { + bytes: slice_ref.as_ptr(), + length: slice_ref.len(), + } +} + #[cfg(test)] mod tests { use std::collections::HashSet; @@ -147,7 +152,7 @@ mod tests { assert!(filter .matches_any(&[not_elem1, not_elem1, elem1, not_elem2, not_elem1, not_elem2, elem2])); - assert!(!filter.matches_any(&[])); + assert!(!filter.matches_any(Vec::<&[u8]>::new().as_ref())); assert!(!filter.matches_any(&[not_elem1, not_elem2])); } @@ -168,8 +173,7 @@ mod tests { hash_set.insert(hash); } - let refs: Vec<&[u8]> = hashes.iter().map(|hash| hash.as_ref()).collect(); - let filter = Bip158Filter::new(&refs); + let filter = Bip158Filter::new(&hashes); let count = 5000000; let hashes: Vec<[u8; 4]> = (0..count)