diff --git a/Cargo.toml b/Cargo.toml index 5e72fdfc..52ed1c03 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ readme = "Readme.md" edition = "2018" [package.metadata.docs.rs] -features = ["experimental", "zstdmt", "doc-cfg"] +features = ["experimental", "zstdmt", "zdict_builder", "doc-cfg"] [badges] travis-ci = { repository = "gyscos/zstd-rs" } @@ -28,7 +28,7 @@ partial-io = "0.5" walkdir = "2.2" [features] -default = ["legacy", "arrays"] +default = ["legacy", "arrays", "zdict_builder"] bindgen = ["zstd-safe/bindgen"] debug = ["zstd-safe/debug"] @@ -41,3 +41,8 @@ thin = ["zstd-safe/thin"] arrays = ["zstd-safe/arrays"] no_asm = ["zstd-safe/no_asm"] doc-cfg = [] +zdict_builder = ["zstd-safe/zdict_builder"] + +[[example]] +name = "train" +required-features = ["zdict_builder"] diff --git a/src/dict.rs b/src/dict.rs index bef6ab60..ba278376 100644 --- a/src/dict.rs +++ b/src/dict.rs @@ -14,11 +14,8 @@ //! [`Encoder::with_dictionary`]: ../struct.Encoder.html#method.with_dictionary //! [`Decoder::with_dictionary`]: ../struct.Decoder.html#method.with_dictionary -use crate::map_error_code; -use std::fs; - +#[cfg(feature = "zdict_builder")] use std::io::{self, Read}; -use std::path; pub use zstd_safe::{CDict, DDict}; @@ -99,11 +96,14 @@ impl<'a> DecoderDictionary<'a> { /// /// This is the most efficient way to train a dictionary, /// since this is directly fed into `zstd`. +#[cfg(feature = "zdict_builder")] pub fn from_continuous( sample_data: &[u8], sample_sizes: &[usize], max_size: usize, ) -> io::Result> { + use crate::map_error_code; + // Complain if the lengths don't add up to the entire data. if sample_sizes.iter().sum::() != sample_data.len() { return Err(io::Error::new( @@ -127,6 +127,7 @@ pub fn from_continuous( /// [`from_continuous`] directly uses the given slice. /// /// [`from_continuous`]: ./fn.from_continuous.html +#[cfg(feature = "zdict_builder")] pub fn from_samples>( samples: &[S], max_size: usize, @@ -140,11 +141,14 @@ pub fn from_samples>( } /// Train a dict from a list of files. +#[cfg(feature = "zdict_builder")] pub fn from_files(filenames: I, max_size: usize) -> io::Result> where - P: AsRef, + P: AsRef, I: IntoIterator, { + use std::fs; + let mut buffer = Vec::new(); let mut sizes = Vec::new(); @@ -158,6 +162,7 @@ where } #[cfg(test)] +#[cfg(feature = "zdict_builder")] mod tests { use std::fs; use std::io; diff --git a/zstd-safe/Cargo.toml b/zstd-safe/Cargo.toml index 9e39fd2c..83c34d17 100644 --- a/zstd-safe/Cargo.toml +++ b/zstd-safe/Cargo.toml @@ -12,14 +12,14 @@ readme = "Readme.md" edition = "2018" [package.metadata.docs.rs] -features = ["experimental", "arrays", "std", "doc-cfg"] +features = ["experimental", "arrays", "std", "zdict_builder", "doc-cfg"] [dependencies] zstd-sys = { path = "zstd-sys", version = "=1.6.3", default-features = false } libc = "0.2.21" [features] -default = ["legacy", "arrays"] +default = ["legacy", "arrays", "zdict_builder"] bindgen = ["zstd-sys/bindgen"] debug = ["zstd-sys/debug"] @@ -32,3 +32,4 @@ thin = ["zstd-sys/thin"] arrays = [] no_asm = ["zstd-sys/no_asm"] doc-cfg = [] +zdict_builder = ["zstd-sys/zdict_builder"] diff --git a/zstd-safe/src/lib.rs b/zstd-safe/src/lib.rs index 15bcd855..8e5758be 100644 --- a/zstd-safe/src/lib.rs +++ b/zstd-safe/src/lib.rs @@ -2041,6 +2041,7 @@ pub fn cctx_set_pledged_src_size( } /// Wraps the `ZDICT_trainFromBuffer()` function. +#[cfg(feature = "zdict_builder")] pub fn train_from_buffer( dict_buffer: &mut C, samples_buffer: &[u8], @@ -2062,6 +2063,7 @@ pub fn train_from_buffer( } /// Wraps the `ZSTD_getDictID_fromDict()` function. +#[cfg(feature = "zdict_builder")] pub fn get_dict_id(dict_buffer: &[u8]) -> Option { let id = unsafe { zstd_sys::ZDICT_getDictID(ptr_void(dict_buffer), dict_buffer.len()) diff --git a/zstd-safe/zstd-sys/Cargo.toml b/zstd-safe/zstd-sys/Cargo.toml index 4e3cc943..d2d388f3 100644 --- a/zstd-safe/zstd-sys/Cargo.toml +++ b/zstd-safe/zstd-sys/Cargo.toml @@ -61,7 +61,7 @@ features = ["parallel"] libc = "0.2.45" [features] -default = ["legacy"] +default = ["legacy", "zdict_builder"] debug = [] # Enable zstd debug logs experimental = [] # Expose experimental ZSTD API @@ -71,3 +71,4 @@ std = [] # Use std types instead of libc in bindgen zstdmt = [] # Enable multi-thread support (with pthread) thin = [] # Optimize binary by size no_asm = [] # Disable ASM files (only on amd64 for decompression) +zdict_builder = [] diff --git a/zstd-safe/zstd-sys/build.rs b/zstd-safe/zstd-sys/build.rs index ce6e7c53..83f2dc28 100644 --- a/zstd-safe/zstd-sys/build.rs +++ b/zstd-safe/zstd-sys/build.rs @@ -5,8 +5,10 @@ use std::{env, fs}; #[cfg(feature = "bindgen")] fn generate_bindings(defs: Vec<&str>, headerpaths: Vec) { let bindings = bindgen::Builder::default() - .header("zstd.h") - .header("zdict.h") + .header("zstd.h"); + #[cfg(feature = "zdict_builder")] + let bindings = bindings.header("zdict.h"); + let bindings = bindings .blocklist_type("max_align_t") .size_t_is_usize(true) .use_core() @@ -19,9 +21,9 @@ fn generate_bindings(defs: Vec<&str>, headerpaths: Vec) { .clang_args(defs.into_iter().map(|def| format!("-D{}", def))); #[cfg(feature = "experimental")] - let bindings = bindings - .clang_arg("-DZSTD_STATIC_LINKING_ONLY") - .clang_arg("-DZDICT_STATIC_LINKING_ONLY"); + let bindings = bindings.clang_arg("-DZSTD_STATIC_LINKING_ONLY"); + #[cfg(all(feature = "experimental", feature = "zdict_builder"))] + let bindings = bindings.clang_arg("-DZDICT_STATIC_LINKING_ONLY"); #[cfg(not(feature = "std"))] let bindings = bindings.ctypes_prefix("libc"); @@ -85,6 +87,7 @@ fn compile_zstd() { "zstd/lib/common", "zstd/lib/compress", "zstd/lib/decompress", + #[cfg(feature = "zdict_builder")] "zstd/lib/dictBuilder", #[cfg(feature = "legacy")] "zstd/lib/legacy", @@ -114,6 +117,16 @@ fn compile_zstd() { config.file("zstd/lib/decompress/huf_decompress_amd64.S"); } + let is_wasm_unknown_unknown = env::var("TARGET").ok() == Some("wasm32-unknown-unknown".into()); + + if is_wasm_unknown_unknown { + println!("cargo:rerun-if-changed=wasm-shim/stdlib.h"); + println!("cargo:rerun-if-changed=wasm-shim/string.h"); + + config.include("wasm-shim/"); + config.define("XXH_STATIC_ASSERT", Some("0")); + } + // Some extra parameters config.opt_level(3); config.include("zstd/lib/"); @@ -137,6 +150,7 @@ fn compile_zstd() { config.flag("-fvisibility=hidden"); config.define("XXH_PRIVATE_API", Some("")); config.define("ZSTDLIB_VISIBILITY", Some("")); + #[cfg(feature = "zdict_builder")] config.define("ZDICTLIB_VISIBILITY", Some("")); config.define("ZSTDERRORLIB_VISIBILITY", Some("")); @@ -152,7 +166,9 @@ fn compile_zstd() { * 7+: events at every position (*very* verbose) */ #[cfg(feature = "debug")] - config.define("DEBUGLEVEL", Some("5")); + if !is_wasm_unknown_unknown { + config.define("DEBUGLEVEL", Some("5")); + } set_pthread(&mut config); set_legacy(&mut config); @@ -168,6 +184,7 @@ fn compile_zstd() { fs::copy(src.join("zstd.h"), include.join("zstd.h")).unwrap(); fs::copy(src.join("zstd_errors.h"), include.join("zstd_errors.h")) .unwrap(); + #[cfg(feature = "zdict_builder")] fs::copy(src.join("zdict.h"), include.join("zdict.h")).unwrap(); println!("cargo:root={}", dst.display()); } diff --git a/zstd-safe/zstd-sys/examples/it_work.rs b/zstd-safe/zstd-sys/examples/it_work.rs new file mode 100644 index 00000000..1de8a829 --- /dev/null +++ b/zstd-safe/zstd-sys/examples/it_work.rs @@ -0,0 +1,50 @@ +use std::convert::TryInto; + +#[no_mangle] +pub extern "C" fn zstd_version() -> u32 { + unsafe { + zstd_sys::ZSTD_versionNumber() + } +} + +macro_rules! zstd_check { + ( $ret:expr ) => {{ + let ret = $ret; + let error_code = unsafe { + zstd_sys::ZSTD_isError(ret) + }; + assert_eq!(error_code, 0); + }} +} + +#[no_mangle] +pub extern "C" fn test_compress() -> bool { + let fbuf = include_bytes!("../Cargo.toml"); + + let cbufsize = unsafe { + zstd_sys::ZSTD_compressBound(fbuf.len()) + }; + let mut cbuf = vec![0; cbufsize]; + + let csize = unsafe { + zstd_sys::ZSTD_compress(cbuf.as_mut_ptr().cast(), cbuf.len(), fbuf.as_ptr().cast(), fbuf.len(), 1) + }; + zstd_check!(csize); + let cbuf = &cbuf[..csize]; + + let rsize = unsafe { + zstd_sys::ZSTD_getFrameContentSize(cbuf.as_ptr().cast(), cbuf.len()) + }; + let rsize = rsize.try_into().unwrap(); + let mut rbuf = vec![0; rsize]; + + let dsize = unsafe { + zstd_sys::ZSTD_decompress(rbuf.as_mut_ptr().cast(), rbuf.len(), cbuf.as_ptr().cast(), cbuf.len()) + }; + zstd_check!(dsize); + assert_eq!(dsize, rsize); + + &fbuf[..] == &rbuf[..] +} + +fn main() {} diff --git a/zstd-safe/zstd-sys/src/lib.rs b/zstd-safe/zstd-sys/src/lib.rs index d6761966..7c41b0a8 100644 --- a/zstd-safe/zstd-sys/src/lib.rs +++ b/zstd-safe/zstd-sys/src/lib.rs @@ -9,6 +9,9 @@ #[cfg(feature = "std")] extern crate std; +#[cfg(target_arch = "wasm32")] +mod wasm_shim; + // If running bindgen, we'll end up with the correct bindings anyway. #[cfg(feature = "bindgen")] include!(concat!(env!("OUT_DIR"), "/bindings.rs")); @@ -26,6 +29,7 @@ include!("bindings_zstd.rs"); #[cfg(all( not(feature = "std"), not(feature = "experimental"), + feature = "zdict_builder", not(feature = "bindgen") ))] include!("bindings_zdict.rs"); @@ -40,6 +44,7 @@ include!("bindings_zstd_experimental.rs"); #[cfg(all( not(feature = "std"), feature = "experimental", + feature = "zdict_builder", not(feature = "bindgen") ))] include!("bindings_zdict_experimental.rs"); @@ -56,6 +61,7 @@ include!("bindings_zstd_std.rs"); #[cfg(all( feature = "std", not(feature = "experimental"), + feature = "zdict_builder", not(feature = "bindgen") ))] include!("bindings_zdict_std.rs"); @@ -70,6 +76,7 @@ include!("bindings_zstd_std_experimental.rs"); #[cfg(all( feature = "std", feature = "experimental", + feature = "zdict_builder", not(feature = "bindgen") ))] include!("bindings_zdict_std_experimental.rs"); diff --git a/zstd-safe/zstd-sys/src/wasm_shim.rs b/zstd-safe/zstd-sys/src/wasm_shim.rs new file mode 100644 index 00000000..875a7440 --- /dev/null +++ b/zstd-safe/zstd-sys/src/wasm_shim.rs @@ -0,0 +1,44 @@ +use std::os::raw::{c_void, c_int}; +use std::alloc::{alloc, dealloc, Layout}; + + +#[no_mangle] +pub extern "C" fn rust_zstd_wasm_shim_malloc(size: usize) -> *mut c_void { + unsafe { + let layout = Layout::from_size_align_unchecked(size, 1); + alloc(layout).cast() + } +} + +#[no_mangle] +pub extern "C" fn rust_zstd_wasm_shim_calloc(nmemb: usize, size: usize) -> *mut c_void { + unsafe { + let layout = Layout::from_size_align_unchecked(size * nmemb, 1); + alloc(layout).cast() + } +} + +#[no_mangle] +pub unsafe extern "C" fn rust_zstd_wasm_shim_free(ptr: *mut c_void) { + // layout is not actually used + let layout = Layout::from_size_align_unchecked(1, 1); + dealloc(ptr.cast(), layout); +} + +#[no_mangle] +pub unsafe extern "C" fn rust_zstd_wasm_shim_memcpy(dest: *mut c_void, src: *const c_void, n: usize) -> *mut c_void { + std::ptr::copy_nonoverlapping(src as *const u8, dest as *mut u8, n); + dest +} + +#[no_mangle] +pub unsafe extern "C" fn rust_zstd_wasm_shim_memmove(dest: *mut c_void, src: *const c_void, n: usize) -> *mut c_void { + std::ptr::copy(src as *const u8, dest as *mut u8, n); + dest +} + +#[no_mangle] +pub unsafe extern "C" fn rust_zstd_wasm_shim_memset(dest: *mut c_void, c: c_int, n: usize) -> *mut c_void { + std::ptr::write_bytes(dest as *mut u8, c as u8, n); + dest +} diff --git a/zstd-safe/zstd-sys/wasm-shim/stdlib.h b/zstd-safe/zstd-sys/wasm-shim/stdlib.h new file mode 100644 index 00000000..be102cc2 --- /dev/null +++ b/zstd-safe/zstd-sys/wasm-shim/stdlib.h @@ -0,0 +1,22 @@ +#include + +#ifndef _STDLIB_H +#define _STDLIB_H 1 + +void *rust_zstd_wasm_shim_malloc(size_t size); +void *rust_zstd_wasm_shim_calloc(size_t nmemb, size_t size); +void rust_zstd_wasm_shim_free(void *ptr); + +inline void *malloc(size_t size) { + return rust_zstd_wasm_shim_malloc(size); +} + +inline void *calloc(size_t nmemb, size_t size) { + return rust_zstd_wasm_shim_calloc(nmemb, size); +} + +inline void free(void *ptr) { + rust_zstd_wasm_shim_free(ptr); +} + +#endif // _STDLIB_H diff --git a/zstd-safe/zstd-sys/wasm-shim/string.h b/zstd-safe/zstd-sys/wasm-shim/string.h new file mode 100644 index 00000000..b0abf79c --- /dev/null +++ b/zstd-safe/zstd-sys/wasm-shim/string.h @@ -0,0 +1,22 @@ +#include + +#ifndef _STRING_H +#define _STRING_H 1 + +void *rust_zstd_wasm_shim_memcpy(void *restrict dest, const void *restrict src, size_t n); +void *rust_zstd_wasm_shim_memmove(void *dest, const void *src, size_t n); +void *rust_zstd_wasm_shim_memset(void *dest, int c, size_t n); + +inline void *memcpy(void *restrict dest, const void *restrict src, size_t n) { + return rust_zstd_wasm_shim_memcpy(dest, src, n); +} + +inline void *memmove(void *dest, const void *src, size_t n) { + return rust_zstd_wasm_shim_memmove(dest, src, n); +} + +inline void *memset(void *dest, int c, size_t n) { + return rust_zstd_wasm_shim_memset(dest, c, n); +} + +#endif // _STRING_H