zstd-sys: impl wasm32-unknown-unknown support (#139)

* impl wasm32-unknown-unknown support

* Add zdict feature for zstd-sys

* Fix comment

* Fix wasm DEBUGLEVEL :'D

* Add zdict_builder feature to zstd/zstd-safe

* Fix test

Cargo.toml

@@ -13,7 +13,7 @@ readme = "Readme.md"
 edition = "2018"
 
 [package.metadata.docs.rs]
-features = ["experimental", "zstdmt", "doc-cfg"]
+features = ["experimental", "zstdmt", "zdict_builder", "doc-cfg"]
 
 [badges]
 travis-ci = { repository = "gyscos/zstd-rs" }
@@ -28,7 +28,7 @@ partial-io = "0.5"
 walkdir = "2.2"
 
 [features]
-default = ["legacy", "arrays"]
+default = ["legacy", "arrays", "zdict_builder"]
 
 bindgen = ["zstd-safe/bindgen"]
 debug = ["zstd-safe/debug"]
@@ -41,3 +41,8 @@ thin = ["zstd-safe/thin"]
 arrays = ["zstd-safe/arrays"]
 no_asm = ["zstd-safe/no_asm"]
 doc-cfg = []
+zdict_builder = ["zstd-safe/zdict_builder"]
+
+[[example]]
+name = "train"
+required-features = ["zdict_builder"]

src/dict.rs

@@ -14,11 +14,8 @@
 //! [`Encoder::with_dictionary`]: ../struct.Encoder.html#method.with_dictionary
 //! [`Decoder::with_dictionary`]: ../struct.Decoder.html#method.with_dictionary
 
-use crate::map_error_code;
-use std::fs;
-
+#[cfg(feature = "zdict_builder")]
 use std::io::{self, Read};
-use std::path;
 
 pub use zstd_safe::{CDict, DDict};
 
@@ -99,11 +96,14 @@ impl<'a> DecoderDictionary<'a> {
 ///
 /// This is the most efficient way to train a dictionary,
 /// since this is directly fed into `zstd`.
+#[cfg(feature = "zdict_builder")]
 pub fn from_continuous(
     sample_data: &[u8],
     sample_sizes: &[usize],
     max_size: usize,
 ) -> io::Result<Vec<u8>> {
+    use crate::map_error_code;
+
     // Complain if the lengths don't add up to the entire data.
     if sample_sizes.iter().sum::<usize>() != sample_data.len() {
         return Err(io::Error::new(
@@ -127,6 +127,7 @@ pub fn from_continuous(
 /// [`from_continuous`] directly uses the given slice.
 ///
 /// [`from_continuous`]: ./fn.from_continuous.html
+#[cfg(feature = "zdict_builder")]
 pub fn from_samples<S: AsRef<[u8]>>(
     samples: &[S],
     max_size: usize,
@@ -140,11 +141,14 @@ pub fn from_samples<S: AsRef<[u8]>>(
 }
 
 /// Train a dict from a list of files.
+#[cfg(feature = "zdict_builder")]
 pub fn from_files<I, P>(filenames: I, max_size: usize) -> io::Result<Vec<u8>>
 where
-    P: AsRef<path::Path>,
+    P: AsRef<std::path::Path>,
     I: IntoIterator<Item = P>,
 {
+    use std::fs;
+
     let mut buffer = Vec::new();
     let mut sizes = Vec::new();
 
@@ -158,6 +162,7 @@ where
 }
 
 #[cfg(test)]
+#[cfg(feature = "zdict_builder")]
 mod tests {
     use std::fs;
     use std::io;

zstd-safe/Cargo.toml

@@ -12,14 +12,14 @@ readme = "Readme.md"
 edition = "2018"
 
 [package.metadata.docs.rs]
-features = ["experimental", "arrays", "std", "doc-cfg"]
+features = ["experimental", "arrays", "std", "zdict_builder", "doc-cfg"]
 
 [dependencies]
 zstd-sys = { path = "zstd-sys", version = "=1.6.3", default-features = false }
 libc = "0.2.21"
 
 [features]
-default = ["legacy", "arrays"]
+default = ["legacy", "arrays", "zdict_builder"]
 
 bindgen = ["zstd-sys/bindgen"]
 debug = ["zstd-sys/debug"]
@@ -32,3 +32,4 @@ thin = ["zstd-sys/thin"]
 arrays = []
 no_asm = ["zstd-sys/no_asm"]
 doc-cfg = []
+zdict_builder = ["zstd-sys/zdict_builder"]

zstd-safe/src/lib.rs

@@ -2041,6 +2041,7 @@ pub fn cctx_set_pledged_src_size(
 }
 
 /// Wraps the `ZDICT_trainFromBuffer()` function.
+#[cfg(feature = "zdict_builder")]
 pub fn train_from_buffer<C: WriteBuf + ?Sized>(
     dict_buffer: &mut C,
     samples_buffer: &[u8],
@@ -2062,6 +2063,7 @@ pub fn train_from_buffer<C: WriteBuf + ?Sized>(
 }
 
 /// Wraps the `ZSTD_getDictID_fromDict()` function.
+#[cfg(feature = "zdict_builder")]
 pub fn get_dict_id(dict_buffer: &[u8]) -> Option<u32> {
     let id = unsafe {
         zstd_sys::ZDICT_getDictID(ptr_void(dict_buffer), dict_buffer.len())

zstd-safe/zstd-sys/Cargo.toml

@@ -61,7 +61,7 @@ features = ["parallel"]
 libc = "0.2.45"
 
 [features]
-default = ["legacy"]
+default = ["legacy", "zdict_builder"]
 
 debug = [] # Enable zstd debug logs
 experimental = [] # Expose experimental ZSTD API
@@ -71,3 +71,4 @@ std = [] # Use std types instead of libc in bindgen
 zstdmt = [] # Enable multi-thread support (with pthread)
 thin = [] # Optimize binary by size
 no_asm = [] # Disable ASM files (only on amd64 for decompression)
+zdict_builder = []

zstd-safe/zstd-sys/build.rs

@@ -5,8 +5,10 @@ use std::{env, fs};
 #[cfg(feature = "bindgen")]
 fn generate_bindings(defs: Vec<&str>, headerpaths: Vec<PathBuf>) {
     let bindings = bindgen::Builder::default()
-        .header("zstd.h")
-        .header("zdict.h")
+        .header("zstd.h");
+    #[cfg(feature = "zdict_builder")]
+    let bindings = bindings.header("zdict.h");
+    let bindings = bindings
         .blocklist_type("max_align_t")
         .size_t_is_usize(true)
         .use_core()
@@ -19,9 +21,9 @@ fn generate_bindings(defs: Vec<&str>, headerpaths: Vec<PathBuf>) {
         .clang_args(defs.into_iter().map(|def| format!("-D{}", def)));
 
     #[cfg(feature = "experimental")]
-    let bindings = bindings
-        .clang_arg("-DZSTD_STATIC_LINKING_ONLY")
-        .clang_arg("-DZDICT_STATIC_LINKING_ONLY");
+    let bindings = bindings.clang_arg("-DZSTD_STATIC_LINKING_ONLY");
+    #[cfg(all(feature = "experimental", feature = "zdict_builder"))]
+    let bindings = bindings.clang_arg("-DZDICT_STATIC_LINKING_ONLY");
 
     #[cfg(not(feature = "std"))]
     let bindings = bindings.ctypes_prefix("libc");
@@ -85,6 +87,7 @@ fn compile_zstd() {
         "zstd/lib/common",
         "zstd/lib/compress",
         "zstd/lib/decompress",
+        #[cfg(feature = "zdict_builder")]
         "zstd/lib/dictBuilder",
         #[cfg(feature = "legacy")]
         "zstd/lib/legacy",
@@ -114,6 +117,16 @@ fn compile_zstd() {
         config.file("zstd/lib/decompress/huf_decompress_amd64.S");
     }
 
+    let is_wasm_unknown_unknown = env::var("TARGET").ok() == Some("wasm32-unknown-unknown".into());
+
+    if is_wasm_unknown_unknown {
+        println!("cargo:rerun-if-changed=wasm-shim/stdlib.h");
+        println!("cargo:rerun-if-changed=wasm-shim/string.h");
+
+        config.include("wasm-shim/");
+        config.define("XXH_STATIC_ASSERT", Some("0"));
+    }
+
     // Some extra parameters
     config.opt_level(3);
     config.include("zstd/lib/");
@@ -137,6 +150,7 @@ fn compile_zstd() {
     config.flag("-fvisibility=hidden");
     config.define("XXH_PRIVATE_API", Some(""));
     config.define("ZSTDLIB_VISIBILITY", Some(""));
+    #[cfg(feature = "zdict_builder")]
     config.define("ZDICTLIB_VISIBILITY", Some(""));
     config.define("ZSTDERRORLIB_VISIBILITY", Some(""));
 
@@ -152,7 +166,9 @@ fn compile_zstd() {
      * 7+: events at every position (*very* verbose)
      */
     #[cfg(feature = "debug")]
-    config.define("DEBUGLEVEL", Some("5"));
+    if !is_wasm_unknown_unknown {
+        config.define("DEBUGLEVEL", Some("5"));
+    }
 
     set_pthread(&mut config);
     set_legacy(&mut config);
@@ -168,6 +184,7 @@ fn compile_zstd() {
     fs::copy(src.join("zstd.h"), include.join("zstd.h")).unwrap();
     fs::copy(src.join("zstd_errors.h"), include.join("zstd_errors.h"))
         .unwrap();
+    #[cfg(feature = "zdict_builder")]
     fs::copy(src.join("zdict.h"), include.join("zdict.h")).unwrap();
     println!("cargo:root={}", dst.display());
 }

zstd-safe/zstd-sys/examples/it_work.rs

@@ -0,0 +1,50 @@
+use std::convert::TryInto;
+
+#[no_mangle]
+pub extern "C" fn zstd_version() -> u32 {
+    unsafe {
+        zstd_sys::ZSTD_versionNumber()
+    }
+}
+
+macro_rules! zstd_check {
+    ( $ret:expr ) => {{
+        let ret = $ret;
+        let error_code = unsafe {
+            zstd_sys::ZSTD_isError(ret)
+        };
+        assert_eq!(error_code, 0);
+    }}
+}
+
+#[no_mangle]
+pub extern "C" fn test_compress() -> bool {
+    let fbuf = include_bytes!("../Cargo.toml");
+
+    let cbufsize = unsafe {
+        zstd_sys::ZSTD_compressBound(fbuf.len())
+    };
+    let mut cbuf = vec![0; cbufsize];
+
+    let csize = unsafe {
+        zstd_sys::ZSTD_compress(cbuf.as_mut_ptr().cast(), cbuf.len(), fbuf.as_ptr().cast(), fbuf.len(), 1)
+    };
+    zstd_check!(csize);
+    let cbuf = &cbuf[..csize];
+
+    let rsize = unsafe {
+        zstd_sys::ZSTD_getFrameContentSize(cbuf.as_ptr().cast(), cbuf.len())
+    };
+    let rsize = rsize.try_into().unwrap();
+    let mut rbuf = vec![0; rsize];
+
+    let dsize = unsafe {
+        zstd_sys::ZSTD_decompress(rbuf.as_mut_ptr().cast(), rbuf.len(), cbuf.as_ptr().cast(), cbuf.len())
+    };
+    zstd_check!(dsize);
+    assert_eq!(dsize, rsize);
+
+    &fbuf[..] == &rbuf[..]
+}
+
+fn main() {}

zstd-safe/zstd-sys/src/lib.rs

@@ -9,6 +9,9 @@
 #[cfg(feature = "std")]
 extern crate std;
 
+#[cfg(target_arch = "wasm32")]
+mod wasm_shim;
+
 // If running bindgen, we'll end up with the correct bindings anyway.
 #[cfg(feature = "bindgen")]
 include!(concat!(env!("OUT_DIR"), "/bindings.rs"));
@@ -26,6 +29,7 @@ include!("bindings_zstd.rs");
 #[cfg(all(
     not(feature = "std"),
     not(feature = "experimental"),
+    feature = "zdict_builder",
     not(feature = "bindgen")
 ))]
 include!("bindings_zdict.rs");
@@ -40,6 +44,7 @@ include!("bindings_zstd_experimental.rs");
 #[cfg(all(
     not(feature = "std"),
     feature = "experimental",
+    feature = "zdict_builder",
     not(feature = "bindgen")
 ))]
 include!("bindings_zdict_experimental.rs");
@@ -56,6 +61,7 @@ include!("bindings_zstd_std.rs");
 #[cfg(all(
     feature = "std",
     not(feature = "experimental"),
+    feature = "zdict_builder",
     not(feature = "bindgen")
 ))]
 include!("bindings_zdict_std.rs");
@@ -70,6 +76,7 @@ include!("bindings_zstd_std_experimental.rs");
 #[cfg(all(
     feature = "std",
     feature = "experimental",
+    feature = "zdict_builder",
     not(feature = "bindgen")
 ))]
 include!("bindings_zdict_std_experimental.rs");

zstd-safe/zstd-sys/src/wasm_shim.rs

@@ -0,0 +1,44 @@
+use std::os::raw::{c_void, c_int};
+use std::alloc::{alloc, dealloc, Layout};
+
+
+#[no_mangle]
+pub extern "C" fn rust_zstd_wasm_shim_malloc(size: usize) -> *mut c_void {
+    unsafe {
+        let layout = Layout::from_size_align_unchecked(size, 1);
+        alloc(layout).cast()
+    }
+}
+
+#[no_mangle]
+pub extern "C" fn rust_zstd_wasm_shim_calloc(nmemb: usize, size: usize) -> *mut c_void {
+    unsafe {
+        let layout = Layout::from_size_align_unchecked(size * nmemb, 1);
+        alloc(layout).cast()
+    }
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn rust_zstd_wasm_shim_free(ptr: *mut c_void) {
+    // layout is not actually used
+    let layout = Layout::from_size_align_unchecked(1, 1);
+    dealloc(ptr.cast(), layout);
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn rust_zstd_wasm_shim_memcpy(dest: *mut c_void, src: *const c_void, n: usize) -> *mut c_void {
+    std::ptr::copy_nonoverlapping(src as *const u8, dest as *mut u8, n);
+    dest
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn rust_zstd_wasm_shim_memmove(dest: *mut c_void, src: *const c_void, n: usize) -> *mut c_void {
+    std::ptr::copy(src as *const u8, dest as *mut u8, n);
+    dest
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn rust_zstd_wasm_shim_memset(dest: *mut c_void, c: c_int, n: usize) -> *mut c_void {
+    std::ptr::write_bytes(dest as *mut u8, c as u8, n);
+    dest
+}

zstd-safe/zstd-sys/wasm-shim/stdlib.h

@@ -0,0 +1,22 @@
+#include <stddef.h>
+
+#ifndef	_STDLIB_H
+#define	_STDLIB_H	1
+
+void *rust_zstd_wasm_shim_malloc(size_t size);
+void *rust_zstd_wasm_shim_calloc(size_t nmemb, size_t size);
+void rust_zstd_wasm_shim_free(void *ptr);
+
+inline void *malloc(size_t size) {
+	return rust_zstd_wasm_shim_malloc(size);
+}
+
+inline void *calloc(size_t nmemb, size_t size) {
+	return rust_zstd_wasm_shim_calloc(nmemb, size);
+}
+
+inline void free(void *ptr) {
+	rust_zstd_wasm_shim_free(ptr);
+}
+
+#endif // _STDLIB_H