zstd-sys: impl wasm32-unknown-unknown support

zstd-safe/zstd-sys/Cargo.toml

@@ -61,7 +61,7 @@ features = ["parallel"]
 libc = "0.2.45"
 
 [features]
-default = ["legacy"]
+default = ["legacy", "zdict"]
 
 debug = [] # Enable zstd debug logs
 experimental = [] # Expose experimental ZSTD API
@@ -71,3 +71,4 @@ std = [] # Use std types instead of libc in bindgen
 zstdmt = [] # Enable multi-thread support (with pthread)
 thin = [] # Optimize binary by size
 no_asm = [] # Disable ASM files (only on amd64 for decompression)
+zdict = []

zstd-safe/zstd-sys/build.rs

@@ -5,8 +5,10 @@ use std::{env, fs};
 #[cfg(feature = "bindgen")]
 fn generate_bindings(defs: Vec<&str>, headerpaths: Vec<PathBuf>) {
     let bindings = bindgen::Builder::default()
-        .header("zstd.h")
-        .header("zdict.h")
+        .header("zstd.h");
+    #[cfg(feature = "zdict")]
+    let bindings = bindings.header("zdict.h");
+    let bindings = bindings
         .blocklist_type("max_align_t")
         .size_t_is_usize(true)
         .use_core()
@@ -19,9 +21,9 @@ fn generate_bindings(defs: Vec<&str>, headerpaths: Vec<PathBuf>) {
         .clang_args(defs.into_iter().map(|def| format!("-D{}", def)));
 
     #[cfg(feature = "experimental")]
-    let bindings = bindings
-        .clang_arg("-DZSTD_STATIC_LINKING_ONLY")
-        .clang_arg("-DZDICT_STATIC_LINKING_ONLY");
+    let bindings = bindings.clang_arg("-DZSTD_STATIC_LINKING_ONLY");
+    #[cfg(all(feature = "experimental", feature = "zdict"))]
+    let bindings = bindings.clang_arg("-DZDICT_STATIC_LINKING_ONLY");
 
     #[cfg(not(feature = "std"))]
     let bindings = bindings.ctypes_prefix("libc");
@@ -85,6 +87,7 @@ fn compile_zstd() {
         "zstd/lib/common",
         "zstd/lib/compress",
         "zstd/lib/decompress",
+        #[cfg(feature = "zdict")]
         "zstd/lib/dictBuilder",
         #[cfg(feature = "legacy")]
         "zstd/lib/legacy",
@@ -114,6 +117,16 @@ fn compile_zstd() {
         config.file("zstd/lib/decompress/huf_decompress_amd64.S");
     }
 
+    let is_wasm_unknown_unknown = env::var("TARGET").ok() == Some("wasm32-unknown-unknown".into());
+
+    if is_wasm_unknown_unknown {
+        println!("cargo:rerun-if-changed=wasm-shim/stdlib.h");
+        println!("cargo:rerun-if-changed=wasm-shim/string.h");
+
+        config.include("wasm-shim/");
+        config.define("XXH_STATIC_ASSERT", Some("0"));
+    }
+
     // Some extra parameters
     config.opt_level(3);
     config.include("zstd/lib/");
@@ -137,6 +150,7 @@ fn compile_zstd() {
     config.flag("-fvisibility=hidden");
     config.define("XXH_PRIVATE_API", Some(""));
     config.define("ZSTDLIB_VISIBILITY", Some(""));
+    #[cfg(feature = "zdict")]
     config.define("ZDICTLIB_VISIBILITY", Some(""));
     config.define("ZSTDERRORLIB_VISIBILITY", Some(""));
 
@@ -152,7 +166,9 @@ fn compile_zstd() {
      * 7+: events at every position (*very* verbose)
      */
     #[cfg(feature = "debug")]
-    config.define("DEBUGLEVEL", Some("5"));
+    if is_wasm_unknown_unknown {
+        config.define("DEBUGLEVEL", Some("5"));
+    }
 
     set_pthread(&mut config);
     set_legacy(&mut config);
@@ -168,6 +184,7 @@ fn compile_zstd() {
     fs::copy(src.join("zstd.h"), include.join("zstd.h")).unwrap();
     fs::copy(src.join("zstd_errors.h"), include.join("zstd_errors.h"))
         .unwrap();
+    #[cfg(feature = "zdict")]
     fs::copy(src.join("zdict.h"), include.join("zdict.h")).unwrap();
     println!("cargo:root={}", dst.display());
 }

zstd-safe/zstd-sys/examples/it_work.rs

@@ -0,0 +1,50 @@
+use std::convert::TryInto;
+
+#[no_mangle]
+pub extern "C" fn zstd_version() -> u32 {
+    unsafe {
+        zstd_sys::ZSTD_versionNumber()
+    }
+}
+
+macro_rules! zstd_check {
+    ( $ret:expr ) => {{
+        let ret = $ret;
+        let error_code = unsafe {
+            zstd_sys::ZSTD_isError(ret)
+        };
+        assert_eq!(error_code, 0);
+    }}
+}
+
+#[no_mangle]
+pub extern "C" fn test_compress() -> bool {
+    let fbuf = include_bytes!("../Cargo.toml");
+
+    let cbufsize = unsafe {
+        zstd_sys::ZSTD_compressBound(fbuf.len())
+    };
+    let mut cbuf = vec![0; cbufsize];
+
+    let csize = unsafe {
+        zstd_sys::ZSTD_compress(cbuf.as_mut_ptr().cast(), cbuf.len(), fbuf.as_ptr().cast(), fbuf.len(), 1)
+    };
+    zstd_check!(csize);
+    let cbuf = &cbuf[..csize];
+
+    let rsize = unsafe {
+        zstd_sys::ZSTD_getFrameContentSize(cbuf.as_ptr().cast(), cbuf.len())
+    };
+    let rsize = rsize.try_into().unwrap();
+    let mut rbuf = vec![0; rsize];
+
+    let dsize = unsafe {
+        zstd_sys::ZSTD_decompress(rbuf.as_mut_ptr().cast(), rbuf.len(), cbuf.as_ptr().cast(), cbuf.len())
+    };
+    zstd_check!(dsize);
+    assert_eq!(dsize, rsize);
+
+    &fbuf[..] == &rbuf[..]
+}
+
+fn main() {}

zstd-safe/zstd-sys/src/lib.rs

@@ -9,6 +9,9 @@
 #[cfg(feature = "std")]
 extern crate std;
 
+#[cfg(target_arch = "wasm32")]
+mod wasm_shim;
+
 // If running bindgen, we'll end up with the correct bindings anyway.
 #[cfg(feature = "bindgen")]
 include!(concat!(env!("OUT_DIR"), "/bindings.rs"));
@@ -26,6 +29,7 @@ include!("bindings_zstd.rs");
 #[cfg(all(
     not(feature = "std"),
     not(feature = "experimental"),
+    feature = "zdict",
     not(feature = "bindgen")
 ))]
 include!("bindings_zdict.rs");
@@ -40,6 +44,7 @@ include!("bindings_zstd_experimental.rs");
 #[cfg(all(
     not(feature = "std"),
     feature = "experimental",
+    feature = "zdict",
     not(feature = "bindgen")
 ))]
 include!("bindings_zdict_experimental.rs");
@@ -56,6 +61,7 @@ include!("bindings_zstd_std.rs");
 #[cfg(all(
     feature = "std",
     not(feature = "experimental"),
+    feature = "zdict",
     not(feature = "bindgen")
 ))]
 include!("bindings_zdict_std.rs");
@@ -70,6 +76,7 @@ include!("bindings_zstd_std_experimental.rs");
 #[cfg(all(
     feature = "std",
     feature = "experimental",
+    feature = "zdict",
     not(feature = "bindgen")
 ))]
 include!("bindings_zdict_std_experimental.rs");

zstd-safe/zstd-sys/src/wasm_shim.rs

@@ -0,0 +1,44 @@
+use std::os::raw::{c_void, c_int};
+use std::alloc::{alloc, dealloc, Layout};
+
+
+#[no_mangle]
+pub extern "C" fn rust_zstd_wasm_shim_malloc(size: usize) -> *mut c_void {
+    unsafe {
+        let layout = Layout::from_size_align_unchecked(size, 1);
+        alloc(layout).cast()
+    }
+}
+
+#[no_mangle]
+pub extern "C" fn rust_zstd_wasm_shim_calloc(nmemb: usize, size: usize) -> *mut c_void {
+    unsafe {
+        let layout = Layout::from_size_align_unchecked(size * nmemb, 1);
+        alloc(layout).cast()
+    }
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn rust_zstd_wasm_shim_free(ptr: *mut c_void) {
+    // layout is not actually used
+    let layout = Layout::from_size_align_unchecked(1, 1);
+    dealloc(ptr.cast(), layout);
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn rust_zstd_wasm_shim_memcpy(dest: *mut c_void, src: *const c_void, n: usize) -> *mut c_void {
+    std::ptr::copy_nonoverlapping(src as *const u8, dest as *mut u8, n);
+    dest
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn rust_zstd_wasm_shim_memmove(dest: *mut c_void, src: *const c_void, n: usize) -> *mut c_void {
+    std::ptr::copy(src as *const u8, dest as *mut u8, n);
+    dest
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn rust_zstd_wasm_shim_memset(dest: *mut c_void, c: c_int, n: usize) -> *mut c_void {
+    std::ptr::write_bytes(dest as *mut u8, c as u8, n);
+    dest
+}

zstd-safe/zstd-sys/wasm-shim/stdlib.h

@@ -0,0 +1,22 @@
+#include <stddef.h>
+
+#ifndef	_STDLIB_H
+#define	_STDLIB_H	1
+
+void *rust_zstd_wasm_shim_malloc(size_t size);
+void *rust_zstd_wasm_shim_calloc(size_t nmemb, size_t size);
+void rust_zstd_wasm_shim_free(void *ptr);
+
+inline void *malloc(size_t size) {
+	return rust_zstd_wasm_shim_malloc(size);
+}
+
+inline void *calloc(size_t nmemb, size_t size) {
+	return rust_zstd_wasm_shim_calloc(nmemb, size);
+}
+
+inline void free(void *ptr) {
+	rust_zstd_wasm_shim_free(ptr);
+}
+
+#endif // _STDLIB_H

zstd-safe/zstd-sys/wasm-shim/string.h

@@ -0,0 +1,22 @@
+#include <stdlib.h>
+
+#ifndef	_STRING_H
+#define	_STRING_H	1
+
+void *rust_zstd_wasm_shim_memcpy(void *restrict dest, const void *restrict src, size_t n);
+void *rust_zstd_wasm_shim_memmove(void *dest, const void *src, size_t n);
+void *rust_zstd_wasm_shim_memset(void *dest, int c, size_t n);
+
+inline void *memcpy(void *restrict dest, const void *restrict src, size_t n) {
+	return rust_zstd_wasm_shim_memcpy(dest, src, n);
+}
+
+inline void *memmove(void *dest, const void *src, size_t n) {
+	return rust_zstd_wasm_shim_memmove(dest, src, n);
+}
+
+inline void *memset(void *dest, int c, size_t n) {
+	return rust_zstd_wasm_shim_memset(dest, c, n);
+}
+
+#endif // _STRING_H