lz4-sys: add libc shim for wasm targets (#49)

* lz4-sys: add libc shim for wasm targets

* include assert.h

* remove libc dep from toplevel crate

* add to CI

* re-include target env wasi

* macos: install llvm

.github/workflows/test.yaml

@@ -22,6 +22,8 @@ jobs:
           toolchain: 1.65.0
           override: true
           components: rustfmt
+      - name: Add wasm32 target
+        run: rustup target add wasm32-unknown-unknown
       - name: Checkout
         uses: actions/checkout@v2
         with:
@@ -32,6 +34,18 @@ jobs:
         run: cargo build --release
       - name: unit tests
         run: cargo test -- --nocapture
+      - name: (MacOS) install LLVM
+        uses: KyleMayes/install-llvm-action@v2
+        if: "${{ matrix.platform == 'macos-latest' }}"
+        with:
+          version: "17"
+      - name: (MacOS) set LLVM as CC
+        if: "${{ matrix.platform == 'macos-latest' }}"
+        run: echo "CC=$(pwd)/llvm/bin/clang-17" >> $GITHUB_ENV
+      - name: build (wasm32)
+        run: cargo build --target wasm32-unknown-unknown
+      - name: check (wasm32)
+        run: cargo check --target wasm32-unknown-unknown
   test-windows:
     runs-on: windows-latest
     steps:

Cargo.toml

@@ -15,7 +15,6 @@ test = false
 doc = false
 
 [dependencies]
-libc = "0.2"
 lz4-sys = { path = "lz4-sys", version = "1.10.0" }
 
 [dev-dependencies]

lz4-sys/build.rs

@@ -40,6 +40,14 @@ fn run() -> Result<(), Box<dyn Error>> {
             }
         }
     }
+    let need_wasm_shim = target == "wasm32-unknown-unknown" || target.starts_with("wasm32-wasi");
+
+    if need_wasm_shim {
+        println!("cargo:rerun-if-changed=wasm-shim/stdlib.h");
+        println!("cargo:rerun-if-changed=wasm-shim/string.h");
+
+        compiler.include("wasm-shim/");
+    }
     compiler.compile("liblz4.a");
 
     let src = env::current_dir()?.join("liblz4").join("lib");

lz4-sys/src/lib.rs

@@ -5,7 +5,19 @@ extern crate libc;
     target_arch = "wasm32",
     not(any(target_env = "wasi", target_os = "wasi"))
 )))]
-use libc::{c_void, c_char, c_uint, size_t, c_int, c_ulonglong};
+pub use libc::{c_char, c_int, c_uint, c_ulonglong, c_void, size_t};
+
+#[cfg(all(
+    target_arch = "wasm32",
+    not(any(target_env = "wasi", target_os = "wasi"))
+))]
+extern crate alloc;
+
+#[cfg(all(
+    target_arch = "wasm32",
+    not(any(target_env = "wasi", target_os = "wasi"))
+))]
+mod wasm_shim;
 
 #[cfg(all(
     target_arch = "wasm32",
@@ -17,14 +29,14 @@ extern crate std;
     target_arch = "wasm32",
     not(any(target_env = "wasi", target_os = "wasi"))
 ))]
-use std::os::raw::{c_void, c_char, c_uint, c_int};
+pub use std::os::raw::{c_char, c_int, c_uint, c_ulonglong, c_void};
 
 #[cfg(all(
     target_arch = "wasm32",
     not(any(target_env = "wasi", target_os = "wasi"))
 ))]
 #[allow(non_camel_case_types)]
-type size_t = usize;
+pub type size_t = usize;
 
 #[derive(Clone, Copy, Debug)]
 #[repr(C)]

lz4-sys/src/wasm_shim.rs

@@ -0,0 +1,123 @@
+//! A shim for the libc functions used in lz4-rs that are not available when building for wasm
+//! targets. Adapted from the shim present in the [zstd](https://github.com/gyscos/zstd-rs) crate.
+//! zstd-rs license here:
+//! The MIT License (MIT)
+//! Copyright (c) 2016 Alexandre Bury
+//!
+//! Permission is hereby granted, free of charge, to any person obtaining a copy of this software
+//! and associated documentation files (the "Software"), to deal in the Software without
+//! restriction, including without limitation the rights to use, copy, modify, merge, publish,
+//! distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the
+//! Software is furnished to do so, subject to the following conditions:
+//!
+//! The above copyright notice and this permission notice shall be included in all copies or
+//! substantial portions of the Software.
+//!
+//! THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+//! BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+//! NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+//! DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+//! OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+use alloc::alloc::{alloc, alloc_zeroed, dealloc, Layout};
+use core::ffi::{c_int, c_void};
+
+const USIZE_ALIGN: usize = core::mem::align_of::<usize>();
+const USIZE_SIZE: usize = core::mem::size_of::<usize>();
+
+#[no_mangle]
+pub extern "C" fn rust_lz4_wasm_shim_malloc(size: usize) -> *mut c_void {
+    wasm_shim_alloc::<false>(size)
+}
+
+#[no_mangle]
+pub extern "C" fn rust_lz4_wasm_shim_memcmp(
+    str1: *const c_void,
+    str2: *const c_void,
+    n: usize,
+) -> i32 {
+    // Safety: function contracts requires str1 and str2 at least `n`-long.
+    unsafe {
+        let str1: &[u8] = core::slice::from_raw_parts(str1 as *const u8, n);
+        let str2: &[u8] = core::slice::from_raw_parts(str2 as *const u8, n);
+        match str1.cmp(str2) {
+            core::cmp::Ordering::Less => -1,
+            core::cmp::Ordering::Equal => 0,
+            core::cmp::Ordering::Greater => 1,
+        }
+    }
+}
+
+#[no_mangle]
+pub extern "C" fn rust_lz4_wasm_shim_calloc(nmemb: usize, size: usize) -> *mut c_void {
+    // note: calloc expects the allocation to be zeroed
+    wasm_shim_alloc::<true>(nmemb * size)
+}
+
+#[inline]
+fn wasm_shim_alloc<const ZEROED: bool>(size: usize) -> *mut c_void {
+    // in order to recover the size upon free, we store the size below the allocation
+    // special alignment is never requested via the malloc API,
+    // so it's not stored, and usize-alignment is used
+    // memory layout: [size] [allocation]
+
+    let full_alloc_size = size + USIZE_SIZE;
+
+    unsafe {
+        let layout = Layout::from_size_align_unchecked(full_alloc_size, USIZE_ALIGN);
+
+        let ptr = if ZEROED {
+            alloc_zeroed(layout)
+        } else {
+            alloc(layout)
+        };
+
+        // SAFETY: ptr is usize-aligned and we've allocated sufficient memory
+        ptr.cast::<usize>().write(full_alloc_size);
+
+        ptr.add(USIZE_SIZE).cast()
+    }
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn rust_lz4_wasm_shim_free(ptr: *mut c_void) {
+    // the layout for the allocation needs to be recovered for dealloc
+    // - the size must be recovered from directly below the allocation
+    // - the alignment will always by USIZE_ALIGN
+
+    let alloc_ptr = ptr.sub(USIZE_SIZE);
+    // SAFETY: the allocation routines must uphold having a valid usize below the provided pointer
+    let full_alloc_size = alloc_ptr.cast::<usize>().read();
+
+    let layout = Layout::from_size_align_unchecked(full_alloc_size, USIZE_ALIGN);
+    dealloc(alloc_ptr.cast(), layout);
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn rust_lz4_wasm_shim_memcpy(
+    dest: *mut c_void,
+    src: *const c_void,
+    n: usize,
+) -> *mut c_void {
+    core::ptr::copy_nonoverlapping(src as *const u8, dest as *mut u8, n);
+    dest
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn rust_lz4_wasm_shim_memmove(
+    dest: *mut c_void,
+    src: *const c_void,
+    n: usize,
+) -> *mut c_void {
+    core::ptr::copy(src as *const u8, dest as *mut u8, n);
+    dest
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn rust_lz4_wasm_shim_memset(
+    dest: *mut c_void,
+    c: c_int,
+    n: usize,
+) -> *mut c_void {
+    core::ptr::write_bytes(dest as *mut u8, c as u8, n);
+    dest
+}

lz4-sys/wasm-shim/assert.h

@@ -0,0 +1,6 @@
+#ifndef _ASSERT_H
+#define _ASSERT_H
+
+#define assert(expr)
+
+#endif // _ASSERT_H

lz4-sys/wasm-shim/stdlib.h

@@ -0,0 +1,14 @@
+#include <stddef.h>
+
+#ifndef _STDLIB_H
+#define _STDLIB_H 1
+
+void *rust_lz4_wasm_shim_malloc(size_t size);
+void *rust_lz4_wasm_shim_calloc(size_t nmemb, size_t size);
+void rust_lz4_wasm_shim_free(void *ptr);
+
+#define malloc(size) rust_lz4_wasm_shim_malloc(size)
+#define calloc(nmemb, size) rust_lz4_wasm_shim_calloc(nmemb, size)
+#define free(ptr) rust_lz4_wasm_shim_free(ptr)
+
+#endif // _STDLIB_H

lz4-sys/wasm-shim/string.h

@@ -0,0 +1,31 @@
+#include <stdlib.h>
+
+#ifndef _STRING_H
+#define _STRING_H 1
+
+int rust_lz4_wasm_shim_memcmp(const void *str1, const void *str2, size_t n);
+void *rust_lz4_wasm_shim_memcpy(void *restrict dest, const void *restrict src, size_t n);
+void *rust_lz4_wasm_shim_memmove(void *dest, const void *src, size_t n);
+void *rust_lz4_wasm_shim_memset(void *dest, int c, size_t n);
+
+inline int memcmp(const void *str1, const void *str2, size_t n)
+{
+	return rust_lz4_wasm_shim_memcmp(str1, str2, n);
+}
+
+inline void *memcpy(void *restrict dest, const void *restrict src, size_t n)
+{
+	return rust_lz4_wasm_shim_memcpy(dest, src, n);
+}
+
+inline void *memmove(void *dest, const void *src, size_t n)
+{
+	return rust_lz4_wasm_shim_memmove(dest, src, n);
+}
+
+inline void *memset(void *dest, int c, size_t n)
+{
+	return rust_lz4_wasm_shim_memset(dest, c, n);
+}
+
+#endif // _STRING_H

src/lib.rs

@@ -1,4 +1,3 @@
-extern crate libc;
 extern crate lz4_sys;
 
 pub mod liblz4;
@@ -16,21 +15,4 @@ pub use crate::liblz4::BlockMode;
 pub use crate::liblz4::BlockSize;
 pub use crate::liblz4::ContentChecksum;
 
-#[cfg(not(all(
-    target_arch = "wasm32",
-    not(any(target_env = "wasi", target_os = "wasi"))
-)))]
-use libc::{c_char, size_t};
-
-#[cfg(all(
-    target_arch = "wasm32",
-    not(any(target_env = "wasi", target_os = "wasi"))
-))]
-use std::os::raw::c_char;
-
-#[cfg(all(
-    target_arch = "wasm32",
-    not(any(target_env = "wasi", target_os = "wasi"))
-))]
-#[allow(non_camel_case_types)]
-type size_t = usize;
+use lz4_sys::{c_char, size_t};