Skip to content

Commit

Permalink
lz4-sys: add libc shim for wasm targets (#49)
Browse files Browse the repository at this point in the history
* lz4-sys: add libc shim for wasm targets

* include assert.h

* remove libc dep from toplevel crate

* add to CI

* re-include target env wasi

* macos: install llvm
  • Loading branch information
james-rms authored Sep 6, 2024
1 parent 14b0071 commit eec50ee
Show file tree
Hide file tree
Showing 9 changed files with 212 additions and 23 deletions.
14 changes: 14 additions & 0 deletions .github/workflows/test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ jobs:
toolchain: 1.65.0
override: true
components: rustfmt
- name: Add wasm32 target
run: rustup target add wasm32-unknown-unknown
- name: Checkout
uses: actions/checkout@v2
with:
Expand All @@ -32,6 +34,18 @@ jobs:
run: cargo build --release
- name: unit tests
run: cargo test -- --nocapture
- name: (MacOS) install LLVM
uses: KyleMayes/install-llvm-action@v2
if: "${{ matrix.platform == 'macos-latest' }}"
with:
version: "17"
- name: (MacOS) set LLVM as CC
if: "${{ matrix.platform == 'macos-latest' }}"
run: echo "CC=$(pwd)/llvm/bin/clang-17" >> $GITHUB_ENV
- name: build (wasm32)
run: cargo build --target wasm32-unknown-unknown
- name: check (wasm32)
run: cargo check --target wasm32-unknown-unknown
test-windows:
runs-on: windows-latest
steps:
Expand Down
1 change: 0 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ test = false
doc = false

[dependencies]
libc = "0.2"
lz4-sys = { path = "lz4-sys", version = "1.10.0" }

[dev-dependencies]
Expand Down
8 changes: 8 additions & 0 deletions lz4-sys/build.rs
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,14 @@ fn run() -> Result<(), Box<dyn Error>> {
}
}
}
let need_wasm_shim = target == "wasm32-unknown-unknown" || target.starts_with("wasm32-wasi");

if need_wasm_shim {
println!("cargo:rerun-if-changed=wasm-shim/stdlib.h");
println!("cargo:rerun-if-changed=wasm-shim/string.h");

compiler.include("wasm-shim/");
}
compiler.compile("liblz4.a");

let src = env::current_dir()?.join("liblz4").join("lib");
Expand Down
18 changes: 15 additions & 3 deletions lz4-sys/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,19 @@ extern crate libc;
target_arch = "wasm32",
not(any(target_env = "wasi", target_os = "wasi"))
)))]
use libc::{c_void, c_char, c_uint, size_t, c_int, c_ulonglong};
pub use libc::{c_char, c_int, c_uint, c_ulonglong, c_void, size_t};

#[cfg(all(
target_arch = "wasm32",
not(any(target_env = "wasi", target_os = "wasi"))
))]
extern crate alloc;

#[cfg(all(
target_arch = "wasm32",
not(any(target_env = "wasi", target_os = "wasi"))
))]
mod wasm_shim;

#[cfg(all(
target_arch = "wasm32",
Expand All @@ -17,14 +29,14 @@ extern crate std;
target_arch = "wasm32",
not(any(target_env = "wasi", target_os = "wasi"))
))]
use std::os::raw::{c_void, c_char, c_uint, c_int};
pub use std::os::raw::{c_char, c_int, c_uint, c_ulonglong, c_void};

#[cfg(all(
target_arch = "wasm32",
not(any(target_env = "wasi", target_os = "wasi"))
))]
#[allow(non_camel_case_types)]
type size_t = usize;
pub type size_t = usize;

#[derive(Clone, Copy, Debug)]
#[repr(C)]
Expand Down
123 changes: 123 additions & 0 deletions lz4-sys/src/wasm_shim.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
//! A shim for the libc functions used in lz4-rs that are not available when building for wasm
//! targets. Adapted from the shim present in the [zstd](https://github.com/gyscos/zstd-rs) crate.
//! zstd-rs license here:
//! The MIT License (MIT)
//! Copyright (c) 2016 Alexandre Bury
//!
//! Permission is hereby granted, free of charge, to any person obtaining a copy of this software
//! and associated documentation files (the "Software"), to deal in the Software without
//! restriction, including without limitation the rights to use, copy, modify, merge, publish,
//! distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the
//! Software is furnished to do so, subject to the following conditions:
//!
//! The above copyright notice and this permission notice shall be included in all copies or
//! substantial portions of the Software.
//!
//! THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
//! BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
//! NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
//! DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
//! OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
use alloc::alloc::{alloc, alloc_zeroed, dealloc, Layout};
use core::ffi::{c_int, c_void};

const USIZE_ALIGN: usize = core::mem::align_of::<usize>();
const USIZE_SIZE: usize = core::mem::size_of::<usize>();

#[no_mangle]
pub extern "C" fn rust_lz4_wasm_shim_malloc(size: usize) -> *mut c_void {
wasm_shim_alloc::<false>(size)
}

#[no_mangle]
pub extern "C" fn rust_lz4_wasm_shim_memcmp(
str1: *const c_void,
str2: *const c_void,
n: usize,
) -> i32 {
// Safety: function contracts requires str1 and str2 at least `n`-long.
unsafe {
let str1: &[u8] = core::slice::from_raw_parts(str1 as *const u8, n);
let str2: &[u8] = core::slice::from_raw_parts(str2 as *const u8, n);
match str1.cmp(str2) {
core::cmp::Ordering::Less => -1,
core::cmp::Ordering::Equal => 0,
core::cmp::Ordering::Greater => 1,
}
}
}

#[no_mangle]
pub extern "C" fn rust_lz4_wasm_shim_calloc(nmemb: usize, size: usize) -> *mut c_void {
// note: calloc expects the allocation to be zeroed
wasm_shim_alloc::<true>(nmemb * size)
}

#[inline]
fn wasm_shim_alloc<const ZEROED: bool>(size: usize) -> *mut c_void {
// in order to recover the size upon free, we store the size below the allocation
// special alignment is never requested via the malloc API,
// so it's not stored, and usize-alignment is used
// memory layout: [size] [allocation]

let full_alloc_size = size + USIZE_SIZE;

unsafe {
let layout = Layout::from_size_align_unchecked(full_alloc_size, USIZE_ALIGN);

let ptr = if ZEROED {
alloc_zeroed(layout)
} else {
alloc(layout)
};

// SAFETY: ptr is usize-aligned and we've allocated sufficient memory
ptr.cast::<usize>().write(full_alloc_size);

ptr.add(USIZE_SIZE).cast()
}
}

#[no_mangle]
pub unsafe extern "C" fn rust_lz4_wasm_shim_free(ptr: *mut c_void) {
// the layout for the allocation needs to be recovered for dealloc
// - the size must be recovered from directly below the allocation
// - the alignment will always by USIZE_ALIGN

let alloc_ptr = ptr.sub(USIZE_SIZE);
// SAFETY: the allocation routines must uphold having a valid usize below the provided pointer
let full_alloc_size = alloc_ptr.cast::<usize>().read();

let layout = Layout::from_size_align_unchecked(full_alloc_size, USIZE_ALIGN);
dealloc(alloc_ptr.cast(), layout);
}

#[no_mangle]
pub unsafe extern "C" fn rust_lz4_wasm_shim_memcpy(
dest: *mut c_void,
src: *const c_void,
n: usize,
) -> *mut c_void {
core::ptr::copy_nonoverlapping(src as *const u8, dest as *mut u8, n);
dest
}

#[no_mangle]
pub unsafe extern "C" fn rust_lz4_wasm_shim_memmove(
dest: *mut c_void,
src: *const c_void,
n: usize,
) -> *mut c_void {
core::ptr::copy(src as *const u8, dest as *mut u8, n);
dest
}

#[no_mangle]
pub unsafe extern "C" fn rust_lz4_wasm_shim_memset(
dest: *mut c_void,
c: c_int,
n: usize,
) -> *mut c_void {
core::ptr::write_bytes(dest as *mut u8, c as u8, n);
dest
}
6 changes: 6 additions & 0 deletions lz4-sys/wasm-shim/assert.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
#ifndef _ASSERT_H
#define _ASSERT_H

#define assert(expr)

#endif // _ASSERT_H
14 changes: 14 additions & 0 deletions lz4-sys/wasm-shim/stdlib.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
#include <stddef.h>

#ifndef _STDLIB_H
#define _STDLIB_H 1

void *rust_lz4_wasm_shim_malloc(size_t size);
void *rust_lz4_wasm_shim_calloc(size_t nmemb, size_t size);
void rust_lz4_wasm_shim_free(void *ptr);

#define malloc(size) rust_lz4_wasm_shim_malloc(size)
#define calloc(nmemb, size) rust_lz4_wasm_shim_calloc(nmemb, size)
#define free(ptr) rust_lz4_wasm_shim_free(ptr)

#endif // _STDLIB_H
31 changes: 31 additions & 0 deletions lz4-sys/wasm-shim/string.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
#include <stdlib.h>

#ifndef _STRING_H
#define _STRING_H 1

int rust_lz4_wasm_shim_memcmp(const void *str1, const void *str2, size_t n);
void *rust_lz4_wasm_shim_memcpy(void *restrict dest, const void *restrict src, size_t n);
void *rust_lz4_wasm_shim_memmove(void *dest, const void *src, size_t n);
void *rust_lz4_wasm_shim_memset(void *dest, int c, size_t n);

inline int memcmp(const void *str1, const void *str2, size_t n)
{
return rust_lz4_wasm_shim_memcmp(str1, str2, n);
}

inline void *memcpy(void *restrict dest, const void *restrict src, size_t n)
{
return rust_lz4_wasm_shim_memcpy(dest, src, n);
}

inline void *memmove(void *dest, const void *src, size_t n)
{
return rust_lz4_wasm_shim_memmove(dest, src, n);
}

inline void *memset(void *dest, int c, size_t n)
{
return rust_lz4_wasm_shim_memset(dest, c, n);
}

#endif // _STRING_H
20 changes: 1 addition & 19 deletions src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
extern crate libc;
extern crate lz4_sys;

pub mod liblz4;
Expand All @@ -16,21 +15,4 @@ pub use crate::liblz4::BlockMode;
pub use crate::liblz4::BlockSize;
pub use crate::liblz4::ContentChecksum;

#[cfg(not(all(
target_arch = "wasm32",
not(any(target_env = "wasi", target_os = "wasi"))
)))]
use libc::{c_char, size_t};

#[cfg(all(
target_arch = "wasm32",
not(any(target_env = "wasi", target_os = "wasi"))
))]
use std::os::raw::c_char;

#[cfg(all(
target_arch = "wasm32",
not(any(target_env = "wasi", target_os = "wasi"))
))]
#[allow(non_camel_case_types)]
type size_t = usize;
use lz4_sys::{c_char, size_t};

0 comments on commit eec50ee

Please sign in to comment.