diff --git a/src/imp/atomic128/x86_64.rs b/src/imp/atomic128/x86_64.rs index 8dd66a1b..8d06a814 100644 --- a/src/imp/atomic128/x86_64.rs +++ b/src/imp/atomic128/x86_64.rs @@ -10,7 +10,7 @@ // - atomic-maybe-uninit https://github.com/taiki-e/atomic-maybe-uninit // // Generated asm: -// - x86_64 (+cmpxchg16b) https://godbolt.org/z/YnYE9qT6b +// - x86_64 (+cmpxchg16b) https://godbolt.org/z/r5x9M8PdK include!("macros.rs"); @@ -124,6 +124,7 @@ unsafe fn cmpxchg16b(dst: *mut u128, old: u128, new: u128) -> (u128, bool) { cmpxchg16b!("edi"); #[cfg(target_pointer_width = "64")] cmpxchg16b!("rdi"); + crate::utils::assert_unchecked(r == 0 || r == 1); // needed to remove extra test (U128 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole, r != 0) } } diff --git a/src/imp/x86.rs b/src/imp/x86.rs index 8e776176..b8748cc4 100644 --- a/src/imp/x86.rs +++ b/src/imp/x86.rs @@ -148,6 +148,7 @@ macro_rules! atomic_bit_opts { // Do not use `preserves_flags` because BTS modifies the CF flag. options(nostack), ); + crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test r != 0 } } @@ -172,6 +173,7 @@ macro_rules! atomic_bit_opts { // Do not use `preserves_flags` because BTR modifies the CF flag. options(nostack), ); + crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test r != 0 } } @@ -196,6 +198,7 @@ macro_rules! atomic_bit_opts { // Do not use `preserves_flags` because BTC modifies the CF flag. options(nostack), ); + crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test r != 0 } } diff --git a/src/utils.rs b/src/utils.rs index 013d29ae..71e8633b 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -246,6 +246,22 @@ macro_rules! items { }; } +#[allow(dead_code)] +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +// Stable version of https://doc.rust-lang.org/nightly/std/hint/fn.assert_unchecked.html. +#[inline(always)] +#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)] +pub(crate) unsafe fn assert_unchecked(cond: bool) { + if !cond { + if cfg!(debug_assertions) { + unreachable!() + } else { + // SAFETY: the caller promised `cond` is true. + unsafe { core::hint::unreachable_unchecked() } + } + } +} + // https://github.com/rust-lang/rust/blob/1.70.0/library/core/src/sync/atomic.rs#L3155 #[inline] #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]