diff --git a/CHANGELOG.md b/CHANGELOG.md index 17cfd97f..950ce0af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ Note: In this file, do not use the hard wrap in the middle of a sentence for com ## [Unreleased] +- Support run-time detection for cmpxchg16b on x86_64 on pre-1.69 rustc. ([#154](https://github.com/taiki-e/portable-atomic/pull/154)) + ## [1.6.0] - 2023-12-06 - Add `cfg_{has,no}_atomic_{8,16,32,64,128,ptr}` macros to enable code when the corresponding atomic implementation is available/unavailable. diff --git a/README.md b/README.md index 92d5de4c..6564e7d5 100644 --- a/README.md +++ b/README.md @@ -170,7 +170,7 @@ RUSTFLAGS="--cfg portable_atomic_no_outline_atomics" cargo ... If dynamic dispatching by run-time CPU feature detection is enabled, it allows maintaining support for older CPUs while using features that are not supported on older CPUs, such as CMPXCHG16B (x86_64) and FEAT_LSE (aarch64). Note: - - Dynamic detection is currently only enabled in Rust 1.59+ for aarch64, in Rust 1.59+ (AVX) or 1.69+ (CMPXCHG16B) for x86_64, nightly only for powerpc64 (disabled by default), otherwise it works the same as when this cfg is set. + - Dynamic detection is currently only enabled in Rust 1.59+ for aarch64 and x86_64, nightly only for powerpc64 (disabled by default), otherwise it works the same as when this cfg is set. - If the required target features are enabled at compile-time, the atomic operations are inlined. - This is compatible with no-std (as with all features except `std`). - On some targets, run-time detection is disabled by default mainly for compatibility with older versions of operating systems or incomplete build environments, and can be enabled by `--cfg portable_atomic_outline_atomics`. (When both cfg are enabled, `*_no_*` cfg is preferred.) diff --git a/build.rs b/build.rs index da09e25e..ab929ee0 100644 --- a/build.rs +++ b/build.rs @@ -170,15 +170,7 @@ fn main() { "x86_64" => { // cmpxchg16b_target_feature stabilized in Rust 1.69 (nightly-2023-03-01): https://github.com/rust-lang/rust/pull/106774 if !version.probe(69, 2023, 2, 28) { - if version.nightly && is_allowed_feature("cmpxchg16b_target_feature") { - // This feature has not been changed since 1.33 - // (https://github.com/rust-lang/rust/commit/fbb56bcf44d28e65a9495decf091b6d0386e540c) - // until it was stabilized in nightly-2023-03-01, so it can be safely enabled in - // nightly, which is older than nightly-2023-03-01. - println!("cargo:rustc-cfg=portable_atomic_unstable_cmpxchg16b_target_feature"); - } else { - println!("cargo:rustc-cfg=portable_atomic_no_cmpxchg16b_target_feature"); - } + println!("cargo:rustc-cfg=portable_atomic_no_cmpxchg16b_target_feature"); } // For Miri and ThreadSanitizer. // https://github.com/rust-lang/rust/pull/109359 (includes https://github.com/rust-lang/stdarch/pull/1358) merged in Rust 1.70 (nightly-2023-03-24). diff --git a/src/cfgs.rs b/src/cfgs.rs index f50acd57..e6f0b3ff 100644 --- a/src/cfgs.rs +++ b/src/cfgs.rs @@ -236,7 +236,6 @@ mod atomic_64_macros { portable_atomic_target_feature = "cmpxchg16b", all( feature = "fallback", - not(portable_atomic_no_cmpxchg16b_target_feature), not(portable_atomic_no_outline_atomics), not(any(target_env = "sgx", miri)), ), @@ -326,7 +325,6 @@ mod atomic_128_macros { portable_atomic_target_feature = "cmpxchg16b", all( feature = "fallback", - not(portable_atomic_no_cmpxchg16b_target_feature), not(portable_atomic_no_outline_atomics), not(any(target_env = "sgx", miri)), ), diff --git a/src/imp/atomic128/README.md b/src/imp/atomic128/README.md index 799a49b3..ef4b0163 100644 --- a/src/imp/atomic128/README.md +++ b/src/imp/atomic128/README.md @@ -6,7 +6,7 @@ Here is the table of targets that support 128-bit atomics and the instructions u | target_arch | load | store | CAS | RMW | note | | ----------- | ---- | ----- | --- | --- | ---- | -| x86_64 | cmpxchg16b or vmovdqa | cmpxchg16b or vmovdqa | cmpxchg16b | cmpxchg16b | cmpxchg16b target feature required. vmovdqa requires Intel or AMD CPU with AVX.
Both compile-time and run-time detection are supported for cmpxchg16b. vmovdqa is currently run-time detection only.
Requires rustc 1.59+ when cmpxchg16b target feature is enabled at compile-time, otherwise requires rustc 1.69+ | +| x86_64 | cmpxchg16b or vmovdqa | cmpxchg16b or vmovdqa | cmpxchg16b | cmpxchg16b | cmpxchg16b target feature required. vmovdqa requires Intel or AMD CPU with AVX.
Both compile-time and run-time detection are supported for cmpxchg16b. vmovdqa is currently run-time detection only.
Requires rustc 1.59+ | | aarch64 | ldxp/stxp or casp or ldp/ldiapp | ldxp/stxp or casp or stp/stilp/swpp | ldxp/stxp or casp | ldxp/stxp or casp/swpp/ldclrp/ldsetp | casp requires lse target feature, ldp/stp requires lse2 target feature, ldiapp/stilp requires lse2 and rcpc3 target features, swpp/ldclrp/ldsetp requires lse128 target feature.
Both compile-time and run-time detection are supported for lse and lse2. Others are currently compile-time detection only.
Requires rustc 1.59+ | | powerpc64 | lq | stq | lqarx/stqcx. | lqarx/stqcx. | Requires target-cpu pwr8+ (powerpc64le is pwr8 by default). Both compile-time and run-time detection are supported (run-time detection is currently disabled by default).
Requires nightly | | s390x | lpq | stpq | cdsg | cdsg | Requires nightly | diff --git a/src/imp/atomic128/intrinsics.rs b/src/imp/atomic128/intrinsics.rs index fc331158..d24ae116 100644 --- a/src/imp/atomic128/intrinsics.rs +++ b/src/imp/atomic128/intrinsics.rs @@ -113,7 +113,7 @@ unsafe fn atomic_compare_exchange( ) -> Result { #[cfg(target_arch = "x86_64")] let (val, ok) = { - #[cfg_attr(not(target_feature = "cmpxchg16b"), target_feature(enable = "cmpxchg16b"))] + #[target_feature(enable = "cmpxchg16b")] #[cfg_attr(target_feature = "cmpxchg16b", inline)] #[cfg_attr(not(target_feature = "cmpxchg16b"), inline(never))] unsafe fn cmpxchg16b( diff --git a/src/imp/atomic128/x86_64.rs b/src/imp/atomic128/x86_64.rs index 9b331712..5950f0b1 100644 --- a/src/imp/atomic128/x86_64.rs +++ b/src/imp/atomic128/x86_64.rs @@ -65,8 +65,15 @@ macro_rules! ptr_modifier { }; } +// Unlike AArch64 and RISC-V, x86's assembler doesn't check instruction +// requirements for the currently enabled target features. In the first place, +// there is no option in the x86 assembly for such case, like ARM .arch_extension, +// RISC-V .option arch, PowerPC .machine, etc. +// However, we set target_feature(enable) when available (Rust 1.69+) in case a +// new codegen backend is added that checks for it in the future, or an option +// is added to the assembler to check for it. #[cfg_attr( - not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")), + not(portable_atomic_no_cmpxchg16b_target_feature), target_feature(enable = "cmpxchg16b") )] #[inline] @@ -260,8 +267,9 @@ unsafe fn atomic_load(src: *mut u128, _order: Ordering) -> u128 { }) } } +// See cmpxchg16b() for target_feature(enable). #[cfg_attr( - not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")), + not(portable_atomic_no_cmpxchg16b_target_feature), target_feature(enable = "cmpxchg16b") )] #[inline] @@ -363,10 +371,12 @@ unsafe fn atomic_store(dst: *mut u128, val: u128, order: Ordering) { } } } +// See cmpxchg16b() for target_feature(enable). #[cfg_attr( - not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")), + not(portable_atomic_no_cmpxchg16b_target_feature), target_feature(enable = "cmpxchg16b") )] +#[inline] unsafe fn atomic_store_cmpxchg16b(dst: *mut u128, val: u128) { // SAFETY: the caller must uphold the safety contract. unsafe { @@ -413,8 +423,9 @@ use atomic_compare_exchange as atomic_compare_exchange_weak; #[cfg(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"))] use atomic_swap_cmpxchg16b as atomic_swap; +// See cmpxchg16b() for target_feature(enable). #[cfg_attr( - not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")), + not(portable_atomic_no_cmpxchg16b_target_feature), target_feature(enable = "cmpxchg16b") )] #[inline] @@ -487,8 +498,9 @@ macro_rules! atomic_rmw_cas_3 { ($name:ident as $reexport_name:ident, $($op:tt)*) => { #[cfg(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"))] use $name as $reexport_name; + // See cmpxchg16b() for target_feature(enable). #[cfg_attr( - not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")), + not(portable_atomic_no_cmpxchg16b_target_feature), target_feature(enable = "cmpxchg16b") )] #[inline] @@ -558,8 +570,9 @@ macro_rules! atomic_rmw_cas_2 { ($name:ident as $reexport_name:ident, $($op:tt)*) => { #[cfg(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"))] use $name as $reexport_name; + // See cmpxchg16b() for target_feature(enable). #[cfg_attr( - not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")), + not(portable_atomic_no_cmpxchg16b_target_feature), target_feature(enable = "cmpxchg16b") )] #[inline] @@ -729,11 +742,9 @@ macro_rules! atomic_rmw_with_ifunc { #[inline] unsafe fn $name($($arg)*, _order: Ordering) $(-> $ret_ty)? { fn_alias! { + // See cmpxchg16b() for target_feature(enable). #[cfg_attr( - not(any( - target_feature = "cmpxchg16b", - portable_atomic_target_feature = "cmpxchg16b", - )), + not(portable_atomic_no_cmpxchg16b_target_feature), target_feature(enable = "cmpxchg16b") )] unsafe fn($($arg)*) $(-> $ret_ty)?; diff --git a/src/imp/fallback/mod.rs b/src/imp/fallback/mod.rs index 0a361c06..87853c5d 100644 --- a/src/imp/fallback/mod.rs +++ b/src/imp/fallback/mod.rs @@ -15,7 +15,6 @@ any( all( target_arch = "x86_64", - not(portable_atomic_no_cmpxchg16b_target_feature), not(portable_atomic_no_outline_atomics), not(any(target_env = "sgx", miri)), ), diff --git a/src/imp/mod.rs b/src/imp/mod.rs index 48216fba..0b805f7f 100644 --- a/src/imp/mod.rs +++ b/src/imp/mod.rs @@ -54,7 +54,6 @@ mod aarch64; portable_atomic_target_feature = "cmpxchg16b", all( feature = "fallback", - not(portable_atomic_no_cmpxchg16b_target_feature), not(portable_atomic_no_outline_atomics), not(any(target_env = "sgx", miri)), ), @@ -340,7 +339,6 @@ items! { portable_atomic_target_feature = "cmpxchg16b", all( feature = "fallback", - not(portable_atomic_no_cmpxchg16b_target_feature), not(portable_atomic_no_outline_atomics), not(any(target_env = "sgx", miri)), ), @@ -405,14 +403,13 @@ pub(crate) use self::aarch64::{AtomicI128, AtomicU128}; // x86_64 & (cmpxchg16b | outline-atomics) #[cfg(all( target_arch = "x86_64", - not(all(any(miri, portable_atomic_sanitize_thread), portable_atomic_no_cmpxchg16b_intrinsic,)), + not(all(any(miri, portable_atomic_sanitize_thread), portable_atomic_no_cmpxchg16b_intrinsic)), any(not(portable_atomic_no_asm), portable_atomic_unstable_asm), any( target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b", all( feature = "fallback", - not(portable_atomic_no_cmpxchg16b_target_feature), not(portable_atomic_no_outline_atomics), not(any(target_env = "sgx", miri)), ), diff --git a/src/lib.rs b/src/lib.rs index a7941471..3a481bb5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -164,7 +164,7 @@ RUSTFLAGS="--cfg portable_atomic_no_outline_atomics" cargo ... If dynamic dispatching by run-time CPU feature detection is enabled, it allows maintaining support for older CPUs while using features that are not supported on older CPUs, such as CMPXCHG16B (x86_64) and FEAT_LSE (aarch64). Note: - - Dynamic detection is currently only enabled in Rust 1.59+ for aarch64, in Rust 1.59+ (AVX) or 1.69+ (CMPXCHG16B) for x86_64, nightly only for powerpc64 (disabled by default), otherwise it works the same as when this cfg is set. + - Dynamic detection is currently only enabled in Rust 1.59+ for aarch64 and x86_64, nightly only for powerpc64 (disabled by default), otherwise it works the same as when this cfg is set. - If the required target features are enabled at compile-time, the atomic operations are inlined. - This is compatible with no-std (as with all features except `std`). - On some targets, run-time detection is disabled by default mainly for compatibility with older versions of operating systems or incomplete build environments, and can be enabled by `--cfg portable_atomic_outline_atomics`. (When both cfg are enabled, `*_no_*` cfg is preferred.) @@ -238,21 +238,10 @@ RUSTFLAGS="--cfg portable_atomic_no_outline_atomics" cargo ... // These features are already stabilized or have already been removed from compilers, // and can safely be enabled for old nightly as long as version detection works. // - cfg(target_has_atomic) -// - #[target_feature(enable = "cmpxchg16b")] on x86_64 // - asm! on ARM, AArch64, RISC-V, x86_64 // - llvm_asm! on AVR (tier 3) and MSP430 (tier 3) // - #[instruction_set] on non-Linux/Android pre-v6 ARM (tier 3) #![cfg_attr(portable_atomic_unstable_cfg_target_has_atomic, feature(cfg_target_has_atomic))] -#![cfg_attr( - all( - target_arch = "x86_64", - portable_atomic_unstable_cmpxchg16b_target_feature, - not(portable_atomic_no_outline_atomics), - not(any(target_env = "sgx", miri)), - feature = "fallback", - ), - feature(cmpxchg16b_target_feature) -)] #![cfg_attr( all( portable_atomic_unstable_asm, diff --git a/src/tests/mod.rs b/src/tests/mod.rs index 63cdbbd5..4d5319a5 100644 --- a/src/tests/mod.rs +++ b/src/tests/mod.rs @@ -143,7 +143,6 @@ fn test_is_lock_free() { { let has_cmpxchg16b = cfg!(all( feature = "fallback", - not(portable_atomic_no_cmpxchg16b_target_feature), not(portable_atomic_no_outline_atomics), not(any(target_env = "sgx", miri)), not(portable_atomic_test_outline_atomics_detect_false),