x86_64: Support run-time detection for cmpxchg16b on pre-1.69 rustc

taiki-e · Mar 9, 2024 · cf87e37 · cf87e37
1 parent 8a4d501
commit cf87e37
Show file tree

Hide file tree

Showing 11 changed files with 29 additions and 42 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,6 +10,8 @@ Note: In this file, do not use the hard wrap in the middle of a sentence for com
 
 ## [Unreleased]
 
+- Support run-time detection for cmpxchg16b on x86_64 on pre-1.69 rustc. ([#154](https://github.com/taiki-e/portable-atomic/pull/154))
+
 ## [1.6.0] - 2023-12-06
 
 - Add `cfg_{has,no}_atomic_{8,16,32,64,128,ptr}` macros to enable code when the corresponding atomic implementation is available/unavailable.

diff --git a/README.md b/README.md
@@ -170,7 +170,7 @@ RUSTFLAGS="--cfg portable_atomic_no_outline_atomics" cargo ...
   If dynamic dispatching by run-time CPU feature detection is enabled, it allows maintaining support for older CPUs while using features that are not supported on older CPUs, such as CMPXCHG16B (x86_64) and FEAT_LSE (aarch64).
 
   Note:
-  - Dynamic detection is currently only enabled in Rust 1.59+ for aarch64, in Rust 1.59+ (AVX) or 1.69+ (CMPXCHG16B) for x86_64, nightly only for powerpc64 (disabled by default), otherwise it works the same as when this cfg is set.
+  - Dynamic detection is currently only enabled in Rust 1.59+ for aarch64 and x86_64, nightly only for powerpc64 (disabled by default), otherwise it works the same as when this cfg is set.
   - If the required target features are enabled at compile-time, the atomic operations are inlined.
   - This is compatible with no-std (as with all features except `std`).
   - On some targets, run-time detection is disabled by default mainly for compatibility with older versions of operating systems or incomplete build environments, and can be enabled by `--cfg portable_atomic_outline_atomics`. (When both cfg are enabled, `*_no_*` cfg is preferred.)

diff --git a/build.rs b/build.rs
@@ -170,15 +170,7 @@ fn main() {
         "x86_64" => {
             // cmpxchg16b_target_feature stabilized in Rust 1.69 (nightly-2023-03-01): https://github.com/rust-lang/rust/pull/106774
             if !version.probe(69, 2023, 2, 28) {
-                if version.nightly && is_allowed_feature("cmpxchg16b_target_feature") {
-                    // This feature has not been changed since 1.33
-                    // (https://github.com/rust-lang/rust/commit/fbb56bcf44d28e65a9495decf091b6d0386e540c)
-                    // until it was stabilized in nightly-2023-03-01, so it can be safely enabled in
-                    // nightly, which is older than nightly-2023-03-01.
-                    println!("cargo:rustc-cfg=portable_atomic_unstable_cmpxchg16b_target_feature");
-                } else {
-                    println!("cargo:rustc-cfg=portable_atomic_no_cmpxchg16b_target_feature");
-                }
+                println!("cargo:rustc-cfg=portable_atomic_no_cmpxchg16b_target_feature");
             }
             // For Miri and ThreadSanitizer.
             // https://github.com/rust-lang/rust/pull/109359 (includes https://github.com/rust-lang/stdarch/pull/1358) merged in Rust 1.70 (nightly-2023-03-24).

diff --git a/src/cfgs.rs b/src/cfgs.rs
@@ -236,7 +236,6 @@ mod atomic_64_macros {
                 portable_atomic_target_feature = "cmpxchg16b",
                 all(
                     feature = "fallback",
-                    not(portable_atomic_no_cmpxchg16b_target_feature),
                     not(portable_atomic_no_outline_atomics),
                     not(any(target_env = "sgx", miri)),
                 ),
@@ -326,7 +325,6 @@ mod atomic_128_macros {
                 portable_atomic_target_feature = "cmpxchg16b",
                 all(
                     feature = "fallback",
-                    not(portable_atomic_no_cmpxchg16b_target_feature),
                     not(portable_atomic_no_outline_atomics),
                     not(any(target_env = "sgx", miri)),
                 ),

diff --git a/src/imp/atomic128/README.md b/src/imp/atomic128/README.md
@@ -6,7 +6,7 @@ Here is the table of targets that support 128-bit atomics and the instructions u
 
 | target_arch | load | store | CAS | RMW | note |
 | ----------- | ---- | ----- | --- | --- | ---- |
-| x86_64 | cmpxchg16b or vmovdqa | cmpxchg16b or vmovdqa | cmpxchg16b | cmpxchg16b | cmpxchg16b target feature required. vmovdqa requires Intel or AMD CPU with AVX. <br> Both compile-time and run-time detection are supported for cmpxchg16b. vmovdqa is currently run-time detection only. <br> Requires rustc 1.59+ when cmpxchg16b target feature is enabled at compile-time, otherwise requires rustc 1.69+ |
+| x86_64 | cmpxchg16b or vmovdqa | cmpxchg16b or vmovdqa | cmpxchg16b | cmpxchg16b | cmpxchg16b target feature required. vmovdqa requires Intel or AMD CPU with AVX. <br> Both compile-time and run-time detection are supported for cmpxchg16b. vmovdqa is currently run-time detection only. <br> Requires rustc 1.59+ |
 | aarch64 | ldxp/stxp or casp or ldp/ldiapp | ldxp/stxp or casp or stp/stilp/swpp | ldxp/stxp or casp | ldxp/stxp or casp/swpp/ldclrp/ldsetp | casp requires lse target feature, ldp/stp requires lse2 target feature, ldiapp/stilp requires lse2 and rcpc3 target features, swpp/ldclrp/ldsetp requires lse128 target feature. <br> Both compile-time and run-time detection are supported for lse and lse2. Others are currently compile-time detection only. <br> Requires rustc 1.59+ |
 | powerpc64 | lq | stq | lqarx/stqcx. | lqarx/stqcx. | Requires target-cpu pwr8+ (powerpc64le is pwr8 by default). Both compile-time and run-time detection are supported (run-time detection is currently disabled by default). <br> Requires nightly |
 | s390x | lpq | stpq | cdsg | cdsg | Requires nightly |

diff --git a/src/imp/atomic128/intrinsics.rs b/src/imp/atomic128/intrinsics.rs
@@ -113,7 +113,7 @@ unsafe fn atomic_compare_exchange(
 ) -> Result<u128, u128> {
     #[cfg(target_arch = "x86_64")]
     let (val, ok) = {
-        #[cfg_attr(not(target_feature = "cmpxchg16b"), target_feature(enable = "cmpxchg16b"))]
+        #[target_feature(enable = "cmpxchg16b")]
         #[cfg_attr(target_feature = "cmpxchg16b", inline)]
         #[cfg_attr(not(target_feature = "cmpxchg16b"), inline(never))]
         unsafe fn cmpxchg16b(

diff --git a/src/imp/atomic128/x86_64.rs b/src/imp/atomic128/x86_64.rs
@@ -65,8 +65,15 @@ macro_rules! ptr_modifier {
     };
 }
 
+// Unlike AArch64 and RISC-V, x86's assembler doesn't check instruction
+// requirements for the currently enabled target features. In the first place,
+// there is no option in the x86 assembly for such case, like ARM .arch_extension,
+// RISC-V .option arch, PowerPC .machine, etc.
+// However, we set target_feature(enable) when available (Rust 1.69+) in case a
+// new codegen backend is added that checks for it in the future, or an option
+// is added to the assembler to check for it.
 #[cfg_attr(
-    not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")),
+    not(portable_atomic_no_cmpxchg16b_target_feature),
     target_feature(enable = "cmpxchg16b")
 )]
 #[inline]
@@ -260,8 +267,9 @@ unsafe fn atomic_load(src: *mut u128, _order: Ordering) -> u128 {
         })
     }
 }
+// See cmpxchg16b() for target_feature(enable).
 #[cfg_attr(
-    not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")),
+    not(portable_atomic_no_cmpxchg16b_target_feature),
     target_feature(enable = "cmpxchg16b")
 )]
 #[inline]
@@ -363,10 +371,12 @@ unsafe fn atomic_store(dst: *mut u128, val: u128, order: Ordering) {
         }
     }
 }
+// See cmpxchg16b() for target_feature(enable).
 #[cfg_attr(
-    not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")),
+    not(portable_atomic_no_cmpxchg16b_target_feature),
     target_feature(enable = "cmpxchg16b")
 )]
+#[inline]
 unsafe fn atomic_store_cmpxchg16b(dst: *mut u128, val: u128) {
     // SAFETY: the caller must uphold the safety contract.
     unsafe {
@@ -413,8 +423,9 @@ use atomic_compare_exchange as atomic_compare_exchange_weak;
 
 #[cfg(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"))]
 use atomic_swap_cmpxchg16b as atomic_swap;
+// See cmpxchg16b() for target_feature(enable).
 #[cfg_attr(
-    not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")),
+    not(portable_atomic_no_cmpxchg16b_target_feature),
     target_feature(enable = "cmpxchg16b")
 )]
 #[inline]
@@ -487,8 +498,9 @@ macro_rules! atomic_rmw_cas_3 {
     ($name:ident as $reexport_name:ident, $($op:tt)*) => {
         #[cfg(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"))]
         use $name as $reexport_name;
+        // See cmpxchg16b() for target_feature(enable).
         #[cfg_attr(
-            not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")),
+            not(portable_atomic_no_cmpxchg16b_target_feature),
             target_feature(enable = "cmpxchg16b")
         )]
         #[inline]
@@ -558,8 +570,9 @@ macro_rules! atomic_rmw_cas_2 {
     ($name:ident as $reexport_name:ident, $($op:tt)*) => {
         #[cfg(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"))]
         use $name as $reexport_name;
+        // See cmpxchg16b() for target_feature(enable).
         #[cfg_attr(
-            not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")),
+            not(portable_atomic_no_cmpxchg16b_target_feature),
             target_feature(enable = "cmpxchg16b")
         )]
         #[inline]
@@ -729,11 +742,9 @@ macro_rules! atomic_rmw_with_ifunc {
         #[inline]
         unsafe fn $name($($arg)*, _order: Ordering) $(-> $ret_ty)? {
             fn_alias! {
+                // See cmpxchg16b() for target_feature(enable).
                 #[cfg_attr(
-                    not(any(
-                        target_feature = "cmpxchg16b",
-                        portable_atomic_target_feature = "cmpxchg16b",
-                    )),
+                    not(portable_atomic_no_cmpxchg16b_target_feature),
                     target_feature(enable = "cmpxchg16b")
                 )]
                 unsafe fn($($arg)*) $(-> $ret_ty)?;

diff --git a/src/imp/fallback/mod.rs b/src/imp/fallback/mod.rs
@@ -15,7 +15,6 @@
     any(
         all(
             target_arch = "x86_64",
-            not(portable_atomic_no_cmpxchg16b_target_feature),
             not(portable_atomic_no_outline_atomics),
             not(any(target_env = "sgx", miri)),
         ),

diff --git a/src/imp/mod.rs b/src/imp/mod.rs
@@ -54,7 +54,6 @@ mod aarch64;
         portable_atomic_target_feature = "cmpxchg16b",
         all(
             feature = "fallback",
-            not(portable_atomic_no_cmpxchg16b_target_feature),
             not(portable_atomic_no_outline_atomics),
             not(any(target_env = "sgx", miri)),
         ),
@@ -340,7 +339,6 @@ items! {
                 portable_atomic_target_feature = "cmpxchg16b",
                 all(
                     feature = "fallback",
-                    not(portable_atomic_no_cmpxchg16b_target_feature),
                     not(portable_atomic_no_outline_atomics),
                     not(any(target_env = "sgx", miri)),
                 ),
@@ -405,14 +403,13 @@ pub(crate) use self::aarch64::{AtomicI128, AtomicU128};
 // x86_64 & (cmpxchg16b | outline-atomics)
 #[cfg(all(
     target_arch = "x86_64",
-    not(all(any(miri, portable_atomic_sanitize_thread), portable_atomic_no_cmpxchg16b_intrinsic,)),
+    not(all(any(miri, portable_atomic_sanitize_thread), portable_atomic_no_cmpxchg16b_intrinsic)),
     any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
     any(
         target_feature = "cmpxchg16b",
         portable_atomic_target_feature = "cmpxchg16b",
         all(
             feature = "fallback",
-            not(portable_atomic_no_cmpxchg16b_target_feature),
             not(portable_atomic_no_outline_atomics),
             not(any(target_env = "sgx", miri)),
         ),

diff --git a/src/lib.rs b/src/lib.rs
@@ -164,7 +164,7 @@ RUSTFLAGS="--cfg portable_atomic_no_outline_atomics" cargo ...
   If dynamic dispatching by run-time CPU feature detection is enabled, it allows maintaining support for older CPUs while using features that are not supported on older CPUs, such as CMPXCHG16B (x86_64) and FEAT_LSE (aarch64).
 
   Note:
-  - Dynamic detection is currently only enabled in Rust 1.59+ for aarch64, in Rust 1.59+ (AVX) or 1.69+ (CMPXCHG16B) for x86_64, nightly only for powerpc64 (disabled by default), otherwise it works the same as when this cfg is set.
+  - Dynamic detection is currently only enabled in Rust 1.59+ for aarch64 and x86_64, nightly only for powerpc64 (disabled by default), otherwise it works the same as when this cfg is set.
   - If the required target features are enabled at compile-time, the atomic operations are inlined.
   - This is compatible with no-std (as with all features except `std`).
   - On some targets, run-time detection is disabled by default mainly for compatibility with older versions of operating systems or incomplete build environments, and can be enabled by `--cfg portable_atomic_outline_atomics`. (When both cfg are enabled, `*_no_*` cfg is preferred.)
@@ -238,21 +238,10 @@ RUSTFLAGS="--cfg portable_atomic_no_outline_atomics" cargo ...
 // These features are already stabilized or have already been removed from compilers,
 // and can safely be enabled for old nightly as long as version detection works.
 // - cfg(target_has_atomic)
-// - #[target_feature(enable = "cmpxchg16b")] on x86_64
 // - asm! on ARM, AArch64, RISC-V, x86_64
 // - llvm_asm! on AVR (tier 3) and MSP430 (tier 3)
 // - #[instruction_set] on non-Linux/Android pre-v6 ARM (tier 3)
 #![cfg_attr(portable_atomic_unstable_cfg_target_has_atomic, feature(cfg_target_has_atomic))]
-#![cfg_attr(
-    all(
-        target_arch = "x86_64",
-        portable_atomic_unstable_cmpxchg16b_target_feature,
-        not(portable_atomic_no_outline_atomics),
-        not(any(target_env = "sgx", miri)),
-        feature = "fallback",
-    ),
-    feature(cmpxchg16b_target_feature)
-)]
 #![cfg_attr(
     all(
         portable_atomic_unstable_asm,

diff --git a/src/tests/mod.rs b/src/tests/mod.rs
@@ -143,7 +143,6 @@ fn test_is_lock_free() {
         {
             let has_cmpxchg16b = cfg!(all(
                 feature = "fallback",
-                not(portable_atomic_no_cmpxchg16b_target_feature),
                 not(portable_atomic_no_outline_atomics),
                 not(any(target_env = "sgx", miri)),
                 not(portable_atomic_test_outline_atomics_detect_false),