x86_64: Add portable_atomic_vmovdqa_atomic cfg

taiki-e · Feb 23, 2023 · 9b567e4 · 9b567e4
1 parent fe84be8
commit 9b567e4
Show file tree

Hide file tree

Showing 4 changed files with 106 additions and 26 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -193,6 +193,31 @@ jobs:
           RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+cmpxchg16b ${{ env.RANDOMIZE_LAYOUT }}
         if: (matrix.target == '' || startsWith(matrix.target, 'x86_64')) && !startsWith(matrix.os, 'macos') && startsWith(matrix.rust, 'nightly') && !(contains(matrix.target, '-musl') || contains(matrix.target, '-android'))
 
+      # +cmpxchg16b,+avx with portable_atomic_vmovdqa_atomic cfg
+      - run: $cargo test -vv --workspace --all-features                   $EXCLUDE $TARGET $DOCTEST_XCOMPILE $BUILD_STD
+        env:
+          RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-feature=+cmpxchg16b,+avx --cfg portable_atomic_vmovdqa_atomic
+          RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+cmpxchg16b,+avx --cfg portable_atomic_vmovdqa_atomic
+        if: matrix.target == '' || startsWith(matrix.target, 'x86_64')
+      - run: $cargo test -vv --workspace --all-features --release         $EXCLUDE $TARGET $DOCTEST_XCOMPILE $BUILD_STD
+        env:
+          RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-feature=+cmpxchg16b,+avx --cfg portable_atomic_vmovdqa_atomic
+          RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+cmpxchg16b,+avx --cfg portable_atomic_vmovdqa_atomic
+        if: matrix.target == '' || startsWith(matrix.target, 'x86_64')
+      # LTO + doctests is very slow on some platforms
+      - run: $cargo test -vv --workspace --all-features --release --tests $EXCLUDE $TARGET $DOCTEST_XCOMPILE $BUILD_STD
+        env:
+          CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 1
+          CARGO_PROFILE_RELEASE_LTO: fat
+          RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-feature=+cmpxchg16b,+avx --cfg portable_atomic_vmovdqa_atomic ${{ env.RANDOMIZE_LAYOUT }}
+          RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+cmpxchg16b,+avx --cfg portable_atomic_vmovdqa_atomic ${{ env.RANDOMIZE_LAYOUT }}
+        if: matrix.target == '' || startsWith(matrix.target, 'x86_64')
+      - run: cargo careful test -vv --workspace --all-features           $EXCLUDE $TARGET $DOCTEST_XCOMPILE
+        env:
+          RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-feature=+cmpxchg16b,+avx --cfg portable_atomic_vmovdqa_atomic ${{ env.RANDOMIZE_LAYOUT }}
+          RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+cmpxchg16b,+avx --cfg portable_atomic_vmovdqa_atomic ${{ env.RANDOMIZE_LAYOUT }}
+        if: (matrix.target == '' || startsWith(matrix.target, 'x86_64')) && startsWith(matrix.rust, 'nightly')
+
       # +lse
       - run: $cargo test -vv --workspace --all-features                   $EXCLUDE $TARGET $DOCTEST_XCOMPILE $BUILD_STD
         env:
@@ -450,6 +475,7 @@ jobs:
       - name: Install Rust
         run: rustup toolchain add nightly --no-self-update --component rust-src && rustup default nightly
       - uses: taiki-e/install-action@valgrind
+
       # doctests on Valgrind are very slow
       - run: cargo test -vv --workspace --all-features           --tests $EXCLUDE
         env:
@@ -465,6 +491,7 @@ jobs:
           CARGO_PROFILE_RELEASE_LTO: fat
           RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} --cfg valgrind
           RUSTFLAGS: ${{ env.RUSTFLAGS }} --cfg valgrind
+
       # +cmpxchg16b
       - run: cargo test -vv --workspace --all-features           --tests $EXCLUDE
         env:
@@ -481,6 +508,22 @@ jobs:
           RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-feature=+cmpxchg16b --cfg valgrind
           RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+cmpxchg16b --cfg valgrind
 
+      # Sandy Bridge (the first Intel chip that introduced AVX) with portable_atomic_vmovdqa_atomic cfg
+      - run: cargo test -vv --workspace --all-features           --tests $EXCLUDE
+        env:
+          RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-feature=+cmpxchg16b,+avx --cfg portable_atomic_vmovdqa_atomic --cfg valgrind
+          RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+cmpxchg16b,+avx --cfg portable_atomic_vmovdqa_atomic --cfg valgrind
+      - run: cargo test -vv --workspace --all-features --release --tests $EXCLUDE
+        env:
+          RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-feature=+cmpxchg16b,+avx --cfg portable_atomic_vmovdqa_atomic --cfg valgrind
+          RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+cmpxchg16b,+avx --cfg portable_atomic_vmovdqa_atomic --cfg valgrind
+      - run: cargo test -vv --workspace --all-features --release --tests $EXCLUDE
+        env:
+          CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 1
+          CARGO_PROFILE_RELEASE_LTO: fat
+          RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-feature=+cmpxchg16b,+avx --cfg portable_atomic_vmovdqa_atomic --cfg valgrind
+          RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+cmpxchg16b,+avx --cfg portable_atomic_vmovdqa_atomic --cfg valgrind
+
   codegen:
     runs-on: ubuntu-latest
     timeout-minutes: 60

diff --git a/src/imp/atomic128/detect/x86_64.rs b/src/imp/atomic128/detect/x86_64.rs
@@ -5,6 +5,7 @@
         portable_atomic_no_outline_atomics,
         not(target_feature = "sse"),
         target_env = "sgx",
+        portable_atomic_vmovdqa_atomic,
         miri,
         portable_atomic_sanitize_thread,
     ),

diff --git a/src/imp/atomic128/x86_64.rs b/src/imp/atomic128/x86_64.rs
@@ -5,6 +5,7 @@
 //
 // Generated asm:
 // - x86_64 (+cmpxchg16b) https://godbolt.org/z/93Grnav1P
+// - x86_64 (+cmpxchg16b,+avx,vmovdqa_atomic) https://godbolt.org/z/K7oxqhjPz
 
 include!("macros.rs");
 
@@ -292,7 +293,10 @@ unsafe fn atomic_load(src: *mut u128, order: Ordering) -> u128 {
     // https://doc.rust-lang.org/nightly/rustc/platform-support/x86_64-unknown-none.html
     // Miri and Sanitizer do not support inline assembly.
     #[cfg(any(
-        portable_atomic_no_outline_atomics,
+        all(
+            portable_atomic_no_outline_atomics,
+            not(all(portable_atomic_vmovdqa_atomic, target_feature = "avx")),
+        ),
         not(target_feature = "sse"),
         miri,
         portable_atomic_sanitize_thread,
@@ -302,22 +306,33 @@ unsafe fn atomic_load(src: *mut u128, order: Ordering) -> u128 {
         _atomic_load_cmpxchg16b(src, order)
     }
     #[cfg(not(any(
-        portable_atomic_no_outline_atomics,
+        all(
+            portable_atomic_no_outline_atomics,
+            not(all(portable_atomic_vmovdqa_atomic, target_feature = "avx")),
+        ),
         not(target_feature = "sse"),
         miri,
         portable_atomic_sanitize_thread,
     )))]
-    // SAFETY: the caller must uphold the safety contract for `atomic_load`.
-    unsafe {
-        ifunc!(unsafe fn(src: *mut u128, order: Ordering) -> u128 {
-            // Check CMPXCHG16B anyway to prevent mixing atomic and non-atomic access.
-            let cpuid = detect::detect();
-            if cpuid.has_cmpxchg16b() && cpuid.has_vmovdqa_atomic() {
-                _atomic_load_vmovdqa
-            } else {
-                _atomic_load_cmpxchg16b
-            }
-        })
+    {
+        #[cfg(all(portable_atomic_vmovdqa_atomic, target_feature = "avx"))]
+        // SAFETY: the caller must uphold the safety contract for `atomic_load`.
+        unsafe {
+            _atomic_load_vmovdqa(src, order)
+        }
+        #[cfg(not(all(portable_atomic_vmovdqa_atomic, target_feature = "avx")))]
+        // SAFETY: the caller must uphold the safety contract for `atomic_load`.
+        unsafe {
+            ifunc!(unsafe fn(src: *mut u128, order: Ordering) -> u128 {
+                // Check CMPXCHG16B anyway to prevent mixing atomic and non-atomic access.
+                let cpuid = detect::detect();
+                if cpuid.has_cmpxchg16b() && cpuid.has_vmovdqa_atomic() {
+                    _atomic_load_vmovdqa
+                } else {
+                    _atomic_load_cmpxchg16b
+                }
+            })
+        }
     }
 }
 
@@ -335,7 +350,10 @@ unsafe fn atomic_store(dst: *mut u128, val: u128, order: Ordering) {
     // https://doc.rust-lang.org/nightly/rustc/platform-support/x86_64-unknown-none.html
     // Miri and Sanitizer do not support inline assembly.
     #[cfg(any(
-        portable_atomic_no_outline_atomics,
+        all(
+            portable_atomic_no_outline_atomics,
+            not(all(portable_atomic_vmovdqa_atomic, target_feature = "avx")),
+        ),
         not(target_feature = "sse"),
         miri,
         portable_atomic_sanitize_thread,
@@ -345,22 +363,33 @@ unsafe fn atomic_store(dst: *mut u128, val: u128, order: Ordering) {
         _atomic_store_cmpxchg16b(dst, val, order);
     }
     #[cfg(not(any(
-        portable_atomic_no_outline_atomics,
+        all(
+            portable_atomic_no_outline_atomics,
+            not(all(portable_atomic_vmovdqa_atomic, target_feature = "avx")),
+        ),
         not(target_feature = "sse"),
         miri,
         portable_atomic_sanitize_thread,
     )))]
-    // SAFETY: the caller must uphold the safety contract for `atomic_store`.
-    unsafe {
-        ifunc!(unsafe fn(dst: *mut u128, val: u128, order: Ordering) {
-            // Check CMPXCHG16B anyway to prevent mixing atomic and non-atomic access.
-            let cpuid = detect::detect();
-            if cpuid.has_cmpxchg16b() && cpuid.has_vmovdqa_atomic() {
-                _atomic_store_vmovdqa
-            } else {
-                _atomic_store_cmpxchg16b
-            }
-        });
+    {
+        #[cfg(all(portable_atomic_vmovdqa_atomic, target_feature = "avx"))]
+        // SAFETY: the caller must uphold the safety contract for `atomic_store`.
+        unsafe {
+            _atomic_store_vmovdqa(dst, val, order);
+        }
+        #[cfg(not(all(portable_atomic_vmovdqa_atomic, target_feature = "avx")))]
+        // SAFETY: the caller must uphold the safety contract for `atomic_store`.
+        unsafe {
+            ifunc!(unsafe fn(dst: *mut u128, val: u128, order: Ordering) {
+                // Check CMPXCHG16B anyway to prevent mixing atomic and non-atomic access.
+                let cpuid = detect::detect();
+                if cpuid.has_cmpxchg16b() && cpuid.has_vmovdqa_atomic() {
+                    _atomic_store_vmovdqa
+                } else {
+                    _atomic_store_cmpxchg16b
+                }
+            });
+        }
     }
 }
 

diff --git a/tools/build.sh b/tools/build.sh
@@ -93,6 +93,7 @@ known_cfgs=(
     portable_atomic_s_mode
     portable_atomic_disable_fiq
     portable_atomic_no_outline_atomics
+    portable_atomic_vmovdqa_atomic
 )
 
 x() {
@@ -339,6 +340,12 @@ build() {
                         x_cargo "${args[@]}" "$@"
                     ;;
             esac
+            # TODO: "LLVM ERROR: Do not know how to split the result of this operator!" on 1.66-stable and 1.67-beta
+            if [[ -n "${nightly}" ]] || [[ "${target}" != "x86_64-unknown-none" ]]; then
+                # +cmpxchg16b,+avx with portable_atomic_vmovdqa_atomic cfg
+                RUSTFLAGS="${target_rustflags} -C target-feature=+cmpxchg16b,+avx --cfg portable_atomic_vmovdqa_atomic" \
+                    x_cargo "${args[@]}" --target-dir target/vmovdqa_atomic "$@"
+            fi
             ;;
         aarch64* | arm64*)
             # macOS is skipped because it is +lse,+lse2 by default