Rollup merge of rust-lang#128192 - mrkajetanp:feature-detect, r=Amanieu

rustc_target: Add various aarch64 features

Add various aarch64 features already supported by LLVM and Linux.
Additionally include some comment fixes to ensure consistency of feature names with the Arm ARM.
Compiler support for features added to stdarch by rust-lang/stdarch#1614.
Tracking issue for unstable aarch64 features is rust-lang#127764.

List of added features:

- FEAT_CSSC
- FEAT_ECV
- FEAT_FAMINMAX
- FEAT_FLAGM2
- FEAT_FP8
- FEAT_FP8DOT2
- FEAT_FP8DOT4
- FEAT_FP8FMA
- FEAT_HBC
- FEAT_LSE128
- FEAT_LSE2
- FEAT_LUT
- FEAT_MOPS
- FEAT_LRCPC3
- FEAT_SVE_B16B16
- FEAT_SVE2p1
- FEAT_WFxT
- FEAT_SME
- FEAT_SME_F16F16
- FEAT_SME_F64F64
- FEAT_SME_F8F16
- FEAT_SME_F8F32
- FEAT_SME_FA64
- FEAT_SME_I16I64
- FEAT_SME_LUTv2
- FEAT_SME2
- FEAT_SME2p1
- FEAT_SSVE_FP8DOT2
- FEAT_SSVE_FP8DOT4
- FEAT_SSVE_FP8FMA

FEAT_FPMR is added in the first commit and then removed in a separate one to highlight it being removed from upstream LLVM 19. The intention is for it to be detectable at runtime through stdarch but not have a corresponding Rust compile-time feature.

compiler/rustc_codegen_llvm/src/attributes.rs

@@ -521,13 +521,20 @@ pub fn llfn_attrs_from_instance<'ll, 'tcx>(
 
     let function_features = function_features
         .iter()
-        .flat_map(|feat| {
-            llvm_util::to_llvm_features(cx.tcx.sess, feat).into_iter().map(|f| format!("+{f}"))
-        })
+        // Convert to LLVMFeatures and filter out unavailable ones
+        .flat_map(|feat| llvm_util::to_llvm_features(cx.tcx.sess, feat))
+        // Convert LLVMFeatures & dependencies to +<feats>s
+        .flat_map(|feat| feat.into_iter().map(|f| format!("+{f}")))
         .chain(codegen_fn_attrs.instruction_set.iter().map(|x| match x {
             InstructionSetAttr::ArmA32 => "-thumb-mode".to_string(),
             InstructionSetAttr::ArmT32 => "+thumb-mode".to_string(),
         }))
+        // HACK: LLVM versions 19+ do not have the FPMR feature and treat it as always enabled
+        // It only exists as a feature in LLVM 18, cannot be passed down for any other version
+        .chain(match &*cx.tcx.sess.target.arch {
+            "aarch64" if llvm_util::get_version().0 == 18 => vec!["+fpmr".to_string()],
+            _ => vec![],
+        })
         .collect::<Vec<String>>();
 
     if cx.tcx.sess.target.is_like_wasm {

compiler/rustc_codegen_llvm/src/llvm_util.rs

@@ -209,7 +209,7 @@ impl<'a> IntoIterator for LLVMFeature<'a> {
 // Though note that Rust can also be build with an external precompiled version of LLVM
 // which might lead to failures if the oldest tested / supported LLVM version
 // doesn't yet support the relevant intrinsics
-pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> LLVMFeature<'a> {
+pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFeature<'a>> {
     let arch = if sess.target.arch == "x86_64" {
         "x86"
     } else if sess.target.arch == "arm64ec" {
@@ -218,40 +218,59 @@ pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> LLVMFeature<'a> {
         &*sess.target.arch
     };
     match (arch, s) {
-        ("x86", "sse4.2") => {
-            LLVMFeature::with_dependency("sse4.2", TargetFeatureFoldStrength::EnableOnly("crc32"))
-        }
-        ("x86", "pclmulqdq") => LLVMFeature::new("pclmul"),
-        ("x86", "rdrand") => LLVMFeature::new("rdrnd"),
-        ("x86", "bmi1") => LLVMFeature::new("bmi"),
-        ("x86", "cmpxchg16b") => LLVMFeature::new("cx16"),
-        ("x86", "lahfsahf") => LLVMFeature::new("sahf"),
-        ("aarch64", "rcpc2") => LLVMFeature::new("rcpc-immo"),
-        ("aarch64", "dpb") => LLVMFeature::new("ccpp"),
-        ("aarch64", "dpb2") => LLVMFeature::new("ccdp"),
-        ("aarch64", "frintts") => LLVMFeature::new("fptoint"),
-        ("aarch64", "fcma") => LLVMFeature::new("complxnum"),
-        ("aarch64", "pmuv3") => LLVMFeature::new("perfmon"),
-        ("aarch64", "paca") => LLVMFeature::new("pauth"),
-        ("aarch64", "pacg") => LLVMFeature::new("pauth"),
+        ("x86", "sse4.2") => Some(LLVMFeature::with_dependency(
+            "sse4.2",
+            TargetFeatureFoldStrength::EnableOnly("crc32"),
+        )),
+        ("x86", "pclmulqdq") => Some(LLVMFeature::new("pclmul")),
+        ("x86", "rdrand") => Some(LLVMFeature::new("rdrnd")),
+        ("x86", "bmi1") => Some(LLVMFeature::new("bmi")),
+        ("x86", "cmpxchg16b") => Some(LLVMFeature::new("cx16")),
+        ("x86", "lahfsahf") => Some(LLVMFeature::new("sahf")),
+        ("aarch64", "rcpc2") => Some(LLVMFeature::new("rcpc-immo")),
+        ("aarch64", "dpb") => Some(LLVMFeature::new("ccpp")),
+        ("aarch64", "dpb2") => Some(LLVMFeature::new("ccdp")),
+        ("aarch64", "frintts") => Some(LLVMFeature::new("fptoint")),
+        ("aarch64", "fcma") => Some(LLVMFeature::new("complxnum")),
+        ("aarch64", "pmuv3") => Some(LLVMFeature::new("perfmon")),
+        ("aarch64", "paca") => Some(LLVMFeature::new("pauth")),
+        ("aarch64", "pacg") => Some(LLVMFeature::new("pauth")),
+        ("aarch64", "sve-b16b16") => Some(LLVMFeature::new("b16b16")),
+        ("aarch64", "flagm2") => Some(LLVMFeature::new("altnzcv")),
         // Rust ties fp and neon together.
         ("aarch64", "neon") => {
-            LLVMFeature::with_dependency("neon", TargetFeatureFoldStrength::Both("fp-armv8"))
+            Some(LLVMFeature::with_dependency("neon", TargetFeatureFoldStrength::Both("fp-armv8")))
         }
         // In LLVM neon implicitly enables fp, but we manually enable
         // neon when a feature only implicitly enables fp
-        ("aarch64", "fhm") => LLVMFeature::new("fp16fml"),
-        ("aarch64", "fp16") => LLVMFeature::new("fullfp16"),
+        ("aarch64", "fhm") => Some(LLVMFeature::new("fp16fml")),
+        ("aarch64", "fp16") => Some(LLVMFeature::new("fullfp16")),
+        // Filter out features that are not supported by the current LLVM version
+        ("aarch64", "faminmax") if get_version().0 < 18 => None,
+        ("aarch64", "fp8") if get_version().0 < 18 => None,
+        ("aarch64", "fp8dot2") if get_version().0 < 18 => None,
+        ("aarch64", "fp8dot4") if get_version().0 < 18 => None,
+        ("aarch64", "fp8fma") if get_version().0 < 18 => None,
+        ("aarch64", "fpmr") if get_version().0 != 18 => None,
+        ("aarch64", "lut") if get_version().0 < 18 => None,
+        ("aarch64", "sme-f8f16") if get_version().0 < 18 => None,
+        ("aarch64", "sme-f8f32") if get_version().0 < 18 => None,
+        ("aarch64", "sme-fa64") if get_version().0 < 18 => None,
+        ("aarch64", "sme-lutv2") if get_version().0 < 18 => None,
+        ("aarch64", "ssve-fp8dot2") if get_version().0 < 18 => None,
+        ("aarch64", "ssve-fp8dot4") if get_version().0 < 18 => None,
+        ("aarch64", "ssve-fp8fma") if get_version().0 < 18 => None,
+        ("aarch64", "v9.5a") if get_version().0 < 18 => None,
         // In LLVM 18, `unaligned-scalar-mem` was merged with `unaligned-vector-mem` into a single feature called
         // `fast-unaligned-access`. In LLVM 19, it was split back out.
         ("riscv32" | "riscv64", "unaligned-scalar-mem") if get_version().0 == 18 => {
-            LLVMFeature::new("fast-unaligned-access")
+            Some(LLVMFeature::new("fast-unaligned-access"))
         }
         // For LLVM 18, enable the evex512 target feature if a avx512 target feature is enabled.
         ("x86", s) if get_version().0 >= 18 && s.starts_with("avx512") => {
-            LLVMFeature::with_dependency(s, TargetFeatureFoldStrength::EnableOnly("evex512"))
+            Some(LLVMFeature::with_dependency(s, TargetFeatureFoldStrength::EnableOnly("evex512")))
         }
-        (_, s) => LLVMFeature::new(s),
+        (_, s) => Some(LLVMFeature::new(s)),
     }
 }
 
@@ -291,13 +310,17 @@ pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> {
                     return true;
                 }
                 // check that all features in a given smallvec are enabled
-                for llvm_feature in to_llvm_features(sess, feature) {
-                    let cstr = SmallCStr::new(llvm_feature);
-                    if !unsafe { llvm::LLVMRustHasFeature(&target_machine, cstr.as_ptr()) } {
-                        return false;
+                if let Some(feat) = to_llvm_features(sess, feature) {
+                    for llvm_feature in feat {
+                        let cstr = SmallCStr::new(llvm_feature);
+                        if !unsafe { llvm::LLVMRustHasFeature(&target_machine, cstr.as_ptr()) } {
+                            return false;
+                        }
                     }
+                    true
+                } else {
+                    false
                 }
-                true
             })
             .map(|(feature, _, _)| Symbol::intern(feature)),
     );
@@ -386,9 +409,9 @@ fn print_target_features(out: &mut String, sess: &Session, tm: &llvm::TargetMach
         .target
         .supported_target_features()
         .iter()
-        .map(|(feature, _gate, _implied)| {
+        .filter_map(|(feature, _gate, _implied)| {
             // LLVM asserts that these are sorted. LLVM and Rust both use byte comparison for these strings.
-            let llvm_feature = to_llvm_features(sess, *feature).llvm_feature_name;
+            let llvm_feature = to_llvm_features(sess, *feature)?.llvm_feature_name;
             let desc =
                 match llvm_target_features.binary_search_by_key(&llvm_feature, |(f, _d)| f).ok() {
                     Some(index) => {
@@ -398,7 +421,7 @@ fn print_target_features(out: &mut String, sess: &Session, tm: &llvm::TargetMach
                     None => "",
                 };
 
-            (*feature, desc)
+            Some((*feature, desc))
         })
         .collect::<Vec<_>>();
 
@@ -595,7 +618,7 @@ pub(crate) fn global_llvm_features(
                     if feature_state.is_none() {
                         let rust_feature =
                             supported_features.iter().find_map(|&(rust_feature, _, _)| {
-                                let llvm_features = to_llvm_features(sess, rust_feature);
+                                let llvm_features = to_llvm_features(sess, rust_feature)?;
                                 if llvm_features.contains(feature)
                                     && !llvm_features.contains(rust_feature)
                                 {
@@ -641,7 +664,7 @@ pub(crate) fn global_llvm_features(
                 // passing requests down to LLVM. This means that all in-language
                 // features also work on the command line instead of having two
                 // different names when the LLVM name and the Rust name differ.
-                let llvm_feature = to_llvm_features(sess, feature);
+                let llvm_feature = to_llvm_features(sess, feature)?;
 
                 Some(
                     std::iter::once(format!(
@@ -691,6 +714,9 @@ fn backend_feature_name<'a>(sess: &Session, s: &'a str) -> Option<&'a str> {
     let feature = s
         .strip_prefix(&['+', '-'][..])
         .unwrap_or_else(|| sess.dcx().emit_fatal(InvalidTargetFeaturePrefix { feature: s }));
+    if s.is_empty() {
+        return None;
+    }
     // Rustc-specific feature requests like `+crt-static` or `-crt-static`
     // are not passed down to LLVM.
     if RUSTC_SPECIFIC_FEATURES.contains(&feature) {

compiler/rustc_feature/src/unstable.rs

@@ -300,6 +300,7 @@ declare_features! (
     // FIXME: Document these and merge with the list below.
 
     // Unstable `#[target_feature]` directives.
+    (unstable, aarch64_unstable_target_feature, "CURRENT_RUSTC_VERSION", Some(44839)),
     (unstable, aarch64_ver_target_feature, "1.27.0", Some(44839)),
     (unstable, arm_target_feature, "1.27.0", Some(44839)),
     (unstable, avx512_target_feature, "1.27.0", Some(44839)),

compiler/rustc_span/src/symbol.rs

@@ -356,6 +356,7 @@ symbols! {
         _task_context,
         a32,
         aarch64_target_feature,
+        aarch64_unstable_target_feature,
         aarch64_ver_target_feature,
         abi,
         abi_amdgpu_kernel,

compiler/rustc_target/src/target_features.rs

@@ -99,6 +99,8 @@ const AARCH64_ALLOWED_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[
     ("bti", Stable, &[]),
     // FEAT_CRC
     ("crc", Stable, &[]),
+    // FEAT_CSSC
+    ("cssc", Unstable(sym::aarch64_unstable_target_feature), &[]),
     // FEAT_DIT
     ("dit", Stable, &[]),
     // FEAT_DotProd
@@ -107,21 +109,37 @@ const AARCH64_ALLOWED_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[
     ("dpb", Stable, &[]),
     // FEAT_DPB2
     ("dpb2", Stable, &["dpb"]),
+    // FEAT_ECV
+    ("ecv", Unstable(sym::aarch64_unstable_target_feature), &[]),
     // FEAT_F32MM
     ("f32mm", Stable, &["sve"]),
     // FEAT_F64MM
     ("f64mm", Stable, &["sve"]),
+    // FEAT_FAMINMAX
+    ("faminmax", Unstable(sym::aarch64_unstable_target_feature), &[]),
     // FEAT_FCMA
     ("fcma", Stable, &["neon"]),
     // FEAT_FHM
     ("fhm", Stable, &["fp16"]),
     // FEAT_FLAGM
     ("flagm", Stable, &[]),
+    // FEAT_FLAGM2
+    ("flagm2", Unstable(sym::aarch64_unstable_target_feature), &[]),
     // FEAT_FP16
     // Rust ties FP and Neon: https://github.com/rust-lang/rust/pull/91608
     ("fp16", Stable, &["neon"]),
+    // FEAT_FP8
+    ("fp8", Unstable(sym::aarch64_unstable_target_feature), &["faminmax", "lut", "bf16"]),
+    // FEAT_FP8DOT2
+    ("fp8dot2", Unstable(sym::aarch64_unstable_target_feature), &["fp8dot4"]),
+    // FEAT_FP8DOT4
+    ("fp8dot4", Unstable(sym::aarch64_unstable_target_feature), &["fp8fma"]),
+    // FEAT_FP8FMA
+    ("fp8fma", Unstable(sym::aarch64_unstable_target_feature), &["fp8"]),
     // FEAT_FRINTTS
     ("frintts", Stable, &[]),
+    // FEAT_HBC
+    ("hbc", Unstable(sym::aarch64_unstable_target_feature), &[]),
     // FEAT_I8MM
     ("i8mm", Stable, &[]),
     // FEAT_JSCVT
@@ -131,6 +149,14 @@ const AARCH64_ALLOWED_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[
     ("lor", Stable, &[]),
     // FEAT_LSE
     ("lse", Stable, &[]),
+    // FEAT_LSE128
+    ("lse128", Unstable(sym::aarch64_unstable_target_feature), &["lse"]),
+    // FEAT_LSE2
+    ("lse2", Unstable(sym::aarch64_unstable_target_feature), &[]),
+    // FEAT_LUT
+    ("lut", Unstable(sym::aarch64_unstable_target_feature), &[]),
+    // FEAT_MOPS
+    ("mops", Unstable(sym::aarch64_unstable_target_feature), &[]),
     // FEAT_MTE & FEAT_MTE2
     ("mte", Stable, &[]),
     // FEAT_AdvSimd & FEAT_FP
@@ -143,14 +169,16 @@ const AARCH64_ALLOWED_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[
     ("pan", Stable, &[]),
     // FEAT_PMUv3
     ("pmuv3", Stable, &[]),
-    // FEAT_RAND
+    // FEAT_RNG
     ("rand", Stable, &[]),
     // FEAT_RAS & FEAT_RASv1p1
     ("ras", Stable, &[]),
-    // FEAT_RCPC
+    // FEAT_LRCPC
     ("rcpc", Stable, &[]),
-    // FEAT_RCPC2
+    // FEAT_LRCPC2
     ("rcpc2", Stable, &["rcpc"]),
+    // FEAT_LRCPC3
+    ("rcpc3", Unstable(sym::aarch64_unstable_target_feature), &["rcpc2"]),
     // FEAT_RDM
     ("rdm", Stable, &["neon"]),
     // FEAT_SB
@@ -161,10 +189,36 @@ const AARCH64_ALLOWED_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[
     ("sha3", Stable, &["sha2"]),
     // FEAT_SM3 & FEAT_SM4
     ("sm4", Stable, &["neon"]),
+    // FEAT_SME
+    ("sme", Unstable(sym::aarch64_unstable_target_feature), &["bf16"]),
+    // FEAT_SME_F16F16
+    ("sme-f16f16", Unstable(sym::aarch64_unstable_target_feature), &["sme2"]),
+    // FEAT_SME_F64F64
+    ("sme-f64f64", Unstable(sym::aarch64_unstable_target_feature), &["sme"]),
+    // FEAT_SME_F8F16
+    ("sme-f8f16", Unstable(sym::aarch64_unstable_target_feature), &["sme-f8f32"]),
+    // FEAT_SME_F8F32
+    ("sme-f8f32", Unstable(sym::aarch64_unstable_target_feature), &["sme2", "fp8"]),
+    // FEAT_SME_FA64
+    ("sme-fa64", Unstable(sym::aarch64_unstable_target_feature), &["sme", "sve2"]),
+    // FEAT_SME_I16I64
+    ("sme-i16i64", Unstable(sym::aarch64_unstable_target_feature), &["sme"]),
+    // FEAT_SME_LUTv2
+    ("sme-lutv2", Unstable(sym::aarch64_unstable_target_feature), &[]),
+    // FEAT_SME2
+    ("sme2", Unstable(sym::aarch64_unstable_target_feature), &["sme"]),
+    // FEAT_SME2p1
+    ("sme2p1", Unstable(sym::aarch64_unstable_target_feature), &["sme2"]),
     // FEAT_SPE
     ("spe", Stable, &[]),
     // FEAT_SSBS & FEAT_SSBS2
     ("ssbs", Stable, &[]),
+    // FEAT_SSVE_FP8FDOT2
+    ("ssve-fp8dot2", Unstable(sym::aarch64_unstable_target_feature), &["ssve-fp8dot4"]),
+    // FEAT_SSVE_FP8FDOT4
+    ("ssve-fp8dot4", Unstable(sym::aarch64_unstable_target_feature), &["ssve-fp8fma"]),
+    // FEAT_SSVE_FP8FMA
+    ("ssve-fp8fma", Unstable(sym::aarch64_unstable_target_feature), &["sme2", "fp8"]),
     // FEAT_SVE
     // It was decided that SVE requires Neon: https://github.com/rust-lang/rust/pull/91608
     //
@@ -173,16 +227,20 @@ const AARCH64_ALLOWED_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[
     //
     // "For backwards compatibility, Neon and VFP are required in the latest architectures."
     ("sve", Stable, &["neon"]),
+    // FEAT_SVE_B16B16 (SVE or SME Instructions)
+    ("sve-b16b16", Unstable(sym::aarch64_unstable_target_feature), &["bf16"]),
     // FEAT_SVE2
     ("sve2", Stable, &["sve"]),
-    // FEAT_SVE2_AES
+    // FEAT_SVE_AES & FEAT_SVE_PMULL128
     ("sve2-aes", Stable, &["sve2", "aes"]),
     // FEAT_SVE2_BitPerm
     ("sve2-bitperm", Stable, &["sve2"]),
     // FEAT_SVE2_SHA3
     ("sve2-sha3", Stable, &["sve2", "sha3"]),
     // FEAT_SVE2_SM4
     ("sve2-sm4", Stable, &["sve2", "sm4"]),
+    // FEAT_SVE2p1
+    ("sve2p1", Unstable(sym::aarch64_unstable_target_feature), &["sve2"]),
     // FEAT_TME
     ("tme", Stable, &[]),
     (
@@ -199,9 +257,19 @@ const AARCH64_ALLOWED_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[
     ("v8.4a", Unstable(sym::aarch64_ver_target_feature), &["v8.3a", "dotprod", "dit", "flagm"]),
     ("v8.5a", Unstable(sym::aarch64_ver_target_feature), &["v8.4a", "ssbs", "sb", "dpb2", "bti"]),
     ("v8.6a", Unstable(sym::aarch64_ver_target_feature), &["v8.5a", "bf16", "i8mm"]),
-    ("v8.7a", Unstable(sym::aarch64_ver_target_feature), &[]),
+    ("v8.7a", Unstable(sym::aarch64_ver_target_feature), &["v8.6a", "wfxt"]),
+    ("v8.8a", Unstable(sym::aarch64_ver_target_feature), &["v8.7a", "hbc", "mops"]),
+    ("v8.9a", Unstable(sym::aarch64_ver_target_feature), &["v8.8a", "cssc"]),
+    ("v9.1a", Unstable(sym::aarch64_ver_target_feature), &["v9a", "v8.6a"]),
+    ("v9.2a", Unstable(sym::aarch64_ver_target_feature), &["v9.1a", "v8.7a"]),
+    ("v9.3a", Unstable(sym::aarch64_ver_target_feature), &["v9.2a", "v8.8a"]),
+    ("v9.4a", Unstable(sym::aarch64_ver_target_feature), &["v9.3a", "v8.9a"]),
+    ("v9.5a", Unstable(sym::aarch64_ver_target_feature), &["v9.4a"]),
+    ("v9a", Unstable(sym::aarch64_ver_target_feature), &["v8.5a", "sve2"]),
     // FEAT_VHE
     ("vh", Stable, &[]),
+    // FEAT_WFxT
+    ("wfxt", Unstable(sym::aarch64_unstable_target_feature), &[]),
     // tidy-alphabetical-end
 ];