diff --git a/compiler/rustc_codegen_cranelift/src/driver/mod.rs b/compiler/rustc_codegen_cranelift/src/driver/mod.rs index fb0eed07c1971..dbc913528f44a 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/mod.rs @@ -30,7 +30,7 @@ fn predefine_mono_items<'tcx>( get_function_sig(tcx, module.target_config().default_call_conv, instance); let linkage = crate::linkage::get_clif_linkage( mono_item, - data.linkage, + data.linkage_info.into_linkage(), data.visibility, is_compiler_builtins, ); diff --git a/compiler/rustc_codegen_gcc/src/base.rs b/compiler/rustc_codegen_gcc/src/base.rs index 18aa32754e1d9..844871a714671 100644 --- a/compiler/rustc_codegen_gcc/src/base.rs +++ b/compiler/rustc_codegen_gcc/src/base.rs @@ -213,7 +213,7 @@ pub fn compile_codegen_unit( let mono_items = cgu.items_in_deterministic_order(tcx); for &(mono_item, data) in &mono_items { - mono_item.predefine::>(&cx, data.linkage, data.visibility); + mono_item.predefine::>(&cx, data.linkage_info, data.visibility); } // ... and now that we have everything pre-defined, fill out those definitions. diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index f62310bd94808..82c3cfe9aa04a 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -86,7 +86,7 @@ pub(crate) fn compile_codegen_unit( let cx = CodegenCx::new(tcx, cgu, &llvm_module); let mono_items = cx.codegen_unit.items_in_deterministic_order(cx.tcx); for &(mono_item, data) in &mono_items { - mono_item.predefine::>(&cx, data.linkage, data.visibility); + mono_item.predefine::>(&cx, data.linkage_info, data.visibility); } // ... and now that we have everything pre-defined, fill out those definitions. diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs index 788a8a13b3ee4..0321a244f8683 100644 --- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs +++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs @@ -283,7 +283,7 @@ fn exported_symbols_provider_local( } if tcx.local_crate_exports_generics() { - use rustc_middle::mir::mono::{Linkage, MonoItem, Visibility}; + use rustc_middle::mir::mono::{MonoItem, Visibility}; use rustc_middle::ty::InstanceKind; // Normally, we require that shared monomorphizations are not hidden, @@ -298,7 +298,7 @@ fn exported_symbols_provider_local( // The symbols created in this loop are sorted below it #[allow(rustc::potential_query_instability)] for (mono_item, data) in cgus.iter().flat_map(|cgu| cgu.items().iter()) { - if data.linkage != Linkage::External { + if !data.linkage_info.is_external() { // We can only re-use things with external linkage, otherwise // we'll get a linker error continue; diff --git a/compiler/rustc_codegen_ssa/src/mono_item.rs b/compiler/rustc_codegen_ssa/src/mono_item.rs index 038c5857face1..04159422e3cb2 100644 --- a/compiler/rustc_codegen_ssa/src/mono_item.rs +++ b/compiler/rustc_codegen_ssa/src/mono_item.rs @@ -1,8 +1,8 @@ use rustc_hir as hir; use rustc_middle::mir::interpret::ErrorHandled; -use rustc_middle::mir::mono::{Linkage, MonoItem, Visibility}; -use rustc_middle::ty::Instance; +use rustc_middle::mir::mono::{LinkageInfo, MonoItem, Visibility}; use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; +use rustc_middle::ty::Instance; use rustc_middle::{span_bug, ty}; use tracing::debug; @@ -14,7 +14,7 @@ pub trait MonoItemExt<'a, 'tcx> { fn predefine>( &self, cx: &'a Bx::CodegenCx, - linkage: Linkage, + linkage_info: LinkageInfo, visibility: Visibility, ); fn to_raw_string(&self) -> String; @@ -116,7 +116,7 @@ impl<'a, 'tcx: 'a> MonoItemExt<'a, 'tcx> for MonoItem<'tcx> { fn predefine>( &self, cx: &'a Bx::CodegenCx, - linkage: Linkage, + linkage_info: LinkageInfo, visibility: Visibility, ) { debug!( @@ -132,10 +132,10 @@ impl<'a, 'tcx: 'a> MonoItemExt<'a, 'tcx> for MonoItem<'tcx> { match *self { MonoItem::Static(def_id) => { - cx.predefine_static(def_id, linkage, visibility, symbol_name); + cx.predefine_static(def_id, linkage_info.into_linkage(), visibility, symbol_name); } MonoItem::Fn(instance) => { - cx.predefine_fn(instance, linkage, visibility, symbol_name); + cx.predefine_fn(instance, linkage_info.into_linkage(), visibility, symbol_name); } MonoItem::GlobalAsm(..) => {} } diff --git a/compiler/rustc_middle/src/mir/mono.rs b/compiler/rustc_middle/src/mir/mono.rs index 161716610fe63..ca230d7fa8584 100644 --- a/compiler/rustc_middle/src/mir/mono.rs +++ b/compiler/rustc_middle/src/mir/mono.rs @@ -266,13 +266,57 @@ pub struct MonoItemData { /// `GloballyShared` maps to `false` and `LocalCopy` maps to `true`. pub inlined: bool, - pub linkage: Linkage, + pub linkage_info: LinkageInfo, pub visibility: Visibility, /// A cached copy of the result of `MonoItem::size_estimate`. pub size_estimate: usize, } +/// Stores how we know what linkage to use +#[derive(Copy, Clone, PartialEq, Debug, TyEncodable, TyDecodable, HashStable)] +pub enum LinkageInfo { + /// The linkage was specified explicitly (e.g. using #[linkage = "..."]) + Explicit(Linkage), + /// Assume the symbol may be used from other CGUs, a safe default + ImplicitExternal, + /// We did not find any uses from other CGUs, so it's fine to make this internal + ImplicitInternal, +} + +impl LinkageInfo { + pub const fn is_external(self) -> bool { + matches!(self.into_linkage(), Linkage::External) + } + + pub const fn into_linkage(self) -> Linkage { + match self { + Self::Explicit(linkage) => linkage, + Self::ImplicitExternal => Linkage::External, + Self::ImplicitInternal => Linkage::Internal, + } + } + + /// Linkage when the MonoItem is a naked function + /// + /// Naked functions are generated as a separate declaration (effectively an extern fn) and + /// definition (using global assembly). To link them together, some flavor of external linkage + /// must be used. + pub const fn into_naked_linkage(self) -> Linkage { + match self { + // promote Weak linkage to ExternalWeak + Self::Explicit(Linkage::WeakAny | Linkage::WeakODR) => Linkage::ExternalWeak, + Self::Explicit(linkage) => linkage, + + // the "implicit" means that linkage is picked by the partitioning algorithm. + // picking external should always be valid (given that we are in fact linking + // to the global assembly in the same CGU) + Self::ImplicitExternal => Linkage::External, + Self::ImplicitInternal => Linkage::External, + } + } +} + /// Specifies the linkage type for a `MonoItem`. /// /// See for more details about these variants. diff --git a/compiler/rustc_monomorphize/src/partitioning.rs b/compiler/rustc_monomorphize/src/partitioning.rs index 7ea4ded2b052c..2fd89e389e547 100644 --- a/compiler/rustc_monomorphize/src/partitioning.rs +++ b/compiler/rustc_monomorphize/src/partitioning.rs @@ -109,8 +109,8 @@ use rustc_middle::bug; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel}; use rustc_middle::mir::mono::{ - CodegenUnit, CodegenUnitNameBuilder, InstantiationMode, Linkage, MonoItem, MonoItemData, - Visibility, + CodegenUnit, CodegenUnitNameBuilder, InstantiationMode, Linkage, LinkageInfo, MonoItem, + MonoItemData, Visibility, }; use rustc_middle::ty::print::{characteristic_def_id_of_type, with_no_trimmed_paths}; use rustc_middle::ty::visit::TypeVisitableExt; @@ -245,7 +245,7 @@ where let cgu = codegen_units.entry(cgu_name).or_insert_with(|| CodegenUnit::new(cgu_name)); let mut can_be_internalized = true; - let (linkage, visibility) = mono_item_linkage_and_visibility( + let (linkage_info, visibility) = mono_item_linkage_info_and_visibility( cx.tcx, &mono_item, &mut can_be_internalized, @@ -259,7 +259,7 @@ where cgu.items_mut().insert(mono_item, MonoItemData { inlined: false, - linkage, + linkage_info, visibility, size_estimate, }); @@ -278,7 +278,7 @@ where // This is a CGU-private copy. cgu.items_mut().entry(inlined_item).or_insert_with(|| MonoItemData { inlined: true, - linkage: Linkage::Internal, + linkage_info: LinkageInfo::ImplicitInternal, visibility: Visibility::Default, size_estimate: inlined_item.size_estimate(cx.tcx), }); @@ -589,7 +589,8 @@ fn internalize_symbols<'tcx>( // If we got here, we did not find any uses from other CGUs, so // it's fine to make this monomorphization internal. - data.linkage = Linkage::Internal; + debug_assert_eq!(data.linkage_info, LinkageInfo::ImplicitExternal); + data.linkage_info = LinkageInfo::ImplicitInternal; data.visibility = Visibility::Default; } } @@ -607,7 +608,7 @@ fn mark_code_coverage_dead_code_cgu<'tcx>(codegen_units: &mut [CodegenUnit<'tcx> // function symbols to be included via `-u` or `/include` linker args. let dead_code_cgu = codegen_units .iter_mut() - .filter(|cgu| cgu.items().iter().any(|(_, data)| data.linkage == Linkage::External)) + .filter(|cgu| cgu.items().iter().any(|(_, data)| data.linkage_info.is_external())) .min_by_key(|cgu| cgu.size_estimate()); // If there are no CGUs that have externally linked items, then we just @@ -736,24 +737,26 @@ fn fallback_cgu_name(name_builder: &mut CodegenUnitNameBuilder<'_>) -> Symbol { name_builder.build_cgu_name(LOCAL_CRATE, &["fallback"], Some("cgu")) } -fn mono_item_linkage_and_visibility<'tcx>( +fn mono_item_linkage_info_and_visibility<'tcx>( tcx: TyCtxt<'tcx>, mono_item: &MonoItem<'tcx>, can_be_internalized: &mut bool, can_export_generics: bool, always_export_generics: bool, -) -> (Linkage, Visibility) { +) -> (LinkageInfo, Visibility) { if let Some(explicit_linkage) = mono_item.explicit_linkage(tcx) { - return (explicit_linkage, Visibility::Default); + (LinkageInfo::Explicit(explicit_linkage), Visibility::Default) + } else { + let vis = mono_item_visibility( + tcx, + mono_item, + can_be_internalized, + can_export_generics, + always_export_generics, + ); + + (LinkageInfo::ImplicitExternal, vis) } - let vis = mono_item_visibility( - tcx, - mono_item, - can_be_internalized, - can_export_generics, - always_export_generics, - ); - (Linkage::External, vis) } type CguNameCache = UnordMap<(DefId, bool), Symbol>; @@ -1033,7 +1036,7 @@ fn debug_dump<'a, 'tcx: 'a>(tcx: TyCtxt<'tcx>, label: &str, cgus: &[CodegenUnit< writeln!(s, " - items: {num_items}, mean size: {mean_size:.1}, sizes: {sizes}",); for (item, data) in cgu.items_in_deterministic_order(tcx) { - let linkage = data.linkage; + let linkage_info = data.linkage_info; let symbol_name = item.symbol_name(tcx).name; let symbol_hash_start = symbol_name.rfind('h'); let symbol_hash = symbol_hash_start.map_or("", |i| &symbol_name[i..]); @@ -1041,7 +1044,7 @@ fn debug_dump<'a, 'tcx: 'a>(tcx: TyCtxt<'tcx>, label: &str, cgus: &[CodegenUnit< let size = data.size_estimate; let _ = with_no_trimmed_paths!(writeln!( s, - " - {item} [{linkage:?}] [{symbol_hash}] ({kind}, size: {size})" + " - {item} [{linkage_info:?}] [{symbol_hash}] ({kind}, size: {size})" )); } @@ -1194,7 +1197,7 @@ fn collect_and_partition_mono_items(tcx: TyCtxt<'_>, (): ()) -> (&DefIdSet, &[Co for cgu in codegen_units { for (&mono_item, &data) in cgu.items() { - item_to_cgus.entry(mono_item).or_default().push((cgu.name(), data.linkage)); + item_to_cgus.entry(mono_item).or_default().push((cgu.name(), data.linkage_info)); } } @@ -1207,11 +1210,11 @@ fn collect_and_partition_mono_items(tcx: TyCtxt<'_>, (): ()) -> (&DefIdSet, &[Co let cgus = item_to_cgus.get_mut(i).unwrap_or(&mut empty); cgus.sort_by_key(|(name, _)| *name); cgus.dedup(); - for &(ref cgu_name, linkage) in cgus.iter() { + for &(ref cgu_name, linkage_info) in cgus.iter() { output.push(' '); output.push_str(cgu_name.as_str()); - let linkage_abbrev = match linkage { + let linkage_abbrev = match linkage_info.into_linkage() { Linkage::External => "External", Linkage::AvailableExternally => "Available", Linkage::LinkOnceAny => "OnceAny",