diff --git a/cranelift/codegen/src/isa/riscv64/inst/emit.rs b/cranelift/codegen/src/isa/riscv64/inst/emit.rs index de059b12dc16..4e52ab7edf3a 100644 --- a/cranelift/codegen/src/isa/riscv64/inst/emit.rs +++ b/cranelift/codegen/src/isa/riscv64/inst/emit.rs @@ -1898,10 +1898,9 @@ impl Inst { // // https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#global-dynamic - // Create a lable that is going to be published to the final binary object. + // Create the lable that is going to be published to the final binary object. let auipc_label = sink.get_label(); sink.bind_label(auipc_label, &mut state.ctrl_plane); - sink.set_label_visiblity(auipc_label, LabelVisibility::Public); // Get the current PC. sink.add_reloc(Reloc::RiscvTlsGdHi20, &**name, 0); diff --git a/cranelift/codegen/src/lib.rs b/cranelift/codegen/src/lib.rs index 6885fd3a5705..c856e1a1b645 100644 --- a/cranelift/codegen/src/lib.rs +++ b/cranelift/codegen/src/lib.rs @@ -56,8 +56,8 @@ pub mod write; pub use crate::entity::packed_option; pub use crate::machinst::buffer::{ - MachCallSite, MachLabelSite, MachReloc, MachSrcLoc, MachStackMap, MachTextSectionBuilder, - MachTrap, RelocTarget, + FinalizedMachReloc, FinalizedRelocTarget, MachCallSite, MachSrcLoc, MachStackMap, + MachTextSectionBuilder, MachTrap, }; pub use crate::machinst::{ CompiledCode, Final, MachBuffer, MachBufferFinalized, MachInst, MachInstEmit, diff --git a/cranelift/codegen/src/machinst/buffer.rs b/cranelift/codegen/src/machinst/buffer.rs index 92a1935efca8..d74c6468a2fd 100644 --- a/cranelift/codegen/src/machinst/buffer.rs +++ b/cranelift/codegen/src/machinst/buffer.rs @@ -229,15 +229,6 @@ enum ForceVeneers { No, } -/// Determines if a label should be emitted to the final binary object. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum LabelVisibility { - /// Public labels are published in the final binary object. - Public, - /// Private labels are internal only. - Private, -} - /// A buffer of output to be produced, fixed up, and then emitted to a CodeSink /// in bulk. /// @@ -276,12 +267,6 @@ pub struct MachBuffer { /// target (iteratively until a non-aliased label); if B is already /// aliased to A, then we cannot alias A back to B. label_aliases: SmallVec<[MachLabel; 16]>, - /// Label Visibility: If a label is visible, we publish it in the final binary - /// object, otherwise it is internal only. - /// - /// Public labels are the exception, so we only keep track of them and assume - /// all others are private. - public_labels: SmallVec<[MachLabel; 4]>, /// Constants that must be emitted at some point. pending_constants: SmallVec<[VCodeConstant; 16]>, /// Byte size of all constants in `pending_constants`. @@ -334,7 +319,6 @@ impl MachBufferFinalized { MachBufferFinalized { data: self.data, relocs: self.relocs, - labels: self.labels, traps: self.traps, call_sites: self.call_sites, srclocs: self @@ -362,9 +346,7 @@ pub struct MachBufferFinalized { /// Any relocations referring to this code. Note that only *external* /// relocations are tracked here; references to labels within the buffer are /// resolved before emission. - pub(crate) relocs: SmallVec<[MachReloc; 16]>, - /// Any published labels referring to this code. - pub(crate) labels: SmallVec<[MachLabelSite; 4]>, + pub(crate) relocs: SmallVec<[FinalizedMachReloc; 16]>, /// Any trap records referring to this code. pub(crate) traps: SmallVec<[MachTrap; 16]>, /// Any call site records referring to this code. @@ -391,10 +373,6 @@ const LABEL_LIST_THRESHOLD: usize = 100; /// for references to the label, and will come back and patch the code /// appropriately when the label's location is eventually known. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -#[cfg_attr( - feature = "enable-serde", - derive(serde_derive::Serialize, serde_derive::Deserialize) -)] pub struct MachLabel(u32); entity_impl!(MachLabel); @@ -448,7 +426,6 @@ impl MachBuffer { cur_srcloc: None, label_offsets: SmallVec::new(), label_aliases: SmallVec::new(), - public_labels: SmallVec::new(), pending_constants: SmallVec::new(), pending_constants_size: 0, pending_traps: SmallVec::new(), @@ -655,20 +632,6 @@ impl MachBuffer { // Post-invariant: by `optimize_branches()` (see argument there). } - /// Set the visibility of a label. The label must have been bound already. - pub fn set_label_visiblity(&mut self, label: MachLabel, visibility: LabelVisibility) { - trace!( - "MachBuffer: set label {:?} visibility to {:?}", - label, - visibility - ); - debug_assert_ne!(self.resolve_label_offset(label), UNKNOWN_LABEL_OFFSET); - match visibility { - LabelVisibility::Public => self.public_labels.push(label), - LabelVisibility::Private => self.public_labels.retain(|l| *l != label), - } - } - /// Lazily clear `labels_at_tail` if the tail offset has moved beyond the /// offset that it applies to. fn lazily_clear_labels_at_tail(&mut self) { @@ -1489,13 +1452,22 @@ impl MachBuffer { let alignment = self.finish_constants(constants); - let labels = self - .public_labels + // Resolve all labels to their offsets. + let finalized_relocs = self + .relocs .iter() - .copied() - .map(|label| MachLabelSite { - offset: self.resolve_label_offset(label), - label, + .map(|reloc| FinalizedMachReloc { + offset: reloc.offset, + kind: reloc.kind, + addend: reloc.addend, + target: match &reloc.target { + RelocTarget::ExternalName(name) => { + FinalizedRelocTarget::ExternalName(name.clone()) + } + RelocTarget::Label(label) => { + FinalizedRelocTarget::Func(self.resolve_label_offset(*label)) + } + }, }) .collect(); @@ -1504,8 +1476,7 @@ impl MachBuffer { MachBufferFinalized { data: self.data, - relocs: self.relocs, - labels, + relocs: finalized_relocs, traps: self.traps, call_sites: self.call_sites, srclocs, @@ -1676,15 +1647,10 @@ impl MachBufferFinalized { } /// Get the list of external relocations for this code. - pub fn relocs(&self) -> &[MachReloc] { + pub fn relocs(&self) -> &[FinalizedMachReloc] { &self.relocs[..] } - /// Get the list of published labels for this code. - pub fn labels(&self) -> &[MachLabelSite] { - &self.labels[..] - } - /// Get the list of trap records for this code. pub fn traps(&self) -> &[MachTrap] { &self.traps[..] @@ -1776,24 +1742,25 @@ impl Ord for MachLabelFixup { feature = "enable-serde", derive(serde_derive::Serialize, serde_derive::Deserialize) )] -pub struct MachReloc { +pub struct MachRelocBase { /// The offset at which the relocation applies, *relative to the /// containing section*. pub offset: CodeOffset, /// The kind of relocation. pub kind: Reloc, /// The external symbol / name to which this relocation refers. - pub target: RelocTarget, + pub target: T, /// The addend to add to the symbol value. pub addend: i64, } +type MachReloc = MachRelocBase; + +/// A relocation resulting from a compilation. +pub type FinalizedMachReloc = MachRelocBase; + /// A Relocation target #[derive(Debug, Clone, PartialEq, Eq, Hash)] -#[cfg_attr( - feature = "enable-serde", - derive(serde_derive::Serialize, serde_derive::Deserialize) -)] pub enum RelocTarget { /// Points to an [ExternalName] outside the current function. ExternalName(ExternalName), @@ -1817,31 +1784,30 @@ impl From for RelocTarget { } } -impl RelocTarget { - /// Returns a display for the current `RelocTarget`, with extra context to prettify the +/// A Relocation target +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[cfg_attr( + feature = "enable-serde", + derive(serde_derive::Serialize, serde_derive::Deserialize) +)] +pub enum FinalizedRelocTarget { + /// Points to an [ExternalName] outside the current function. + ExternalName(ExternalName), + /// Points to a [CodeOffset] from the start of the current function. + Func(CodeOffset), +} + +impl FinalizedRelocTarget { + /// Returns a display for the current [FinalizedRelocTarget], with extra context to prettify the /// output. pub fn display<'a>(&'a self, params: Option<&'a FunctionParameters>) -> String { match self { - RelocTarget::ExternalName(name) => format!("{}", name.display(params)), - RelocTarget::Label(label) => format!(".L{}", label.as_u32()), + FinalizedRelocTarget::ExternalName(name) => format!("{}", name.display(params)), + FinalizedRelocTarget::Func(offset) => format!("func+{offset}"), } } } -/// A label resulting from a compilation. -#[derive(Clone, Debug, PartialEq)] -#[cfg_attr( - feature = "enable-serde", - derive(serde_derive::Serialize, serde_derive::Deserialize) -)] -pub struct MachLabelSite { - /// The offset at which the relocation applies, *relative to the - /// containing section*. - pub offset: CodeOffset, - /// The label. - pub label: MachLabel, -} - /// A trap record resulting from a compilation. #[derive(Clone, Debug, PartialEq)] #[cfg_attr( diff --git a/cranelift/filetests/filetests/isa/riscv64/tls-elf.clif b/cranelift/filetests/filetests/isa/riscv64/tls-elf.clif index c82c9513169e..18e781b2fc0d 100644 --- a/cranelift/filetests/filetests/isa/riscv64/tls-elf.clif +++ b/cranelift/filetests/filetests/isa/riscv64/tls-elf.clif @@ -40,7 +40,7 @@ block0(v0: i32): ; block1: ; offset 0x18 ; mv s1, a0 ; auipc a0, 0 ; reloc_external RiscvTlsGdHi20 u1:0 0 -; mv a0, a0 ; reloc_external RiscvPCRelLo12I .L1 0 +; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+28 0 ; auipc t5, 0 ; ld t5, 0xc(t5) ; j 0xc diff --git a/cranelift/jit/src/backend.rs b/cranelift/jit/src/backend.rs index cb2b668b5da0..438c60d49e27 100644 --- a/cranelift/jit/src/backend.rs +++ b/cranelift/jit/src/backend.rs @@ -4,7 +4,7 @@ use crate::{compiled_blob::CompiledBlob, memory::BranchProtection, memory::Memor use cranelift_codegen::binemit::Reloc; use cranelift_codegen::isa::{OwnedTargetIsa, TargetIsa}; use cranelift_codegen::settings::Configurable; -use cranelift_codegen::{self, ir, settings, MachLabelSite, MachReloc}; +use cranelift_codegen::{self, ir, settings, FinalizedMachReloc}; use cranelift_control::ControlPlane; use cranelift_entity::SecondaryMap; use cranelift_module::{ @@ -758,8 +758,7 @@ impl Module for JITModule { func: &ir::Function, alignment: u64, bytes: &[u8], - relocs: &[MachReloc], - _labels: &[MachLabelSite], + relocs: &[FinalizedMachReloc], ) -> ModuleResult<()> { info!("defining function {} with bytes", id); let decl = self.declarations.get_function_decl(id); diff --git a/cranelift/module/src/module.rs b/cranelift/module/src/module.rs index 66c378a1e692..cae49ef17e97 100644 --- a/cranelift/module/src/module.rs +++ b/cranelift/module/src/module.rs @@ -13,8 +13,9 @@ use cranelift_codegen::entity::{entity_impl, PrimaryMap}; use cranelift_codegen::ir::function::{Function, VersionMarker}; use cranelift_codegen::ir::{ExternalName, UserFuncName}; use cranelift_codegen::settings::SetError; -use cranelift_codegen::{ir, isa, CodegenError, CompileError, Context, MachLabel, RelocTarget}; -use cranelift_codegen::{MachLabelSite, MachReloc}; +use cranelift_codegen::{ + ir, isa, CodegenError, CompileError, Context, FinalizedMachReloc, FinalizedRelocTarget, +}; use cranelift_control::ControlPlane; use std::borrow::{Cow, ToOwned}; use std::string::String; @@ -34,21 +35,23 @@ pub struct ModuleReloc { } impl ModuleReloc { - /// Converts a `MachReloc` produced from a `Function` into a `ModuleReloc`. - pub fn from_mach_reloc(mach_reloc: &MachReloc, func: &Function) -> Self { + /// Converts a `FinalizedMachReloc` produced from a `Function` into a `ModuleReloc`. + pub fn from_mach_reloc(mach_reloc: &FinalizedMachReloc, func: &Function) -> Self { let name = match mach_reloc.target { - RelocTarget::ExternalName(ExternalName::User(reff)) => { + FinalizedRelocTarget::ExternalName(ExternalName::User(reff)) => { let name = &func.params.user_named_funcs()[reff]; ModuleRelocTarget::user(name.namespace, name.index) } - RelocTarget::ExternalName(ExternalName::TestCase(_)) => unimplemented!(), - RelocTarget::ExternalName(ExternalName::LibCall(libcall)) => { + FinalizedRelocTarget::ExternalName(ExternalName::TestCase(_)) => unimplemented!(), + FinalizedRelocTarget::ExternalName(ExternalName::LibCall(libcall)) => { ModuleRelocTarget::LibCall(libcall) } - RelocTarget::ExternalName(ExternalName::KnownSymbol(ks)) => { + FinalizedRelocTarget::ExternalName(ExternalName::KnownSymbol(ks)) => { ModuleRelocTarget::KnownSymbol(ks) } - RelocTarget::Label(label) => ModuleRelocTarget::FunctionLabel(func.name.clone(), label), + FinalizedRelocTarget::Func(offset) => { + ModuleRelocTarget::FunctionOffset(func.name.clone(), offset) + } }; Self { offset: mach_reloc.offset, @@ -426,8 +429,8 @@ pub enum ModuleRelocTarget { LibCall(ir::LibCall), /// Symbols known to the linker. KnownSymbol(ir::KnownSymbol), - /// A label inside a function - FunctionLabel(UserFuncName, MachLabel), + /// A offset inside a function + FunctionOffset(UserFuncName, CodeOffset), } impl ModuleRelocTarget { @@ -443,7 +446,7 @@ impl Display for ModuleRelocTarget { Self::User { namespace, index } => write!(f, "u{}:{}", namespace, index), Self::LibCall(lc) => write!(f, "%{}", lc), Self::KnownSymbol(ks) => write!(f, "{}", ks), - Self::FunctionLabel(fname, label) => write!(f, ".L{}_{}", fname, label.as_u32()), + Self::FunctionOffset(fname, offset) => write!(f, "{fname}+{offset}"), } } } @@ -700,7 +703,7 @@ impl ModuleDeclarations { ModuleRelocTarget::User { namespace, .. } => *namespace == 0, ModuleRelocTarget::LibCall(_) | ModuleRelocTarget::KnownSymbol(_) - | ModuleRelocTarget::FunctionLabel(..) => { + | ModuleRelocTarget::FunctionOffset(..) => { panic!("unexpected module ext name") } } @@ -976,8 +979,7 @@ pub trait Module { func: &ir::Function, alignment: u64, bytes: &[u8], - relocs: &[MachReloc], - labels: &[MachLabelSite], + relocs: &[FinalizedMachReloc], ) -> ModuleResult<()>; /// Define a data object, producing the data contents from the given `DataContext`. @@ -1079,10 +1081,9 @@ impl Module for &mut M { func: &ir::Function, alignment: u64, bytes: &[u8], - relocs: &[MachReloc], - labels: &[MachLabelSite], + relocs: &[FinalizedMachReloc], ) -> ModuleResult<()> { - (**self).define_function_bytes(func_id, func, alignment, bytes, relocs, labels) + (**self).define_function_bytes(func_id, func, alignment, bytes, relocs) } fn define_data(&mut self, data_id: DataId, data: &DataDescription) -> ModuleResult<()> { diff --git a/cranelift/object/src/backend.rs b/cranelift/object/src/backend.rs index a50439c7b957..d083d84ce12f 100644 --- a/cranelift/object/src/backend.rs +++ b/cranelift/object/src/backend.rs @@ -5,7 +5,7 @@ use cranelift_codegen::binemit::{Addend, CodeOffset, Reloc}; use cranelift_codegen::entity::SecondaryMap; use cranelift_codegen::ir::UserFuncName; use cranelift_codegen::isa::{OwnedTargetIsa, TargetIsa}; -use cranelift_codegen::{self, ir, MachLabel, MachLabelSite, MachReloc}; +use cranelift_codegen::{self, ir, FinalizedMachReloc}; use cranelift_control::ControlPlane; use cranelift_module::{ DataDescription, DataId, FuncId, Init, Linkage, Module, ModuleDeclarations, ModuleError, @@ -18,6 +18,7 @@ use object::write::{ use object::{ RelocationEncoding, RelocationKind, SectionKind, SymbolFlags, SymbolKind, SymbolScope, }; +use std::collections::hash_map::Entry; use std::collections::HashMap; use std::mem; use target_lexicon::PointerWidth; @@ -131,7 +132,7 @@ pub struct ObjectModule { libcalls: HashMap, libcall_names: Box String + Send + Sync>, known_symbols: HashMap, - known_labels: HashMap<(UserFuncName, MachLabel), SymbolId>, + known_labels: HashMap<(UserFuncName, CodeOffset), SymbolId>, per_function_section: bool, } @@ -327,7 +328,6 @@ impl Module for ObjectModule { alignment, &code, ctx.compiled_code().unwrap().buffer.relocs(), - ctx.compiled_code().unwrap().buffer.labels(), ) } @@ -337,8 +337,7 @@ impl Module for ObjectModule { func: &ir::Function, alignment: u64, bytes: &[u8], - relocs: &[MachReloc], - labels: &[MachLabelSite], + relocs: &[FinalizedMachReloc], ) -> ModuleResult<()> { info!("defining function {} with bytes", func_id); let decl = self.declarations.get_function_decl(func_id); @@ -382,23 +381,6 @@ impl Module for ObjectModule { }); } - for label_site in labels { - let name = format!(".L{}_{}", func_id.as_u32(), label_site.label.as_u32()); - let symbol_id = self.object.add_symbol(Symbol { - name: name.as_bytes().to_vec(), - value: offset + label_site.offset as u64, - size: 0, - kind: SymbolKind::Label, - scope: SymbolScope::Compilation, - weak: false, - section: SymbolSection::Section(section), - flags: SymbolFlags::None, - }); - - self.known_labels - .insert((func.name.clone(), label_site.label), symbol_id); - } - Ok(()) } @@ -612,13 +594,35 @@ impl ObjectModule { } } - // All of these function labels should exist, since they are inserted when the function is - // first defined. - ModuleRelocTarget::FunctionLabel(ref fname, label) => self - .known_labels - .get(&(fname.clone(), label)) - .copied() - .expect("Unknown function label"), + ModuleRelocTarget::FunctionOffset(ref fname, offset) => { + match self.known_labels.entry((fname.clone(), offset)) { + Entry::Occupied(o) => *o.get(), + Entry::Vacant(v) => { + let func_user_name = fname.get_user().unwrap(); + let func_id = FuncId::from_name(&ModuleRelocTarget::user( + func_user_name.namespace, + func_user_name.index, + )); + let func_symbol_id = self.functions[func_id].unwrap().0; + let func_symbol = self.object.symbol(func_symbol_id); + + let name = format!(".L{}_{}", func_id.as_u32(), offset); + let symbol_id = self.object.add_symbol(Symbol { + name: name.as_bytes().to_vec(), + value: func_symbol.value + offset as u64, + size: 0, + kind: SymbolKind::Label, + scope: SymbolScope::Compilation, + weak: false, + section: SymbolSection::Section(func_symbol.section.id().unwrap()), + flags: SymbolFlags::None, + }); + + v.insert(symbol_id); + symbol_id + } + } + } } } diff --git a/cranelift/src/disasm.rs b/cranelift/src/disasm.rs index 156e866ce681..0b0235b67959 100644 --- a/cranelift/src/disasm.rs +++ b/cranelift/src/disasm.rs @@ -2,12 +2,12 @@ use anyhow::Result; use cfg_if::cfg_if; use cranelift_codegen::ir::function::FunctionParameters; use cranelift_codegen::isa::TargetIsa; -use cranelift_codegen::{MachReloc, MachStackMap, MachTrap}; +use cranelift_codegen::{FinalizedMachReloc, MachStackMap, MachTrap}; use std::fmt::Write; -fn print_relocs(func_params: &FunctionParameters, relocs: &[MachReloc]) -> String { +fn print_relocs(func_params: &FunctionParameters, relocs: &[FinalizedMachReloc]) -> String { let mut text = String::new(); - for &MachReloc { + for &FinalizedMachReloc { kind, offset, ref target, @@ -121,7 +121,7 @@ pub fn print_all( mem: &[u8], code_size: u32, print: bool, - relocs: &[MachReloc], + relocs: &[FinalizedMachReloc], traps: &[MachTrap], stack_maps: &[MachStackMap], ) -> Result<()> {