diff --git a/cranelift/codegen/src/binemit/mod.rs b/cranelift/codegen/src/binemit/mod.rs index 9f00b93791ac..d5582d87f3be 100644 --- a/cranelift/codegen/src/binemit/mod.rs +++ b/cranelift/codegen/src/binemit/mod.rs @@ -92,6 +92,18 @@ pub enum Reloc { /// jalr ra, ra, 0 RiscvCall, + /// RISC-V TLS GD: High 20 bits of 32-bit PC-relative TLS GD GOT reference, + /// + /// This is the `R_RISCV_TLS_GD_HI20` relocation from the RISC-V ELF psABI document. + /// https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#global-dynamic + RiscvTlsGdHi20, + + /// Low 12 bits of a 32-bit PC-relative relocation (I-Type instruction) + /// + /// This is the `R_RISCV_PCREL_LO12_I` relocation from the RISC-V ELF psABI document. + /// https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses + RiscvPCRelLo12I, + /// s390x TLS GD64 - 64-bit offset of tls_index for GD symbol in GOT S390xTlsGd64, /// s390x TLS GDCall - marker to enable optimization of TLS calls @@ -114,7 +126,8 @@ impl fmt::Display for Reloc { Self::X86SecRel => write!(f, "SecRel"), Self::Arm32Call | Self::Arm64Call => write!(f, "Call"), Self::RiscvCall => write!(f, "RiscvCall"), - + Self::RiscvTlsGdHi20 => write!(f, "RiscvTlsGdHi20"), + Self::RiscvPCRelLo12I => write!(f, "RiscvPCRelLo12I"), Self::ElfX86_64TlsGd => write!(f, "ElfX86_64TlsGd"), Self::MachOX86_64Tlv => write!(f, "MachOX86_64Tlv"), Self::MachOAarch64TlsAdrPage21 => write!(f, "MachOAarch64TlsAdrPage21"), diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs index 2a3c9b8b9b75..43bdd4a3e981 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs @@ -3174,7 +3174,7 @@ impl MachInstEmit for Inst { // Note: this is not `Inst::Jump { .. }.emit(..)` because we // have different metadata in this case: we don't have a label // for the target, but rather a function relocation. - sink.add_reloc(Reloc::Arm64Call, callee, 0); + sink.add_reloc(Reloc::Arm64Call, &**callee, 0); sink.put4(enc_jump26(0b000101, 0)); sink.add_call_site(ir::Opcode::ReturnCall); @@ -3382,12 +3382,12 @@ impl MachInstEmit for Inst { // ldr rd, [rd, :got_lo12:X] // adrp rd, symbol - sink.add_reloc(Reloc::Aarch64AdrGotPage21, name, 0); + sink.add_reloc(Reloc::Aarch64AdrGotPage21, &**name, 0); let inst = Inst::Adrp { rd, off: 0 }; inst.emit(&[], sink, emit_info, state); // ldr rd, [rd, :got_lo12:X] - sink.add_reloc(Reloc::Aarch64Ld64GotLo12Nc, name, 0); + sink.add_reloc(Reloc::Aarch64Ld64GotLo12Nc, &**name, 0); let inst = Inst::ULoad64 { rd, mem: AMode::reg(rd.to_reg()), @@ -3415,7 +3415,7 @@ impl MachInstEmit for Inst { dest: BranchTarget::ResolvedOffset(12), }; inst.emit(&[], sink, emit_info, state); - sink.add_reloc(Reloc::Abs8, name, offset); + sink.add_reloc(Reloc::Abs8, &**name, offset); sink.put8(0); } } diff --git a/cranelift/codegen/src/isa/riscv64/inst.isle b/cranelift/codegen/src/isa/riscv64/inst.isle index e14adf2818f1..2fe4fa6f8e02 100644 --- a/cranelift/codegen/src/isa/riscv64/inst.isle +++ b/cranelift/codegen/src/isa/riscv64/inst.isle @@ -148,6 +148,11 @@ (name BoxExternalName) (offset i64)) + ;; Load a TLS symbol address + (ElfTlsGetAddr + (rd WritableReg) + (name BoxExternalName)) + ;; Load address referenced by `mem` into `rd`. (LoadAddr (rd WritableReg) @@ -2625,6 +2630,12 @@ (decl load_ext_name (ExternalName i64) Reg) (extern constructor load_ext_name load_ext_name) +(decl elf_tls_get_addr (ExternalName) Reg) +(rule (elf_tls_get_addr name) + (let ((dst WritableReg (temp_writable_reg $I64)) + (_ Unit (emit (MInst.ElfTlsGetAddr dst name)))) + dst)) + (decl int_convert_2_float_op (Type bool Type) FpuOPRR) (extern constructor int_convert_2_float_op int_convert_2_float_op) diff --git a/cranelift/codegen/src/isa/riscv64/inst/emit.rs b/cranelift/codegen/src/isa/riscv64/inst/emit.rs index 7effa2b4cb2d..3f8ef674ff3a 100644 --- a/cranelift/codegen/src/isa/riscv64/inst/emit.rs +++ b/cranelift/codegen/src/isa/riscv64/inst/emit.rs @@ -1,7 +1,7 @@ //! Riscv64 ISA: binary code emission. use crate::binemit::StackMap; -use crate::ir::{self, RelSourceLoc, TrapCode}; +use crate::ir::{self, LibCall, RelSourceLoc, TrapCode}; use crate::isa::riscv64::inst::*; use crate::isa::riscv64::lower::isle::generated_code::{CaOp, CrOp}; use crate::machinst::{AllocationConsumer, Reg, Writable}; @@ -343,6 +343,7 @@ impl Inst { | Inst::Jal { .. } | Inst::CondBr { .. } | Inst::LoadExtName { .. } + | Inst::ElfTlsGetAddr { .. } | Inst::LoadAddr { .. } | Inst::VirtualSPOffsetAdj { .. } | Inst::Mov { .. } @@ -978,7 +979,7 @@ impl Inst { ); sink.add_call_site(ir::Opcode::ReturnCall); - sink.add_reloc(Reloc::RiscvCall, &callee, 0); + sink.add_reloc(Reloc::RiscvCall, &**callee, 0); Inst::construct_auipc_and_jalr(None, writable_spilltmp_reg(), 0) .into_iter() .for_each(|i| i.emit_uncompressed(sink, emit_info, state, start_off)); @@ -1958,6 +1959,60 @@ impl Inst { sink.bind_label(label_end, &mut state.ctrl_plane); } + + &Inst::ElfTlsGetAddr { rd, ref name } => { + // RISC-V's TLS GD model is slightly different from other arches. + // + // We have a relocation (R_RISCV_TLS_GD_HI20) that loads the high 20 bits + // of the address relative to the GOT entry. This relocation points to + // the symbol as usual. + // + // However when loading the bottom 12bits of the address, we need to + // use a label that points to the previous AUIPC instruction. + // + // label: + // auipc a0,0 # R_RISCV_TLS_GD_HI20 (symbol) + // addi a0,a0,0 # R_RISCV_PCREL_LO12_I (label) + // + // https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#global-dynamic + + // Create the lable that is going to be published to the final binary object. + let auipc_label = sink.get_label(); + sink.bind_label(auipc_label, &mut state.ctrl_plane); + + // Get the current PC. + sink.add_reloc(Reloc::RiscvTlsGdHi20, &**name, 0); + Inst::Auipc { + rd: rd, + imm: Imm20::from_i32(0), + } + .emit_uncompressed(sink, emit_info, state, start_off); + + // The `addi` here, points to the `auipc` label instead of directly to the symbol. + sink.add_reloc(Reloc::RiscvPCRelLo12I, &auipc_label, 0); + Inst::AluRRImm12 { + alu_op: AluOPRRI::Addi, + rd: rd, + rs: rd.to_reg(), + imm12: Imm12::from_i16(0), + } + .emit_uncompressed(sink, emit_info, state, start_off); + + Inst::Call { + info: Box::new(CallInfo { + dest: ExternalName::LibCall(LibCall::ElfTlsGetAddr), + uses: smallvec![], + defs: smallvec![], + opcode: crate::ir::Opcode::TlsValue, + caller_callconv: CallConv::SystemV, + callee_callconv: CallConv::SystemV, + callee_pop_size: 0, + clobbers: PRegSet::empty(), + }), + } + .emit_uncompressed(sink, emit_info, state, start_off); + } + &Inst::TrapIfC { rs1, rs2, @@ -3273,6 +3328,12 @@ impl Inst { offset, }, + Inst::ElfTlsGetAddr { rd, name } => { + let rd = allocs.next_writable(rd); + debug_assert_eq!(a0(), rd.to_reg()); + Inst::ElfTlsGetAddr { rd, name } + } + Inst::TrapIfC { rs1, rs2, diff --git a/cranelift/codegen/src/isa/riscv64/inst/mod.rs b/cranelift/codegen/src/isa/riscv64/inst/mod.rs index 334af1c53419..5a12644f82e0 100644 --- a/cranelift/codegen/src/isa/riscv64/inst/mod.rs +++ b/cranelift/codegen/src/isa/riscv64/inst/mod.rs @@ -473,6 +473,13 @@ fn riscv64_get_operands VReg>(inst: &Inst, collector: &mut Operan &Inst::LoadExtName { rd, .. } => { collector.reg_def(rd); } + &Inst::ElfTlsGetAddr { rd, .. } => { + // x10 is a0 which is both the first argument and the first return value. + collector.reg_fixed_def(rd, a0()); + let mut clobbers = Riscv64MachineDeps::get_regs_clobbered_by_call(CallConv::SystemV); + clobbers.remove(px_reg(10)); + collector.reg_clobbers(clobbers); + } &Inst::LoadAddr { rd, mem } => { if let Some(r) = mem.get_allocatable_register() { collector.reg_use(r); @@ -1691,6 +1698,10 @@ impl Inst { let rd = format_reg(rd.to_reg(), allocs); format!("load_sym {},{}{:+}", rd, name.display(None), offset) } + &Inst::ElfTlsGetAddr { rd, ref name } => { + let rd = format_reg(rd.to_reg(), allocs); + format!("elf_tls_get_addr {rd},{}", name.display(None)) + } &MInst::LoadAddr { ref rd, ref mem } => { let rs = mem.to_string_with_alloc(allocs); let rd = format_reg(rd.to_reg(), allocs); diff --git a/cranelift/codegen/src/isa/riscv64/lower.isle b/cranelift/codegen/src/isa/riscv64/lower.isle index 26b375f3cdf3..ec264a221094 100644 --- a/cranelift/codegen/src/isa/riscv64/lower.isle +++ b/cranelift/codegen/src/isa/riscv64/lower.isle @@ -1675,6 +1675,11 @@ (lower (symbol_value (symbol_value_data name _ offset))) (load_ext_name name offset)) +;;;;; Rules for `tls_value` ;;;;;;;;;;;;;; + +(rule (lower (has_type (tls_model (TlsModel.ElfGd)) (tls_value (symbol_value_data name _ _)))) + (elf_tls_get_addr name)) + ;;;;; Rules for `bitcast`;;;;;;;;; (rule (lower (has_type out_ty (bitcast _ v @ (value_type in_ty)))) diff --git a/cranelift/codegen/src/lib.rs b/cranelift/codegen/src/lib.rs index 250b36cc897b..c856e1a1b645 100644 --- a/cranelift/codegen/src/lib.rs +++ b/cranelift/codegen/src/lib.rs @@ -56,7 +56,8 @@ pub mod write; pub use crate::entity::packed_option; pub use crate::machinst::buffer::{ - MachCallSite, MachReloc, MachSrcLoc, MachStackMap, MachTextSectionBuilder, MachTrap, + FinalizedMachReloc, FinalizedRelocTarget, MachCallSite, MachSrcLoc, MachStackMap, + MachTextSectionBuilder, MachTrap, }; pub use crate::machinst::{ CompiledCode, Final, MachBuffer, MachBufferFinalized, MachInst, MachInstEmit, diff --git a/cranelift/codegen/src/machinst/buffer.rs b/cranelift/codegen/src/machinst/buffer.rs index 8bf7cfdb4fc3..aa69feef4929 100644 --- a/cranelift/codegen/src/machinst/buffer.rs +++ b/cranelift/codegen/src/machinst/buffer.rs @@ -171,6 +171,7 @@ //! all longer-range fixups to later. use crate::binemit::{Addend, CodeOffset, Reloc, StackMap}; +use crate::ir::function::FunctionParameters; use crate::ir::{ExternalName, Opcode, RelSourceLoc, SourceLoc, TrapCode}; use crate::isa::unwind::UnwindInst; use crate::machinst::{ @@ -345,7 +346,7 @@ pub struct MachBufferFinalized { /// Any relocations referring to this code. Note that only *external* /// relocations are tracked here; references to labels within the buffer are /// resolved before emission. - pub(crate) relocs: SmallVec<[MachReloc; 16]>, + pub(crate) relocs: SmallVec<[FinalizedMachReloc; 16]>, /// Any trap records referring to this code. pub(crate) traps: SmallVec<[MachTrap; 16]>, /// Any call site records referring to this code. @@ -1451,12 +1452,31 @@ impl MachBuffer { let alignment = self.finish_constants(constants); + // Resolve all labels to their offsets. + let finalized_relocs = self + .relocs + .iter() + .map(|reloc| FinalizedMachReloc { + offset: reloc.offset, + kind: reloc.kind, + addend: reloc.addend, + target: match &reloc.target { + RelocTarget::ExternalName(name) => { + FinalizedRelocTarget::ExternalName(name.clone()) + } + RelocTarget::Label(label) => { + FinalizedRelocTarget::Func(self.resolve_label_offset(*label)) + } + }, + }) + .collect(); + let mut srclocs = self.srclocs; srclocs.sort_by_key(|entry| entry.start); MachBufferFinalized { data: self.data, - relocs: self.relocs, + relocs: finalized_relocs, traps: self.traps, call_sites: self.call_sites, srclocs, @@ -1467,8 +1487,13 @@ impl MachBuffer { } /// Add an external relocation at the current offset. - pub fn add_reloc(&mut self, kind: Reloc, name: &ExternalName, addend: Addend) { - let name = name.clone(); + pub fn add_reloc + Clone>( + &mut self, + kind: Reloc, + target: &T, + addend: Addend, + ) { + let target: RelocTarget = target.clone().into(); // FIXME(#3277): This should use `I::LabelUse::from_reloc` to optionally // generate a label-use statement to track whether an island is possibly // needed to escape this function to actually get to the external name. @@ -1505,7 +1530,7 @@ impl MachBuffer { self.relocs.push(MachReloc { offset: self.data.len() as CodeOffset, kind, - name, + target, addend, }); } @@ -1622,7 +1647,7 @@ impl MachBufferFinalized { } /// Get the list of external relocations for this code. - pub fn relocs(&self) -> &[MachReloc] { + pub fn relocs(&self) -> &[FinalizedMachReloc] { &self.relocs[..] } @@ -1717,18 +1742,72 @@ impl Ord for MachLabelFixup { feature = "enable-serde", derive(serde_derive::Serialize, serde_derive::Deserialize) )] -pub struct MachReloc { +pub struct MachRelocBase { /// The offset at which the relocation applies, *relative to the /// containing section*. pub offset: CodeOffset, /// The kind of relocation. pub kind: Reloc, /// The external symbol / name to which this relocation refers. - pub name: ExternalName, + pub target: T, /// The addend to add to the symbol value. pub addend: i64, } +type MachReloc = MachRelocBase; + +/// A relocation resulting from a compilation. +pub type FinalizedMachReloc = MachRelocBase; + +/// A Relocation target +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum RelocTarget { + /// Points to an [ExternalName] outside the current function. + ExternalName(ExternalName), + /// Points to a [MachLabel] inside this function. + /// This is different from [MachLabelFixup] in that both the relocation and the + /// label will be emitted and are only resolved at link time. + /// + /// There is no reason to prefer this over [MachLabelFixup] unless the ABI requires it. + Label(MachLabel), +} + +impl From for RelocTarget { + fn from(name: ExternalName) -> Self { + Self::ExternalName(name) + } +} + +impl From for RelocTarget { + fn from(label: MachLabel) -> Self { + Self::Label(label) + } +} + +/// A Relocation target +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[cfg_attr( + feature = "enable-serde", + derive(serde_derive::Serialize, serde_derive::Deserialize) +)] +pub enum FinalizedRelocTarget { + /// Points to an [ExternalName] outside the current function. + ExternalName(ExternalName), + /// Points to a [CodeOffset] from the start of the current function. + Func(CodeOffset), +} + +impl FinalizedRelocTarget { + /// Returns a display for the current [FinalizedRelocTarget], with extra context to prettify the + /// output. + pub fn display<'a>(&'a self, params: Option<&'a FunctionParameters>) -> String { + match self { + FinalizedRelocTarget::ExternalName(name) => format!("{}", name.display(params)), + FinalizedRelocTarget::Func(offset) => format!("func+{offset}"), + } + } +} + /// A trap record resulting from a compilation. #[derive(Clone, Debug, PartialEq)] #[cfg_attr( diff --git a/cranelift/codegen/src/machinst/mod.rs b/cranelift/codegen/src/machinst/mod.rs index fd08b6be7988..a8d3ce376d12 100644 --- a/cranelift/codegen/src/machinst/mod.rs +++ b/cranelift/codegen/src/machinst/mod.rs @@ -430,7 +430,7 @@ impl CompiledCodeBase { buf, " ; reloc_external {} {} {}", reloc.kind, - reloc.name.display(params), + reloc.target.display(params), reloc.addend, )?; } diff --git a/cranelift/filetests/filetests/isa/riscv64/tls-elf.clif b/cranelift/filetests/filetests/isa/riscv64/tls-elf.clif new file mode 100644 index 000000000000..18e781b2fc0d --- /dev/null +++ b/cranelift/filetests/filetests/isa/riscv64/tls-elf.clif @@ -0,0 +1,58 @@ +test compile precise-output +set tls_model=elf_gd +target riscv64 + +function u0:0(i32) -> i32, i64 { +gv0 = symbol colocated tls u1:0 + +block0(v0: i32): + v1 = global_value.i64 gv0 + return v0, v1 +} + +; VCode: +; add sp,-16 +; sd ra,8(sp) +; sd fp,0(sp) +; mv fp,sp +; sd s1,-8(sp) +; add sp,-16 +; block0: +; mv s1,a0 +; elf_tls_get_addr a0,userextname0 +; mv a1,a0 +; mv a0,s1 +; add sp,+16 +; ld s1,-8(sp) +; ld ra,8(sp) +; ld fp,0(sp) +; add sp,+16 +; ret +; +; Disassembled: +; block0: ; offset 0x0 +; addi sp, sp, -0x10 +; sd ra, 8(sp) +; sd s0, 0(sp) +; mv s0, sp +; sd s1, -8(sp) +; addi sp, sp, -0x10 +; block1: ; offset 0x18 +; mv s1, a0 +; auipc a0, 0 ; reloc_external RiscvTlsGdHi20 u1:0 0 +; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+28 0 +; auipc t5, 0 +; ld t5, 0xc(t5) +; j 0xc +; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %ElfTlsGetAddr 0 +; .byte 0x00, 0x00, 0x00, 0x00 +; jalr t5 +; mv a1, a0 +; mv a0, s1 +; addi sp, sp, 0x10 +; ld s1, -8(sp) +; ld ra, 8(sp) +; ld s0, 0(sp) +; addi sp, sp, 0x10 +; ret + diff --git a/cranelift/jit/src/backend.rs b/cranelift/jit/src/backend.rs index b564b82b6eb4..438c60d49e27 100644 --- a/cranelift/jit/src/backend.rs +++ b/cranelift/jit/src/backend.rs @@ -4,12 +4,12 @@ use crate::{compiled_blob::CompiledBlob, memory::BranchProtection, memory::Memor use cranelift_codegen::binemit::Reloc; use cranelift_codegen::isa::{OwnedTargetIsa, TargetIsa}; use cranelift_codegen::settings::Configurable; -use cranelift_codegen::{self, ir, settings, MachReloc}; +use cranelift_codegen::{self, ir, settings, FinalizedMachReloc}; use cranelift_control::ControlPlane; use cranelift_entity::SecondaryMap; use cranelift_module::{ DataDescription, DataId, FuncId, Init, Linkage, Module, ModuleDeclarations, ModuleError, - ModuleExtName, ModuleReloc, ModuleResult, + ModuleReloc, ModuleRelocTarget, ModuleResult, }; use log::info; use std::cell::RefCell; @@ -300,9 +300,9 @@ impl JITModule { std::ptr::write(plt_ptr, plt_val); } - fn get_address(&self, name: &ModuleExtName) -> *const u8 { + fn get_address(&self, name: &ModuleRelocTarget) -> *const u8 { match *name { - ModuleExtName::User { .. } => { + ModuleRelocTarget::User { .. } => { let (name, linkage) = if ModuleDeclarations::is_function(name) { if self.hotswap_enabled { return self.get_plt_address(name); @@ -337,7 +337,7 @@ impl JITModule { panic!("can't resolve symbol {}", name); } } - ModuleExtName::LibCall(ref libcall) => { + ModuleRelocTarget::LibCall(ref libcall) => { let sym = (self.libcall_names)(*libcall); self.lookup_symbol(&sym) .unwrap_or_else(|| panic!("can't resolve libcall {}", sym)) @@ -354,9 +354,9 @@ impl JITModule { unsafe { got_entry.as_ref() }.load(Ordering::SeqCst) } - fn get_got_address(&self, name: &ModuleExtName) -> NonNull> { + fn get_got_address(&self, name: &ModuleRelocTarget) -> NonNull> { match *name { - ModuleExtName::User { .. } => { + ModuleRelocTarget::User { .. } => { if ModuleDeclarations::is_function(name) { let func_id = FuncId::from_name(name); self.function_got_entries[func_id].unwrap() @@ -365,7 +365,7 @@ impl JITModule { self.data_object_got_entries[data_id].unwrap() } } - ModuleExtName::LibCall(ref libcall) => *self + ModuleRelocTarget::LibCall(ref libcall) => *self .libcall_got_entries .get(libcall) .unwrap_or_else(|| panic!("can't resolve libcall {}", libcall)), @@ -373,9 +373,9 @@ impl JITModule { } } - fn get_plt_address(&self, name: &ModuleExtName) -> *const u8 { + fn get_plt_address(&self, name: &ModuleRelocTarget) -> *const u8 { match *name { - ModuleExtName::User { .. } => { + ModuleRelocTarget::User { .. } => { if ModuleDeclarations::is_function(name) { let func_id = FuncId::from_name(name); self.function_plt_entries[func_id] @@ -386,7 +386,7 @@ impl JITModule { unreachable!("PLT relocations can only have functions as target"); } } - ModuleExtName::LibCall(ref libcall) => self + ModuleRelocTarget::LibCall(ref libcall) => self .libcall_plt_entries .get(libcall) .unwrap_or_else(|| panic!("can't resolve libcall {}", libcall)) @@ -731,10 +731,10 @@ impl Module for JITModule { .unwrap() .perform_relocations( |name| match *name { - ModuleExtName::User { .. } => { + ModuleRelocTarget::User { .. } => { unreachable!("non GOT or PLT relocation in function {} to {}", id, name) } - ModuleExtName::LibCall(ref libcall) => self + ModuleRelocTarget::LibCall(ref libcall) => self .libcall_plt_entries .get(libcall) .unwrap_or_else(|| panic!("can't resolve libcall {}", libcall)) @@ -758,7 +758,7 @@ impl Module for JITModule { func: &ir::Function, alignment: u64, bytes: &[u8], - relocs: &[MachReloc], + relocs: &[FinalizedMachReloc], ) -> ModuleResult<()> { info!("defining function {} with bytes", id); let decl = self.declarations.get_function_decl(id); diff --git a/cranelift/jit/src/compiled_blob.rs b/cranelift/jit/src/compiled_blob.rs index f8f8bcd70d8f..de01855c6d7b 100644 --- a/cranelift/jit/src/compiled_blob.rs +++ b/cranelift/jit/src/compiled_blob.rs @@ -1,6 +1,6 @@ use cranelift_codegen::binemit::Reloc; -use cranelift_module::ModuleExtName; use cranelift_module::ModuleReloc; +use cranelift_module::ModuleRelocTarget; use std::convert::TryFrom; /// Reads a 32bit instruction at `iptr`, and writes it again after @@ -21,9 +21,9 @@ pub(crate) struct CompiledBlob { impl CompiledBlob { pub(crate) fn perform_relocations( &self, - get_address: impl Fn(&ModuleExtName) -> *const u8, - get_got_entry: impl Fn(&ModuleExtName) -> *const u8, - get_plt_entry: impl Fn(&ModuleExtName) -> *const u8, + get_address: impl Fn(&ModuleRelocTarget) -> *const u8, + get_got_entry: impl Fn(&ModuleRelocTarget) -> *const u8, + get_plt_entry: impl Fn(&ModuleRelocTarget) -> *const u8, ) { use std::ptr::write_unaligned; diff --git a/cranelift/module/src/data_context.rs b/cranelift/module/src/data_context.rs index 2b262eca38b3..b55f96fcd3c5 100644 --- a/cranelift/module/src/data_context.rs +++ b/cranelift/module/src/data_context.rs @@ -9,7 +9,7 @@ use std::string::String; use std::vec::Vec; use crate::module::ModuleReloc; -use crate::ModuleExtName; +use crate::ModuleRelocTarget; /// This specifies how data is to be initialized. #[derive(Clone, PartialEq, Eq, Debug)] @@ -53,9 +53,9 @@ pub struct DataDescription { /// How the data should be initialized. pub init: Init, /// External function declarations. - pub function_decls: PrimaryMap, + pub function_decls: PrimaryMap, /// External data object declarations. - pub data_decls: PrimaryMap, + pub data_decls: PrimaryMap, /// Function addresses to write at specified offsets. pub function_relocs: Vec<(CodeOffset, ir::FuncRef)>, /// Data addresses to write at specified offsets. @@ -122,7 +122,7 @@ impl DataDescription { /// Users of the `Module` API generally should call /// `Module::declare_func_in_data` instead, as it takes care of generating /// the appropriate `ExternalName`. - pub fn import_function(&mut self, name: ModuleExtName) -> ir::FuncRef { + pub fn import_function(&mut self, name: ModuleRelocTarget) -> ir::FuncRef { self.function_decls.push(name) } @@ -133,7 +133,7 @@ impl DataDescription { /// Users of the `Module` API generally should call /// `Module::declare_data_in_data` instead, as it takes care of generating /// the appropriate `ExternalName`. - pub fn import_global_value(&mut self, name: ModuleExtName) -> ir::GlobalValue { + pub fn import_global_value(&mut self, name: ModuleRelocTarget) -> ir::GlobalValue { self.data_decls.push(name) } @@ -176,7 +176,7 @@ impl DataDescription { #[cfg(test)] mod tests { - use crate::ModuleExtName; + use crate::ModuleRelocTarget; use super::{DataDescription, Init}; @@ -191,11 +191,11 @@ mod tests { data.define_zeroinit(256); - let _func_a = data.import_function(ModuleExtName::user(0, 0)); - let func_b = data.import_function(ModuleExtName::user(0, 1)); - let func_c = data.import_function(ModuleExtName::user(0, 2)); - let _data_a = data.import_global_value(ModuleExtName::user(0, 3)); - let data_b = data.import_global_value(ModuleExtName::user(0, 4)); + let _func_a = data.import_function(ModuleRelocTarget::user(0, 0)); + let func_b = data.import_function(ModuleRelocTarget::user(0, 1)); + let func_c = data.import_function(ModuleRelocTarget::user(0, 2)); + let _data_a = data.import_global_value(ModuleRelocTarget::user(0, 3)); + let data_b = data.import_global_value(ModuleRelocTarget::user(0, 4)); data.write_function_addr(8, func_b); data.write_function_addr(16, func_c); diff --git a/cranelift/module/src/lib.rs b/cranelift/module/src/lib.rs index b2bbe088e4af..88313c3a224b 100644 --- a/cranelift/module/src/lib.rs +++ b/cranelift/module/src/lib.rs @@ -29,7 +29,7 @@ mod traps; pub use crate::data_context::{DataDescription, Init}; pub use crate::module::{ DataDeclaration, DataId, FuncId, FuncOrDataId, FunctionDeclaration, Linkage, Module, - ModuleDeclarations, ModuleError, ModuleExtName, ModuleReloc, ModuleResult, + ModuleDeclarations, ModuleError, ModuleReloc, ModuleRelocTarget, ModuleResult, }; pub use crate::traps::TrapSite; diff --git a/cranelift/module/src/module.rs b/cranelift/module/src/module.rs index 917b8ca688c8..cae49ef17e97 100644 --- a/cranelift/module/src/module.rs +++ b/cranelift/module/src/module.rs @@ -11,9 +11,11 @@ use core::fmt::Display; use cranelift_codegen::binemit::{CodeOffset, Reloc}; use cranelift_codegen::entity::{entity_impl, PrimaryMap}; use cranelift_codegen::ir::function::{Function, VersionMarker}; +use cranelift_codegen::ir::{ExternalName, UserFuncName}; use cranelift_codegen::settings::SetError; -use cranelift_codegen::MachReloc; -use cranelift_codegen::{ir, isa, CodegenError, CompileError, Context}; +use cranelift_codegen::{ + ir, isa, CodegenError, CompileError, Context, FinalizedMachReloc, FinalizedRelocTarget, +}; use cranelift_control::ControlPlane; use std::borrow::{Cow, ToOwned}; use std::string::String; @@ -27,22 +29,29 @@ pub struct ModuleReloc { /// The kind of relocation. pub kind: Reloc, /// The external symbol / name to which this relocation refers. - pub name: ModuleExtName, + pub name: ModuleRelocTarget, /// The addend to add to the symbol value. pub addend: i64, } impl ModuleReloc { - /// Converts a `MachReloc` produced from a `Function` into a `ModuleReloc`. - pub fn from_mach_reloc(mach_reloc: &MachReloc, func: &Function) -> Self { - let name = match mach_reloc.name { - ir::ExternalName::User(reff) => { + /// Converts a `FinalizedMachReloc` produced from a `Function` into a `ModuleReloc`. + pub fn from_mach_reloc(mach_reloc: &FinalizedMachReloc, func: &Function) -> Self { + let name = match mach_reloc.target { + FinalizedRelocTarget::ExternalName(ExternalName::User(reff)) => { let name = &func.params.user_named_funcs()[reff]; - ModuleExtName::user(name.namespace, name.index) + ModuleRelocTarget::user(name.namespace, name.index) + } + FinalizedRelocTarget::ExternalName(ExternalName::TestCase(_)) => unimplemented!(), + FinalizedRelocTarget::ExternalName(ExternalName::LibCall(libcall)) => { + ModuleRelocTarget::LibCall(libcall) + } + FinalizedRelocTarget::ExternalName(ExternalName::KnownSymbol(ks)) => { + ModuleRelocTarget::KnownSymbol(ks) + } + FinalizedRelocTarget::Func(offset) => { + ModuleRelocTarget::FunctionOffset(func.name.clone(), offset) } - ir::ExternalName::TestCase(_) => unimplemented!(), - ir::ExternalName::LibCall(libcall) => ModuleExtName::LibCall(libcall), - ir::ExternalName::KnownSymbol(ks) => ModuleExtName::KnownSymbol(ks), }; Self { offset: mach_reloc.offset, @@ -63,7 +72,7 @@ pub struct FuncId(u32); entity_impl!(FuncId, "funcid"); /// Function identifiers are namespace 0 in `ir::ExternalName` -impl From for ModuleExtName { +impl From for ModuleRelocTarget { fn from(id: FuncId) -> Self { Self::User { namespace: 0, @@ -74,8 +83,8 @@ impl From for ModuleExtName { impl FuncId { /// Get the `FuncId` for the function named by `name`. - pub fn from_name(name: &ModuleExtName) -> FuncId { - if let ModuleExtName::User { namespace, index } = name { + pub fn from_name(name: &ModuleRelocTarget) -> FuncId { + if let ModuleRelocTarget::User { namespace, index } = name { debug_assert_eq!(*namespace, 0); FuncId::from_u32(*index) } else { @@ -94,7 +103,7 @@ pub struct DataId(u32); entity_impl!(DataId, "dataid"); /// Data identifiers are namespace 1 in `ir::ExternalName` -impl From for ModuleExtName { +impl From for ModuleRelocTarget { fn from(id: DataId) -> Self { Self::User { namespace: 1, @@ -105,8 +114,8 @@ impl From for ModuleExtName { impl DataId { /// Get the `DataId` for the data object named by `name`. - pub fn from_name(name: &ModuleExtName) -> DataId { - if let ModuleExtName::User { namespace, index } = name { + pub fn from_name(name: &ModuleRelocTarget) -> DataId { + if let ModuleRelocTarget::User { namespace, index } = name { debug_assert_eq!(*namespace, 1); DataId::from_u32(*index) } else { @@ -191,7 +200,7 @@ pub enum FuncOrDataId { } /// Mapping to `ModuleExtName` is trivial based on the `FuncId` and `DataId` mapping. -impl From for ModuleExtName { +impl From for ModuleRelocTarget { fn from(id: FuncOrDataId) -> Self { match id { FuncOrDataId::Func(funcid) => Self::from(funcid), @@ -408,7 +417,7 @@ impl DataDeclaration { feature = "enable-serde", derive(serde_derive::Serialize, serde_derive::Deserialize) )] -pub enum ModuleExtName { +pub enum ModuleRelocTarget { /// User defined function, converted from `ExternalName::User`. User { /// Arbitrary. @@ -420,21 +429,24 @@ pub enum ModuleExtName { LibCall(ir::LibCall), /// Symbols known to the linker. KnownSymbol(ir::KnownSymbol), + /// A offset inside a function + FunctionOffset(UserFuncName, CodeOffset), } -impl ModuleExtName { +impl ModuleRelocTarget { /// Creates a user-defined external name. pub fn user(namespace: u32, index: u32) -> Self { Self::User { namespace, index } } } -impl Display for ModuleExtName { +impl Display for ModuleRelocTarget { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { match self { Self::User { namespace, index } => write!(f, "u{}:{}", namespace, index), Self::LibCall(lc) => write!(f, "%{}", lc), Self::KnownSymbol(ks) => write!(f, "{}", ks), + Self::FunctionOffset(fname, offset) => write!(f, "{fname}+{offset}"), } } } @@ -686,10 +698,12 @@ impl ModuleDeclarations { } /// Return whether `name` names a function, rather than a data object. - pub fn is_function(name: &ModuleExtName) -> bool { + pub fn is_function(name: &ModuleRelocTarget) -> bool { match name { - ModuleExtName::User { namespace, .. } => *namespace == 0, - ModuleExtName::LibCall(_) | ModuleExtName::KnownSymbol(_) => { + ModuleRelocTarget::User { namespace, .. } => *namespace == 0, + ModuleRelocTarget::LibCall(_) + | ModuleRelocTarget::KnownSymbol(_) + | ModuleRelocTarget::FunctionOffset(..) => { panic!("unexpected module ext name") } } @@ -918,12 +932,12 @@ pub trait Module { /// TODO: Same as above. fn declare_func_in_data(&self, func_id: FuncId, data: &mut DataDescription) -> ir::FuncRef { - data.import_function(ModuleExtName::user(0, func_id.as_u32())) + data.import_function(ModuleRelocTarget::user(0, func_id.as_u32())) } /// TODO: Same as above. fn declare_data_in_data(&self, data_id: DataId, data: &mut DataDescription) -> ir::GlobalValue { - data.import_global_value(ModuleExtName::user(1, data_id.as_u32())) + data.import_global_value(ModuleRelocTarget::user(1, data_id.as_u32())) } /// Define a function, producing the function body from the given `Context`. @@ -965,7 +979,7 @@ pub trait Module { func: &ir::Function, alignment: u64, bytes: &[u8], - relocs: &[MachReloc], + relocs: &[FinalizedMachReloc], ) -> ModuleResult<()>; /// Define a data object, producing the data contents from the given `DataContext`. @@ -1067,7 +1081,7 @@ impl Module for &mut M { func: &ir::Function, alignment: u64, bytes: &[u8], - relocs: &[MachReloc], + relocs: &[FinalizedMachReloc], ) -> ModuleResult<()> { (**self).define_function_bytes(func_id, func, alignment, bytes, relocs) } diff --git a/cranelift/object/src/backend.rs b/cranelift/object/src/backend.rs index 453936aeffb3..d083d84ce12f 100644 --- a/cranelift/object/src/backend.rs +++ b/cranelift/object/src/backend.rs @@ -3,12 +3,13 @@ use anyhow::anyhow; use cranelift_codegen::binemit::{Addend, CodeOffset, Reloc}; use cranelift_codegen::entity::SecondaryMap; +use cranelift_codegen::ir::UserFuncName; use cranelift_codegen::isa::{OwnedTargetIsa, TargetIsa}; -use cranelift_codegen::{self, ir, MachReloc}; +use cranelift_codegen::{self, ir, FinalizedMachReloc}; use cranelift_control::ControlPlane; use cranelift_module::{ DataDescription, DataId, FuncId, Init, Linkage, Module, ModuleDeclarations, ModuleError, - ModuleExtName, ModuleReloc, ModuleResult, + ModuleReloc, ModuleRelocTarget, ModuleResult, }; use log::info; use object::write::{ @@ -17,6 +18,7 @@ use object::write::{ use object::{ RelocationEncoding, RelocationKind, SectionKind, SymbolFlags, SymbolKind, SymbolScope, }; +use std::collections::hash_map::Entry; use std::collections::HashMap; use std::mem; use target_lexicon::PointerWidth; @@ -130,6 +132,7 @@ pub struct ObjectModule { libcalls: HashMap, libcall_names: Box String + Send + Sync>, known_symbols: HashMap, + known_labels: HashMap<(UserFuncName, CodeOffset), SymbolId>, per_function_section: bool, } @@ -149,6 +152,7 @@ impl ObjectModule { libcalls: HashMap::new(), libcall_names: builder.libcall_names, known_symbols: HashMap::new(), + known_labels: HashMap::new(), per_function_section: builder.per_function_section, } } @@ -333,21 +337,18 @@ impl Module for ObjectModule { func: &ir::Function, alignment: u64, bytes: &[u8], - relocs: &[MachReloc], + relocs: &[FinalizedMachReloc], ) -> ModuleResult<()> { info!("defining function {} with bytes", func_id); let decl = self.declarations.get_function_decl(func_id); + let decl_name = decl.linkage_name(func_id); if !decl.linkage.is_definable() { - return Err(ModuleError::InvalidImportDefinition( - decl.linkage_name(func_id).into_owned(), - )); + return Err(ModuleError::InvalidImportDefinition(decl_name.into_owned())); } let &mut (symbol, ref mut defined) = self.functions[func_id].as_mut().unwrap(); if *defined { - return Err(ModuleError::DuplicateDefinition( - decl.linkage_name(func_id).into_owned(), - )); + return Err(ModuleError::DuplicateDefinition(decl_name.into_owned())); } *defined = true; @@ -528,9 +529,9 @@ impl ObjectModule { /// This should only be called during finish because it creates /// symbols for missing libcalls. - fn get_symbol(&mut self, name: &ModuleExtName) -> SymbolId { + fn get_symbol(&mut self, name: &ModuleRelocTarget) -> SymbolId { match *name { - ModuleExtName::User { .. } => { + ModuleRelocTarget::User { .. } => { if ModuleDeclarations::is_function(name) { let id = FuncId::from_name(name); self.functions[id].unwrap().0 @@ -539,7 +540,7 @@ impl ObjectModule { self.data_objects[id].unwrap().0 } } - ModuleExtName::LibCall(ref libcall) => { + ModuleRelocTarget::LibCall(ref libcall) => { let name = (self.libcall_names)(*libcall); if let Some(symbol) = self.object.symbol_id(name.as_bytes()) { symbol @@ -562,7 +563,7 @@ impl ObjectModule { } // These are "magic" names well-known to the linker. // They require special treatment. - ModuleExtName::KnownSymbol(ref known_symbol) => { + ModuleRelocTarget::KnownSymbol(ref known_symbol) => { if let Some(symbol) = self.known_symbols.get(known_symbol) { *symbol } else { @@ -592,6 +593,36 @@ impl ObjectModule { symbol } } + + ModuleRelocTarget::FunctionOffset(ref fname, offset) => { + match self.known_labels.entry((fname.clone(), offset)) { + Entry::Occupied(o) => *o.get(), + Entry::Vacant(v) => { + let func_user_name = fname.get_user().unwrap(); + let func_id = FuncId::from_name(&ModuleRelocTarget::user( + func_user_name.namespace, + func_user_name.index, + )); + let func_symbol_id = self.functions[func_id].unwrap().0; + let func_symbol = self.object.symbol(func_symbol_id); + + let name = format!(".L{}_{}", func_id.as_u32(), offset); + let symbol_id = self.object.add_symbol(Symbol { + name: name.as_bytes().to_vec(), + value: func_symbol.value + offset as u64, + size: 0, + kind: SymbolKind::Label, + scope: SymbolScope::Compilation, + weak: false, + section: SymbolSection::Section(func_symbol.section.id().unwrap()), + flags: SymbolFlags::None, + }); + + v.insert(symbol_id); + symbol_id + } + } + } } } @@ -776,6 +807,30 @@ impl ObjectModule { 0, ) } + Reloc::RiscvTlsGdHi20 => { + assert_eq!( + self.object.format(), + object::BinaryFormat::Elf, + "RiscvTlsGdHi20 is not supported for this file format" + ); + ( + RelocationKind::Elf(object::elf::R_RISCV_TLS_GD_HI20), + RelocationEncoding::Generic, + 0, + ) + } + Reloc::RiscvPCRelLo12I => { + assert_eq!( + self.object.format(), + object::BinaryFormat::Elf, + "RiscvPCRelLo12I is not supported for this file format" + ); + ( + RelocationKind::Elf(object::elf::R_RISCV_PCREL_LO12_I), + RelocationEncoding::Generic, + 0, + ) + } // FIXME reloc => unimplemented!("{:?}", reloc), }; @@ -846,7 +901,7 @@ struct SymbolRelocs { #[derive(Clone)] struct ObjectRelocRecord { offset: CodeOffset, - name: ModuleExtName, + name: ModuleRelocTarget, kind: RelocationKind, encoding: RelocationEncoding, size: u8, diff --git a/cranelift/src/disasm.rs b/cranelift/src/disasm.rs index 1739f526b8c5..0b0235b67959 100644 --- a/cranelift/src/disasm.rs +++ b/cranelift/src/disasm.rs @@ -2,15 +2,15 @@ use anyhow::Result; use cfg_if::cfg_if; use cranelift_codegen::ir::function::FunctionParameters; use cranelift_codegen::isa::TargetIsa; -use cranelift_codegen::{MachReloc, MachStackMap, MachTrap}; +use cranelift_codegen::{FinalizedMachReloc, MachStackMap, MachTrap}; use std::fmt::Write; -fn print_relocs(func_params: &FunctionParameters, relocs: &[MachReloc]) -> String { +fn print_relocs(func_params: &FunctionParameters, relocs: &[FinalizedMachReloc]) -> String { let mut text = String::new(); - for &MachReloc { + for &FinalizedMachReloc { kind, offset, - ref name, + ref target, addend, } in relocs { @@ -18,7 +18,7 @@ fn print_relocs(func_params: &FunctionParameters, relocs: &[MachReloc]) -> Strin text, "reloc_external: {} {} {} at {}", kind, - name.display(Some(func_params)), + target.display(Some(func_params)), addend, offset ) @@ -121,7 +121,7 @@ pub fn print_all( mem: &[u8], code_size: u32, print: bool, - relocs: &[MachReloc], + relocs: &[FinalizedMachReloc], traps: &[MachTrap], stack_maps: &[MachStackMap], ) -> Result<()> { diff --git a/crates/cranelift-shared/src/lib.rs b/crates/cranelift-shared/src/lib.rs index add380288703..b508e04a79e6 100644 --- a/crates/cranelift-shared/src/lib.rs +++ b/crates/cranelift-shared/src/lib.rs @@ -1,7 +1,7 @@ use cranelift_codegen::{ binemit, ir::{self, ExternalName, UserExternalNameRef}, - settings, MachReloc, MachTrap, + settings, FinalizedMachReloc, FinalizedRelocTarget, MachTrap, }; use std::collections::BTreeMap; use wasmtime_environ::{FlagValue, FuncIndex, Trap, TrapInformation}; @@ -101,24 +101,26 @@ pub fn mach_trap_to_trap(trap: &MachTrap) -> Option { /// Converts machine relocations to relocation information /// to perform. -fn mach_reloc_to_reloc(reloc: &MachReloc, transform_user_func_ref: F) -> Relocation +fn mach_reloc_to_reloc(reloc: &FinalizedMachReloc, transform_user_func_ref: F) -> Relocation where F: Fn(UserExternalNameRef) -> (u32, u32), { - let &MachReloc { + let &FinalizedMachReloc { offset, kind, - ref name, + ref target, addend, } = reloc; - let reloc_target = if let ExternalName::User(user_func_ref) = *name { - let (namespace, index) = transform_user_func_ref(user_func_ref); - debug_assert_eq!(namespace, 0); - RelocationTarget::UserFunc(FuncIndex::from_u32(index)) - } else if let ExternalName::LibCall(libcall) = *name { - RelocationTarget::LibCall(libcall) - } else { - panic!("unrecognized external name") + let reloc_target = match *target { + FinalizedRelocTarget::ExternalName(ExternalName::User(user_func_ref)) => { + let (namespace, index) = transform_user_func_ref(user_func_ref); + debug_assert_eq!(namespace, 0); + RelocationTarget::UserFunc(FuncIndex::from_u32(index)) + } + FinalizedRelocTarget::ExternalName(ExternalName::LibCall(libcall)) => { + RelocationTarget::LibCall(libcall) + } + _ => panic!("unrecognized external name"), }; Relocation { reloc: kind,