riscv64: Implement ELF TLS GD Relocations

cranelift/codegen/src/binemit/mod.rs

@@ -92,6 +92,18 @@ pub enum Reloc {
     /// jalr ra, ra, 0
     RiscvCall,
 
+    /// RISC-V TLS GD: High 20 bits of 32-bit PC-relative TLS GD GOT reference,
+    ///
+    /// This is the `R_RISCV_TLS_GD_HI20` relocation from the RISC-V ELF psABI document.
+    /// https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#global-dynamic
+    RiscvTlsGdHi20,
+
+    /// Low 12 bits of a 32-bit PC-relative relocation (I-Type instruction)
+    ///
+    /// This is the `R_RISCV_PCREL_LO12_I` relocation from the RISC-V ELF psABI document.
+    /// https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses
+    RiscvPCRelLo12I,
+
     /// s390x TLS GD64 - 64-bit offset of tls_index for GD symbol in GOT
     S390xTlsGd64,
     /// s390x TLS GDCall - marker to enable optimization of TLS calls
@@ -114,7 +126,8 @@ impl fmt::Display for Reloc {
             Self::X86SecRel => write!(f, "SecRel"),
             Self::Arm32Call | Self::Arm64Call => write!(f, "Call"),
             Self::RiscvCall => write!(f, "RiscvCall"),
-
+            Self::RiscvTlsGdHi20 => write!(f, "RiscvTlsGdHi20"),
+            Self::RiscvPCRelLo12I => write!(f, "RiscvPCRelLo12I"),
             Self::ElfX86_64TlsGd => write!(f, "ElfX86_64TlsGd"),
             Self::MachOX86_64Tlv => write!(f, "MachOX86_64Tlv"),
             Self::MachOAarch64TlsAdrPage21 => write!(f, "MachOAarch64TlsAdrPage21"),

cranelift/codegen/src/isa/aarch64/inst/emit.rs

@@ -3174,7 +3174,7 @@ impl MachInstEmit for Inst {
                 // Note: this is not `Inst::Jump { .. }.emit(..)` because we
                 // have different metadata in this case: we don't have a label
                 // for the target, but rather a function relocation.
-                sink.add_reloc(Reloc::Arm64Call, callee, 0);
+                sink.add_reloc(Reloc::Arm64Call, &**callee, 0);
                 sink.put4(enc_jump26(0b000101, 0));
                 sink.add_call_site(ir::Opcode::ReturnCall);
 
@@ -3382,12 +3382,12 @@ impl MachInstEmit for Inst {
                     //   ldr     rd, [rd, :got_lo12:X]
 
                     // adrp rd, symbol
-                    sink.add_reloc(Reloc::Aarch64AdrGotPage21, name, 0);
+                    sink.add_reloc(Reloc::Aarch64AdrGotPage21, &**name, 0);
                     let inst = Inst::Adrp { rd, off: 0 };
                     inst.emit(&[], sink, emit_info, state);
 
                     // ldr rd, [rd, :got_lo12:X]
-                    sink.add_reloc(Reloc::Aarch64Ld64GotLo12Nc, name, 0);
+                    sink.add_reloc(Reloc::Aarch64Ld64GotLo12Nc, &**name, 0);
                     let inst = Inst::ULoad64 {
                         rd,
                         mem: AMode::reg(rd.to_reg()),
@@ -3415,7 +3415,7 @@ impl MachInstEmit for Inst {
                         dest: BranchTarget::ResolvedOffset(12),
                     };
                     inst.emit(&[], sink, emit_info, state);
-                    sink.add_reloc(Reloc::Abs8, name, offset);
+                    sink.add_reloc(Reloc::Abs8, &**name, offset);
                     sink.put8(0);
                 }
             }

cranelift/codegen/src/isa/riscv64/inst.isle

@@ -148,6 +148,11 @@
       (name BoxExternalName)
       (offset i64))
 
+    ;; Load a TLS symbol address
+    (ElfTlsGetAddr
+      (rd WritableReg)
+      (name BoxExternalName))
+
     ;; Load address referenced by `mem` into `rd`.
     (LoadAddr
       (rd WritableReg)
@@ -2625,6 +2630,12 @@
 (decl load_ext_name (ExternalName i64) Reg)
 (extern constructor load_ext_name load_ext_name)
 
+(decl elf_tls_get_addr (ExternalName) Reg)
+(rule (elf_tls_get_addr name)
+      (let ((dst WritableReg (temp_writable_reg $I64))
+            (_ Unit (emit (MInst.ElfTlsGetAddr dst name))))
+        dst))
+
 (decl int_convert_2_float_op (Type bool Type) FpuOPRR)
 (extern constructor int_convert_2_float_op int_convert_2_float_op)
 

cranelift/codegen/src/isa/riscv64/inst/emit.rs

@@ -1,7 +1,7 @@
 //! Riscv64 ISA: binary code emission.
 
 use crate::binemit::StackMap;
-use crate::ir::{self, RelSourceLoc, TrapCode};
+use crate::ir::{self, LibCall, RelSourceLoc, TrapCode};
 use crate::isa::riscv64::inst::*;
 use crate::isa::riscv64::lower::isle::generated_code::{CaOp, CrOp};
 use crate::machinst::{AllocationConsumer, Reg, Writable};
@@ -343,6 +343,7 @@ impl Inst {
             | Inst::Jal { .. }
             | Inst::CondBr { .. }
             | Inst::LoadExtName { .. }
+            | Inst::ElfTlsGetAddr { .. }
             | Inst::LoadAddr { .. }
             | Inst::VirtualSPOffsetAdj { .. }
             | Inst::Mov { .. }
@@ -978,7 +979,7 @@ impl Inst {
                 );
 
                 sink.add_call_site(ir::Opcode::ReturnCall);
-                sink.add_reloc(Reloc::RiscvCall, &callee, 0);
+                sink.add_reloc(Reloc::RiscvCall, &**callee, 0);
                 Inst::construct_auipc_and_jalr(None, writable_spilltmp_reg(), 0)
                     .into_iter()
                     .for_each(|i| i.emit_uncompressed(sink, emit_info, state, start_off));
@@ -1958,6 +1959,60 @@ impl Inst {
 
                 sink.bind_label(label_end, &mut state.ctrl_plane);
             }
+
+            &Inst::ElfTlsGetAddr { rd, ref name } => {
+                // RISC-V's TLS GD model is slightly different from other arches.
+                //
+                // We have a relocation (R_RISCV_TLS_GD_HI20) that loads the high 20 bits
+                // of the address relative to the GOT entry. This relocation points to
+                // the symbol as usual.
+                //
+                // However when loading the bottom 12bits of the address, we need to
+                // use a label that points to the previous AUIPC instruction.
+                //
+                // label:
+                //    auipc a0,0                    # R_RISCV_TLS_GD_HI20 (symbol)
+                //    addi  a0,a0,0                 # R_RISCV_PCREL_LO12_I (label)
+                //
+                // https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#global-dynamic
+
+                // Create the lable that is going to be published to the final binary object.
+                let auipc_label = sink.get_label();
+                sink.bind_label(auipc_label, &mut state.ctrl_plane);
+
+                // Get the current PC.
+                sink.add_reloc(Reloc::RiscvTlsGdHi20, &**name, 0);
+                Inst::Auipc {
+                    rd: rd,
+                    imm: Imm20::from_i32(0),
+                }
+                .emit_uncompressed(sink, emit_info, state, start_off);
+
+                // The `addi` here, points to the `auipc` label instead of directly to the symbol.
+                sink.add_reloc(Reloc::RiscvPCRelLo12I, &auipc_label, 0);
+                Inst::AluRRImm12 {
+                    alu_op: AluOPRRI::Addi,
+                    rd: rd,
+                    rs: rd.to_reg(),
+                    imm12: Imm12::from_i16(0),
+                }
+                .emit_uncompressed(sink, emit_info, state, start_off);
+
+                Inst::Call {
+                    info: Box::new(CallInfo {
+                        dest: ExternalName::LibCall(LibCall::ElfTlsGetAddr),
+                        uses: smallvec![],
+                        defs: smallvec![],
+                        opcode: crate::ir::Opcode::TlsValue,
+                        caller_callconv: CallConv::SystemV,
+                        callee_callconv: CallConv::SystemV,
+                        callee_pop_size: 0,
+                        clobbers: PRegSet::empty(),
+                    }),
+                }
+                .emit_uncompressed(sink, emit_info, state, start_off);
+            }
+
             &Inst::TrapIfC {
                 rs1,
                 rs2,
@@ -3273,6 +3328,12 @@ impl Inst {
                 offset,
             },
 
+            Inst::ElfTlsGetAddr { rd, name } => {
+                let rd = allocs.next_writable(rd);
+                debug_assert_eq!(a0(), rd.to_reg());
+                Inst::ElfTlsGetAddr { rd, name }
+            }
+
             Inst::TrapIfC {
                 rs1,
                 rs2,

cranelift/codegen/src/isa/riscv64/inst/mod.rs

@@ -473,6 +473,13 @@ fn riscv64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut Operan
         &Inst::LoadExtName { rd, .. } => {
             collector.reg_def(rd);
         }
+        &Inst::ElfTlsGetAddr { rd, .. } => {
+            // x10 is a0 which is both the first argument and the first return value.
+            collector.reg_fixed_def(rd, a0());
+            let mut clobbers = Riscv64MachineDeps::get_regs_clobbered_by_call(CallConv::SystemV);
+            clobbers.remove(px_reg(10));
+            collector.reg_clobbers(clobbers);
+        }
         &Inst::LoadAddr { rd, mem } => {
             if let Some(r) = mem.get_allocatable_register() {
                 collector.reg_use(r);
@@ -1691,6 +1698,10 @@ impl Inst {
                 let rd = format_reg(rd.to_reg(), allocs);
                 format!("load_sym {},{}{:+}", rd, name.display(None), offset)
             }
+            &Inst::ElfTlsGetAddr { rd, ref name } => {
+                let rd = format_reg(rd.to_reg(), allocs);
+                format!("elf_tls_get_addr {rd},{}", name.display(None))
+            }
             &MInst::LoadAddr { ref rd, ref mem } => {
                 let rs = mem.to_string_with_alloc(allocs);
                 let rd = format_reg(rd.to_reg(), allocs);

cranelift/codegen/src/isa/riscv64/lower.isle

@@ -1675,6 +1675,11 @@
    (lower (symbol_value (symbol_value_data name _ offset)))
    (load_ext_name name offset))
 
+;;;;;  Rules for `tls_value` ;;;;;;;;;;;;;;
+
+(rule (lower (has_type (tls_model (TlsModel.ElfGd)) (tls_value (symbol_value_data name _ _))))
+      (elf_tls_get_addr name))
+
 ;;;;;  Rules for `bitcast`;;;;;;;;;
 (rule
    (lower (has_type out_ty (bitcast _ v @ (value_type in_ty))))

cranelift/codegen/src/lib.rs

@@ -56,7 +56,8 @@ pub mod write;
 
 pub use crate::entity::packed_option;
 pub use crate::machinst::buffer::{
-    MachCallSite, MachReloc, MachSrcLoc, MachStackMap, MachTextSectionBuilder, MachTrap,
+    FinalizedMachReloc, FinalizedRelocTarget, MachCallSite, MachSrcLoc, MachStackMap,
+    MachTextSectionBuilder, MachTrap,
 };
 pub use crate::machinst::{
     CompiledCode, Final, MachBuffer, MachBufferFinalized, MachInst, MachInstEmit,

cranelift/codegen/src/machinst/buffer.rs

@@ -171,6 +171,7 @@
 //!   all longer-range fixups to later.
 
 use crate::binemit::{Addend, CodeOffset, Reloc, StackMap};
+use crate::ir::function::FunctionParameters;
 use crate::ir::{ExternalName, Opcode, RelSourceLoc, SourceLoc, TrapCode};
 use crate::isa::unwind::UnwindInst;
 use crate::machinst::{
@@ -345,7 +346,7 @@ pub struct MachBufferFinalized<T: CompilePhase> {
     /// Any relocations referring to this code. Note that only *external*
     /// relocations are tracked here; references to labels within the buffer are
     /// resolved before emission.
-    pub(crate) relocs: SmallVec<[MachReloc; 16]>,
+    pub(crate) relocs: SmallVec<[FinalizedMachReloc; 16]>,
     /// Any trap records referring to this code.
     pub(crate) traps: SmallVec<[MachTrap; 16]>,
     /// Any call site records referring to this code.
@@ -1451,12 +1452,31 @@ impl<I: VCodeInst> MachBuffer<I> {
 
         let alignment = self.finish_constants(constants);
 
+        // Resolve all labels to their offsets.
+        let finalized_relocs = self
+            .relocs
+            .iter()
+            .map(|reloc| FinalizedMachReloc {
+                offset: reloc.offset,
+                kind: reloc.kind,
+                addend: reloc.addend,
+                target: match &reloc.target {
+                    RelocTarget::ExternalName(name) => {
+                        FinalizedRelocTarget::ExternalName(name.clone())
+                    }
+                    RelocTarget::Label(label) => {
+                        FinalizedRelocTarget::Func(self.resolve_label_offset(*label))
+                    }
+                },
+            })
+            .collect();
+
         let mut srclocs = self.srclocs;
         srclocs.sort_by_key(|entry| entry.start);
 
         MachBufferFinalized {
             data: self.data,
-            relocs: self.relocs,
+            relocs: finalized_relocs,
             traps: self.traps,
             call_sites: self.call_sites,
             srclocs,
@@ -1467,8 +1487,13 @@ impl<I: VCodeInst> MachBuffer<I> {
     }
 
     /// Add an external relocation at the current offset.
-    pub fn add_reloc(&mut self, kind: Reloc, name: &ExternalName, addend: Addend) {
-        let name = name.clone();
+    pub fn add_reloc<T: Into<RelocTarget> + Clone>(
+        &mut self,
+        kind: Reloc,
+        target: &T,
+        addend: Addend,
+    ) {
+        let target: RelocTarget = target.clone().into();
         // FIXME(#3277): This should use `I::LabelUse::from_reloc` to optionally
         // generate a label-use statement to track whether an island is possibly
         // needed to escape this function to actually get to the external name.
@@ -1505,7 +1530,7 @@ impl<I: VCodeInst> MachBuffer<I> {
         self.relocs.push(MachReloc {
             offset: self.data.len() as CodeOffset,
             kind,
-            name,
+            target,
             addend,
         });
     }
@@ -1622,7 +1647,7 @@ impl<T: CompilePhase> MachBufferFinalized<T> {
     }
 
     /// Get the list of external relocations for this code.
-    pub fn relocs(&self) -> &[MachReloc] {
+    pub fn relocs(&self) -> &[FinalizedMachReloc] {
         &self.relocs[..]
     }
 
@@ -1717,18 +1742,72 @@ impl<I: VCodeInst> Ord for MachLabelFixup<I> {
     feature = "enable-serde",
     derive(serde_derive::Serialize, serde_derive::Deserialize)
 )]
-pub struct MachReloc {
+pub struct MachRelocBase<T> {
     /// The offset at which the relocation applies, *relative to the
     /// containing section*.
     pub offset: CodeOffset,
     /// The kind of relocation.
     pub kind: Reloc,
     /// The external symbol / name to which this relocation refers.
-    pub name: ExternalName,
+    pub target: T,
     /// The addend to add to the symbol value.
     pub addend: i64,
 }
 
+type MachReloc = MachRelocBase<RelocTarget>;
+
+/// A relocation resulting from a compilation.
+pub type FinalizedMachReloc = MachRelocBase<FinalizedRelocTarget>;
+
+/// A Relocation target
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub enum RelocTarget {
+    /// Points to an [ExternalName] outside the current function.
+    ExternalName(ExternalName),
+    /// Points to a [MachLabel] inside this function.
+    /// This is different from [MachLabelFixup] in that both the relocation and the
+    /// label will be emitted and are only resolved at link time.
+    ///
+    /// There is no reason to prefer this over [MachLabelFixup] unless the ABI requires it.
+    Label(MachLabel),
+}
+
+impl From<ExternalName> for RelocTarget {
+    fn from(name: ExternalName) -> Self {
+        Self::ExternalName(name)
+    }
+}
+
+impl From<MachLabel> for RelocTarget {
+    fn from(label: MachLabel) -> Self {
+        Self::Label(label)
+    }
+}
+
+/// A Relocation target
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[cfg_attr(
+    feature = "enable-serde",
+    derive(serde_derive::Serialize, serde_derive::Deserialize)
+)]
+pub enum FinalizedRelocTarget {
+    /// Points to an [ExternalName] outside the current function.
+    ExternalName(ExternalName),
+    /// Points to a [CodeOffset] from the start of the current function.
+    Func(CodeOffset),
+}
+
+impl FinalizedRelocTarget {
+    /// Returns a display for the current [FinalizedRelocTarget], with extra context to prettify the
+    /// output.
+    pub fn display<'a>(&'a self, params: Option<&'a FunctionParameters>) -> String {
+        match self {
+            FinalizedRelocTarget::ExternalName(name) => format!("{}", name.display(params)),
+            FinalizedRelocTarget::Func(offset) => format!("func+{offset}"),
+        }
+    }
+}
+
 /// A trap record resulting from a compilation.
 #[derive(Clone, Debug, PartialEq)]
 #[cfg_attr(