Skip to content

Commit

Permalink
Unrolled build for rust-lang#124251
Browse files Browse the repository at this point in the history
Rollup merge of rust-lang#124251 - scottmcm:unop-ptr-metadata, r=oli-obk

Add an intrinsic for `ptr::metadata`

The follow-up to rust-lang#123840, so we can remove `PtrComponents` and `PtrRepr` from libcore entirely (well, after a bootstrap update).

As discussed in <https://rust-lang.zulipchat.com/#narrow/stream/189540-t-compiler.2Fwg-mir-opt/topic/.60ptr_metadata.60.20in.20MIR/near/435637808>, this introduces `UnOp::PtrMetadata` taking a raw pointer and returning the associated metadata value.

By no longer going through a `union`, this should also help future PRs better optimize pointer operations.

r? ``@oli-obk``
  • Loading branch information
rust-timer authored May 29, 2024
2 parents da159eb + 57948c8 commit 058425c
Show file tree
Hide file tree
Showing 42 changed files with 600 additions and 53 deletions.
38 changes: 25 additions & 13 deletions compiler/rustc_codegen_cranelift/src/base.rs
Original file line number Diff line number Diff line change
Expand Up @@ -616,22 +616,34 @@ fn codegen_stmt<'tcx>(
Rvalue::UnaryOp(un_op, ref operand) => {
let operand = codegen_operand(fx, operand);
let layout = operand.layout();
let val = operand.load_scalar(fx);
let res = match un_op {
UnOp::Not => match layout.ty.kind() {
ty::Bool => {
let res = fx.bcx.ins().icmp_imm(IntCC::Equal, val, 0);
CValue::by_val(res, layout)
UnOp::Not => {
let val = operand.load_scalar(fx);
match layout.ty.kind() {
ty::Bool => {
let res = fx.bcx.ins().icmp_imm(IntCC::Equal, val, 0);
CValue::by_val(res, layout)
}
ty::Uint(_) | ty::Int(_) => {
CValue::by_val(fx.bcx.ins().bnot(val), layout)
}
_ => unreachable!("un op Not for {:?}", layout.ty),
}
ty::Uint(_) | ty::Int(_) => {
CValue::by_val(fx.bcx.ins().bnot(val), layout)
}
UnOp::Neg => {
let val = operand.load_scalar(fx);
match layout.ty.kind() {
ty::Int(_) => CValue::by_val(fx.bcx.ins().ineg(val), layout),
ty::Float(_) => CValue::by_val(fx.bcx.ins().fneg(val), layout),
_ => unreachable!("un op Neg for {:?}", layout.ty),
}
_ => unreachable!("un op Not for {:?}", layout.ty),
},
UnOp::Neg => match layout.ty.kind() {
ty::Int(_) => CValue::by_val(fx.bcx.ins().ineg(val), layout),
ty::Float(_) => CValue::by_val(fx.bcx.ins().fneg(val), layout),
_ => unreachable!("un op Neg for {:?}", layout.ty),
}
UnOp::PtrMetadata => match layout.abi {
Abi::Scalar(_) => CValue::zst(dest_layout),
Abi::ScalarPair(_, _) => {
CValue::by_val(operand.load_scalar_pair(fx).1, dest_layout)
}
_ => bug!("Unexpected `PtrToMetadata` operand: {operand:?}"),
},
};
lval.write_cvalue(fx, res);
Expand Down
2 changes: 1 addition & 1 deletion compiler/rustc_codegen_cranelift/src/constant.rs
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ pub(crate) fn codegen_const_value<'tcx>(
assert!(layout.is_sized(), "unsized const value");

if layout.is_zst() {
return CValue::by_ref(crate::Pointer::dangling(layout.align.pref), layout);
return CValue::zst(layout);
}

match const_val {
Expand Down
8 changes: 8 additions & 0 deletions compiler/rustc_codegen_cranelift/src/value_and_place.rs
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,14 @@ impl<'tcx> CValue<'tcx> {
CValue(CValueInner::ByValPair(value, extra), layout)
}

/// Create an instance of a ZST
///
/// The is represented by a dangling pointer of suitable alignment.
pub(crate) fn zst(layout: TyAndLayout<'tcx>) -> CValue<'tcx> {
assert!(layout.is_zst());
CValue::by_ref(crate::Pointer::dangling(layout.align.pref), layout)
}

pub(crate) fn layout(&self) -> TyAndLayout<'tcx> {
self.1
}
Expand Down
5 changes: 5 additions & 0 deletions compiler/rustc_codegen_ssa/src/mir/operand.rs
Original file line number Diff line number Diff line change
Expand Up @@ -565,6 +565,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
for elem in place_ref.projection.iter() {
match elem {
mir::ProjectionElem::Field(ref f, _) => {
debug_assert!(
!o.layout.ty.is_any_ptr(),
"Bad PlaceRef: destructing pointers should use cast/PtrMetadata, \
but tried to access field {f:?} of pointer {o:?}",
);
o = o.extract_field(bx, f.index());
}
mir::ProjectionElem::Index(_)
Expand Down
5 changes: 5 additions & 0 deletions compiler/rustc_codegen_ssa/src/mir/place.rs
Original file line number Diff line number Diff line change
Expand Up @@ -480,6 +480,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
cg_base = match *elem {
mir::ProjectionElem::Deref => bx.load_operand(cg_base).deref(bx.cx()),
mir::ProjectionElem::Field(ref field, _) => {
debug_assert!(
!cg_base.layout.ty.is_any_ptr(),
"Bad PlaceRef: destructing pointers should use cast/PtrMetadata, \
but tried to access field {field:?} of pointer {cg_base:?}",
);
cg_base.project_field(bx, field.index())
}
mir::ProjectionElem::OpaqueCast(ty) => {
Expand Down
34 changes: 26 additions & 8 deletions compiler/rustc_codegen_ssa/src/mir/rvalue.rs
Original file line number Diff line number Diff line change
Expand Up @@ -623,19 +623,36 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {

mir::Rvalue::UnaryOp(op, ref operand) => {
let operand = self.codegen_operand(bx, operand);
let lloperand = operand.immediate();
let is_float = operand.layout.ty.is_floating_point();
let llval = match op {
mir::UnOp::Not => bx.not(lloperand),
let (val, layout) = match op {
mir::UnOp::Not => {
let llval = bx.not(operand.immediate());
(OperandValue::Immediate(llval), operand.layout)
}
mir::UnOp::Neg => {
if is_float {
bx.fneg(lloperand)
let llval = if is_float {
bx.fneg(operand.immediate())
} else {
bx.neg(operand.immediate())
};
(OperandValue::Immediate(llval), operand.layout)
}
mir::UnOp::PtrMetadata => {
debug_assert!(operand.layout.ty.is_unsafe_ptr());
let (_, meta) = operand.val.pointer_parts();
assert_eq!(operand.layout.fields.count() > 1, meta.is_some());
if let Some(meta) = meta {
(OperandValue::Immediate(meta), operand.layout.field(self.cx, 1))
} else {
bx.neg(lloperand)
(OperandValue::ZeroSized, bx.cx().layout_of(bx.tcx().types.unit))
}
}
};
OperandRef { val: OperandValue::Immediate(llval), layout: operand.layout }
debug_assert!(
val.is_expected_variant_for_type(self.cx, layout),
"Made wrong variant {val:?} for type {layout:?}",
);
OperandRef { val, layout }
}

mir::Rvalue::Discriminant(ref place) => {
Expand Down Expand Up @@ -718,8 +735,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
let values = op.val.immediates_or_place().left_or_else(|p| {
bug!("Field {field_idx:?} is {p:?} making {layout:?}");
});
inputs.extend(values);
let scalars = self.value_kind(op.layout).scalars().unwrap();
debug_assert_eq!(values.len(), scalars.len());
inputs.extend(values);
input_scalars.extend(scalars);
}

Expand Down
2 changes: 1 addition & 1 deletion compiler/rustc_const_eval/src/interpret/cast.rs
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
assert!(cast_to.ty.is_unsafe_ptr());
// Handle casting any ptr to raw ptr (might be a fat ptr).
if cast_to.size == src.layout.size {
// Thin or fat pointer that just hast the ptr kind of target type changed.
// Thin or fat pointer that just has the ptr kind of target type changed.
return Ok(ImmTy::from_immediate(**src, cast_to));
} else {
// Casting the metadata away from a fat ptr.
Expand Down
28 changes: 24 additions & 4 deletions compiler/rustc_const_eval/src/interpret/operator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use rustc_middle::{bug, span_bug};
use rustc_span::symbol::sym;
use tracing::trace;

use super::{err_ub, throw_ub, ImmTy, InterpCx, Machine};
use super::{err_ub, throw_ub, ImmTy, InterpCx, Machine, MemPlaceMeta};

impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
fn three_way_compare<T: Ord>(&self, lhs: T, rhs: T) -> ImmTy<'tcx, M::Provenance> {
Expand Down Expand Up @@ -415,11 +415,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
use rustc_middle::mir::UnOp::*;

let layout = val.layout;
let val = val.to_scalar();
trace!("Running unary op {:?}: {:?} ({})", un_op, val, layout.ty);

match layout.ty.kind() {
ty::Bool => {
let val = val.to_scalar();
let val = val.to_bool()?;
let res = match un_op {
Not => !val,
Expand All @@ -428,6 +428,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
Ok(ImmTy::from_bool(res, *self.tcx))
}
ty::Float(fty) => {
let val = val.to_scalar();
// No NaN adjustment here, `-` is a bitwise operation!
let res = match (un_op, fty) {
(Neg, FloatTy::F32) => Scalar::from_f32(-val.to_f32()?),
Expand All @@ -436,8 +437,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
};
Ok(ImmTy::from_scalar(res, layout))
}
_ => {
assert!(layout.ty.is_integral());
_ if layout.ty.is_integral() => {
let val = val.to_scalar();
let val = val.to_bits(layout.size)?;
let res = match un_op {
Not => self.truncate(!val, layout), // bitwise negation, then truncate
Expand All @@ -450,9 +451,28 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
// Truncate to target type.
self.truncate(res, layout)
}
_ => span_bug!(self.cur_span(), "Invalid integer op {:?}", un_op),
};
Ok(ImmTy::from_uint(res, layout))
}
ty::RawPtr(..) => {
assert_eq!(un_op, PtrMetadata);
let (_, meta) = val.to_scalar_and_meta();
Ok(match meta {
MemPlaceMeta::Meta(scalar) => {
let ty = un_op.ty(*self.tcx, val.layout.ty);
let layout = self.layout_of(ty)?;
ImmTy::from_scalar(scalar, layout)
}
MemPlaceMeta::None => {
let unit_layout = self.layout_of(self.tcx.types.unit)?;
ImmTy::uninit(unit_layout)
}
})
}
_ => {
bug!("Unexpected unary op argument {val:?}")
}
}
}
}
2 changes: 2 additions & 0 deletions compiler/rustc_hir_analysis/src/check/intrinsic.rs
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,7 @@ pub fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -
| sym::is_val_statically_known
| sym::ptr_mask
| sym::aggregate_raw_ptr
| sym::ptr_metadata
| sym::ub_checks
| sym::fadd_algebraic
| sym::fsub_algebraic
Expand Down Expand Up @@ -576,6 +577,7 @@ pub fn check_intrinsic_type(
// This type check is not particularly useful, but the `where` bounds
// on the definition in `core` do the heavy lifting for checking it.
sym::aggregate_raw_ptr => (3, 1, vec![param(1), param(2)], param(0)),
sym::ptr_metadata => (2, 1, vec![Ty::new_imm_ptr(tcx, param(0))], param(1)),

sym::ub_checks => (0, 1, Vec::new(), tcx.types.bool),

Expand Down
7 changes: 7 additions & 0 deletions compiler/rustc_middle/src/mir/syntax.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1434,6 +1434,13 @@ pub enum UnOp {
Not,
/// The `-` operator for negation
Neg,
/// Get the metadata `M` from a `*const/mut impl Pointee<Metadata = M>`.
///
/// For example, this will give a `()` from `*const i32`, a `usize` from
/// `*mut [u8]`, or a pointer to a vtable from a `*const dyn Foo`.
///
/// Allowed only in [`MirPhase::Runtime`]; earlier it's an intrinsic.
PtrMetadata,
}

#[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Ord, Eq, Hash)]
Expand Down
26 changes: 25 additions & 1 deletion compiler/rustc_middle/src/mir/tcx.rs
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,10 @@ impl<'tcx> Rvalue<'tcx> {
let rhs_ty = rhs.ty(local_decls, tcx);
op.ty(tcx, lhs_ty, rhs_ty)
}
Rvalue::UnaryOp(UnOp::Not | UnOp::Neg, ref operand) => operand.ty(local_decls, tcx),
Rvalue::UnaryOp(op, ref operand) => {
let arg_ty = operand.ty(local_decls, tcx);
op.ty(tcx, arg_ty)
}
Rvalue::Discriminant(ref place) => place.ty(local_decls, tcx).ty.discriminant_ty(tcx),
Rvalue::NullaryOp(NullOp::SizeOf | NullOp::AlignOf | NullOp::OffsetOf(..), _) => {
tcx.types.usize
Expand Down Expand Up @@ -282,6 +285,27 @@ impl<'tcx> BinOp {
}
}

impl<'tcx> UnOp {
pub fn ty(&self, tcx: TyCtxt<'tcx>, arg_ty: Ty<'tcx>) -> Ty<'tcx> {
match self {
UnOp::Not | UnOp::Neg => arg_ty,
UnOp::PtrMetadata => {
let pointee_ty = arg_ty
.builtin_deref(true)
.unwrap_or_else(|| bug!("PtrMetadata of non-dereferenceable ty {arg_ty:?}"));
if pointee_ty.is_trivially_sized(tcx) {
tcx.types.unit
} else {
let Some(metadata_def_id) = tcx.lang_items().metadata_type() else {
bug!("No metadata_type lang item while looking at {arg_ty:?}")
};
Ty::new_projection(tcx, metadata_def_id, [pointee_ty])
}
}
}
}
}

impl BorrowKind {
pub fn to_mutbl_lossy(self) -> hir::Mutability {
match self {
Expand Down
2 changes: 2 additions & 0 deletions compiler/rustc_middle/src/ty/print/pretty.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1579,8 +1579,10 @@ pub trait PrettyPrinter<'tcx>: Printer<'tcx> + fmt::Write {
let formatted_op = match op {
UnOp::Not => "!",
UnOp::Neg => "-",
UnOp::PtrMetadata => "PtrMetadata",
};
let parenthesized = match ct.kind() {
_ if op == UnOp::PtrMetadata => true,
ty::ConstKind::Expr(Expr::UnOp(c_op, ..)) => c_op != op,
ty::ConstKind::Expr(_) => true,
_ => false,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,7 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
Ok(Rvalue::BinaryOp(BinOp::Offset, Box::new((ptr, offset))))
},
@call(mir_len, args) => Ok(Rvalue::Len(self.parse_place(args[0])?)),
@call(mir_ptr_metadata, args) => Ok(Rvalue::UnaryOp(UnOp::PtrMetadata, self.parse_operand(args[0])?)),
@call(mir_copy_for_deref, args) => Ok(Rvalue::CopyForDeref(self.parse_place(args[0])?)),
ExprKind::Borrow { borrow_kind, arg } => Ok(
Rvalue::Ref(self.tcx.lifetimes.re_erased, *borrow_kind, self.parse_place(*arg)?)
Expand Down
17 changes: 17 additions & 0 deletions compiler/rustc_mir_transform/src/lower_intrinsics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -316,6 +316,23 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics {

terminator.kind = TerminatorKind::Goto { target };
}
sym::ptr_metadata => {
let Ok([ptr]) = <[_; 1]>::try_from(std::mem::take(args)) else {
span_bug!(
terminator.source_info.span,
"Wrong number of arguments for ptr_metadata intrinsic",
);
};
let target = target.unwrap();
block.statements.push(Statement {
source_info: terminator.source_info,
kind: StatementKind::Assign(Box::new((
*destination,
Rvalue::UnaryOp(UnOp::PtrMetadata, ptr.node),
))),
});
terminator.kind = TerminatorKind::Goto { target };
}
_ => {}
}
}
Expand Down
2 changes: 1 addition & 1 deletion compiler/rustc_mir_transform/src/promote_consts.rs
Original file line number Diff line number Diff line change
Expand Up @@ -464,7 +464,7 @@ impl<'tcx> Validator<'_, 'tcx> {
Rvalue::UnaryOp(op, operand) => {
match op {
// These operations can never fail.
UnOp::Neg | UnOp::Not => {}
UnOp::Neg | UnOp::Not | UnOp::PtrMetadata => {}
}

self.validate_operand(operand)?;
Expand Down
10 changes: 10 additions & 0 deletions compiler/rustc_mir_transform/src/validate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1109,6 +1109,16 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
ty::Int(..) | ty::Uint(..) | ty::Bool
);
}
UnOp::PtrMetadata => {
if !matches!(self.mir_phase, MirPhase::Runtime(_)) {
// It would probably be fine to support this in earlier phases,
// but at the time of writing it's only ever introduced from intrinsic lowering,
// so earlier things can just `bug!` on it.
self.fail(location, "PtrMetadata should be in runtime MIR only");
}

check_kinds!(a, "Cannot PtrMetadata non-pointer type {:?}", ty::RawPtr(..));
}
}
}
Rvalue::ShallowInitBox(operand, _) => {
Expand Down
14 changes: 13 additions & 1 deletion compiler/rustc_smir/src/rustc_internal/internal.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ use rustc_span::Symbol;
use stable_mir::abi::Layout;
use stable_mir::mir::alloc::AllocId;
use stable_mir::mir::mono::{Instance, MonoItem, StaticDef};
use stable_mir::mir::{BinOp, Mutability, Place, ProjectionElem, Safety};
use stable_mir::mir::{BinOp, Mutability, Place, ProjectionElem, Safety, UnOp};
use stable_mir::ty::{
Abi, AdtDef, Binder, BoundRegionKind, BoundTyKind, BoundVariableKind, ClosureKind, Const,
DynKind, ExistentialPredicate, ExistentialProjection, ExistentialTraitRef, FloatTy, FnSig,
Expand Down Expand Up @@ -582,6 +582,18 @@ impl RustcInternal for BinOp {
}
}

impl RustcInternal for UnOp {
type T<'tcx> = rustc_middle::mir::UnOp;

fn internal<'tcx>(&self, _tables: &mut Tables<'_>, _tcx: TyCtxt<'tcx>) -> Self::T<'tcx> {
match self {
UnOp::Not => rustc_middle::mir::UnOp::Not,
UnOp::Neg => rustc_middle::mir::UnOp::Neg,
UnOp::PtrMetadata => rustc_middle::mir::UnOp::PtrMetadata,
}
}
}

impl<T> RustcInternal for &T
where
T: RustcInternal,
Expand Down
Loading

0 comments on commit 058425c

Please sign in to comment.