Rollup merge of #123840 - scottmcm:aggregate-kind-rawptr, r=cjgillot

Add an intrinsic for `ptr::from_raw_parts(_mut)`

Fixes #123174
cc `@CAD97` `@saethlin`
r? `@cjgillot`

As suggested in #123190 (comment), this adds a new `AggregateKind::RawPtr` for creating a pointer from its data pointer and its metadata.

That means that `slice::from_raw_parts` and friends no longer need to hard-code pointer layout into `libcore`, and because it no longer does union hacks the MIR is shorter and more amenable to optimizations.

compiler/rustc_borrowck/src/lib.rs

@@ -1315,7 +1315,8 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                     }
                     AggregateKind::Adt(..)
                     | AggregateKind::Array(..)
-                    | AggregateKind::Tuple { .. } => (),
+                    | AggregateKind::Tuple { .. }
+                    | AggregateKind::RawPtr(..) => (),
                 }
 
                 for operand in operands {

compiler/rustc_borrowck/src/type_check/mod.rs

@@ -1921,7 +1921,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                 }
             }
             AggregateKind::Array(ty) => Ok(ty),
-            AggregateKind::Tuple => {
+            AggregateKind::Tuple | AggregateKind::RawPtr(..) => {
                 unreachable!("This should have been covered in check_rvalues");
             }
         }
@@ -2518,6 +2518,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                 AggregateKind::Closure(_, _) => None,
                 AggregateKind::Coroutine(_, _) => None,
                 AggregateKind::CoroutineClosure(_, _) => None,
+                AggregateKind::RawPtr(_, _) => None,
             },
         }
     }
@@ -2539,6 +2540,10 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             return;
         }
 
+        if let AggregateKind::RawPtr(..) = aggregate_kind {
+            bug!("RawPtr should only be in runtime MIR");
+        }
+
         for (i, operand) in operands.iter_enumerated() {
             let field_ty = match self.aggregate_field_ty(aggregate_kind, i, location) {
                 Ok(field_ty) => field_ty,
@@ -2757,7 +2762,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                 ),
             ),
 
-            AggregateKind::Array(_) | AggregateKind::Tuple => {
+            AggregateKind::Array(_) | AggregateKind::Tuple | AggregateKind::RawPtr(..) => {
                 (CRATE_DEF_ID.to_def_id(), ty::InstantiatedPredicates::empty())
             }
         };

compiler/rustc_codegen_cranelift/src/base.rs

@@ -813,6 +813,19 @@ fn codegen_stmt<'tcx>(
                     );
                     lval.write_cvalue(fx, val);
                 }
+                Rvalue::Aggregate(ref kind, ref operands)
+                    if matches!(**kind, AggregateKind::RawPtr(..)) =>
+                {
+                    let ty = to_place_and_rval.1.ty(&fx.mir.local_decls, fx.tcx);
+                    let layout = fx.layout_of(fx.monomorphize(ty));
+                    let [data, meta] = &*operands.raw else {
+                        bug!("RawPtr fields: {operands:?}");
+                    };
+                    let data = codegen_operand(fx, data);
+                    let meta = codegen_operand(fx, meta);
+                    let ptr_val = CValue::pointer_from_data_and_meta(data, meta, layout);
+                    lval.write_cvalue(fx, ptr_val);
+                }
                 Rvalue::Aggregate(ref kind, ref operands) => {
                     let (variant_index, variant_dest, active_field_index) = match **kind {
                         mir::AggregateKind::Adt(_, variant_index, _, _, active_field_index) => {

compiler/rustc_codegen_cranelift/src/value_and_place.rs

@@ -94,6 +94,23 @@ impl<'tcx> CValue<'tcx> {
         CValue(CValueInner::ByValPair(value, extra), layout)
     }
 
+    /// For `AggregateKind::RawPtr`, create a pointer from its parts.
+    ///
+    /// Panics if the `layout` is not a raw pointer.
+    pub(crate) fn pointer_from_data_and_meta(
+        data: CValue<'tcx>,
+        meta: CValue<'tcx>,
+        layout: TyAndLayout<'tcx>,
+    ) -> CValue<'tcx> {
+        assert!(layout.ty.is_unsafe_ptr());
+        let inner = match (data.0, meta.0) {
+            (CValueInner::ByVal(p), CValueInner::ByVal(m)) => CValueInner::ByValPair(p, m),
+            (p @ CValueInner::ByVal(_), CValueInner::ByRef(..)) if meta.1.is_zst() => p,
+            _ => bug!("RawPtr operands {data:?} {meta:?}"),
+        };
+        CValue(inner, layout)
+    }
+
     pub(crate) fn layout(&self) -> TyAndLayout<'tcx> {
         self.1
     }

compiler/rustc_codegen_ssa/src/mir/rvalue.rs

@@ -9,7 +9,7 @@ use crate::MemFlags;
 
 use rustc_hir as hir;
 use rustc_middle::mir;
-use rustc_middle::mir::Operand;
+use rustc_middle::mir::{AggregateKind, Operand};
 use rustc_middle::ty::cast::{CastTy, IntTy};
 use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, TyAndLayout};
 use rustc_middle::ty::{self, adjustment::PointerCoercion, Instance, Ty, TyCtxt};
@@ -720,6 +720,24 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 OperandRef { val: OperandValue::Immediate(static_), layout }
             }
             mir::Rvalue::Use(ref operand) => self.codegen_operand(bx, operand),
+            mir::Rvalue::Aggregate(box mir::AggregateKind::RawPtr(..), ref fields) => {
+                let ty = rvalue.ty(self.mir, self.cx.tcx());
+                let layout = self.cx.layout_of(self.monomorphize(ty));
+                let [data, meta] = &*fields.raw else {
+                    bug!("RawPtr fields: {fields:?}");
+                };
+                let data = self.codegen_operand(bx, data);
+                let meta = self.codegen_operand(bx, meta);
+                match (data.val, meta.val) {
+                    (p @ OperandValue::Immediate(_), OperandValue::ZeroSized) => {
+                        OperandRef { val: p, layout }
+                    }
+                    (OperandValue::Immediate(p), OperandValue::Immediate(m)) => {
+                        OperandRef { val: OperandValue::Pair(p, m), layout }
+                    }
+                    _ => bug!("RawPtr operands {data:?} {meta:?}"),
+                }
+            }
             mir::Rvalue::Repeat(..) | mir::Rvalue::Aggregate(..) => {
                 // According to `rvalue_creates_operand`, only ZST
                 // aggregate rvalues are allowed to be operands.
@@ -1032,6 +1050,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             mir::Rvalue::ThreadLocalRef(_) |
             mir::Rvalue::Use(..) => // (*)
                 true,
+            // This always produces a `ty::RawPtr`, so will be Immediate or Pair
+            mir::Rvalue::Aggregate(box AggregateKind::RawPtr(..), ..) => true,
             mir::Rvalue::Repeat(..) |
             mir::Rvalue::Aggregate(..) => {
                 let ty = rvalue.ty(self.mir, self.cx.tcx());

compiler/rustc_const_eval/src/interpret/step.rs

@@ -9,7 +9,9 @@ use rustc_middle::mir;
 use rustc_middle::ty::layout::LayoutOf;
 use rustc_target::abi::{FieldIdx, FIRST_VARIANT};
 
-use super::{ImmTy, InterpCx, InterpResult, Machine, PlaceTy, Projectable, Scalar};
+use super::{
+    ImmTy, Immediate, InterpCx, InterpResult, Machine, MemPlaceMeta, PlaceTy, Projectable, Scalar,
+};
 use crate::util;
 
 impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
@@ -303,6 +305,27 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 let variant_dest = self.project_downcast(dest, variant_index)?;
                 (variant_index, variant_dest, active_field_index)
             }
+            mir::AggregateKind::RawPtr(..) => {
+                // Pointers don't have "fields" in the normal sense, so the
+                // projection-based code below would either fail in projection
+                // or in type mismatches. Instead, build an `Immediate` from
+                // the parts and write that to the destination.
+                let [data, meta] = &operands.raw else {
+                    bug!("{kind:?} should have 2 operands, had {operands:?}");
+                };
+                let data = self.eval_operand(data, None)?;
+                let data = self.read_pointer(&data)?;
+                let meta = self.eval_operand(meta, None)?;
+                let meta = if meta.layout.is_zst() {
+                    MemPlaceMeta::None
+                } else {
+                    MemPlaceMeta::Meta(self.read_scalar(&meta)?)
+                };
+                let ptr_imm = Immediate::new_pointer_with_meta(data, meta, self);
+                let ptr = ImmTy::from_immediate(ptr_imm, dest.layout);
+                self.copy_op(&ptr, dest)?;
+                return Ok(());
+            }
             _ => (FIRST_VARIANT, dest.clone(), None),
         };
         if active_field_index.is_some() {

compiler/rustc_const_eval/src/transform/validate.rs

@@ -923,6 +923,47 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                         }
                     }
                 }
+                AggregateKind::RawPtr(pointee_ty, mutability) => {
+                    if !matches!(self.mir_phase, MirPhase::Runtime(_)) {
+                        // It would probably be fine to support this in earlier phases,
+                        // but at the time of writing it's only ever introduced from intrinsic lowering,
+                        // so earlier things just `bug!` on it.
+                        self.fail(location, "RawPtr should be in runtime MIR only");
+                    }
+
+                    if fields.len() != 2 {
+                        self.fail(location, "raw pointer aggregate must have 2 fields");
+                    } else {
+                        let data_ptr_ty = fields.raw[0].ty(self.body, self.tcx);
+                        let metadata_ty = fields.raw[1].ty(self.body, self.tcx);
+                        if let ty::RawPtr(in_pointee, in_mut) = data_ptr_ty.kind() {
+                            if *in_mut != mutability {
+                                self.fail(location, "input and output mutability must match");
+                            }
+
+                            // FIXME: check `Thin` instead of `Sized`
+                            if !in_pointee.is_sized(self.tcx, self.param_env) {
+                                self.fail(location, "input pointer must be thin");
+                            }
+                        } else {
+                            self.fail(
+                                location,
+                                "first operand to raw pointer aggregate must be a raw pointer",
+                            );
+                        }
+
+                        // FIXME: Check metadata more generally
+                        if pointee_ty.is_slice() {
+                            if !self.mir_assign_valid_types(metadata_ty, self.tcx.types.usize) {
+                                self.fail(location, "slice metadata must be usize");
+                            }
+                        } else if pointee_ty.is_sized(self.tcx, self.param_env) {
+                            if metadata_ty != self.tcx.types.unit {
+                                self.fail(location, "metadata for pointer-to-thin must be unit");
+                            }
+                        }
+                    }
+                }
             },
             Rvalue::Ref(_, BorrowKind::Fake, _) => {
                 if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) {

compiler/rustc_hir_analysis/src/check/intrinsic.rs

@@ -128,6 +128,7 @@ pub fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -
         | sym::variant_count
         | sym::is_val_statically_known
         | sym::ptr_mask
+        | sym::aggregate_raw_ptr
         | sym::ub_checks
         | sym::fadd_algebraic
         | sym::fsub_algebraic
@@ -574,6 +575,10 @@ pub fn check_intrinsic_type(
                 (0, 0, vec![Ty::new_imm_ptr(tcx, Ty::new_unit(tcx))], tcx.types.usize)
             }
 
+            // This type check is not particularly useful, but the `where` bounds
+            // on the definition in `core` do the heavy lifting for checking it.
+            sym::aggregate_raw_ptr => (3, 1, vec![param(1), param(2)], param(0)),
+
             sym::ub_checks => (0, 1, Vec::new(), tcx.types.bool),
 
             sym::simd_eq

compiler/rustc_middle/src/mir/pretty.rs

@@ -1094,6 +1094,15 @@ impl<'tcx> Debug for Rvalue<'tcx> {
 
                         struct_fmt.finish()
                     }),
+
+                    AggregateKind::RawPtr(pointee_ty, mutability) => {
+                        let kind_str = match mutability {
+                            Mutability::Mut => "mut",
+                            Mutability::Not => "const",
+                        };
+                        with_no_trimmed_paths!(write!(fmt, "*{kind_str} {pointee_ty} from "))?;
+                        fmt_tuple(fmt, "")
+                    }
                 }
             }
 

compiler/rustc_middle/src/mir/syntax.rs

@@ -1351,6 +1351,21 @@ pub enum AggregateKind<'tcx> {
     Closure(DefId, GenericArgsRef<'tcx>),
     Coroutine(DefId, GenericArgsRef<'tcx>),
     CoroutineClosure(DefId, GenericArgsRef<'tcx>),
+
+    /// Construct a raw pointer from the data pointer and metadata.
+    ///
+    /// The `Ty` here is the type of the *pointee*, not the pointer itself.
+    /// The `Mutability` indicates whether this produces a `*const` or `*mut`.
+    ///
+    /// The [`Rvalue::Aggregate`] operands for thus must be
+    ///
+    /// 0. A raw pointer of matching mutability with any [`core::ptr::Thin`] pointee
+    /// 1. A value of the appropriate [`core::ptr::Pointee::Metadata`] type
+    ///
+    /// *Both* operands must always be included, even the unit value if this is
+    /// creating a thin pointer. If you're just converting between thin pointers,
+    /// you may want an [`Rvalue::Cast`] with [`CastKind::PtrToPtr`] instead.
+    RawPtr(Ty<'tcx>, Mutability),
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq, TyEncodable, TyDecodable, Hash, HashStable)]

compiler/rustc_middle/src/mir/tcx.rs

@@ -206,6 +206,7 @@ impl<'tcx> Rvalue<'tcx> {
                 AggregateKind::CoroutineClosure(did, args) => {
                     Ty::new_coroutine_closure(tcx, did, args)
                 }
+                AggregateKind::RawPtr(ty, mutability) => Ty::new_ptr(tcx, ty, mutability),
             },
             Rvalue::ShallowInitBox(_, ty) => Ty::new_box(tcx, ty),
             Rvalue::CopyForDeref(ref place) => place.ty(local_decls, tcx).ty,

compiler/rustc_middle/src/mir/visit.rs

@@ -751,6 +751,9 @@ macro_rules! make_mir_visitor {
                             ) => {
                                 self.visit_args(coroutine_closure_args, location);
                             }
+                            AggregateKind::RawPtr(ty, _) => {
+                                self.visit_ty($(& $mutability)? *ty, TyContext::Location(location));
+                            }
                         }
 
                         for operand in operands {

compiler/rustc_mir_transform/src/gvn.rs

@@ -885,6 +885,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
                 AggregateKind::Adt(did, ..) => tcx.def_kind(did) != DefKind::Enum,
                 // Coroutines are never ZST, as they at least contain the implicit states.
                 AggregateKind::Coroutine(..) => false,
+                AggregateKind::RawPtr(..) => bug!("MIR for RawPtr aggregate must have 2 fields"),
             };
 
             if is_zst {
@@ -910,6 +911,8 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
             }
             // Do not track unions.
             AggregateKind::Adt(_, _, _, _, Some(_)) => return None,
+            // FIXME: Do the extra work to GVN `from_raw_parts`
+            AggregateKind::RawPtr(..) => return None,
         };
 
         let fields: Option<Vec<_>> = fields

compiler/rustc_mir_transform/src/instsimplify.rs

@@ -36,6 +36,7 @@ impl<'tcx> MirPass<'tcx> for InstSimplify {
                         ctx.simplify_bool_cmp(&statement.source_info, rvalue);
                         ctx.simplify_ref_deref(&statement.source_info, rvalue);
                         ctx.simplify_len(&statement.source_info, rvalue);
+                        ctx.simplify_ptr_aggregate(&statement.source_info, rvalue);
                         ctx.simplify_cast(rvalue);
                     }
                     _ => {}
@@ -58,8 +59,17 @@ struct InstSimplifyContext<'tcx, 'a> {
 
 impl<'tcx> InstSimplifyContext<'tcx, '_> {
     fn should_simplify(&self, source_info: &SourceInfo, rvalue: &Rvalue<'tcx>) -> bool {
+        self.should_simplify_custom(source_info, "Rvalue", rvalue)
+    }
+
+    fn should_simplify_custom(
+        &self,
+        source_info: &SourceInfo,
+        label: &str,
+        value: impl std::fmt::Debug,
+    ) -> bool {
         self.tcx.consider_optimizing(|| {
-            format!("InstSimplify - Rvalue: {rvalue:?} SourceInfo: {source_info:?}")
+            format!("InstSimplify - {label}: {value:?} SourceInfo: {source_info:?}")
         })
     }
 
@@ -111,7 +121,7 @@ impl<'tcx> InstSimplifyContext<'tcx, '_> {
         if a.const_.ty().is_bool() { a.const_.try_to_bool() } else { None }
     }
 
-    /// Transform "&(*a)" ==> "a".
+    /// Transform `&(*a)` ==> `a`.
     fn simplify_ref_deref(&self, source_info: &SourceInfo, rvalue: &mut Rvalue<'tcx>) {
         if let Rvalue::Ref(_, _, place) = rvalue {
             if let Some((base, ProjectionElem::Deref)) = place.as_ref().last_projection() {
@@ -131,7 +141,7 @@ impl<'tcx> InstSimplifyContext<'tcx, '_> {
         }
     }
 
-    /// Transform "Len([_; N])" ==> "N".
+    /// Transform `Len([_; N])` ==> `N`.
     fn simplify_len(&self, source_info: &SourceInfo, rvalue: &mut Rvalue<'tcx>) {
         if let Rvalue::Len(ref place) = *rvalue {
             let place_ty = place.ty(self.local_decls, self.tcx).ty;
@@ -147,6 +157,30 @@ impl<'tcx> InstSimplifyContext<'tcx, '_> {
         }
     }
 
+    /// Transform `Aggregate(RawPtr, [p, ()])` ==> `Cast(PtrToPtr, p)`.
+    fn simplify_ptr_aggregate(&self, source_info: &SourceInfo, rvalue: &mut Rvalue<'tcx>) {
+        if let Rvalue::Aggregate(box AggregateKind::RawPtr(pointee_ty, mutability), fields) = rvalue
+        {
+            let meta_ty = fields.raw[1].ty(self.local_decls, self.tcx);
+            if meta_ty.is_unit() {
+                // The mutable borrows we're holding prevent printing `rvalue` here
+                if !self.should_simplify_custom(
+                    source_info,
+                    "Aggregate::RawPtr",
+                    (&pointee_ty, *mutability, &fields),
+                ) {
+                    return;
+                }
+
+                let mut fields = std::mem::take(fields);
+                let _meta = fields.pop().unwrap();
+                let data = fields.pop().unwrap();
+                let ptr_ty = Ty::new_ptr(self.tcx, *pointee_ty, *mutability);
+                *rvalue = Rvalue::Cast(CastKind::PtrToPtr, data, ptr_ty);
+            }
+        }
+    }
+
     fn simplify_ub_check(&self, source_info: &SourceInfo, rvalue: &mut Rvalue<'tcx>) {
         if let Rvalue::NullaryOp(NullOp::UbChecks, _) = *rvalue {
             let const_ = Const::from_bool(self.tcx, self.tcx.sess.ub_checks());

compiler/rustc_mir_transform/src/known_panics_lint.rs

@@ -603,6 +603,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
                         AggregateKind::Adt(_, variant, _, _, _) => variant,
                         AggregateKind::Array(_)
                         | AggregateKind::Tuple
+                        | AggregateKind::RawPtr(_, _)
                         | AggregateKind::Closure(_, _)
                         | AggregateKind::Coroutine(_, _)
                         | AggregateKind::CoroutineClosure(_, _) => VariantIdx::ZERO,