Add intrinsics for bigint helper methods

compiler/rustc_codegen_gcc/src/intrinsic/mod.rs

@@ -192,6 +192,12 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
             | sym::prefetch_write_instruction => {
                 unimplemented!();
             }
+            #[cfg(not(bootstrap))]
+            sym::add_with_carry
+            | sym::sub_with_carry
+            | sym::mul_double
+            | sym::mul_double_add
+            | sym::mul_double_add2 => unimplemented!(),
             sym::ctlz
             | sym::ctlz_nonzero
             | sym::cttz

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -347,7 +347,12 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
             | sym::rotate_left
             | sym::rotate_right
             | sym::saturating_add
-            | sym::saturating_sub => {
+            | sym::saturating_sub
+            | sym::add_with_carry
+            | sym::sub_with_carry
+            | sym::mul_double
+            | sym::mul_double_add
+            | sym::mul_double_add2 => {
                 let ty = arg_tys[0];
                 match int_type_width_signed(ty, self) {
                     Some((width, signed)) => match name {
@@ -417,6 +422,76 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                             );
                             self.call_intrinsic(llvm_name, &[lhs, rhs])
                         }
+                        sym::add_with_carry | sym::sub_with_carry => {
+                            let llty = self.type_ix(width);
+                            let is_add = name == sym::add_with_carry;
+                            let lhs = args[0].immediate();
+                            let rhs = args[1].immediate();
+
+                            // sign-extending the carry would treat it as -1, not 1
+                            let carry = self.intcast(args[2].immediate(), llty, false);
+
+                            let llvm_name = &format!(
+                                "llvm.{}{}.with.overflow.i{}",
+                                if signed { 's' } else { 'u' },
+                                if is_add { "add" } else { "sub" },
+                                width,
+                            );
+
+                            let ret = self.call_intrinsic(llvm_name, &[lhs, rhs]);
+                            let agg = self.extract_value(ret, 0);
+                            let overflow1 = self.extract_value(ret, 1);
+
+                            let ret = self.call_intrinsic(llvm_name, &[agg, carry]);
+                            let agg = self.extract_value(ret, 0);
+                            let overflow2 = self.extract_value(ret, 1);
+
+                            let overflow = if signed {
+                                self.icmp(IntPredicate::IntNE, overflow1, overflow2)
+                            } else {
+                                self.or(overflow1, overflow2)
+                            };
+
+                            let holder = self.const_struct(
+                                &[self.const_undef(llty), self.const_undef(self.type_i1())],
+                                false,
+                            );
+                            let holder = self.insert_value(holder, agg, 0);
+                            let holder = self.insert_value(holder, overflow, 1);
+                            holder
+                        }
+                        sym::mul_double | sym::mul_double_add | sym::mul_double_add2 => {
+                            let single_ty = self.type_ix(width);
+                            let double_ty = self.type_ix(width * 2);
+                            let lhs = self.intcast(args[0].immediate(), double_ty, signed);
+                            let rhs = self.intcast(args[1].immediate(), double_ty, signed);
+                            let mut ret = self.mul(lhs, rhs);
+                            if name == sym::mul_double_add || name == sym::mul_double_add2 {
+                                let carry = self.intcast(args[2].immediate(), double_ty, signed);
+                                ret = self.add(ret, carry)
+                            }
+                            if name == sym::mul_double_add2 {
+                                let carry2 = self.intcast(args[3].immediate(), double_ty, signed);
+                                ret = self.add(ret, carry2);
+                            }
+
+                            // note: insignificant part is always treated as unsigned, even if we
+                            //   coerce it to signed in the final result to make the intrinsic
+                            //   signature simpler
+                            let lo = self.intcast(ret, single_ty, signed);
+
+                            let bits = self.const_uint(double_ty, width);
+                            let hi = self.ashr(ret, bits);
+                            let hi = self.intcast(hi, single_ty, signed);
+
+                            let holder = self.const_struct(
+                                &[self.const_undef(single_ty), self.const_undef(single_ty)],
+                                false,
+                            );
+                            let holder = self.insert_value(holder, lo, 0);
+                            let holder = self.insert_value(holder, hi, 1);
+                            holder
+                        }
                         _ => bug!(),
                     },
                     None => {

compiler/rustc_const_eval/src/interpret/intrinsics.rs

@@ -178,6 +178,34 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 )?;
                 self.write_scalar(val, dest)?;
             }
+            sym::add_with_carry | sym::sub_with_carry => {
+                let l = self.read_immediate(&args[0])?;
+                let r = self.read_immediate(&args[1])?;
+                let c = self.read_immediate(&args[2])?;
+                let (val, overflowed) = self.carrying_arith(
+                    if intrinsic_name == sym::add_with_carry { BinOp::Add } else { BinOp::Sub },
+                    &l,
+                    &r,
+                    &c,
+                )?;
+                self.write_scalar_pair(val, overflowed, dest)?;
+            }
+            sym::mul_double | sym::mul_double_add | sym::mul_double_add2 => {
+                let l = self.read_immediate(&args[0])?;
+                let r = self.read_immediate(&args[1])?;
+                let c1 = if intrinsic_name != sym::mul_double {
+                    Some(self.read_immediate(&args[2])?)
+                } else {
+                    None
+                };
+                let c2 = if intrinsic_name == sym::mul_double_add2 {
+                    Some(self.read_immediate(&args[3])?)
+                } else {
+                    None
+                };
+                let (lo, hi) = self.mul_double_add2(&l, &r, c1.as_ref(), c2.as_ref())?;
+                self.write_scalar_pair(lo, hi, dest)?;
+            }
             sym::discriminant_value => {
                 let place = self.deref_pointer(&args[0])?;
                 let variant = self.read_discriminant(&place)?;
@@ -573,6 +601,106 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         })
     }
 
+    pub fn carrying_arith(
+        &self,
+        mir_op: BinOp,
+        l: &ImmTy<'tcx, M::Provenance>,
+        r: &ImmTy<'tcx, M::Provenance>,
+        c: &ImmTy<'tcx, M::Provenance>,
+    ) -> InterpResult<'tcx, (Scalar<M::Provenance>, Scalar<M::Provenance>)> {
+        assert_eq!(l.layout.ty, r.layout.ty);
+        assert_matches!(l.layout.ty.kind(), ty::Int(..) | ty::Uint(..));
+        assert_matches!(c.layout.ty.kind(), ty::Bool);
+        assert_matches!(mir_op, BinOp::Add | BinOp::Sub);
+
+        let mir_op = mir_op.wrapping_to_overflowing().unwrap();
+
+        let (val, overflowed1) = self.binary_op(mir_op, l, r)?.to_scalar_pair();
+
+        let val = ImmTy::from_scalar(val, l.layout);
+        let c = ImmTy::from_scalar(c.to_scalar(), l.layout);
+
+        let (val, overflowed2) = self.binary_op(mir_op, &val, &c)?.to_scalar_pair();
+
+        let overflowed1 = overflowed1.to_bool()?;
+        let overflowed2 = overflowed2.to_bool()?;
+
+        let overflowed = Scalar::from_bool(if l.layout.abi.is_signed() {
+            overflowed1 != overflowed2
+        } else {
+            overflowed1 | overflowed2
+        });
+
+        interp_ok((val, overflowed))
+    }
+
+    pub fn mul_double_add2(
+        &self,
+        l: &ImmTy<'tcx, M::Provenance>,
+        r: &ImmTy<'tcx, M::Provenance>,
+        c1: Option<&ImmTy<'tcx, M::Provenance>>,
+        c2: Option<&ImmTy<'tcx, M::Provenance>>,
+    ) -> InterpResult<'tcx, (Scalar<M::Provenance>, Scalar<M::Provenance>)> {
+        assert_eq!(l.layout.ty, r.layout.ty);
+        assert_matches!(l.layout.ty.kind(), ty::Int(..) | ty::Uint(..));
+
+        let is_signed = l.layout.abi.is_signed();
+        let size = l.layout.size;
+        let bits = size.bits();
+        let l = l.to_scalar_int()?;
+        let r = r.to_scalar_int()?;
+
+        interp_ok(if is_signed {
+            let l = l.to_int(size);
+            let r = r.to_int(size);
+            let c1 = c1.map_or(interp_ok(0), |c1| interp_ok(c1.to_scalar_int()?.to_int(size)))?;
+            let c2 = c2.map_or(interp_ok(0), |c2| interp_ok(c2.to_scalar_int()?.to_int(size)))?;
+            if bits == 128 {
+                #[cfg(bootstrap)]
+                {
+                    let _ = (l, r, c1, c2);
+                    unimplemented!()
+                }
+                #[cfg(not(bootstrap))]
+                {
+                    let (lo, hi) = l.carrying2_mul(r, c1, c2);
+                    let lo = Scalar::from_uint(lo, size);
+                    let hi = Scalar::from_int(hi, size);
+                    (lo, hi)
+                }
+            } else {
+                let prod = l * r + c1 + c2;
+                let lo = Scalar::from_int(prod, size);
+                let hi = Scalar::from_int(prod >> size.bits(), size);
+                (lo, hi)
+            }
+        } else {
+            let l = l.to_uint(size);
+            let r = r.to_uint(size);
+            let c1 = c1.map_or(interp_ok(0), |c1| interp_ok(c1.to_scalar_int()?.to_uint(size)))?;
+            let c2 = c2.map_or(interp_ok(0), |c2| interp_ok(c2.to_scalar_int()?.to_uint(size)))?;
+            if bits == 128 {
+                #[cfg(bootstrap)]
+                {
+                    let _ = (l, r, c1, c2);
+                    unimplemented!()
+                }
+                #[cfg(not(bootstrap))]
+                {
+                    let (lo, hi) = l.carrying2_mul(r, c1, c2);
+                    let lo = Scalar::from_uint(lo, size);
+                    let hi = Scalar::from_uint(hi, size);
+                    (lo, hi)
+                }
+            } else {
+                let prod = l * r + c1 + c2;
+                let lo = Scalar::from_uint(prod, size);
+                let hi = Scalar::from_uint(prod >> size.bits(), size);
+                (lo, hi)
+            }
+        })
+    }
+
     /// Offsets a pointer by some multiple of its type, returning an error if the pointer leaves its
     /// allocation.
     pub fn ptr_offset_inbounds(

compiler/rustc_const_eval/src/interpret/place.rs

@@ -626,6 +626,17 @@ where
         self.write_immediate(Immediate::Scalar(val.into()), dest)
     }
 
+    /// Write a scalar pair to a place
+    #[inline(always)]
+    pub fn write_scalar_pair(
+        &mut self,
+        val1: impl Into<Scalar<M::Provenance>>,
+        val2: impl Into<Scalar<M::Provenance>>,
+        dest: &impl Writeable<'tcx, M::Provenance>,
+    ) -> InterpResult<'tcx> {
+        self.write_immediate(Immediate::ScalarPair(val1.into(), val2.into()), dest)
+    }
+
     /// Write a pointer to a place
     #[inline(always)]
     pub fn write_pointer(

compiler/rustc_const_eval/src/lib.rs

@@ -5,6 +5,7 @@
 #![cfg_attr(not(bootstrap), warn(unqualified_local_imports))]
 #![doc(rust_logo)]
 #![feature(assert_matches)]
+#![feature(bigint_helper_methods)]
 #![feature(box_patterns)]
 #![feature(decl_macro)]
 #![feature(if_let_guard)]

compiler/rustc_hir_analysis/src/check/intrinsic.rs

@@ -103,6 +103,11 @@ pub fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -
         | sym::add_with_overflow
         | sym::sub_with_overflow
         | sym::mul_with_overflow
+        | sym::add_with_carry
+        | sym::sub_with_carry
+        | sym::mul_double
+        | sym::mul_double_add
+        | sym::mul_double_add2
         | sym::wrapping_add
         | sym::wrapping_sub
         | sym::wrapping_mul
@@ -433,6 +438,28 @@ pub fn check_intrinsic_type(
                 (1, 0, vec![param(0), param(0)], Ty::new_tup(tcx, &[param(0), tcx.types.bool]))
             }
 
+            sym::add_with_carry | sym::sub_with_carry => (
+                1,
+                0,
+                vec![param(0), param(0), tcx.types.bool],
+                Ty::new_tup(tcx, &[param(0), tcx.types.bool]),
+            ),
+
+            sym::mul_double => {
+                (1, 0, vec![param(0), param(0)], Ty::new_tup(tcx, &[param(0), param(0)]))
+            }
+
+            sym::mul_double_add => {
+                (1, 0, vec![param(0), param(0), param(0)], Ty::new_tup(tcx, &[param(0), param(0)]))
+            }
+
+            sym::mul_double_add2 => (
+                1,
+                0,
+                vec![param(0), param(0), param(0), param(0)],
+                Ty::new_tup(tcx, &[param(0), param(0)]),
+            ),
+
             sym::ptr_guaranteed_cmp => (
                 1,
                 0,

compiler/rustc_span/src/symbol.rs

@@ -377,6 +377,7 @@ symbols! {
         abort,
         add,
         add_assign,
+        add_with_carry,
         add_with_overflow,
         address,
         adt_const_params,
@@ -1276,6 +1277,9 @@ symbols! {
         move_size_limit,
         mul,
         mul_assign,
+        mul_double,
+        mul_double_add,
+        mul_double_add2,
         mul_with_overflow,
         multiple_supertrait_upcastable,
         must_not_suspend,
@@ -1918,6 +1922,7 @@ symbols! {
         structural_peq,
         sub,
         sub_assign,
+        sub_with_carry,
         sub_with_overflow,
         suggestion,
         surface_async_drop_in_place,