Rollup of 9 pull requests

compiler/rustc_borrowck/src/invalidation.rs

@@ -289,6 +289,10 @@ impl<'cx, 'tcx> InvalidationGenerator<'cx, 'tcx> {
             | Rvalue::ShallowInitBox(ref operand, _ /*ty*/) => {
                 self.consume_operand(location, operand)
             }
+            Rvalue::VirtualRef(ref place) => {
+                let op = &Operand::Copy(*place);
+                self.consume_operand(location, op);
+            }
 
             Rvalue::Len(place) | Rvalue::Discriminant(place) => {
                 let af = match *rvalue {

compiler/rustc_borrowck/src/lib.rs

@@ -1224,6 +1224,23 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
             | Rvalue::ShallowInitBox(ref operand, _ /*ty*/) => {
                 self.consume_operand(location, (operand, span), flow_state)
             }
+            Rvalue::VirtualRef(place) => {
+                self.access_place(
+                    location,
+                    (place, span),
+                    (Deep, Read(ReadKind::Copy)),
+                    LocalMutationIsAllowed::No,
+                    flow_state,
+                );
+
+                // Finally, check if path was already moved.
+                self.check_if_path_or_subpath_is_moved(
+                    location,
+                    InitializationRequiringAction::Use,
+                    (place.as_ref(), span),
+                    flow_state,
+                );
+            }
 
             Rvalue::Len(place) | Rvalue::Discriminant(place) => {
                 let af = match *rvalue {

compiler/rustc_borrowck/src/type_check/mod.rs

@@ -2278,6 +2278,10 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             Rvalue::Use(operand) | Rvalue::UnaryOp(_, operand) => {
                 self.check_operand(operand, location);
             }
+            Rvalue::VirtualRef(place) => {
+                let op = &Operand::Copy(*place);
+                self.check_operand(op, location);
+            }
 
             Rvalue::BinaryOp(_, box (left, right))
             | Rvalue::CheckedBinaryOp(_, box (left, right)) => {
@@ -2308,6 +2312,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             | Rvalue::BinaryOp(..)
             | Rvalue::CheckedBinaryOp(..)
             | Rvalue::NullaryOp(..)
+            | Rvalue::VirtualRef(..)
             | Rvalue::UnaryOp(..)
             | Rvalue::Discriminant(..) => None,
 

compiler/rustc_builtin_macros/src/assert/context.rs

@@ -1,11 +1,10 @@
-use crate::assert::expr_if_not;
 use rustc_ast::{
     attr,
     ptr::P,
     token,
     tokenstream::{DelimSpan, TokenStream, TokenTree},
-    BorrowKind, Expr, ExprKind, ItemKind, MacArgs, MacCall, MacDelimiter, Mutability, Path,
-    PathSegment, Stmt, StructRest, UseTree, UseTreeKind, DUMMY_NODE_ID,
+    BinOpKind, BorrowKind, Expr, ExprKind, ItemKind, MacArgs, MacCall, MacDelimiter, Mutability,
+    Path, PathSegment, Stmt, StructRest, UnOp, UseTree, UseTreeKind, DUMMY_NODE_ID,
 };
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::fx::FxHashSet;
@@ -16,11 +15,19 @@ use rustc_span::{
 };
 
 pub(super) struct Context<'cx, 'a> {
+    // An optimization.
+    //
+    // Elements that aren't consumed (PartialEq, PartialOrd, ...) can be copied **after** the
+    // `assert!` expression fails rather than copied on-the-fly.
+    best_case_captures: Vec<Stmt>,
     // Top-level `let captureN = Capture::new()` statements
     capture_decls: Vec<Capture>,
     cx: &'cx ExtCtxt<'a>,
     // Formatting string used for debugging
     fmt_string: String,
+    // If the current expression being visited consumes itself. Used to construct
+    // `best_case_captures`.
+    is_consumed: bool,
     // Top-level `let __local_bindN = &expr` statements
     local_bind_decls: Vec<Stmt>,
     // Used to avoid capturing duplicated paths
@@ -36,9 +43,11 @@ pub(super) struct Context<'cx, 'a> {
 impl<'cx, 'a> Context<'cx, 'a> {
     pub(super) fn new(cx: &'cx ExtCtxt<'a>, span: Span) -> Self {
         Self {
+            best_case_captures: <_>::default(),
             capture_decls: <_>::default(),
             cx,
             fmt_string: <_>::default(),
+            is_consumed: true,
             local_bind_decls: <_>::default(),
             paths: <_>::default(),
             span,
@@ -69,14 +78,22 @@ impl<'cx, 'a> Context<'cx, 'a> {
         self.manage_cond_expr(&mut cond_expr);
         let initial_imports = self.build_initial_imports();
         let panic = self.build_panic(&expr_str, panic_path);
+        let cond_expr_with_unlikely = self.build_unlikely(cond_expr);
+
+        let Self { best_case_captures, capture_decls, cx, local_bind_decls, span, .. } = self;
 
-        let Self { capture_decls, cx, local_bind_decls, span, .. } = self;
+        let mut assert_then_stmts = Vec::with_capacity(2);
+        assert_then_stmts.extend(best_case_captures);
+        assert_then_stmts.push(self.cx.stmt_expr(panic));
+        let assert_then = self.cx.block(span, assert_then_stmts);
 
         let mut stmts = Vec::with_capacity(4);
         stmts.push(initial_imports);
         stmts.extend(capture_decls.into_iter().map(|c| c.decl));
         stmts.extend(local_bind_decls);
-        stmts.push(cx.stmt_expr(expr_if_not(cx, span, cond_expr, panic, None)));
+        stmts.push(
+            cx.stmt_expr(cx.expr(span, ExprKind::If(cond_expr_with_unlikely, assert_then, None))),
+        );
         cx.expr_block(cx.block(span, stmts))
     }
 
@@ -115,6 +132,16 @@ impl<'cx, 'a> Context<'cx, 'a> {
         )
     }
 
+    /// Takes the conditional expression of `assert!` and then wraps it inside `unlikely`
+    fn build_unlikely(&self, cond_expr: P<Expr>) -> P<Expr> {
+        let unlikely_path = self.cx.std_path(&[sym::intrinsics, sym::unlikely]);
+        self.cx.expr_call(
+            self.span,
+            self.cx.expr_path(self.cx.path(self.span, unlikely_path)),
+            vec![self.cx.expr(self.span, ExprKind::Unary(UnOp::Not, cond_expr))],
+        )
+    }
+
     /// The necessary custom `panic!(...)` expression.
     ///
     /// panic!(
@@ -167,17 +194,39 @@ impl<'cx, 'a> Context<'cx, 'a> {
     /// See [Self::manage_initial_capture] and [Self::manage_try_capture]
     fn manage_cond_expr(&mut self, expr: &mut P<Expr>) {
         match (*expr).kind {
-            ExprKind::AddrOf(_, _, ref mut local_expr) => {
-                self.manage_cond_expr(local_expr);
+            ExprKind::AddrOf(_, mutability, ref mut local_expr) => {
+                self.with_is_consumed_management(
+                    matches!(mutability, Mutability::Mut),
+                    |this| this.manage_cond_expr(local_expr)
+                );
             }
             ExprKind::Array(ref mut local_exprs) => {
                 for local_expr in local_exprs {
                     self.manage_cond_expr(local_expr);
                 }
             }
-            ExprKind::Binary(_, ref mut lhs, ref mut rhs) => {
-                self.manage_cond_expr(lhs);
-                self.manage_cond_expr(rhs);
+            ExprKind::Binary(ref op, ref mut lhs, ref mut rhs) => {
+                self.with_is_consumed_management(
+                    matches!(
+                        op.node,
+                        BinOpKind::Add
+                            | BinOpKind::And
+                            | BinOpKind::BitAnd
+                            | BinOpKind::BitOr
+                            | BinOpKind::BitXor
+                            | BinOpKind::Div
+                            | BinOpKind::Mul
+                            | BinOpKind::Or
+                            | BinOpKind::Rem
+                            | BinOpKind::Shl
+                            | BinOpKind::Shr
+                            | BinOpKind::Sub
+                    ),
+                    |this| {
+                        this.manage_cond_expr(lhs);
+                        this.manage_cond_expr(rhs);
+                    }
+                );
             }
             ExprKind::Call(_, ref mut local_exprs) => {
                 for local_expr in local_exprs {
@@ -228,8 +277,11 @@ impl<'cx, 'a> Context<'cx, 'a> {
                     self.manage_cond_expr(local_expr);
                 }
             }
-            ExprKind::Unary(_, ref mut local_expr) => {
-                self.manage_cond_expr(local_expr);
+            ExprKind::Unary(un_op, ref mut local_expr) => {
+                self.with_is_consumed_management(
+                    matches!(un_op, UnOp::Neg | UnOp::Not),
+                    |this| this.manage_cond_expr(local_expr)
+                );
             }
             // Expressions that are not worth or can not be captured.
             //
@@ -337,9 +389,23 @@ impl<'cx, 'a> Context<'cx, 'a> {
             ))
             .add_trailing_semicolon();
         let local_bind_path = self.cx.expr_path(Path::from_ident(local_bind));
-        let ret = self.cx.stmt_expr(local_bind_path);
-        let block = self.cx.expr_block(self.cx.block(self.span, vec![try_capture_call, ret]));
-        *expr = self.cx.expr_deref(self.span, block);
+        let rslt = if self.is_consumed {
+            let ret = self.cx.stmt_expr(local_bind_path);
+            self.cx.expr_block(self.cx.block(self.span, vec![try_capture_call, ret]))
+        } else {
+            self.best_case_captures.push(try_capture_call);
+            local_bind_path
+        };
+        *expr = self.cx.expr_deref(self.span, rslt);
+    }
+
+    // Calls `f` with the internal `is_consumed` set to `curr_is_consumed` and then
+    // sets the internal `is_consumed` back to its original value.
+    fn with_is_consumed_management(&mut self, curr_is_consumed: bool, f: impl FnOnce(&mut Self)) {
+        let prev_is_consumed = self.is_consumed;
+        self.is_consumed = curr_is_consumed;
+        f(self);
+        self.is_consumed = prev_is_consumed;
     }
 }
 

compiler/rustc_codegen_cranelift/src/base.rs

@@ -503,6 +503,11 @@ fn codegen_stmt<'tcx>(
                     let val = codegen_operand(fx, operand);
                     lval.write_cvalue(fx, val);
                 }
+                Rvalue::VirtualRef(place) => {
+                    let cplace = codegen_place(fx, place);
+                    let val = cplace.to_cvalue(fx);
+                    lval.write_cvalue(fx, val)
+                }
                 Rvalue::Ref(_, _, place) | Rvalue::AddressOf(_, place) => {
                     let place = codegen_place(fx, place);
                     let ref_ = place.place_ref(fx, lval.layout());

compiler/rustc_codegen_ssa/src/mir/rvalue.rs

@@ -8,6 +8,7 @@ use crate::traits::*;
 use crate::MemFlags;
 
 use rustc_middle::mir;
+use rustc_middle::mir::Operand;
 use rustc_middle::ty::cast::{CastTy, IntTy};
 use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
 use rustc_middle::ty::{self, adjustment::PointerCast, Instance, Ty, TyCtxt};
@@ -401,6 +402,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 self.codegen_place_to_pointer(bx, place, mk_ref)
             }
 
+            mir::Rvalue::VirtualRef(place) => {
+                let operand = self.codegen_operand(&mut bx, &Operand::Copy(place));
+                (bx, operand)
+            }
             mir::Rvalue::AddressOf(mutability, place) => {
                 let mk_ptr = move |tcx: TyCtxt<'tcx>, ty: Ty<'tcx>| {
                     tcx.mk_ptr(ty::TypeAndMut { ty, mutbl: mutability })
@@ -755,6 +760,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     pub fn rvalue_creates_operand(&self, rvalue: &mir::Rvalue<'tcx>, span: Span) -> bool {
         match *rvalue {
             mir::Rvalue::Ref(..) |
+            mir::Rvalue::VirtualRef(..) |
             mir::Rvalue::AddressOf(..) |
             mir::Rvalue::Len(..) |
             mir::Rvalue::Cast(..) | // (*)

compiler/rustc_const_eval/src/interpret/step.rs

@@ -172,6 +172,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 self.copy_op(&op, &dest)?;
             }
 
+            VirtualRef(ref place) => {
+                let op = self.eval_place_to_op(*place, Some(dest.layout))?;
+                self.copy_op(&op, &dest)?;
+            }
+
             BinaryOp(bin_op, box (ref left, ref right)) => {
                 let layout = binop_left_homogeneous(bin_op).then_some(dest.layout);
                 let left = self.read_immediate(&self.eval_operand(left, layout)?)?;

compiler/rustc_const_eval/src/transform/check_consts/check.rs

@@ -445,6 +445,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
             Rvalue::ThreadLocalRef(_) => self.check_op(ops::ThreadLocalAccess),
 
             Rvalue::Use(_)
+            | Rvalue::VirtualRef(..)
             | Rvalue::Repeat(..)
             | Rvalue::Discriminant(..)
             | Rvalue::Len(_)

compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs

@@ -260,6 +260,8 @@ where
             in_place::<Q, _>(cx, in_local, place.as_ref())
         }
 
+        Rvalue::VirtualRef(place) => in_place::<Q, _>(cx, in_local, place.as_ref()),
+
         Rvalue::Use(operand)
         | Rvalue::Repeat(operand, _)
         | Rvalue::UnaryOp(_, operand)

compiler/rustc_const_eval/src/transform/check_consts/resolver.rs

@@ -199,6 +199,7 @@ where
             mir::Rvalue::Cast(..)
             | mir::Rvalue::ShallowInitBox(..)
             | mir::Rvalue::Use(..)
+            | mir::Rvalue::VirtualRef(..)
             | mir::Rvalue::ThreadLocalRef(..)
             | mir::Rvalue::Repeat(..)
             | mir::Rvalue::Len(..)

compiler/rustc_const_eval/src/transform/promote_consts.rs

@@ -494,6 +494,10 @@ impl<'tcx> Validator<'_, 'tcx> {
             Rvalue::Use(operand) | Rvalue::Repeat(operand, _) => {
                 self.validate_operand(operand)?;
             }
+            Rvalue::VirtualRef(place) => {
+                let op = &Operand::Copy(*place);
+                self.validate_operand(op)?
+            }
 
             Rvalue::Discriminant(place) | Rvalue::Len(place) => {
                 self.validate_place(place.as_ref())?

compiler/rustc_infer/src/infer/resolve.rs

@@ -92,7 +92,7 @@ impl<'a, 'tcx> TypeFolder<'tcx> for OpportunisticRegionResolver<'a, 'tcx> {
                     .borrow_mut()
                     .unwrap_region_constraints()
                     .opportunistic_resolve_var(rid);
-                self.tcx().reuse_or_mk_region(r, ty::ReVar(resolved))
+                TypeFolder::tcx(self).reuse_or_mk_region(r, ty::ReVar(resolved))
             }
             _ => r,
         }
@@ -179,15 +179,13 @@ struct FullTypeResolver<'a, 'tcx> {
     infcx: &'a InferCtxt<'a, 'tcx>,
 }
 
-impl<'a, 'tcx> TypeFolder<'tcx> for FullTypeResolver<'a, 'tcx> {
+impl<'a, 'tcx> FallibleTypeFolder<'tcx> for FullTypeResolver<'a, 'tcx> {
     type Error = FixupError<'tcx>;
 
     fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
         self.infcx.tcx
     }
-}
 
-impl<'a, 'tcx> FallibleTypeFolder<'tcx> for FullTypeResolver<'a, 'tcx> {
     fn try_fold_ty(&mut self, t: Ty<'tcx>) -> Result<Ty<'tcx>, Self::Error> {
         if !t.needs_infer() {
             Ok(t) // micro-optimize -- if there is nothing in this type that this fold affects...