Make TLS accesses explicit in MIR

src/librustc_codegen_llvm/common.rs

@@ -259,6 +259,7 @@ impl ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
                     GlobalAlloc::Function(fn_instance) => self.get_fn_addr(fn_instance),
                     GlobalAlloc::Static(def_id) => {
                         assert!(self.tcx.is_static(def_id));
+                        assert!(!self.tcx.is_thread_local_static(def_id));
                         self.get_static(def_id)
                     }
                 };

src/librustc_codegen_ssa/mir/rvalue.rs

@@ -522,6 +522,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 let operand = OperandRef { val: OperandValue::Immediate(val), layout: box_layout };
                 (bx, operand)
             }
+            mir::Rvalue::ThreadLocalRef(def_id) => {
+                assert!(bx.cx().tcx().is_static(def_id));
+                let static_ = bx.get_static(def_id);
+                let layout = bx.layout_of(bx.cx().tcx().static_ptr_ty(def_id));
+                let operand = OperandRef::from_immediate_or_packed_pair(&mut bx, static_, layout);
+                (bx, operand)
+            }
             mir::Rvalue::Use(ref operand) => {
                 let operand = self.codegen_operand(&mut bx, operand);
                 (bx, operand)
@@ -745,6 +752,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             mir::Rvalue::UnaryOp(..) |
             mir::Rvalue::Discriminant(..) |
             mir::Rvalue::NullaryOp(..) |
+            mir::Rvalue::ThreadLocalRef(_) |
             mir::Rvalue::Use(..) => // (*)
                 true,
             mir::Rvalue::Repeat(..) |

src/librustc_middle/mir/interpret/error.rs

@@ -513,6 +513,8 @@ pub enum UnsupportedOpInfo {
     //
     /// Encountered raw bytes where we needed a pointer.
     ReadBytesAsPointer,
+    /// Accessing thread local statics
+    ThreadLocalStatic(DefId),
 }
 
 impl fmt::Display for UnsupportedOpInfo {
@@ -526,6 +528,7 @@ impl fmt::Display for UnsupportedOpInfo {
             NoMirFor(did) => write!(f, "no MIR body is available for {:?}", did),
             ReadPointerAsBytes => write!(f, "unable to turn pointer into raw bytes",),
             ReadBytesAsPointer => write!(f, "unable to turn bytes into a pointer"),
+            ThreadLocalStatic(did) => write!(f, "accessing thread local static {:?}", did),
         }
     }
 }

src/librustc_middle/mir/interpret/mod.rs

@@ -209,6 +209,7 @@ pub fn specialized_encode_alloc_id<'tcx, E: Encoder>(
             fn_instance.encode(encoder)?;
         }
         GlobalAlloc::Static(did) => {
+            assert!(!tcx.is_thread_local_static(did));
             // References to statics doesn't need to know about their allocations,
             // just about its `DefId`.
             AllocDiscriminant::Static.encode(encoder)?;

src/librustc_middle/mir/mod.rs

@@ -2209,6 +2209,11 @@ pub enum Rvalue<'tcx> {
     /// &x or &mut x
     Ref(Region<'tcx>, BorrowKind, Place<'tcx>),
 
+    /// Accessing a thread local static. This is inherently a runtime operation, even if llvm
+    /// treats it as an access to a static. This `Rvalue` yields a reference to the thread local
+    /// static.
+    ThreadLocalRef(DefId),
+
     /// Create a raw pointer to the given place
     /// Can be generated by raw address of expressions (`&raw const x`),
     /// or when casting a reference to a raw pointer.
@@ -2348,6 +2353,10 @@ impl<'tcx> Debug for Rvalue<'tcx> {
             UnaryOp(ref op, ref a) => write!(fmt, "{:?}({:?})", op, a),
             Discriminant(ref place) => write!(fmt, "discriminant({:?})", place),
             NullaryOp(ref op, ref t) => write!(fmt, "{:?}({:?})", op, t),
+            ThreadLocalRef(did) => ty::tls::with(|tcx| {
+                let muta = tcx.static_mutability(did).unwrap().prefix_str();
+                write!(fmt, "&/*tls*/ {}{}", muta, tcx.def_path_str(did))
+            }),
             Ref(region, borrow_kind, ref place) => {
                 let kind_str = match borrow_kind {
                     BorrowKind::Shared => "",
@@ -2501,7 +2510,10 @@ impl Constant<'tcx> {
     pub fn check_static_ptr(&self, tcx: TyCtxt<'_>) -> Option<DefId> {
         match self.literal.val.try_to_scalar() {
             Some(Scalar::Ptr(ptr)) => match tcx.global_alloc(ptr.alloc_id) {
-                GlobalAlloc::Static(def_id) => Some(def_id),
+                GlobalAlloc::Static(def_id) => {
+                    assert!(!tcx.is_thread_local_static(def_id));
+                    Some(def_id)
+                }
                 _ => None,
             },
             _ => None,

src/librustc_middle/mir/tcx.rs

@@ -152,6 +152,13 @@ impl<'tcx> Rvalue<'tcx> {
             Rvalue::Repeat(ref operand, count) => {
                 tcx.mk_ty(ty::Array(operand.ty(local_decls, tcx), count))
             }
+            Rvalue::ThreadLocalRef(did) => {
+                if tcx.is_mutable_static(did) {
+                    tcx.mk_mut_ptr(tcx.type_of(did))
+                } else {
+                    tcx.mk_imm_ref(tcx.lifetimes.re_static, tcx.type_of(did))
+                }
+            }
             Rvalue::Ref(reg, bk, ref place) => {
                 let place_ty = place.ty(local_decls, tcx).ty;
                 tcx.mk_ref(reg, ty::TypeAndMut { ty: place_ty, mutbl: bk.to_mutbl_lossy() })

src/librustc_middle/mir/type_foldable.rs

@@ -173,6 +173,7 @@ impl<'tcx> TypeFoldable<'tcx> for Rvalue<'tcx> {
         match *self {
             Use(ref op) => Use(op.fold_with(folder)),
             Repeat(ref op, len) => Repeat(op.fold_with(folder), len),
+            ThreadLocalRef(did) => ThreadLocalRef(did.fold_with(folder)),
             Ref(region, bk, ref place) => {
                 Ref(region.fold_with(folder), bk, place.fold_with(folder))
             }
@@ -216,6 +217,7 @@ impl<'tcx> TypeFoldable<'tcx> for Rvalue<'tcx> {
         match *self {
             Use(ref op) => op.visit_with(visitor),
             Repeat(ref op, _) => op.visit_with(visitor),
+            ThreadLocalRef(did) => did.visit_with(visitor),
             Ref(region, _, ref place) => region.visit_with(visitor) || place.visit_with(visitor),
             AddressOf(_, ref place) => place.visit_with(visitor),
             Len(ref place) => place.visit_with(visitor),

src/librustc_middle/mir/visit.rs

@@ -600,6 +600,8 @@ macro_rules! make_mir_visitor {
                         self.visit_operand(value, location);
                     }
 
+                    Rvalue::ThreadLocalRef(_) => {}
+
                     Rvalue::Ref(r, bk, path) => {
                         self.visit_region(r, location);
                         let ctx = match bk {

src/librustc_mir/borrow_check/invalidation.rs

@@ -295,6 +295,8 @@ impl<'cx, 'tcx> InvalidationGenerator<'cx, 'tcx> {
                 self.access_place(location, place, access_kind, LocalMutationIsAllowed::No);
             }
 
+            Rvalue::ThreadLocalRef(_) => {}
+
             Rvalue::Use(ref operand)
             | Rvalue::Repeat(ref operand, _)
             | Rvalue::UnaryOp(_ /*un_op*/, ref operand)

src/librustc_mir/borrow_check/mod.rs

@@ -1290,6 +1290,8 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                 );
             }
 
+            Rvalue::ThreadLocalRef(_) => {}
+
             Rvalue::Use(ref operand)
             | Rvalue::Repeat(ref operand, _)
             | Rvalue::UnaryOp(_ /*un_op*/, ref operand)

src/librustc_mir/borrow_check/type_check/mod.rs

@@ -2353,6 +2353,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             }
 
             Rvalue::AddressOf(..)
+            | Rvalue::ThreadLocalRef(..)
             | Rvalue::Use(..)
             | Rvalue::Len(..)
             | Rvalue::BinaryOp(..)
@@ -2368,6 +2369,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
     fn rvalue_user_ty(&self, rvalue: &Rvalue<'tcx>) -> Option<UserTypeAnnotationIndex> {
         match rvalue {
             Rvalue::Use(_)
+            | Rvalue::ThreadLocalRef(_)
             | Rvalue::Repeat(..)
             | Rvalue::Ref(..)
             | Rvalue::AddressOf(..)

src/librustc_mir/dataflow/impls/borrowed_locals.rs

@@ -176,6 +176,7 @@ where
 
             mir::Rvalue::Cast(..)
             | mir::Rvalue::Use(..)
+            | mir::Rvalue::ThreadLocalRef(..)
             | mir::Rvalue::Repeat(..)
             | mir::Rvalue::Len(..)
             | mir::Rvalue::BinaryOp(..)

src/librustc_mir/dataflow/move_paths/builder.rs

@@ -324,6 +324,7 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> {
 
     fn gather_rvalue(&mut self, rvalue: &Rvalue<'tcx>) {
         match *rvalue {
+            Rvalue::ThreadLocalRef(_) => {} // not-a-move
             Rvalue::Use(ref operand)
             | Rvalue::Repeat(ref operand, _)
             | Rvalue::Cast(_, ref operand, _)

src/librustc_mir/interpret/machine.rs

@@ -358,6 +358,13 @@ pub trait Machine<'mir, 'tcx>: Sized {
         _mem: &Memory<'mir, 'tcx, Self>,
         _ptr: Pointer<Self::PointerTag>,
     ) -> InterpResult<'tcx, u64>;
+
+    fn thread_local_alloc_id(
+        _ecx: &mut InterpCx<'mir, 'tcx, Self>,
+        did: DefId,
+    ) -> InterpResult<'tcx, AllocId> {
+        throw_unsup!(ThreadLocalStatic(did))
+    }
 }
 
 // A lot of the flexibility above is just needed for `Miri`, but all "compile-time" machines

src/librustc_mir/interpret/memory.rs

@@ -437,6 +437,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
             Some(GlobalAlloc::Function(..)) => throw_ub!(DerefFunctionPointer(id)),
             None => throw_ub!(PointerUseAfterFree(id)),
             Some(GlobalAlloc::Static(def_id)) => {
+                assert!(!tcx.is_thread_local_static(def_id));
                 // Notice that every static has two `AllocId` that will resolve to the same
                 // thing here: one maps to `GlobalAlloc::Static`, this is the "lazy" ID,
                 // and the other one is maps to `GlobalAlloc::Memory`, this is returned by
@@ -592,6 +593,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
         // be held throughout the match.
         match self.tcx.get_global_alloc(id) {
             Some(GlobalAlloc::Static(did)) => {
+                assert!(!self.tcx.is_thread_local_static(did));
                 // Use size and align of the type.
                 let ty = self.tcx.type_of(did);
                 let layout = self.tcx.layout_of(ParamEnv::empty().and(ty)).unwrap();

src/librustc_mir/interpret/step.rs

@@ -141,6 +141,12 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
 
         use rustc_middle::mir::Rvalue::*;
         match *rvalue {
+            ThreadLocalRef(did) => {
+                let id = M::thread_local_alloc_id(self, did)?;
+                let val = Scalar::Ptr(self.tag_global_base_pointer(id.into()));
+                self.write_scalar(val, dest)?;
+            }
+
             Use(ref operand) => {
                 // Avoid recomputing the layout
                 let op = self.eval_operand(operand, Some(dest.layout))?;

src/librustc_mir/interpret/validity.rs

@@ -418,6 +418,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
                 // Skip validation entirely for some external statics
                 let alloc_kind = self.ecx.tcx.get_global_alloc(ptr.alloc_id);
                 if let Some(GlobalAlloc::Static(did)) = alloc_kind {
+                    assert!(!self.ecx.tcx.is_thread_local_static(did));
                     // See const_eval::machine::MemoryExtra::can_access_statics for why
                     // this check is so important.
                     // This check is reachable when the const just referenced the static,

src/librustc_mir/monomorphize/collector.rs

@@ -1136,6 +1136,7 @@ fn create_mono_items_for_default_impls<'tcx>(
 fn collect_miri<'tcx>(tcx: TyCtxt<'tcx>, alloc_id: AllocId, output: &mut Vec<MonoItem<'tcx>>) {
     match tcx.global_alloc(alloc_id) {
         GlobalAlloc::Static(def_id) => {
+            assert!(!tcx.is_thread_local_static(def_id));
             let instance = Instance::mono(tcx, def_id);
             if should_monomorphize_locally(tcx, &instance) {
                 trace!("collecting static {:?}", def_id);

src/librustc_mir/transform/check_consts/qualifs.rs

@@ -153,7 +153,9 @@ where
     F: FnMut(Local) -> bool,
 {
     match rvalue {
-        Rvalue::NullaryOp(..) => Q::in_any_value_of_ty(cx, rvalue.ty(cx.body, cx.tcx)),
+        Rvalue::ThreadLocalRef(_) | Rvalue::NullaryOp(..) => {
+            Q::in_any_value_of_ty(cx, rvalue.ty(cx.body, cx.tcx))
+        }
 
         Rvalue::Discriminant(place) | Rvalue::Len(place) => {
             in_place::<Q, _>(cx, in_local, place.as_ref())

src/librustc_mir/transform/check_consts/validation.rs

@@ -263,11 +263,11 @@ impl Validator<'mir, 'tcx> {
     }
 
     fn check_static(&mut self, def_id: DefId, span: Span) {
-        if self.tcx.is_thread_local_static(def_id) {
-            self.check_op_spanned(ops::ThreadLocalAccess, span)
-        } else {
-            self.check_op_spanned(ops::StaticAccess, span)
-        }
+        assert!(
+            !self.tcx.is_thread_local_static(def_id),
+            "tls access is checked in `Rvalue::ThreadLocalRef"
+        );
+        self.check_op_spanned(ops::StaticAccess, span)
     }
 }
 
@@ -332,6 +332,8 @@ impl Visitor<'tcx> for Validator<'mir, 'tcx> {
         self.super_rvalue(rvalue, location);
 
         match *rvalue {
+            Rvalue::ThreadLocalRef(_) => self.check_op(ops::ThreadLocalAccess),
+
             Rvalue::Use(_)
             | Rvalue::Repeat(..)
             | Rvalue::UnaryOp(UnOp::Neg, _)

src/librustc_mir/transform/promote_consts.rs

@@ -603,6 +603,8 @@ impl<'tcx> Validator<'_, 'tcx> {
         }
 
         match rvalue {
+            Rvalue::ThreadLocalRef(_) => Err(Unpromotable),
+
             Rvalue::NullaryOp(..) => Ok(()),
 
             Rvalue::Discriminant(place) | Rvalue::Len(place) => self.validate_place(place.as_ref()),

src/librustc_mir/transform/qualify_min_const_fn.rs

@@ -156,6 +156,9 @@ fn check_rvalue(
     span: Span,
 ) -> McfResult {
     match rvalue {
+        Rvalue::ThreadLocalRef(_) => {
+            Err((span, "cannot access thread local storage in const fn".into()))
+        }
         Rvalue::Repeat(operand, _) | Rvalue::Use(operand) => {
             check_operand(tcx, operand, span, def_id, body)
         }

src/librustc_mir_build/build/expr/as_place.rs

@@ -258,6 +258,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             | ExprKind::InlineAsm { .. }
             | ExprKind::LlvmInlineAsm { .. }
             | ExprKind::Yield { .. }
+            | ExprKind::ThreadLocalRef(_)
             | ExprKind::Call { .. } => {
                 // these are not places, so we need to make a temporary.
                 debug_assert!(match Category::of(&expr.kind) {

src/librustc_mir_build/build/expr/as_rvalue.rs

@@ -53,6 +53,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         let source_info = this.source_info(expr_span);
 
         match expr.kind {
+            ExprKind::ThreadLocalRef(did) => block.and(Rvalue::ThreadLocalRef(did)),
             ExprKind::Scope { region_scope, lint_level, value } => {
                 let region_scope = (region_scope, source_info);
                 this.in_scope(region_scope, lint_level, |this| this.as_rvalue(block, scope, value))

src/librustc_mir_build/build/expr/as_temp.rs

@@ -63,10 +63,18 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             if let Some(tail_info) = this.block_context.currently_in_block_tail() {
                 local_decl = local_decl.block_tail(tail_info);
             }
-            if let ExprKind::StaticRef { def_id, .. } = expr.kind {
-                let is_thread_local = this.hir.tcx().is_thread_local_static(def_id);
-                local_decl.internal = true;
-                local_decl.local_info = Some(box LocalInfo::StaticRef { def_id, is_thread_local });
+            match expr.kind {
+                ExprKind::StaticRef { def_id, .. } => {
+                    assert!(!this.hir.tcx().is_thread_local_static(def_id));
+                    local_decl.internal = true;
+                    local_decl.local_info = Some(box LocalInfo::StaticRef { def_id, is_thread_local: false });
+                }
+                ExprKind::ThreadLocalRef(def_id) => {
+                    assert!(this.hir.tcx().is_thread_local_static(def_id));
+                    local_decl.internal = true;
+                    local_decl.local_info = Some(box LocalInfo::StaticRef { def_id, is_thread_local: true });
+                }
+                _ => {}
             }
             this.local_decls.push(local_decl)
         };

src/librustc_mir_build/build/expr/category.rs

@@ -65,6 +65,7 @@ impl Category {
             | ExprKind::Repeat { .. }
             | ExprKind::Assign { .. }
             | ExprKind::AssignOp { .. }
+            | ExprKind::ThreadLocalRef(_)
             | ExprKind::LlvmInlineAsm { .. } => Some(Category::Rvalue(RvalueFunc::AsRvalue)),
 
             ExprKind::Literal { .. } | ExprKind::StaticRef { .. } => Some(Category::Constant),

src/librustc_mir_build/build/expr/into.rs

@@ -444,6 +444,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             | ExprKind::Tuple { .. }
             | ExprKind::Closure { .. }
             | ExprKind::Literal { .. }
+            | ExprKind::ThreadLocalRef(_)
             | ExprKind::StaticRef { .. } => {
                 debug_assert!(match Category::of(&expr.kind).unwrap() {
                     // should be handled above

src/librustc_mir_build/hair/cx/expr.rs

@@ -855,20 +855,17 @@ fn convert_path_expr<'a, 'tcx>(
         // a constant reference (or constant raw pointer for `static mut`) in MIR
         Res::Def(DefKind::Static, id) => {
             let ty = cx.tcx.static_ptr_ty(id);
-            let ptr = cx.tcx.create_static_alloc(id);
             let temp_lifetime = cx.region_scope_tree.temporary_scope(expr.hir_id.local_id);
-            ExprKind::Deref {
-                arg: Expr {
-                    ty,
-                    temp_lifetime,
-                    span: expr.span,
-                    kind: ExprKind::StaticRef {
-                        literal: ty::Const::from_scalar(cx.tcx, Scalar::Ptr(ptr.into()), ty),
-                        def_id: id,
-                    },
+            let kind = if cx.tcx.is_thread_local_static(id) {
+                ExprKind::ThreadLocalRef(id)
+            } else {
+                let ptr = cx.tcx.create_static_alloc(id);
+                ExprKind::StaticRef {
+                    literal: ty::Const::from_scalar(cx.tcx, Scalar::Ptr(ptr.into()), ty),
+                    def_id: id,
                 }
-                .to_ref(),
-            }
+            };
+            ExprKind::Deref { arg: Expr { ty, temp_lifetime, span: expr.span, kind }.to_ref() }
         }
 
         Res::Local(var_hir_id) => convert_var(cx, expr, var_hir_id),

src/librustc_mir_build/hair/mod.rs

@@ -284,6 +284,8 @@ crate enum ExprKind<'tcx> {
         operands: Vec<InlineAsmOperand<'tcx>>,
         options: InlineAsmOptions,
     },
+    /// An expression taking a reference to a thread local.
+    ThreadLocalRef(DefId),
     LlvmInlineAsm {
         asm: &'tcx hir::LlvmInlineAsmInner,
         outputs: Vec<ExprRef<'tcx>>,