Add atomic load/store intrinsics to the compiler.

src/libcore/unstable/intrinsics.rs

@@ -20,6 +20,16 @@ pub extern "rust-intrinsic" {
     pub fn atomic_cxchg_acq(dst: &mut int, old: int, src: int) -> int;
     pub fn atomic_cxchg_rel(dst: &mut int, old: int, src: int) -> int;
 
+    #[cfg(not(stage0))]
+    pub fn atomic_load(src: &int) -> int;
+    #[cfg(not(stage0))]
+    pub fn atomic_load_acq(src: &int) -> int;
+
+    #[cfg(not(stage0))]
+    pub fn atomic_store(dst: &mut int, val: int);
+    #[cfg(not(stage0))]
+    pub fn atomic_store_rel(dst: &mut int, val: int);
+
     pub fn atomic_xchg(dst: &mut int, src: int) -> int;
     pub fn atomic_xchg_acq(dst: &mut int, src: int) -> int;
     pub fn atomic_xchg_rel(dst: &mut int, src: int) -> int;

src/librustc/lib/llvm.rs

@@ -1339,13 +1339,16 @@ pub mod llvm {
                                     PointerVal: ValueRef) -> ValueRef;
         #[fast_ffi]
         pub unsafe fn LLVMBuildLoad(B: BuilderRef,
-                                PointerVal: ValueRef,
-                                Name: *c_char)
-                             -> ValueRef;
+                                    PointerVal: ValueRef,
+                                    Name: *c_char)
+                                 -> ValueRef;
+
         #[fast_ffi]
         pub unsafe fn LLVMBuildStore(B: BuilderRef,
                                      Val: ValueRef,
-                                     Ptr: ValueRef) -> ValueRef;
+                                     Ptr: ValueRef)
+                                  -> ValueRef;
+
         #[fast_ffi]
         pub unsafe fn LLVMBuildGEP(B: BuilderRef,
                                Pointer: ValueRef,
@@ -1561,6 +1564,17 @@ pub mod llvm {
                                        Name: *c_char) -> ValueRef;
 
         /* Atomic Operations */
+        pub unsafe fn LLVMBuildAtomicLoad(B: BuilderRef,
+                                          PointerVal: ValueRef,
+                                          Order: AtomicOrdering)
+                                       -> ValueRef;
+
+        pub unsafe fn LLVMBuildAtomicStore(B: BuilderRef,
+                                           Val: ValueRef,
+                                           Ptr: ValueRef,
+                                           Order: AtomicOrdering)
+                                        -> ValueRef;
+
         pub unsafe fn LLVMBuildAtomicCmpXchg(B: BuilderRef,
                                              LHS: ValueRef,
                                              CMP: ValueRef,

src/librustc/middle/trans/build.rs

@@ -537,6 +537,18 @@ pub fn Load(cx: block, PointerVal: ValueRef) -> ValueRef {
     }
 }
 
+pub fn AtomicLoad(cx: block, PointerVal: ValueRef, order: AtomicOrdering) -> ValueRef {
+    unsafe {
+        let ccx = cx.fcx.ccx;
+        if cx.unreachable {
+            return llvm::LLVMGetUndef(ccx.int_type);
+        }
+        count_insn(cx, "load.atomic");
+        return llvm::LLVMBuildAtomicLoad(B(cx), PointerVal, order);
+    }
+}
+
+
 pub fn LoadRangeAssert(cx: block, PointerVal: ValueRef, lo: c_ulonglong,
                        hi: c_ulonglong, signed: lib::llvm::Bool) -> ValueRef {
     let value = Load(cx, PointerVal);
@@ -567,6 +579,17 @@ pub fn Store(cx: block, Val: ValueRef, Ptr: ValueRef) {
     }
 }
 
+pub fn AtomicStore(cx: block, Val: ValueRef, Ptr: ValueRef, order: AtomicOrdering) {
+    unsafe {
+        if cx.unreachable { return; }
+        debug!("Store %s -> %s",
+               val_str(cx.ccx().tn, Val),
+               val_str(cx.ccx().tn, Ptr));
+        count_insn(cx, "store.atomic");
+        llvm::LLVMBuildAtomicStore(B(cx), Val, Ptr, order);
+    }
+}
+
 pub fn GEP(cx: block, Pointer: ValueRef, Indices: &[ValueRef]) -> ValueRef {
     unsafe {
         if cx.unreachable { return llvm::LLVMGetUndef(T_ptr(T_nil())); }

src/librustc/middle/trans/foreign.rs

@@ -592,6 +592,30 @@ pub fn trans_intrinsic(ccx: @CrateContext,
                                     Release);
             Store(bcx, old, fcx.llretptr.get());
         }
+        ~"atomic_load" => {
+            let old = AtomicLoad(bcx,
+                                 get_param(decl, first_real_arg),
+                                 SequentiallyConsistent);
+            Store(bcx, old, fcx.llretptr.get());
+        }
+        ~"atomic_load_acq" => {
+            let old = AtomicLoad(bcx,
+                                 get_param(decl, first_real_arg),
+                                 Acquire);
+            Store(bcx, old, fcx.llretptr.get());
+        }
+        ~"atomic_store" => {
+            AtomicStore(bcx,
+                        get_param(decl, first_real_arg + 1u),
+                        get_param(decl, first_real_arg),
+                        SequentiallyConsistent);
+        }
+        ~"atomic_store_rel" => {
+            AtomicStore(bcx,
+                        get_param(decl, first_real_arg + 1u),
+                        get_param(decl, first_real_arg),
+                        Release);
+        }
         ~"atomic_xchg" => {
             let old = AtomicRMW(bcx, Xchg,
                                 get_param(decl, first_real_arg),

src/librustc/middle/trans/type_use.rs

@@ -124,7 +124,9 @@ pub fn type_uses_for(ccx: @CrateContext, fn_id: def_id, n_tps: uint)
                 ~"get_tydesc" | ~"needs_drop" => use_tydesc,
 
                 ~"atomic_cxchg"    | ~"atomic_cxchg_acq"|
-                ~"atomic_cxchg_rel"| ~"atomic_xchg"     |
+                ~"atomic_cxchg_rel"| ~"atomic_load"     |
+                ~"atomic_load_acq" | ~"atomic_store"    |
+                ~"atomic_store_rel"| ~"atomic_xchg"     |
                 ~"atomic_xadd"     | ~"atomic_xsub"     |
                 ~"atomic_xchg_acq" | ~"atomic_xadd_acq" |
                 ~"atomic_xsub_acq" | ~"atomic_xchg_rel" |

src/librustc/middle/typeck/check/mod.rs

@@ -3486,6 +3486,25 @@ pub fn check_intrinsic_type(ccx: @mut CrateCtxt, it: @ast::foreign_item) {
          ],
          ty::mk_int())
       }
+      ~"atomic_load"     | ~"atomic_load_acq" => {
+        (0,
+         ~[
+            arg(ty::mk_imm_rptr(tcx,
+                            ty::re_bound(ty::br_anon(0)),
+                            ty::mk_int()))
+         ],
+        ty::mk_int())
+      }
+      ~"atomic_store"    | ~"atomic_store_rel" => {
+        (0,
+         ~[
+            arg(ty::mk_mut_rptr(tcx,
+                                ty::re_bound(ty::br_anon(0)),
+                                ty::mk_int())),
+            arg(ty::mk_int())
+         ],
+         ty::mk_nil())
+      }
       ~"atomic_xchg"     | ~"atomic_xadd"     | ~"atomic_xsub"     |
       ~"atomic_xchg_acq" | ~"atomic_xadd_acq" | ~"atomic_xsub_acq" |
       ~"atomic_xchg_rel" | ~"atomic_xadd_rel" | ~"atomic_xsub_rel" => {

src/rustllvm/RustWrapper.cpp

@@ -545,6 +545,28 @@ extern "C" LLVMTypeRef LLVMMetadataType(void) {
   return LLVMMetadataTypeInContext(LLVMGetGlobalContext());
 }
 
+extern "C" LLVMValueRef LLVMBuildAtomicLoad(LLVMBuilderRef B,
+                                            LLVMValueRef source,
+                                            const char* Name,
+                                            AtomicOrdering order) {
+    LoadInst* li = new LoadInst(unwrap(source),0);
+    li->setVolatile(true);
+    li->setAtomic(order);
+    li->setAlignment(sizeof(intptr_t));
+    return wrap(unwrap(B)->Insert(li));
+}
+
+extern "C" LLVMValueRef LLVMBuildAtomicStore(LLVMBuilderRef B,
+                                            LLVMValueRef val,
+                                            LLVMValueRef target,
+                                            AtomicOrdering order) {
+    StoreInst* si = new StoreInst(unwrap(val),unwrap(target));
+    si->setVolatile(true);
+    si->setAtomic(order);
+    si->setAlignment(sizeof(intptr_t));
+    return wrap(unwrap(B)->Insert(si));
+}
+
 extern "C" LLVMValueRef LLVMBuildAtomicCmpXchg(LLVMBuilderRef B,
                                                LLVMValueRef target,
                                                LLVMValueRef old,

src/rustllvm/rustllvm.def.in

@@ -84,6 +84,8 @@ LLVMArrayType
 LLVMBasicBlockAsValue
 LLVMBlockAddress
 LLVMBuildAShr
+LLVMBuildAtomicLoad
+LLVMBuildAtomicStore
 LLVMBuildAtomicCmpXchg
 LLVMBuildAtomicRMW
 LLVMBuildAdd

src/test/run-pass/intrinsic-atomics.rs

@@ -15,6 +15,12 @@ mod rusti {
         pub fn atomic_cxchg_acq(dst: &mut int, old: int, src: int) -> int;
         pub fn atomic_cxchg_rel(dst: &mut int, old: int, src: int) -> int;
 
+        pub fn atomic_load(src: &int) -> int;
+        pub fn atomic_load_acq(src: &int) -> int;
+
+        pub fn atomic_store(dst: &mut int, val: int);
+        pub fn atomic_store_rel(dst: &mut int, val: int);
+
         pub fn atomic_xchg(dst: &mut int, src: int) -> int;
         pub fn atomic_xchg_acq(dst: &mut int, src: int) -> int;
         pub fn atomic_xchg_rel(dst: &mut int, src: int) -> int;
@@ -33,6 +39,15 @@ pub fn main() {
     unsafe {
         let mut x = ~1;
 
+        assert!(rusti::atomic_load(x) == 1);
+        *x = 5;
+        assert!(rusti::atomic_load_acq(x) == 5);
+
+        rusti::atomic_store(x,3);
+        assert!(*x == 3);
+        rusti::atomic_store_rel(x,1);
+        assert!(*x == 1);
+
         assert!(rusti::atomic_cxchg(x, 1, 2) == 1);
         assert!(*x == 2);