[SYCL][CUDA][libclc] Added atomics with scopes and memory orders (#4820)

Added libclc implementations for CUDA atomics, including for various scopes and memory orders. They are implemented using LLVM intrinsics and exposed as clang builtins, which are than used to implement functions in libclc.
intel · Nov 18, 2021 · 2ebde5f · 2ebde5f
1 parent 53c3268
commit 2ebde5f
Show file tree

Hide file tree

Showing 23 changed files with 9,278 additions and 334 deletions.
diff --git a/clang/include/clang/Basic/BuiltinsNVPTX.def b/clang/include/clang/Basic/BuiltinsNVPTX.def
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
diff --git a/clang/test/CodeGen/builtins-nvptx.c b/clang/test/CodeGen/builtins-nvptx.c
diff --git a/libclc/ptx-nvidiacl/libspirv/SOURCES b/libclc/ptx-nvidiacl/libspirv/SOURCES
@@ -85,5 +85,13 @@ images/image_helpers.ll
 images/image.cl
 group/collectives_helpers.ll
 group/collectives.cl
-SPV_EXT_shader_atomic_float_add/atomicfaddext.cl
-SPV_EXT_shader_atomic_float_add/faddext_helpers.ll
+atomic/atomic_add.cl
+atomic/atomic_and.cl
+atomic/atomic_cmpxchg.cl
+atomic/atomic_dec.cl
+atomic/atomic_inc.cl
+atomic/atomic_max.cl
+atomic/atomic_min.cl
+atomic/atomic_xchg.cl
+atomic/atomic_or.cl
+atomic/atomic_xor.cl
diff --git a/libclc/ptx-nvidiacl/libspirv/SPV_EXT_shader_atomic_float_add/atomicfaddext.cl b/libclc/ptx-nvidiacl/libspirv/SPV_EXT_shader_atomic_float_add/atomicfaddext.cl
diff --git a/libclc/ptx-nvidiacl/libspirv/SPV_EXT_shader_atomic_float_add/faddext_helpers.ll b/libclc/ptx-nvidiacl/libspirv/SPV_EXT_shader_atomic_float_add/faddext_helpers.ll
diff --git a/libclc/ptx-nvidiacl/libspirv/atomic/atomic_add.cl b/libclc/ptx-nvidiacl/libspirv/atomic/atomic_add.cl
@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <atomic_helpers.h>
+#include <spirv/spirv.h>
+#include <spirv/spirv_types.h>
+
+__CLC_NVVM_ATOMIC(int, i, int, i, add, _Z18__spirv_AtomicIAdd)
+__CLC_NVVM_ATOMIC(uint, j, int, i, add, _Z18__spirv_AtomicIAdd)
+__CLC_NVVM_ATOMIC(long, l, long, l, add, _Z18__spirv_AtomicIAdd)
+__CLC_NVVM_ATOMIC(ulong, m, long, l, add, _Z18__spirv_AtomicIAdd)
+
+__CLC_NVVM_ATOMIC(float, f, float, f, add, _Z21__spirv_AtomicFAddEXT)
+#ifdef cl_khr_int64_base_atomics
+__CLC_NVVM_ATOMIC(double, d, double, d, add, _Z21__spirv_AtomicFAddEXT)
+#endif
+
+#undef __CLC_NVVM_ATOMIC_TYPES
+#undef __CLC_NVVM_ATOMIC
+#undef __CLC_NVVM_ATOMIC_IMPL
diff --git a/libclc/ptx-nvidiacl/libspirv/atomic/atomic_and.cl b/libclc/ptx-nvidiacl/libspirv/atomic/atomic_and.cl
@@ -0,0 +1,20 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <atomic_helpers.h>
+#include <spirv/spirv.h>
+#include <spirv/spirv_types.h>
+
+__CLC_NVVM_ATOMIC(int, i, int, i, and, _Z17__spirv_AtomicAnd)
+__CLC_NVVM_ATOMIC(long, l, long, l, and, _Z17__spirv_AtomicAnd)
+__CLC_NVVM_ATOMIC(unsigned int, j, int, i, and, _Z17__spirv_AtomicAnd)
+__CLC_NVVM_ATOMIC(unsigned long, m, long, l, and, _Z17__spirv_AtomicAnd)
+
+#undef __CLC_NVVM_ATOMIC_TYPES
+#undef __CLC_NVVM_ATOMIC
+#undef __CLC_NVVM_ATOMIC_IMPL