Fix translation of device execution built-ins

* Change SPIR-V Writer/Reader to consume/produce LLVM IR with blocks and
device side enqueue built-ins as it is produced by clang
* Improve function pointers removal related to OpenCL 2.0 blocks
* Improve tests
* Fix translation of literal structs

include/LLVMSPIRVLib.h

@@ -56,6 +56,7 @@ void initializeOCLTypeToSPIRVPass(PassRegistry &);
 void initializeSPIRVLowerBoolPass(PassRegistry &);
 void initializeSPIRVLowerConstExprPass(PassRegistry &);
 void initializeSPIRVLowerSPIRBlocksPass(PassRegistry &);
+void initializeSPIRVLowerOCLBlocksPass(PassRegistry &);
 void initializeSPIRVLowerMemmovePass(PassRegistry &);
 void initializeSPIRVRegularizeLLVMPass(PassRegistry &);
 void initializeSPIRVToOCL20Pass(PassRegistry &);
@@ -136,6 +137,9 @@ ModulePass *createSPIRVLowerConstExpr();
 /// Create a pass for lowering SPIR 2.0 blocks to functions calls.
 ModulePass *createSPIRVLowerSPIRBlocks();
 
+/// Create a pass for removing function pointers related to OCL 2.0 blocks
+ModulePass *createSPIRVLowerOCLBlocks();
+
 /// Create a pass for lowering llvm.memmove to llvm.memcpys with a temporary
 /// variable.
 ModulePass *createSPIRVLowerMemmove();

lib/SPIRV/CMakeLists.txt

@@ -4,16 +4,6 @@ if(SPIRV_USE_LLVM_API)
 endif(SPIRV_USE_LLVM_API)
 
 add_llvm_library(LLVMSPIRVLib
-  libSPIRV/SPIRVBasicBlock.cpp
-  libSPIRV/SPIRVDebug.cpp
-  libSPIRV/SPIRVDecorate.cpp
-  libSPIRV/SPIRVEntry.cpp
-  libSPIRV/SPIRVFunction.cpp
-  libSPIRV/SPIRVInstruction.cpp
-  libSPIRV/SPIRVModule.cpp
-  libSPIRV/SPIRVStream.cpp
-  libSPIRV/SPIRVType.cpp
-  libSPIRV/SPIRVValue.cpp
   Mangler/FunctionDescriptor.cpp
   Mangler/Mangler.cpp
   Mangler/ManglingUtils.cpp
@@ -25,15 +15,26 @@ add_llvm_library(LLVMSPIRVLib
   OCLUtil.cpp
   SPIRVLowerBool.cpp
   SPIRVLowerConstExpr.cpp
-  SPIRVLowerSPIRBlocks.cpp
   SPIRVLowerMemmove.cpp
+  SPIRVLowerOCLBlocks.cpp
+  SPIRVLowerSPIRBlocks.cpp
   SPIRVReader.cpp
   SPIRVRegularizeLLVM.cpp
   SPIRVToOCL20.cpp
   SPIRVUtil.cpp
   SPIRVWriter.cpp
   SPIRVWriterPass.cpp
   TransOCLMD.cpp
+  libSPIRV/SPIRVBasicBlock.cpp
+  libSPIRV/SPIRVDebug.cpp
+  libSPIRV/SPIRVDecorate.cpp
+  libSPIRV/SPIRVEntry.cpp
+  libSPIRV/SPIRVFunction.cpp
+  libSPIRV/SPIRVInstruction.cpp
+  libSPIRV/SPIRVModule.cpp
+  libSPIRV/SPIRVStream.cpp
+  libSPIRV/SPIRVType.cpp
+  libSPIRV/SPIRVValue.cpp
   LINK_COMPONENTS
     Analysis
     BitWriter

lib/SPIRV/OCL20ToSPIRV.cpp

@@ -42,6 +42,7 @@
 #include "SPIRVInternal.h"
 
 #include "llvm/ADT/StringSwitch.h"
+#include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InstVisitor.h"
 #include "llvm/IR/Instruction.h"
@@ -259,6 +260,12 @@ class OCL20ToSPIRV : public ModulePass, public InstVisitor<OCL20ToSPIRV> {
                                 const std::string &DemangledName,
                                 unsigned int Offset);
 
+  /// Transform enqueue_kernel and kernel query built-in functions to
+  /// spirv-friendly format filling arguments, required for device-side enqueue
+  /// instructions, but missed in the original call
+  void visitCallEnqueueKernel(CallInst *CI, const std::string &DemangledName);
+  void visitCallKernelQuery(CallInst *CI, const std::string &DemangledName);
+
   /// For cl_intel_subgroups block read built-ins:
   void visitSubgroupBlockReadINTEL(CallInst *CI, StringRef MangledName,
                                    const std::string &DemangledName);
@@ -351,6 +358,7 @@ bool OCL20ToSPIRV::runOnModule(Module &Module) {
     if (auto GV = dyn_cast<GlobalValue>(I))
       GV->eraseFromParent();
 
+  eraseUselessFunctions(M); // remove unused functions declarations
   LLVM_DEBUG(dbgs() << "After OCL20ToSPIRV:\n" << *M);
 
   std::string Err;
@@ -512,6 +520,14 @@ void OCL20ToSPIRV::visitCallInst(CallInst &CI) {
                              OCLImageChannelOrderOffset);
     return;
   }
+  if (isEnqueueKernelBI(MangledName)) {
+    visitCallEnqueueKernel(&CI, DemangledName);
+    return;
+  }
+  if (isKernelQueryBI(MangledName)) {
+    visitCallKernelQuery(&CI, DemangledName);
+    return;
+  }
   if (DemangledName.find(kOCLBuiltinName::SubgroupBlockReadINTELPrefix) == 0) {
     visitSubgroupBlockReadINTEL(&CI, MangledName, DemangledName);
     return;
@@ -1449,6 +1465,110 @@ void OCL20ToSPIRV::visitCallGetImageChannel(CallInst *CI, StringRef MangledName,
                       },
                       &Attrs);
 }
+void OCL20ToSPIRV::visitCallEnqueueKernel(CallInst *CI,
+                                          const std::string &DemangledName) {
+  const DataLayout &DL = M->getDataLayout();
+  bool HasEvents = DemangledName.find("events") != std::string::npos;
+
+  // SPIRV OpEnqueueKernel instruction has 10+ arguments.
+  SmallVector<Value *, 16> Args;
+
+  // Copy all arguments before block invoke function pointer
+  // which match with what Clang 6.0 produced
+  const unsigned BlockFIdx = HasEvents ? 6 : 3;
+  Args.assign(CI->arg_begin(), CI->arg_begin() + BlockFIdx);
+
+  // If no event arguments in original call, add dummy ones
+  if (!HasEvents) {
+    Args.push_back(getInt32(M, 0));           // dummy num events
+    Args.push_back(getOCLNullClkEventPtr(M)); // dummy wait events
+    Args.push_back(getOCLNullClkEventPtr(M)); // dummy ret event
+  }
+
+  // Invoke: Pointer to invoke function
+  Value *BlockFunc = CI->getArgOperand(BlockFIdx);
+  Args.push_back(cast<Function>(GetUnderlyingObject(BlockFunc, DL)));
+
+  // Param: Pointer to block literal
+  Value *BlockLiteral = CI->getArgOperand(BlockFIdx + 1);
+  Args.push_back(BlockLiteral);
+
+  // Param Size: Size of block literal structure
+  // Param Aligment: Aligment of block literal structure
+  // TODO: these numbers should be obtained from block literal structure
+  Type *ParamType = GetUnderlyingObject(BlockLiteral, DL)->getType();
+  if (PointerType *PT = dyn_cast<PointerType>(ParamType))
+    ParamType = PT->getElementType();
+  Args.push_back(getInt32(M, DL.getTypeStoreSize(ParamType)));
+  Args.push_back(getInt32(M, DL.getPrefTypeAlignment(ParamType)));
+
+  // Local sizes arguments: Sizes of block invoke arguments
+  // Clang 6.0 and higher generates local size operands as an array,
+  // so we need to unpack them
+  if (DemangledName.find("_varargs") != std::string::npos) {
+    const unsigned LocalSizeArrayIdx = HasEvents ? 9 : 6;
+    auto *LocalSizeArray =
+        cast<GetElementPtrInst>(CI->getArgOperand(LocalSizeArrayIdx));
+    auto *LocalSizeArrayTy =
+        cast<ArrayType>(LocalSizeArray->getSourceElementType());
+    const uint64_t LocalSizeNum = LocalSizeArrayTy->getNumElements();
+    for (unsigned I = 0; I < LocalSizeNum; ++I)
+      Args.push_back(GetElementPtrInst::Create(
+          LocalSizeArray->getSourceElementType(), // Pointee type
+          LocalSizeArray->getPointerOperand(),    // Alloca
+          {getInt32(M, 0), getInt32(M, I)},       // Indices
+          "", CI));
+  }
+
+  StringRef NewName = "__spirv_EnqueueKernel__";
+  FunctionType *FT =
+      FunctionType::get(CI->getType(), getTypes(Args), false /*isVarArg*/);
+  Function *NewF =
+      Function::Create(FT, GlobalValue::ExternalLinkage, NewName, M);
+  NewF->setCallingConv(CallingConv::SPIR_FUNC);
+  CallInst *NewCall = CallInst::Create(NewF, Args, "", CI);
+  NewCall->setCallingConv(NewF->getCallingConv());
+  CI->replaceAllUsesWith(NewCall);
+  CI->eraseFromParent();
+}
+
+void OCL20ToSPIRV::visitCallKernelQuery(CallInst *CI,
+                                        const std::string &DemangledName) {
+  const DataLayout &DL = M->getDataLayout();
+  bool HasNDRange =
+      DemangledName.find("_for_ndrange_impl") != std::string::npos;
+  // BIs with "_for_ndrange_impl" suffix has NDRange argument first, and
+  // Invoke argument following. For other BIs Invoke function is the first arg
+  const unsigned BlockFIdx = HasNDRange ? 1 : 0;
+  Value *BlockFVal = CI->getArgOperand(BlockFIdx)->stripPointerCasts();
+
+  auto *BlockF = cast<Function>(GetUnderlyingObject(BlockFVal, DL));
+
+  AttributeList Attrs = CI->getCalledFunction()->getAttributes();
+  mutateCallInst(M, CI,
+                 [=](CallInst *CI, std::vector<Value *> &Args) {
+                   Value *Param = *Args.rbegin();
+                   Type *ParamType = GetUnderlyingObject(Param, DL)->getType();
+                   if (PointerType *PT = dyn_cast<PointerType>(ParamType)) {
+                     ParamType = PT->getElementType();
+                   }
+                   // Last arg corresponds to SPIRV Param operand.
+                   // Insert Invoke in front of Param.
+                   // Add Param Size and Param Align at the end.
+                   Args[BlockFIdx] = BlockF;
+                   Args.push_back(getInt32(M, DL.getTypeStoreSize(ParamType)));
+                   Args.push_back(
+                       getInt32(M, DL.getPrefTypeAlignment(ParamType)));
+
+                   Op Opcode = OCLSPIRVBuiltinMap::map(DemangledName);
+                   // Adding "__" postfix, so in case we have multiple such
+                   // functions and their names will have numerical postfix,
+                   // then the numerical postfix will be droped and we will get
+                   // correct function name.
+                   return getSPIRVFuncName(Opcode, kSPIRVName::Postfix);
+                 },
+                 /*BuiltinFuncMangleInfo*/ nullptr, &Attrs);
+}
 
 // The intel_sub_group_block_read built-ins are overloaded to support both
 // buffers and images, but need to be mapped to distinct SPIR-V instructions.

lib/SPIRV/OCLUtil.cpp

@@ -605,6 +605,19 @@ bool isSpecialTypeInitializer(Instruction *Inst) {
   return isSamplerInitializer(Inst) || isPipeStorageInitializer(Inst);
 }
 
+bool isEnqueueKernelBI(const StringRef MangledName) {
+  return MangledName == "__enqueue_kernel_basic" ||
+         MangledName == "__enqueue_kernel_basic_events" ||
+         MangledName == "__enqueue_kernel_varargs" ||
+         MangledName == "__enqueue_kernel_events_varargs";
+}
+
+bool isKernelQueryBI(const StringRef MangledName) {
+  return MangledName == "__get_kernel_work_group_size_impl" ||
+         MangledName == "__get_kernel_sub_group_count_for_ndrange_impl" ||
+         MangledName == "__get_kernel_max_sub_group_size_for_ndrange_impl" ||
+         MangledName == "__get_kernel_preferred_work_group_size_multiple_impl";
+}
 } // namespace OCLUtil
 
 void llvm::mangleOpenClBuiltin(const std::string &UniqName,

lib/SPIRV/OCLUtil.h

@@ -365,6 +365,9 @@ bool isPipeStorageInitializer(Instruction *Inst);
 /// Check (isSamplerInitializer || isPipeStorageInitializer)
 bool isSpecialTypeInitializer(Instruction *Inst);
 
+bool isEnqueueKernelBI(const StringRef MangledName);
+bool isKernelQueryBI(const StringRef MangledName);
+
 } // namespace OCLUtil
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -542,11 +545,12 @@ template <> inline void SPIRVMap<std::string, Op, SPIRVInstruction>::init() {
   // CL 2.0 kernel enqueue builtins
   _SPIRV_OP(enqueue_marker, EnqueueMarker)
   _SPIRV_OP(enqueue_kernel, EnqueueKernel)
-  _SPIRV_OP(get_kernel_ndrange_subgroup_count, GetKernelNDrangeSubGroupCount)
-  _SPIRV_OP(get_kernel_ndrange_max_subgroup_count,
+  _SPIRV_OP(get_kernel_sub_group_count_for_ndrange_impl,
+            GetKernelNDrangeSubGroupCount)
+  _SPIRV_OP(get_kernel_max_sub_group_size_for_ndrange_impl,
             GetKernelNDrangeMaxSubGroupSize)
-  _SPIRV_OP(get_kernel_work_group_size, GetKernelWorkGroupSize)
-  _SPIRV_OP(get_kernel_preferred_work_group_size_multiple,
+  _SPIRV_OP(get_kernel_work_group_size_impl, GetKernelWorkGroupSize)
+  _SPIRV_OP(get_kernel_preferred_work_group_size_multiple_impl,
             GetKernelPreferredWorkGroupSizeMultiple)
   _SPIRV_OP(retain_event, RetainEvent)
   _SPIRV_OP(release_event, ReleaseEvent)

lib/SPIRV/SPIRVInternal.h

@@ -533,6 +533,10 @@ Function *getOrCreateFunction(Module *M, Type *RetTy, ArrayRef<Type *> ArgTypes,
                               AttributeList *Attrs = nullptr,
                               bool TakeName = true);
 
+PointerType *getOCLClkEventType(Module *M);
+PointerType *getOCLClkEventPtrType(Module *M);
+Constant *getOCLNullClkEventPtr(Module *M);
+
 /// Get function call arguments.
 /// \param Start Starting index.
 /// \param End Ending index.