Merge from 'master' to 'xmain-web' (KhronosGroup#9)

lib/SPIRV/OCL20ToSPIRV.cpp

@@ -953,17 +953,33 @@ void OCL20ToSPIRV::visitCallGroupBuiltin(CallInst *CI,
     return;
 
   if (DemangledName != kOCLBuiltinName::WaitGroupEvent) {
-    StringRef GroupOp = DemangledName;
-    GroupOp = GroupOp.drop_front(strlen(kSPIRVName::GroupPrefix));
+    StringRef FuncName = DemangledName;
+    FuncName = FuncName.drop_front(strlen(kSPIRVName::GroupPrefix));
     SPIRSPIRVGroupOperationMap::foreachConditional(
         [&](const std::string &S, SPIRVGroupOperationKind G) {
-          if (!GroupOp.startswith(S))
+          if (!FuncName.startswith(S))
             return true; // continue
           PreOps.push_back(G);
-          StringRef Op = GroupOp.drop_front(S.size() + 1);
-          assert(!Op.empty() && "Invalid OpenCL group builtin function");
+          StringRef Op = StringSwitch<StringRef>(FuncName)
+            .StartsWith("ballot", "group_ballot_bit_count_")
+            .StartsWith("non_uniform", kSPIRVName::GroupNonUniformPrefix)
+            .Default(kSPIRVName::GroupPrefix);
+          StringRef ClusteredOp =
+            FuncName.contains("clustered_") ?
+            "clustered_" : "";
+          StringRef LogicalOp =
+            FuncName.contains("logical_") ?
+            "logical_" : "";
+          StringRef GroupOp = StringSwitch<StringRef>(FuncName)
+            .Case("ballot_bit_count", "add")
+            .Case("ballot_inclusive_scan", "add")
+            .Case("ballot_exclusive_scan", "add")
+            .Default(FuncName.take_back(3));    // assumes op is three characters
+          if (GroupOp.startswith("_"))
+            GroupOp = GroupOp.take_back(2);     // when op is two characters
+          assert(!GroupOp.empty() && "Invalid OpenCL group builtin function");
           char OpTyC = 0;
-          auto NeedSign = Op == "max" || Op == "min";
+          auto NeedSign = GroupOp == "max" || GroupOp == "min";
           auto OpTy = F->getReturnType();
           if (OpTy->isFloatingPointTy())
             OpTyC = 'f';
@@ -979,23 +995,27 @@ void OCL20ToSPIRV::visitCallGroupBuiltin(CallInst *CI,
           } else
             llvm_unreachable("Invalid OpenCL group builtin argument type");
 
-          DemangledName =
-              std::string(kSPIRVName::GroupPrefix) + OpTyC + Op.str();
+          DemangledName = Op.str() + ClusteredOp.str() + LogicalOp.str() +
+            OpTyC + GroupOp.str();
           return false; // break out of loop
         });
   }
 
-  bool IsGroupAllAny = (DemangledName.find("_all") != std::string::npos ||
-                        DemangledName.find("_any") != std::string::npos);
-  bool IsGroupAllEqual = DemangledName.find("_all_equal") != std::string::npos;
+  const bool IsElect = DemangledName == "group_elect";
+  const bool IsAllOrAny = (DemangledName.find("_all") != std::string::npos ||
+                           DemangledName.find("_any") != std::string::npos);
+  const bool IsAllEqual = DemangledName.find("_all_equal") != std::string::npos;
+  const bool IsBallot = DemangledName == "group_ballot";
+  const bool IsInverseBallot = DemangledName == "group_inverse_ballot";
+  const bool IsLogical = DemangledName.find("_logical") != std::string::npos;
 
   auto Consts = getInt32(M, PreOps);
   OCLBuiltinTransInfo Info;
-  if (IsGroupAllAny || IsGroupAllEqual)
+  if (IsElect || IsAllOrAny || IsAllEqual || IsInverseBallot || IsLogical)
     Info.RetTy = Type::getInt1Ty(*Ctx);
   Info.UniqName = DemangledName;
   Info.PostProc = [=](std::vector<Value *> &Ops) {
-    if (IsGroupAllAny && !IsGroupAllEqual) {
+    if ((IsAllOrAny && !IsAllEqual) || IsBallot || IsLogical) {
       IRBuilder<> IRB(CI);
       Ops[0] =
           IRB.CreateICmpNE(Ops[0], ConstantInt::get(Type::getInt32Ty(*Ctx), 0));

lib/SPIRV/OCLUtil.h

@@ -639,6 +639,16 @@ template <> inline void SPIRVMap<std::string, SPIRVGroupOperationKind>::init() {
   add("reduce", GroupOperationReduce);
   add("scan_inclusive", GroupOperationInclusiveScan);
   add("scan_exclusive", GroupOperationExclusiveScan);
+  add("ballot_bit_count", GroupOperationReduce);
+  add("ballot_inclusive_scan", GroupOperationInclusiveScan);
+  add("ballot_exclusive_scan", GroupOperationExclusiveScan);
+  add("non_uniform_reduce", GroupOperationReduce);
+  add("non_uniform_scan_inclusive", GroupOperationInclusiveScan);
+  add("non_uniform_scan_exclusive", GroupOperationExclusiveScan);
+  add("non_uniform_reduce_logical", GroupOperationReduce);
+  add("non_uniform_scan_inclusive_logical", GroupOperationInclusiveScan);
+  add("non_uniform_scan_exclusive_logical", GroupOperationExclusiveScan);
+  add("clustered_reduce", GroupOperationClusteredReduce);
 }
 
 template <> inline void SPIRVMap<std::string, SPIRVFPRoundingModeKind>::init() {
@@ -723,12 +733,19 @@ inline void SPIRVMap<std::string, SPIRVBuiltinVariableKind>::init() {
   add("get_group_id", BuiltInWorkgroupId);
   add("get_global_linear_id", BuiltInGlobalLinearId);
   add("get_local_linear_id", BuiltInLocalInvocationIndex);
+  // cl_khr_subgroups
   add("get_sub_group_size", BuiltInSubgroupSize);
   add("get_max_sub_group_size", BuiltInSubgroupMaxSize);
   add("get_num_sub_groups", BuiltInNumSubgroups);
   add("get_enqueued_num_sub_groups", BuiltInNumEnqueuedSubgroups);
   add("get_sub_group_id", BuiltInSubgroupId);
   add("get_sub_group_local_id", BuiltInSubgroupLocalInvocationId);
+  // cl_khr_subgroup_ballot
+  add("get_sub_group_eq_mask", BuiltInSubgroupEqMask);
+  add("get_sub_group_ge_mask", BuiltInSubgroupGeMask);
+  add("get_sub_group_gt_mask", BuiltInSubgroupGtMask);
+  add("get_sub_group_le_mask", BuiltInSubgroupLeMask);
+  add("get_sub_group_lt_mask", BuiltInSubgroupLtMask);
 }
 
 // Maps uniqued OCL builtin function name to SPIR-V op code.
@@ -872,6 +889,61 @@ template <> inline void SPIRVMap<std::string, Op, SPIRVInstruction>::init() {
   _SPIRV_OP(intel_sub_group_media_block_read, SubgroupImageMediaBlockReadINTEL)
   _SPIRV_OP(intel_sub_group_media_block_write,
             SubgroupImageMediaBlockWriteINTEL)
+  // cl_khr_subgroup_non_uniform_vote
+  _SPIRV_OP(group_elect, GroupNonUniformElect)
+  _SPIRV_OP(group_non_uniform_all, GroupNonUniformAll)
+  _SPIRV_OP(group_non_uniform_any, GroupNonUniformAny)
+  _SPIRV_OP(group_non_uniform_all_equal, GroupNonUniformAllEqual)
+  // cl_khr_subgroup_ballot
+  _SPIRV_OP(group_non_uniform_broadcast, GroupNonUniformBroadcast)
+  _SPIRV_OP(group_broadcast_first, GroupNonUniformBroadcastFirst)
+  _SPIRV_OP(group_ballot, GroupNonUniformBallot)
+  _SPIRV_OP(group_inverse_ballot, GroupNonUniformInverseBallot)
+  _SPIRV_OP(group_ballot_bit_extract, GroupNonUniformBallotBitExtract)
+  _SPIRV_OP(group_ballot_bit_count_iadd, GroupNonUniformBallotBitCount)
+  _SPIRV_OP(group_ballot_find_lsb, GroupNonUniformBallotFindLSB)
+  _SPIRV_OP(group_ballot_find_msb, GroupNonUniformBallotFindMSB)
+  // cl_khr_subgroup_non_uniform_arithmetic
+  _SPIRV_OP(group_non_uniform_iadd, GroupNonUniformIAdd)
+  _SPIRV_OP(group_non_uniform_fadd, GroupNonUniformFAdd)
+  _SPIRV_OP(group_non_uniform_imul, GroupNonUniformIMul)
+  _SPIRV_OP(group_non_uniform_fmul, GroupNonUniformFMul)
+  _SPIRV_OP(group_non_uniform_smin, GroupNonUniformSMin)
+  _SPIRV_OP(group_non_uniform_umin, GroupNonUniformUMin)
+  _SPIRV_OP(group_non_uniform_fmin, GroupNonUniformFMin)
+  _SPIRV_OP(group_non_uniform_smax, GroupNonUniformSMax)
+  _SPIRV_OP(group_non_uniform_umax, GroupNonUniformUMax)
+  _SPIRV_OP(group_non_uniform_fmax, GroupNonUniformFMax)
+  _SPIRV_OP(group_non_uniform_iand, GroupNonUniformBitwiseAnd)
+  _SPIRV_OP(group_non_uniform_ior, GroupNonUniformBitwiseOr)
+  _SPIRV_OP(group_non_uniform_ixor, GroupNonUniformBitwiseXor)
+  _SPIRV_OP(group_non_uniform_logical_iand, GroupNonUniformLogicalAnd)
+  _SPIRV_OP(group_non_uniform_logical_ior, GroupNonUniformLogicalOr)
+  _SPIRV_OP(group_non_uniform_logical_ixor, GroupNonUniformLogicalXor)
+  // cl_khr_subgroup_shuffle
+  _SPIRV_OP(group_shuffle, GroupNonUniformShuffle)
+  _SPIRV_OP(group_shuffle_xor, GroupNonUniformShuffleXor)
+  // cl_khr_subgroup_shuffle_relative
+  _SPIRV_OP(group_shuffle_up, GroupNonUniformShuffleUp)
+  _SPIRV_OP(group_shuffle_down, GroupNonUniformShuffleDown)
+  // cl_khr_subgroup_clustered_reduce
+  _SPIRV_OP(group_clustered_iadd, GroupNonUniformIAdd)
+  _SPIRV_OP(group_clustered_iadd, GroupNonUniformIAdd)
+  _SPIRV_OP(group_clustered_fadd, GroupNonUniformFAdd)
+  _SPIRV_OP(group_clustered_imul, GroupNonUniformIMul)
+  _SPIRV_OP(group_clustered_fmul, GroupNonUniformFMul)
+  _SPIRV_OP(group_clustered_smin, GroupNonUniformSMin)
+  _SPIRV_OP(group_clustered_umin, GroupNonUniformUMin)
+  _SPIRV_OP(group_clustered_fmin, GroupNonUniformFMin)
+  _SPIRV_OP(group_clustered_smax, GroupNonUniformSMax)
+  _SPIRV_OP(group_clustered_umax, GroupNonUniformUMax)
+  _SPIRV_OP(group_clustered_fmax, GroupNonUniformFMax)
+  _SPIRV_OP(group_clustered_iand, GroupNonUniformBitwiseAnd)
+  _SPIRV_OP(group_clustered_ior, GroupNonUniformBitwiseOr)
+  _SPIRV_OP(group_clustered_ixor, GroupNonUniformBitwiseXor)
+  _SPIRV_OP(group_clustered_logical_iand, GroupNonUniformLogicalAnd)
+  _SPIRV_OP(group_clustered_logical_ior, GroupNonUniformLogicalOr)
+  _SPIRV_OP(group_clustered_logical_ixor, GroupNonUniformLogicalXor)
 #undef _SPIRV_OP
 }
 

lib/SPIRV/SPIRVInternal.h

@@ -331,6 +331,7 @@ const static char AtomicPrefixInternal[] = "atomic_";
 
 namespace kSPIRVName {
 const static char GroupPrefix[] = "group_";
+const static char GroupNonUniformPrefix[] = "group_non_uniform_";
 const static char Prefix[] = "__spirv_";
 const static char Postfix[] = "__";
 const static char ImageQuerySize[] = "ImageQuerySize";

lib/SPIRV/libSPIRV/SPIRVEnum.h

@@ -407,6 +407,11 @@ template <> inline void SPIRVMap<BuiltIn, SPIRVCapVec>::init() {
   ADD_VEC_INIT(BuiltInNumEnqueuedSubgroups, {CapabilityKernel});
   ADD_VEC_INIT(BuiltInSubgroupId, {CapabilityKernel});
   ADD_VEC_INIT(BuiltInSubgroupLocalInvocationId, {CapabilityKernel});
+  ADD_VEC_INIT(BuiltInSubgroupEqMask, {CapabilityGroupNonUniformBallot});
+  ADD_VEC_INIT(BuiltInSubgroupGeMask, {CapabilityGroupNonUniformBallot});
+  ADD_VEC_INIT(BuiltInSubgroupGtMask, {CapabilityGroupNonUniformBallot});
+  ADD_VEC_INIT(BuiltInSubgroupLeMask, {CapabilityGroupNonUniformBallot});
+  ADD_VEC_INIT(BuiltInSubgroupLtMask, {CapabilityGroupNonUniformBallot});
   ADD_VEC_INIT(BuiltInVertexIndex, {CapabilityShader});
   ADD_VEC_INIT(BuiltInInstanceIndex, {CapabilityShader});
 }

lib/SPIRV/libSPIRV/SPIRVInstruction.h

@@ -2525,29 +2525,29 @@ _SPIRV_OP(GroupNonUniformBroadcastFirst, true, 5)
 _SPIRV_OP(GroupNonUniformBallot, true, 5)
 _SPIRV_OP(GroupNonUniformInverseBallot, true, 5)
 _SPIRV_OP(GroupNonUniformBallotBitExtract, true, 6)
-_SPIRV_OP(GroupNonUniformBallotBitCount, true, 6)
+_SPIRV_OP(GroupNonUniformBallotBitCount, true, 6, false, 1)
 _SPIRV_OP(GroupNonUniformBallotFindLSB, true, 5)
 _SPIRV_OP(GroupNonUniformBallotFindMSB, true, 5)
 _SPIRV_OP(GroupNonUniformShuffle, true, 6)
 _SPIRV_OP(GroupNonUniformShuffleXor, true, 6)
 _SPIRV_OP(GroupNonUniformShuffleUp, true, 6)
 _SPIRV_OP(GroupNonUniformShuffleDown, true, 6)
-_SPIRV_OP(GroupNonUniformIAdd, true, 6, true)
-_SPIRV_OP(GroupNonUniformFAdd, true, 6, true)
-_SPIRV_OP(GroupNonUniformIMul, true, 6, true)
-_SPIRV_OP(GroupNonUniformFMul, true, 6, true)
-_SPIRV_OP(GroupNonUniformSMin, true, 6, true)
-_SPIRV_OP(GroupNonUniformUMin, true, 6, true)
-_SPIRV_OP(GroupNonUniformFMin, true, 6, true)
-_SPIRV_OP(GroupNonUniformSMax, true, 6, true)
-_SPIRV_OP(GroupNonUniformUMax, true, 6, true)
-_SPIRV_OP(GroupNonUniformFMax, true, 6, true)
-_SPIRV_OP(GroupNonUniformBitwiseAnd, true, 6, true)
-_SPIRV_OP(GroupNonUniformBitwiseOr, true, 6, true)
-_SPIRV_OP(GroupNonUniformBitwiseXor, true, 6, true)
-_SPIRV_OP(GroupNonUniformLogicalAnd, true, 6, true)
-_SPIRV_OP(GroupNonUniformLogicalOr, true, 6, true)
-_SPIRV_OP(GroupNonUniformLogicalXor, true, 6, true)
+_SPIRV_OP(GroupNonUniformIAdd, true, 6, true, 1)
+_SPIRV_OP(GroupNonUniformFAdd, true, 6, true, 1)
+_SPIRV_OP(GroupNonUniformIMul, true, 6, true, 1)
+_SPIRV_OP(GroupNonUniformFMul, true, 6, true, 1)
+_SPIRV_OP(GroupNonUniformSMin, true, 6, true, 1)
+_SPIRV_OP(GroupNonUniformUMin, true, 6, true, 1)
+_SPIRV_OP(GroupNonUniformFMin, true, 6, true, 1)
+_SPIRV_OP(GroupNonUniformSMax, true, 6, true, 1)
+_SPIRV_OP(GroupNonUniformUMax, true, 6, true, 1)
+_SPIRV_OP(GroupNonUniformFMax, true, 6, true, 1)
+_SPIRV_OP(GroupNonUniformBitwiseAnd, true, 6, true, 1)
+_SPIRV_OP(GroupNonUniformBitwiseOr, true, 6, true, 1)
+_SPIRV_OP(GroupNonUniformBitwiseXor, true, 6, true, 1)
+_SPIRV_OP(GroupNonUniformLogicalAnd, true, 6, true, 1)
+_SPIRV_OP(GroupNonUniformLogicalOr, true, 6, true, 1)
+_SPIRV_OP(GroupNonUniformLogicalXor, true, 6, true, 1)
 
 #undef _SPIRV_OP
 

lib/SPIRV/libSPIRV/SPIRVIsValidEnum.h

@@ -459,6 +459,11 @@ inline bool isValid(spv::BuiltIn V) {
   case BuiltInNumEnqueuedSubgroups:
   case BuiltInSubgroupId:
   case BuiltInSubgroupLocalInvocationId:
+  case BuiltInSubgroupEqMask:
+  case BuiltInSubgroupGeMask:
+  case BuiltInSubgroupGtMask:
+  case BuiltInSubgroupLeMask:
+  case BuiltInSubgroupLtMask:
   case BuiltInVertexIndex:
   case BuiltInInstanceIndex:
     return true;
@@ -563,6 +568,14 @@ inline bool isValid(spv::Capability V) {
   case CapabilitySubgroupDispatch:
   case CapabilityNamedBarrier:
   case CapabilityPipeStorage:
+  case CapabilityGroupNonUniform:
+  case CapabilityGroupNonUniformVote:
+  case CapabilityGroupNonUniformArithmetic:
+  case CapabilityGroupNonUniformBallot:
+  case CapabilityGroupNonUniformShuffle:
+  case CapabilityGroupNonUniformShuffleRelative:
+  case CapabilityGroupNonUniformClustered:
+  case CapabilityGroupNonUniformQuad:
   case CapabilityFPGAMemoryAttributesINTEL:
   case CapabilityArbitraryPrecisionIntegersINTEL:
   case CapabilityFPGALoopControlsINTEL:

lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h

@@ -394,6 +394,11 @@ template <> inline void SPIRVMap<BuiltIn, std::string>::init() {
   add(BuiltInNumEnqueuedSubgroups, "BuiltInNumEnqueuedSubgroups");
   add(BuiltInSubgroupId, "BuiltInSubgroupId");
   add(BuiltInSubgroupLocalInvocationId, "BuiltInSubgroupLocalInvocationId");
+  add(BuiltInSubgroupEqMask, "BuiltInSubgroupEqMask");
+  add(BuiltInSubgroupGeMask, "BuiltInSubgroupGeMask");
+  add(BuiltInSubgroupGtMask, "BuiltInSubgroupGtMask");
+  add(BuiltInSubgroupLeMask, "BuiltInSubgroupLeMask");
+  add(BuiltInSubgroupLtMask, "BuiltInSubgroupLtMask");
   add(BuiltInVertexIndex, "BuiltInVertexIndex");
   add(BuiltInInstanceIndex, "BuiltInInstanceIndex");
 }

lib/SPIRV/libSPIRV/SPIRVOpCode.h

@@ -131,7 +131,9 @@ inline bool hasExecScope(Op OpCode) {
 
 inline bool hasGroupOperation(Op OpCode) {
   unsigned OC = OpCode;
-  return OpGroupIAdd <= OC && OC <= OpGroupSMax;
+  return (OpGroupIAdd <= OC && OC <= OpGroupSMax) ||
+         (OpGroupNonUniformBallotBitCount == OC) ||
+         (OpGroupNonUniformIAdd <= OC && OC <= OpGroupNonUniformLogicalXor);
 }
 
 inline bool isGroupOpCode(Op OpCode) {