Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Backport to 14] fpbuiltin-max-error support #2421

Merged
merged 3 commits into from
Mar 21, 2024
Merged

[Backport to 14] fpbuiltin-max-error support #2421

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f8816dc
[Backport to 14] fpbuiltin-max-error support
MiloszSkobejko Mar 11, 2024
6cc84c9
fixed EOF and attributes
MiloszSkobejko Mar 20, 2024
5798044
Merge branch 'llvm_release_140' into fpbuiltin-max-error-backport-llv…
MiloszSkobejko Mar 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions include/LLVMSPIRVExtensions.inc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,4 +63,5 @@ EXT(SPV_INTEL_masked_gather_scatter)
EXT(SPV_INTEL_tensor_float32_conversion) // TODO: to remove old extension
EXT(SPV_INTEL_tensor_float32_rounding)
EXT(SPV_EXT_relaxed_printf_string_address_space)
EXT(SPV_INTEL_fp_max_error)
EXT(SPV_INTEL_cache_controls)
41 changes: 41 additions & 0 deletions lib/SPIRV/SPIRVReader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3819,7 +3819,48 @@ void SPIRVToLLVM::transDecorationsToMetadata(SPIRVValue *BV, Value *V) {
SetDecorationsMetadata(I);
}

namespace {

static float convertSPIRVWordToFloat(SPIRVWord Spir) {
union {
float F;
SPIRVWord Spir;
} FPMaxError;
FPMaxError.Spir = Spir;
return FPMaxError.F;
}

static bool transFPMaxErrorDecoration(SPIRVValue *BV, Value *V,
LLVMContext *Context) {
SPIRVWord ID;
if (Instruction *I = dyn_cast<Instruction>(V))
if (BV->hasDecorate(DecorationFPMaxErrorDecorationINTEL, 0, &ID)) {
auto Literals =
BV->getDecorationLiterals(DecorationFPMaxErrorDecorationINTEL);
assert(Literals.size() == 1 &&
"FP Max Error decoration shall have 1 operand");
auto F = convertSPIRVWordToFloat(Literals[0]);
if (CallInst *CI = dyn_cast<CallInst>(I)) {
// Add attribute
auto A = llvm::Attribute::get(*Context, "fpbuiltin-max-error",
std::to_string(F));
CI->addFnAttr(A);
} else {
// Add metadata
MDNode *N =
MDNode::get(*Context, MDString::get(*Context, std::to_string(F)));
I->setMetadata("fpbuiltin-max-error", N);
}
return true;
}
return false;
}
} // namespace

bool SPIRVToLLVM::transDecoration(SPIRVValue *BV, Value *V) {
if (transFPMaxErrorDecoration(BV, V, Context))
return true;

if (!transAlign(BV, V))
return false;

Expand Down
7 changes: 7 additions & 0 deletions lib/SPIRV/SPIRVUtil.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -766,6 +766,13 @@ CallInst *mutateCallInst(
auto NewCI = addCallInst(M, NewName, CI->getType(), Args, Attrs, CI, Mangle,
InstName, TakeFuncName);
NewCI->setDebugLoc(CI->getDebugLoc());
NewCI->copyMetadata(*CI);
NewCI->setAttributes(CI->getAttributes());
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it a cherry-pick or a stand-alone fix?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's a stand-alone fix

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also there were spaces at the end previously

NewCI->setTailCall(CI->isTailCall());
if (CI->hasFnAttr("fpbuiltin-max-error")) {
auto Attr = CI->getFnAttr("fpbuiltin-max-error");
NewCI->addFnAttr(Attr);
}
LLVM_DEBUG(dbgs() << " => " << *NewCI << '\n');
CI->replaceAllUsesWith(NewCI);
CI->eraseFromParent();
Expand Down
153 changes: 153 additions & 0 deletions lib/SPIRV/SPIRVWriter.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,19 @@ using namespace llvm;
using namespace SPIRV;
using namespace OCLUtil;

namespace {

static SPIRVWord convertFloatToSPIRVWord(float F) {
union {
float F;
SPIRVWord Spir;
} FPMaxError;
FPMaxError.F = F;
return FPMaxError.Spir;
}

} // namespace

namespace SPIRV {

static void foreachKernelArgMD(
Expand Down Expand Up @@ -3396,6 +3409,26 @@ bool LLVMToSPIRVBase::isKnownIntrinsic(Intrinsic::ID Id) {
}
}

// Add decoration if needed
SPIRVInstruction *addFPBuiltinDecoration(SPIRVModule *BM, IntrinsicInst *II,
SPIRVInstruction *I) {
const bool AllowFPMaxError =
BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_fp_max_error);
assert(II->getCalledFunction()->getName().startswith("llvm.fpbuiltin"));
// Add a new decoration for llvm.builtin intrinsics, if needed
if (AllowFPMaxError)
if (II->getAttributes().hasFnAttr("fpbuiltin-max-error")) {
double F = 0.0;
II->getAttributes()
.getFnAttr("fpbuiltin-max-error")
.getValueAsString()
.getAsDouble(F);
I->addDecorate(DecorationFPMaxErrorDecorationINTEL,
convertFloatToSPIRVWord(F));
}
return I;
}

// Performs mapping of LLVM IR rounding mode to SPIR-V rounding mode
// Value *V is metadata <rounding mode> argument of
// llvm.experimental.constrained.* intrinsics
Expand Down Expand Up @@ -4090,6 +4123,8 @@ SPIRVValue *LLVMToSPIRVBase::transIntrinsicInst(IntrinsicInst *II,
}

default:
if (auto *BVar = transFPBuiltinIntrinsicInst(II, BB))
return BVar;
if (BM->isUnknownIntrinsicAllowed(II))
return BM->addCallInst(
transFunctionDecl(II->getCalledFunction()),
Expand All @@ -4105,6 +4140,124 @@ SPIRVValue *LLVMToSPIRVBase::transIntrinsicInst(IntrinsicInst *II,
return nullptr;
}

LLVMToSPIRVBase::FPBuiltinType
LLVMToSPIRVBase::getFPBuiltinType(IntrinsicInst *II, StringRef &OpName) {
StringRef Name = II->getCalledFunction()->getName();
if (!Name.startswith("llvm.fpbuiltin"))
return FPBuiltinType::UNKNOWN;
Name.consume_front("llvm.fpbuiltin.");
OpName = Name.split('.').first;
FPBuiltinType Type =
StringSwitch<FPBuiltinType>(OpName)
.Cases("fadd", "fsub", "fmul", "fdiv", "frem",
FPBuiltinType::REGULAR_MATH)
.Cases("sin", "cos", "tan", FPBuiltinType::EXT_1OPS)
.Cases("sinh", "cosh", "tanh", FPBuiltinType::EXT_1OPS)
.Cases("asin", "acos", "atan", FPBuiltinType::EXT_1OPS)
.Cases("asinh", "acosh", "atanh", FPBuiltinType::EXT_1OPS)
.Cases("exp", "exp2", "exp10", "expm1", FPBuiltinType::EXT_1OPS)
.Cases("log", "log2", "log10", "log1p", FPBuiltinType::EXT_1OPS)
.Cases("sqrt", "rsqrt", "erf", "erfc", FPBuiltinType::EXT_1OPS)
.Cases("atan2", "pow", "hypot", "ldexp", FPBuiltinType::EXT_2OPS)
.Case("sincos", FPBuiltinType::EXT_3OPS)
.Default(FPBuiltinType::UNKNOWN);
return Type;
}

SPIRVValue *LLVMToSPIRVBase::transFPBuiltinIntrinsicInst(IntrinsicInst *II,
SPIRVBasicBlock *BB) {
StringRef OpName;
auto FPBuiltinTypeVal = getFPBuiltinType(II, OpName);
if (FPBuiltinTypeVal == FPBuiltinType::UNKNOWN)
return nullptr;
switch (FPBuiltinTypeVal) {
case FPBuiltinType::REGULAR_MATH: {
auto BinOp = StringSwitch<Op>(OpName)
.Case("fadd", OpFAdd)
.Case("fsub", OpFSub)
.Case("fmul", OpFMul)
.Case("fdiv", OpFDiv)
.Case("frem", OpFRem)
.Default(OpUndef);
auto *BI = BM->addBinaryInst(BinOp, transType(II->getType()),
transValue(II->getArgOperand(0), BB),
transValue(II->getArgOperand(1), BB), BB);
return addFPBuiltinDecoration(BM, II, BI);
}
case FPBuiltinType::EXT_1OPS: {
if (!checkTypeForSPIRVExtendedInstLowering(II, BM))
break;
SPIRVType *STy = transType(II->getType());
std::vector<SPIRVValue *> Ops(1, transValue(II->getArgOperand(0), BB));
auto ExtOp = StringSwitch<SPIRVWord>(OpName)
.Case("sin", OpenCLLIB::Sin)
.Case("cos", OpenCLLIB::Cos)
.Case("tan", OpenCLLIB::Tan)
.Case("sinh", OpenCLLIB::Sinh)
.Case("cosh", OpenCLLIB::Cosh)
.Case("tanh", OpenCLLIB::Tanh)
.Case("asin", OpenCLLIB::Asin)
.Case("acos", OpenCLLIB::Acos)
.Case("atan", OpenCLLIB::Atan)
.Case("asinh", OpenCLLIB::Asinh)
.Case("acosh", OpenCLLIB::Acosh)
.Case("atanh", OpenCLLIB::Atanh)
.Case("exp", OpenCLLIB::Exp)
.Case("exp2", OpenCLLIB::Exp2)
.Case("exp10", OpenCLLIB::Exp10)
.Case("expm1", OpenCLLIB::Expm1)
.Case("log", OpenCLLIB::Log)
.Case("log2", OpenCLLIB::Log2)
.Case("log10", OpenCLLIB::Log10)
.Case("log1p", OpenCLLIB::Log1p)
.Case("sqrt", OpenCLLIB::Sqrt)
.Case("rsqrt", OpenCLLIB::Rsqrt)
.Case("erf", OpenCLLIB::Erf)
.Case("erfc", OpenCLLIB::Erfc)
.Default(SPIRVWORD_MAX);
assert(ExtOp != SPIRVWORD_MAX);
auto *BI = BM->addExtInst(STy, BM->getExtInstSetId(SPIRVEIS_OpenCL), ExtOp,
Ops, BB);
return addFPBuiltinDecoration(BM, II, BI);
}
case FPBuiltinType::EXT_2OPS: {
if (!checkTypeForSPIRVExtendedInstLowering(II, BM))
break;
SPIRVType *STy = transType(II->getType());
std::vector<SPIRVValue *> Ops{transValue(II->getArgOperand(0), BB),
transValue(II->getArgOperand(1), BB)};
auto ExtOp = StringSwitch<SPIRVWord>(OpName)
.Case("atan2", OpenCLLIB::Atan2)
.Case("hypot", OpenCLLIB::Hypot)
.Case("pow", OpenCLLIB::Pow)
.Case("ldexp", OpenCLLIB::Ldexp)
.Default(SPIRVWORD_MAX);
assert(ExtOp != SPIRVWORD_MAX);
auto *BI = BM->addExtInst(STy, BM->getExtInstSetId(SPIRVEIS_OpenCL), ExtOp,
Ops, BB);
return addFPBuiltinDecoration(BM, II, BI);
}
case FPBuiltinType::EXT_3OPS: {
if (!checkTypeForSPIRVExtendedInstLowering(II, BM))
break;
SPIRVType *STy = transType(II->getType());
std::vector<SPIRVValue *> Ops{transValue(II->getArgOperand(0), BB),
transValue(II->getArgOperand(1), BB),
transValue(II->getArgOperand(2), BB)};
auto ExtOp = StringSwitch<SPIRVWord>(OpName)
.Case("sincos", OpenCLLIB::Sincos)
.Default(SPIRVWORD_MAX);
assert(ExtOp != SPIRVWORD_MAX);
auto *BI = BM->addExtInst(STy, BM->getExtInstSetId(SPIRVEIS_OpenCL), ExtOp,
Ops, BB);
return addFPBuiltinDecoration(BM, II, BI);
}
default:
return nullptr;
}
return nullptr;
}

SPIRVValue *LLVMToSPIRVBase::transFenceInst(FenceInst *FI,
SPIRVBasicBlock *BB) {
SPIRVWord MemorySemantics;
Expand Down
10 changes: 10 additions & 0 deletions lib/SPIRV/SPIRVWriter.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,16 @@ class LLVMToSPIRVBase {
bool transWorkItemBuiltinCallsToVariables();
bool isKnownIntrinsic(Intrinsic::ID Id);
SPIRVValue *transIntrinsicInst(IntrinsicInst *Intrinsic, SPIRVBasicBlock *BB);
enum class FPBuiltinType {
REGULAR_MATH,
EXT_1OPS,
EXT_2OPS,
EXT_3OPS,
UNKNOWN
};
FPBuiltinType getFPBuiltinType(IntrinsicInst *II, StringRef &);
SPIRVValue *transFPBuiltinIntrinsicInst(IntrinsicInst *II,
SPIRVBasicBlock *BB);
SPIRVValue *transFenceInst(FenceInst *FI, SPIRVBasicBlock *BB);
SPIRVValue *transCallInst(CallInst *Call, SPIRVBasicBlock *BB);
SPIRVValue *transDirectCallInst(CallInst *Call, SPIRVBasicBlock *BB);
Expand Down
2 changes: 2 additions & 0 deletions lib/SPIRV/libSPIRV/SPIRVDecorate.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,8 @@ class SPIRVDecorate : public SPIRVDecorateGeneric {
case internal::DecorationInitModeINTEL:
case internal::DecorationImplementInCSRINTEL:
return ExtensionID::SPV_INTEL_global_variable_decorations;
case DecorationFPMaxErrorDecorationINTEL:
return ExtensionID::SPV_INTEL_fp_max_error;
case internal::DecorationCacheControlLoadINTEL:
case internal::DecorationCacheControlStoreINTEL:
return ExtensionID::SPV_INTEL_cache_controls;
Expand Down
2 changes: 2 additions & 0 deletions lib/SPIRV/libSPIRV/SPIRVEnum.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -470,6 +470,8 @@ template <> inline void SPIRVMap<Decoration, SPIRVCapVec>::init() {
{internal::CapabilityCacheControlsINTEL});
ADD_VEC_INIT(internal::DecorationCacheControlStoreINTEL,
{internal::CapabilityCacheControlsINTEL});
ADD_VEC_INIT(DecorationFPMaxErrorDecorationINTEL,
{CapabilityFPMaxErrorINTEL});
}

template <> inline void SPIRVMap<BuiltIn, SPIRVCapVec>::init() {
Expand Down
2 changes: 2 additions & 0 deletions lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,7 @@ template <> inline void SPIRVMap<Decoration, std::string>::init() {
add(DecorationMediaBlockIOINTEL, "MediaBlockIOINTEL");
add(DecorationAliasScopeINTEL, "AliasScopeINTEL");
add(DecorationNoAliasINTEL, "NoAliasINTEL");
add(DecorationFPMaxErrorDecorationINTEL, "FPMaxErrorDecorationINTEL");

// From spirv_internal.hpp
add(internal::DecorationCallableFunctionINTEL, "CallableFunctionINTEL");
Expand Down Expand Up @@ -601,6 +602,7 @@ template <> inline void SPIRVMap<Capability, std::string>::init() {
add(CapabilityDebugInfoModuleINTEL, "DebugInfoModuleINTEL");
add(CapabilitySplitBarrierINTEL, "SplitBarrierINTEL");
add(CapabilityGroupUniformArithmeticKHR, "GroupUniformArithmeticKHR");
add(CapabilityFPMaxErrorINTEL, "FPMaxErrorINTEL");

// From spirv_internal.hpp
add(internal::CapabilityFPGADSPControlINTEL, "FPGADSPControlINTEL");
Expand Down
Loading