From 5afc020fbdbf6e9f9139fbc4919430af3a652285 Mon Sep 17 00:00:00 2001
From: "Unknown W. Brackets" <checkins@unknownbrackets.org>
Date: Sun, 5 Apr 2020 16:35:13 -0700
Subject: [PATCH] x86jit: Force INF * 0 to +NAN.

See #12519 - this is needed for some graphics to render properly.  Seems
to already happen on ARM, so no change to armjit.
---
 Core/MIPS/IR/IRInterpreter.cpp |  6 +++++-
 Core/MIPS/MIPSInt.cpp          |  9 ++++++++-
 Core/MIPS/x86/CompFPU.cpp      | 25 ++++++++++++++++++++++++-
 3 files changed, 37 insertions(+), 3 deletions(-)

diff --git a/Core/MIPS/IR/IRInterpreter.cpp b/Core/MIPS/IR/IRInterpreter.cpp
index 13753c78ac00..9a4e49122825 100644
--- a/Core/MIPS/IR/IRInterpreter.cpp
+++ b/Core/MIPS/IR/IRInterpreter.cpp
@@ -708,7 +708,11 @@ u32 IRInterpret(MIPSState *mips, const IRInst *inst, int count) {
 			mips->f[inst->dest] = mips->f[inst->src1] - mips->f[inst->src2];
 			break;
 		case IROp::FMul:
-			mips->f[inst->dest] = mips->f[inst->src1] * mips->f[inst->src2];
+			if ((my_isinf(mips->f[inst->src1]) && mips->f[inst->src2] == 0.0f) || (my_isinf(mips->f[inst->src2]) && mips->f[inst->src1] == 0.0f)) {
+				mips->fi[inst->dest] = 0x7fc00000;
+			} else {
+				mips->f[inst->dest] = mips->f[inst->src1] * mips->f[inst->src2];
+			}
 			break;
 		case IROp::FDiv:
 			mips->f[inst->dest] = mips->f[inst->src1] / mips->f[inst->src2];
diff --git a/Core/MIPS/MIPSInt.cpp b/Core/MIPS/MIPSInt.cpp
index 903386a08e5a..93372391c898 100644
--- a/Core/MIPS/MIPSInt.cpp
+++ b/Core/MIPS/MIPSInt.cpp
@@ -974,7 +974,14 @@ namespace MIPSInt
 		{
 		case 0: F(fd) = F(fs) + F(ft); break; // add.s
 		case 1: F(fd) = F(fs) - F(ft); break; // sub.s
-		case 2: F(fd) = F(fs) * F(ft); break; // mul.s
+		case 2: // mul.s
+			if ((my_isinf(F(fs)) && F(ft) == 0.0f) || (my_isinf(F(ft)) && F(fs) == 0.0f)) {
+				// Must be positive NAN, see #12519.
+				FI(fd) = 0x7fc00000;
+			} else {
+				F(fd) = F(fs) * F(ft);
+			}
+			break;
 		case 3: F(fd) = F(fs) / F(ft); break; // div.s
 		default:
 			_dbg_assert_msg_(CPU,0,"Trying to interpret FPU3Op instruction that can't be interpreted");
diff --git a/Core/MIPS/x86/CompFPU.cpp b/Core/MIPS/x86/CompFPU.cpp
index 1220f2a38242..806d631047f8 100644
--- a/Core/MIPS/x86/CompFPU.cpp
+++ b/Core/MIPS/x86/CompFPU.cpp
@@ -50,6 +50,8 @@ namespace MIPSComp {
 using namespace Gen;
 using namespace X64JitConstants;
 
+alignas(16) const u32 reverseQNAN[4] = { 0x803FFFFF, 0x803FFFFF, 0x803FFFFF, 0x803FFFFF };
+
 void Jit::CopyFPReg(X64Reg dst, OpArg src) {
 	if (src.IsSimpleReg()) {
 		MOVAPS(dst, src);
@@ -90,7 +92,28 @@ void Jit::Comp_FPU3op(MIPSOpcode op) {
 	switch (op & 0x3f) {
 	case 0: CompFPTriArith(op, &XEmitter::ADDSS, false); break; //F(fd) = F(fs) + F(ft); //add
 	case 1: CompFPTriArith(op, &XEmitter::SUBSS, true); break;  //F(fd) = F(fs) - F(ft); //sub
-	case 2: CompFPTriArith(op, &XEmitter::MULSS, false); break; //F(fd) = F(fs) * F(ft); //mul
+	case 2: //F(fd) = F(fs) * F(ft); //mul
+		// XMM1 = !my_isnan(fs) && !my_isnan(ft)
+		MOVSS(XMM1, fpr.R(_FS));
+		CMPORDSS(XMM1, fpr.R(_FT));
+		CompFPTriArith(op, &XEmitter::MULSS, false);
+
+		// fd must still be in a reg, save it in XMM0 for now.
+		MOVAPS(XMM0, fpr.R(_FD));
+		// fd = my_isnan(fd) && !my_isnan(fs) && !my_isnan(ft)
+		CMPUNORDSS(fpr.RX(_FD), fpr.R(_FD));
+		ANDPS(fpr.RX(_FD), R(XMM1));
+		// At this point fd = FFFFFFFF if non-NAN inputs produced a NAN output.
+		// We'll AND it with the inverse QNAN bits to clear (00000000 means no change.)
+		if (RipAccessible(&reverseQNAN)) {
+			ANDPS(fpr.RX(_FD), M(&reverseQNAN));  // rip accessible
+		} else {
+			MOV(PTRBITS, R(TEMPREG), ImmPtr(&reverseQNAN));
+			ANDPS(fpr.RX(_FD), MatR(TEMPREG));
+		}
+		// ANDN is backwards, which is why we saved XMM0 to start.  Now put it back.
+		ANDNPS(fpr.RX(_FD), R(XMM0));
+		break;
 	case 3: CompFPTriArith(op, &XEmitter::DIVSS, true); break;  //F(fd) = F(fs) / F(ft); //div
 	default:
 		_dbg_assert_msg_(CPU,0,"Trying to compile FPU3Op instruction that can't be interpreted");