GPU: Correct depth clamp range in range cull

GPU/Common/GPUStateUtils.cpp

@@ -520,7 +520,7 @@ float DepthSliceFactor() {
 }
 
 // This is used for float values which might not be integers, but are in the integer scale of 65535.
-static float ToScaledDepthFromInteger(float z) {
+float ToScaledDepthFromIntegerScale(float z) {
 	if (!gstate_c.Supports(GPU_SUPPORTS_ACCURATE_DEPTH)) {
 		return z * (1.0f / 65535.0f);
 	}
@@ -536,10 +536,6 @@ static float ToScaledDepthFromInteger(float z) {
 	}
 }
 
-float ToScaledDepth(u16 z) {
-	return ToScaledDepthFromInteger((float)(int)z);
-}
-
 float FromScaledDepth(float z) {
 	if (!gstate_c.Supports(GPU_SUPPORTS_ACCURATE_DEPTH)) {
 		return z * 65535.0f;
@@ -605,8 +601,8 @@ void ConvertViewportAndScissor(bool useBufferedRendering, float renderWidth, flo
 		out.viewportY = renderY + displayOffsetY;
 		out.viewportW = curRTWidth * renderWidthFactor;
 		out.viewportH = curRTHeight * renderHeightFactor;
-		out.depthRangeMin = ToScaledDepthFromInteger(0);
-		out.depthRangeMax = ToScaledDepthFromInteger(65536);
+		out.depthRangeMin = ToScaledDepthFromIntegerScale(0);
+		out.depthRangeMax = ToScaledDepthFromIntegerScale(65536);
 	} else {
 		// These we can turn into a glViewport call, offset by offsetX and offsetY. Math after.
 		float vpXScale = gstate.getViewportXScale();
@@ -715,11 +711,11 @@ void ConvertViewportAndScissor(bool useBufferedRendering, float renderWidth, flo
 		if (!gstate_c.Supports(GPU_SUPPORTS_ACCURATE_DEPTH)) {
 			zScale = 1.0f;
 			zOffset = 0.0f;
-			out.depthRangeMin = ToScaledDepthFromInteger(vpZCenter - vpZScale);
-			out.depthRangeMax = ToScaledDepthFromInteger(vpZCenter + vpZScale);
+			out.depthRangeMin = ToScaledDepthFromIntegerScale(vpZCenter - vpZScale);
+			out.depthRangeMax = ToScaledDepthFromIntegerScale(vpZCenter + vpZScale);
 		} else {
-			out.depthRangeMin = ToScaledDepthFromInteger(minz);
-			out.depthRangeMax = ToScaledDepthFromInteger(maxz);
+			out.depthRangeMin = ToScaledDepthFromIntegerScale(minz);
+			out.depthRangeMax = ToScaledDepthFromIntegerScale(maxz);
 		}
 
 		// OpenGL will clamp these for us anyway, and Direct3D will error if not clamped.

GPU/Common/GPUStateUtils.h

@@ -72,7 +72,7 @@ struct ViewportAndScissor {
 	bool dirtyDepth;
 };
 void ConvertViewportAndScissor(bool useBufferedRendering, float renderWidth, float renderHeight, int bufferWidth, int bufferHeight, ViewportAndScissor &out);
-float ToScaledDepth(u16 z);
+float ToScaledDepthFromIntegerScale(float z);
 float FromScaledDepth(float z);
 float DepthSliceFactor();
 

GPU/Common/ShaderUniforms.cpp

@@ -42,9 +42,27 @@ void CalcCullRange(float minValues[4], float maxValues[4], bool flipViewport, bo
 		return (pspViewport - gstate_c.vpYOffset) * heightScale;
 	};
 	auto reverseViewportZ = [hasNegZ](float z) {
-		float pspViewport = (z - gstate.getViewportZCenter()) * (1.0f / gstate.getViewportZScale());
-		// Differs from GLES: depth is 0 to 1, not -1 to 1.
-		float realViewport = (pspViewport - gstate_c.vpZOffset) * gstate_c.vpDepthScale;
+		float vpZScale = gstate.getViewportZScale();
+		float vpZCenter = gstate.getViewportZCenter();
+
+		float scale, center;
+		if (gstate_c.Supports(GPU_SUPPORTS_ACCURATE_DEPTH)) {
+			// These are just the reverse of the formulas in GPUStateUtils.
+			float halfActualZRange = vpZScale * (1.0f / gstate_c.vpDepthScale);
+			float minz = -((gstate_c.vpZOffset * halfActualZRange) - vpZCenter) - halfActualZRange;
+
+			// In accurate depth mode, we're comparing against a value scaled to (minz, maxz).
+			// And minz might be very negative, (e.g. if we're clamping in that direction.)
+			scale = halfActualZRange;
+			center = minz + halfActualZRange;
+		} else {
+			// In old-style depth mode, we're comparing against a value scaled to viewport.
+			// (and possibly incorrectly clipped against it.)
+			scale = vpZScale;
+			center = vpZCenter;
+		}
+
+		float realViewport = (z - center) * (1.0f / scale);
 		return hasNegZ ? realViewport : (realViewport * 0.5f + 0.5f);
 	};
 	auto sortPair = [](float a, float b) {

GPU/Common/SoftwareTransformCommon.cpp

@@ -460,7 +460,7 @@ void SoftwareTransform(
 		if (params->allowSeparateAlphaClear || (alphaMatchesColor && depthMatchesStencil)) {
 			result->color = transformed[1].color0_32;
 			// Need to rescale from a [0, 1] float.  This is the final transformed value.
-			result->depth = ToScaledDepth((s16)(int)(transformed[1].z * 65535.0f));
+			result->depth = ToScaledDepthFromIntegerScale((int)(transformed[1].z * 65535.0f));
 			result->action = SW_CLEAR;
 			gpuStats.numClears++;
 			return;