Special case depal lookups for Test Drive's strange usage.

This implements the hack I suggested in #13355, where instead of first
reducing the color to RGB565 as the real game does, we just take each
channel at full precision and do the lookup according to the mask,
linearly filtering the palette.

This makes the game look a lot nicer and is also a small optimization,
but the hack is very specific so kinda ugly in a way.

Core/Compatibility.cpp

@@ -96,6 +96,7 @@ void Compatibility::CheckSettings(IniFile &iniFile, const std::string &gameID) {
 	CheckSetting(iniFile, gameID, "MaliDepthStencilBugWorkaround", &flags_.MaliDepthStencilBugWorkaround);
 	CheckSetting(iniFile, gameID, "ZZT3SelectHack", &flags_.ZZT3SelectHack);
 	CheckSetting(iniFile, gameID, "AllowLargeFBTextureOffsets", &flags_.AllowLargeFBTextureOffsets);
+	CheckSetting(iniFile, gameID, "SmoothedRedDepal", &flags_.SmoothedRedDepal);
 }
 
 void Compatibility::CheckSetting(IniFile &iniFile, const std::string &gameID, const char *option, bool *flag) {

Core/Compatibility.h

@@ -86,6 +86,7 @@ struct CompatFlags {
 	bool MaliDepthStencilBugWorkaround;
 	bool ZZT3SelectHack;
 	bool AllowLargeFBTextureOffsets;
+	bool SmoothedRedDepal;
 };
 
 class IniFile;

GPU/Common/FragmentShaderGenerator.cpp

@@ -23,8 +23,10 @@
 #include "Common/GPU/OpenGL/GLFeatures.h"
 #include "Common/GPU/ShaderWriter.h"
 #include "Common/GPU/thin3d.h"
+#include "Core/Compatibility.h"
 #include "Core/Reporting.h"
 #include "Core/Config.h"
+#include "Core/System.h"
 #include "GPU/Common/GPUStateUtils.h"
 #include "GPU/Common/ShaderId.h"
 #include "GPU/Common/ShaderUniforms.h"
@@ -83,6 +85,7 @@ bool GenerateFragmentShader(const FShaderID &id, char *buffer, const ShaderLangu
 
 	bool doFlatShading = id.Bit(FS_BIT_FLATSHADE) && !flatBug;
 	bool shaderDepal = id.Bit(FS_BIT_SHADER_DEPAL);
+	bool smoothedRedDepal = PSP_CoreParameter().compat.flags().SmoothedRedDepal;
 	bool bgraTexture = id.Bit(FS_BIT_BGRA_TEXTURE);
 	bool colorWriteMask = id.Bit(FS_BIT_COLOR_WRITEMASK) && compat.bitwiseOps;
 
@@ -564,6 +567,28 @@ bool GenerateFragmentShader(const FShaderID &id, char *buffer, const ShaderLangu
 						WRITE(p, "  vec4 t = %s(tex, %s.xy);\n", compat.texture, texcoord);
 					}
 				} 
+			} else if (shaderDepal && smoothedRedDepal) {
+				// Specific mode for Test Drive. Fixes the banding.
+				if (doTextureProjection) {
+					// We don't use textureProj because we need better control and it's probably not much of a savings anyway.
+					// However it is good for precision on older hardware like PowerVR.
+					WRITE(p, "  vec2 uv = %s.xy/%s.z;\n  vec2 uv_round;\n", texcoord, texcoord);
+				} else {
+					WRITE(p, "  vec2 uv = %s.xy;\n  vec2 uv_round;\n", texcoord);
+				}
+				WRITE(p, "  vec4 t = %s(tex, %s.xy);\n", compat.texture, texcoord);
+				WRITE(p, "  uint depalShift = (u_depal_mask_shift_off_fmt >> 8) & 0xFFU;\n");
+				WRITE(p, "  uint depalFmt = (u_depal_mask_shift_off_fmt >> 24) & 0x3U;\n");
+				WRITE(p, "  float index0 = t.r;\n");
+				WRITE(p, "  float mul = 32.0 / 256.0;\n");
+				WRITE(p, "  if (depalFmt == 0) {\n");  // yes, different versions of Test Drive use different formats. Could do compile time by adding more compat flags but meh.
+				WRITE(p, "    if (depalShift == 5) { index0 = t.g; mul = 64.0 / 256.0; }\n");
+				WRITE(p, "    else if (depalShift == 11) { index0 = t.b; }\n");
+				WRITE(p, "  } else {\n");
+				WRITE(p, "    if (depalShift == 5) { index0 = t.g; }\n");
+				WRITE(p, "    else if (depalShift == 10) { index0 = t.b; }\n");
+				WRITE(p, "  }\n");
+				WRITE(p, "  t = %s(pal, vec2(index0 * mul, 0.0));\n", compat.texture);
 			} else {
 				if (doTextureProjection) {
 					// We don't use textureProj because we need better control and it's probably not much of a savings anyway.

GPU/Vulkan/DrawEngineVulkan.cpp

@@ -185,8 +185,8 @@ void DrawEngineVulkan::InitDeviceObjects() {
 	samp.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
 	samp.mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST;
 	samp.flags = 0;
-	samp.magFilter = VK_FILTER_NEAREST;
-	samp.minFilter = VK_FILTER_NEAREST;
+	samp.magFilter = VK_FILTER_LINEAR;
+	samp.minFilter = VK_FILTER_LINEAR;
 	res = vkCreateSampler(device, &samp, nullptr, &samplerSecondary_);
 	_dbg_assert_(VK_SUCCESS == res);
 	res = vkCreateSampler(device, &samp, nullptr, &nullSampler_);

GPU/Vulkan/DrawEngineVulkan.h

@@ -218,7 +218,7 @@ class DrawEngineVulkan : public DrawEngineCommon {
 	// Secondary texture for shader blending
 	VkImageView boundSecondary_ = VK_NULL_HANDLE;
 	VkImageView boundDepal_ = VK_NULL_HANDLE;
-	VkSampler samplerSecondary_ = VK_NULL_HANDLE;  // This one is actually never used since we use fetch.
+	VkSampler samplerSecondary_ = VK_NULL_HANDLE;  // This one is actually never used since we use fetch (except in SmoothedRedDepal mode for Test Drive).
 
 	PrehashMap<VertexArrayInfoVulkan *, nullptr> vai_;
 	VulkanPushBuffer *vertexCache_;

assets/compat.ini

@@ -1252,3 +1252,10 @@ ULKS46143 = true
 ULES00981 = true
 ULES00982 = true
 LBSW10345 = true  # Some modded version found in our report logs
+
+[SmoothedRedDepal]
+# Test Drive Unlimited smoothed CLUT lookups. See comments in #13355
+ULET00386 = true
+ULES00637 = true
+ULKS46126 = true
+ULUS10249 = true