WIP vulkan: add MSAA support for visibility pass

Vis will render at half the res but 4x MSAA which should essentially
result in a no-op.
Also add legacy path for devices with no depth write in compute.

src/renderer/CMakeLists.txt

@@ -186,9 +186,12 @@ set(REAPER_SHADER_SRCS
     ${REAPER_SHADER_DIR}/tiled_lighting/tile_depth_downsample.comp.hlsl
     ${REAPER_SHADER_DIR}/tiled_lighting/tiled_lighting.comp.hlsl
     ${REAPER_SHADER_DIR}/tiled_lighting/tiled_lighting_debug.comp.hlsl
+    ${REAPER_SHADER_DIR}/vis_buffer/resolve_depth_legacy.frag.hlsl
     ${REAPER_SHADER_DIR}/vis_buffer/vis_buffer_raster.frag.hlsl
     ${REAPER_SHADER_DIR}/vis_buffer/vis_buffer_raster.vert.hlsl
     ${REAPER_SHADER_DIR}/vis_buffer/fill_gbuffer.comp.hlsl
+    ${REAPER_SHADER_DIR}/vis_buffer/fill_gbuffer_msaa.comp.hlsl
+    ${REAPER_SHADER_DIR}/vis_buffer/fill_gbuffer_msaa_with_depth_resolve.comp.hlsl
 )
 
 # Shader compilation

src/renderer/shader/vis_buffer/fill_gbuffer.comp.hlsl

@@ -13,9 +13,23 @@
 #include "vis_buffer.hlsl"
 #include "fill_gbuffer.share.hlsl"
 
+#if defined(ENABLE_MSAA_DEPTH_RESOLVE) && !defined(ENABLE_MSAA_VIS_BUFFER)
+    #error
+#endif
+
 VK_PUSH_CONSTANT_HELPER(FillGBufferPushConstants) push;
 
+#if defined(ENABLE_MSAA_VIS_BUFFER)
+VK_BINDING(0, Slot_VisBuffer) Texture2DMS<VisBufferRawType> VisBufferMS;
+#else
 VK_BINDING(0, Slot_VisBuffer) Texture2D<VisBufferRawType> VisBuffer;
+#endif
+
+#if defined(ENABLE_MSAA_DEPTH_RESOLVE)
+VK_BINDING(0, Slot_VisBufferDepthMS) Texture2DMS<float> VisBufferDepthMS;
+VK_BINDING(0, Slot_ResolvedDepth) RWTexture2D<float> ResolvedDepth;
+#endif
+
 VK_BINDING(0, Slot_GBuffer0) RWTexture2D<GBuffer0Type> GBuffer0;
 VK_BINDING(0, Slot_GBuffer1) RWTexture2D<GBuffer1Type> GBuffer1;
 VK_BINDING(0, Slot_instance_params) StructuredBuffer<MeshInstance> instance_params;
@@ -49,7 +63,18 @@ void main(uint3 gtid : SV_GroupThreadID,
         return;
     }
 
+#if defined(ENABLE_MSAA_VIS_BUFFER)
+    uint sample_location = (position_ts.x & 1) + (position_ts.y & 1) * 2;
+    VisBufferRawType vis_buffer_raw = VisBufferMS.Load(uint3(position_ts / 2, 0), sample_location);
+
+#if defined(ENABLE_MSAA_DEPTH_RESOLVE)
+    // Resolve depth in this pass as well
+    const float depth = VisBufferDepthMS.Load(uint3(position_ts / 2, 0), sample_location);
+    ResolvedDepth[position_ts] = depth;
+#endif
+#else
     VisBufferRawType vis_buffer_raw = VisBuffer.Load(uint3(position_ts, 0));
+#endif
 
     if (!is_vis_buffer_valid(vis_buffer_raw))
     {

src/renderer/shader/vis_buffer/fill_gbuffer.share.hlsl

@@ -11,17 +11,19 @@
 #include "shared_types.hlsl"
 
 #define Slot_VisBuffer              0
-#define Slot_GBuffer0               1
-#define Slot_GBuffer1               2
-#define Slot_instance_params        3
-#define Slot_visible_index_buffer   4
-#define Slot_buffer_position_ms     5
-#define Slot_buffer_normal_ms       6
-#define Slot_buffer_tangent_ms      7
-#define Slot_buffer_uv              8
-#define Slot_visible_meshlets       9
-#define Slot_diffuse_map_sampler    10
-#define Slot_material_maps          11
+#define Slot_VisBufferDepthMS       1
+#define Slot_ResolvedDepth          2
+#define Slot_GBuffer0               3
+#define Slot_GBuffer1               4
+#define Slot_instance_params        5
+#define Slot_visible_index_buffer   6
+#define Slot_buffer_position_ms     7
+#define Slot_buffer_normal_ms       8
+#define Slot_buffer_tangent_ms      9
+#define Slot_buffer_uv              10
+#define Slot_visible_meshlets       11
+#define Slot_diffuse_map_sampler    12
+#define Slot_material_maps          13
 
 static const hlsl_uint GBufferFillThreadCountX = 16;
 static const hlsl_uint GBufferFillThreadCountY = 16;

src/renderer/shader/vis_buffer/fill_gbuffer_msaa.comp.hlsl

@@ -0,0 +1,2 @@
+#define ENABLE_MSAA_VIS_BUFFER
+#include "fill_gbuffer.comp.hlsl"

src/renderer/shader/vis_buffer/fill_gbuffer_msaa_with_depth_resolve.comp.hlsl

@@ -0,0 +1,3 @@
+#define ENABLE_MSAA_VIS_BUFFER
+#define ENABLE_MSAA_DEPTH_RESOLVE
+#include "fill_gbuffer.comp.hlsl"

src/renderer/shader/vis_buffer/resolve_depth_legacy.frag.hlsl

@@ -0,0 +1,24 @@
+#include "lib/base.hlsl"
+
+VK_BINDING(0, 0) Texture2DMS<float> DepthMS;
+
+struct PS_INPUT
+{
+    float4 PositionCS : SV_Position;
+    float2 PositionUV : TEXCOORD0;
+};
+
+struct PS_OUTPUT
+{
+    float depth : SV_Depth;
+};
+
+void main(in PS_INPUT input, out PS_OUTPUT output)
+{
+    uint2 position_ts = input.PositionCS.xy;
+    uint sample_location = (position_ts.x & 1) + (position_ts.y & 1) * 2;
+
+    const float depth = DepthMS.Load(uint3(position_ts / 2, 0), sample_location);
+
+    output.depth = depth;
+}

src/renderer/vulkan/Backend.cpp

@@ -208,8 +208,8 @@ namespace
         vkGetDeviceQueue(backend.device, backend.physical_device.present_queue_family_index, 0, &backend.present_queue);
     }
 
-    void vulkan_check_physical_device_supported_extensions(VkPhysicalDevice       physical_device,
-                                                           std::span<const char*> checked_extensions)
+    void vulkan_check_physical_device_supported_extensions(
+        VkPhysicalDevice physical_device, std::span<const char*> checked_extensions)
     {
         if (checked_extensions.empty())
             return;

src/renderer/vulkan/Backend.h

@@ -84,6 +84,7 @@ struct REAPER_RENDERER_API VulkanBackend
     {
         bool freeze_meshlet_culling = false; // FIXME using the framegraph we can't have persistent resources yet
         bool enable_debug_tile_lighting = true;
+        bool enable_msaa_visibility = false;
     } options;
 
     BackendResources* resources = nullptr;

src/renderer/vulkan/PhysicalDevice.cpp

@@ -123,6 +123,47 @@ namespace
             }
         }
     }
+
+    bool check_support_for_compute_writable_depth_format(VkPhysicalDevice physical_device)
+    {
+        // FIXME This way of detection is a bit brittle.
+        // We're not exactly matching what the render pass code will do
+        // NOTE: Maybe we could have detection code per-renderpass instead.
+        const VkPhysicalDeviceImageFormatInfo2 compute_writable_depth_info = {
+            .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2,
+            .pNext = nullptr,
+            .format = VK_FORMAT_D16_UNORM, // Only checking one format here
+            .type = VK_IMAGE_TYPE_2D,
+            .tiling = VK_IMAGE_TILING_OPTIMAL,
+            .usage = VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT,
+            .flags = VK_FLAGS_NONE,
+        };
+
+        VkImageFormatProperties2 image_properties;
+        image_properties.sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2;
+        image_properties.pNext = nullptr;
+
+        const VkResult result =
+            vkGetPhysicalDeviceImageFormatProperties2(physical_device, &compute_writable_depth_info, &image_properties);
+
+        switch (result)
+        {
+        case VK_SUCCESS:
+            return true;
+        case VK_ERROR_FORMAT_NOT_SUPPORTED:
+            return false;
+        default:
+            AssertVk(result);
+            return false;
+        }
+    }
+
+    PhysicalDeviceInfo::MacroFeatures fill_physical_device_info_features(VkPhysicalDevice handle)
+    {
+        return PhysicalDeviceInfo::MacroFeatures{
+            .compute_stores_to_depth = check_support_for_compute_writable_depth_format(handle),
+        };
+    }
 } // namespace
 
 PhysicalDeviceInfo create_physical_device_info(VkPhysicalDevice handle, IWindow* window,
@@ -136,6 +177,8 @@ PhysicalDeviceInfo create_physical_device_info(VkPhysicalDevice handle, IWindow*
     fill_physical_device_supported_features(physical_device, handle);
     fill_physical_device_supported_queues(physical_device, handle, window, presentationSurface);
 
+    physical_device.macro_features = fill_physical_device_info_features(handle);
+
     return physical_device;
 }
 } // namespace Reaper

src/renderer/vulkan/PhysicalDevice.h

@@ -33,6 +33,11 @@ struct PhysicalDeviceInfo
     // These can point to the same object!
     uint32_t graphics_queue_family_index;
     uint32_t present_queue_family_index;
+
+    struct MacroFeatures
+    {
+        bool compute_stores_to_depth = false;
+    } macro_features;
 };
 
 class IWindow;

src/renderer/vulkan/ShaderModules.cpp

@@ -80,6 +80,11 @@ ShaderModules create_shader_modules(VulkanBackend& backend)
     modules.vis_buffer_raster_fs = create_shader_module(backend.device, "vis_buffer/vis_buffer_raster.frag.spv");
     modules.vis_buffer_raster_vs = create_shader_module(backend.device, "vis_buffer/vis_buffer_raster.vert.spv");
     modules.vis_fill_gbuffer_cs = create_shader_module(backend.device, "vis_buffer/fill_gbuffer.comp.spv");
+    modules.vis_fill_gbuffer_msaa_cs = create_shader_module(backend.device, "vis_buffer/fill_gbuffer_msaa.comp.spv");
+    modules.vis_fill_gbuffer_msaa_with_depth_resolve_cs =
+        create_shader_module(backend.device, "vis_buffer/fill_gbuffer_msaa_with_depth_resolve.comp.spv");
+    modules.vis_resolve_depth_legacy_fs =
+        create_shader_module(backend.device, "vis_buffer/resolve_depth_legacy.frag.spv");
 
     return modules;
 }
@@ -116,5 +121,8 @@ void destroy_shader_modules(VulkanBackend& backend, ShaderModules& shader_module
     vkDestroyShaderModule(backend.device, shader_modules.vis_buffer_raster_fs, nullptr);
     vkDestroyShaderModule(backend.device, shader_modules.vis_buffer_raster_vs, nullptr);
     vkDestroyShaderModule(backend.device, shader_modules.vis_fill_gbuffer_cs, nullptr);
+    vkDestroyShaderModule(backend.device, shader_modules.vis_fill_gbuffer_msaa_cs, nullptr);
+    vkDestroyShaderModule(backend.device, shader_modules.vis_fill_gbuffer_msaa_with_depth_resolve_cs, nullptr);
+    vkDestroyShaderModule(backend.device, shader_modules.vis_resolve_depth_legacy_fs, nullptr);
 }
 } // namespace Reaper

src/renderer/vulkan/ShaderModules.h

@@ -43,6 +43,9 @@ struct ShaderModules
     VkShaderModule vis_buffer_raster_fs;
     VkShaderModule vis_buffer_raster_vs;
     VkShaderModule vis_fill_gbuffer_cs;
+    VkShaderModule vis_fill_gbuffer_msaa_cs;
+    VkShaderModule vis_fill_gbuffer_msaa_with_depth_resolve_cs;
+    VkShaderModule vis_resolve_depth_legacy_fs;
 };
 
 struct VulkanBackend;

src/renderer/vulkan/api/AssertHelper.h

@@ -13,10 +13,10 @@
 #include "core/Assert.h"
 
 // NOTE: It's really important to evaluate 'vk_call' only once
-#define AssertVk(vk_call)                                                                     \
-    do                                                                                        \
-    {                                                                                         \
-        const VkResult result = vk_call;                                                      \
-        AssertImpl(__FILE__, __FUNCTION__, __LINE__, result == VK_SUCCESS,                    \
-                   fmt::format("vulkan call failed with result '{}'", vk_to_string(result))); \
+#define AssertVk(vk_call)                                                                        \
+    do                                                                                           \
+    {                                                                                            \
+        const VkResult vk_result = vk_call;                                                      \
+        AssertImpl(__FILE__, __FUNCTION__, __LINE__, vk_result == VK_SUCCESS,                    \
+                   fmt::format("vulkan call failed with result '{}'", vk_to_string(vk_result))); \
     } while (false)

src/renderer/vulkan/renderpass/TestGraphics.cpp

@@ -107,6 +107,7 @@ void backend_debug_ui(VulkanBackend& backend)
     {
         ImGui::Checkbox("Freeze culling [BROKEN]", &backend.options.freeze_meshlet_culling); // FIXME
         ImGui::Checkbox("Enable debug tile culling", &backend.options.enable_debug_tile_lighting);
+        ImGui::Checkbox("Enable MSAA-based visibility", &backend.options.enable_msaa_visibility);
         ImGui::SliderFloat("Tonemap min (nits)", &backend.presentInfo.tonemap_min_nits, 0.0001f, 1.f);
         ImGui::SliderFloat("Tonemap max (nits)", &backend.presentInfo.tonemap_max_nits, 80.f, 2000.f);
         ImGui::SliderFloat("SDR UI max brightness (nits)", &backend.presentInfo.sdr_ui_max_brightness_nits, 20.f,
@@ -198,10 +199,11 @@ void backend_execute_frame(ReaperRoot& root, VulkanBackend& backend, CommandBuff
     const ShadowFrameGraphRecord shadow = create_shadow_map_pass_record(builder, meshlet_pass, prepared);
 
     const VisBufferFrameGraphRecord vis_buffer_record =
-        create_vis_buffer_pass_record(builder, meshlet_pass, render_extent);
+        create_vis_buffer_pass_record(builder, meshlet_pass, render_extent, backend.options.enable_msaa_visibility,
+                                      backend.physical_device.macro_features.compute_stores_to_depth);
 
     const HZBReduceFrameGraphRecord hzb_reduce =
-        create_hzb_pass_record(builder, vis_buffer_record.render.depth, tiled_lighting_frame);
+        create_hzb_pass_record(builder, vis_buffer_record.depth, tiled_lighting_frame);
 
     const LightRasterFrameGraphRecord light_raster_record = create_tiled_lighting_raster_pass_record(
         builder, tiled_lighting_frame, hzb_reduce.hzb_properties, hzb_reduce.hzb_texture);
@@ -213,7 +215,7 @@ void backend_execute_frame(ReaperRoot& root, VulkanBackend& backend, CommandBuff
         create_tiled_lighting_debug_pass_record(builder, tiled_lighting, render_extent);
 
     const ForwardFrameGraphRecord forward =
-        create_forward_pass_record(builder, meshlet_pass, shadow, vis_buffer_record.render.depth, render_extent);
+        create_forward_pass_record(builder, meshlet_pass, shadow, vis_buffer_record.depth, render_extent);
 
     const GUIFrameGraphRecord gui = create_gui_pass_record(builder, backend.presentInfo.surface_extent);
 
@@ -260,10 +262,11 @@ void backend_execute_frame(ReaperRoot& root, VulkanBackend& backend, CommandBuff
         update_shadow_map_resources(descriptor_write_helper, resources.frame_storage_allocator, prepared,
                                     resources.shadow_map_resources, resources.mesh_cache.vertexBufferPosition);
 
-        update_vis_buffer_pass_resources(framegraph, resources.framegraph_resources, vis_buffer_record,
-                                         descriptor_write_helper, resources.frame_storage_allocator,
-                                         resources.vis_buffer_pass_resources, prepared, resources.samplers_resources,
-                                         resources.material_resources, resources.mesh_cache);
+        update_vis_buffer_pass_resources(
+            framegraph, resources.framegraph_resources, vis_buffer_record, descriptor_write_helper,
+            resources.frame_storage_allocator, resources.vis_buffer_pass_resources, prepared,
+            resources.samplers_resources, resources.material_resources, resources.mesh_cache,
+            backend.options.enable_msaa_visibility, backend.physical_device.macro_features.compute_stores_to_depth);
 
         update_tiled_lighting_raster_pass_resources(framegraph, resources.framegraph_resources, light_raster_record,
                                                     descriptor_write_helper, resources.frame_storage_allocator,
@@ -420,19 +423,25 @@ void backend_execute_frame(ReaperRoot& root, VulkanBackend& backend, CommandBuff
                                          resources.shadow_map_resources);
 
         record_vis_buffer_pass_command_buffer(frame_graph_helper, vis_buffer_record.render, cmdBuffer,
-                                              resources.pipeline_factory, prepared,
-                                              resources.vis_buffer_pass_resources);
+                                              resources.pipeline_factory, prepared, resources.vis_buffer_pass_resources,
+                                              backend.options.enable_msaa_visibility);
+
+        record_fill_gbuffer_pass_command_buffer(frame_graph_helper, vis_buffer_record.fill_gbuffer, cmdBuffer,
+                                                resources.pipeline_factory, resources.vis_buffer_pass_resources,
+                                                render_extent, backend.options.enable_msaa_visibility,
+                                                backend.physical_device.macro_features.compute_stores_to_depth);
+
+        record_legacy_depth_resolve_pass_command_buffer(
+            frame_graph_helper, vis_buffer_record.legacy_depth_resolve, cmdBuffer, resources.pipeline_factory,
+            resources.vis_buffer_pass_resources, backend.options.enable_msaa_visibility,
+            backend.physical_device.macro_features.compute_stores_to_depth);
 
         record_hzb_command_buffer(
             frame_graph_helper, hzb_reduce, cmdBuffer, resources.pipeline_factory, resources.hzb_pass_resources,
             VkExtent2D{.width = vis_buffer_record.scene_depth_properties.width,
                        .height = vis_buffer_record.scene_depth_properties.height},
             VkExtent2D{.width = hzb_reduce.hzb_properties.width, .height = hzb_reduce.hzb_properties.height});
 
-        record_fill_gbuffer_pass_command_buffer(frame_graph_helper, vis_buffer_record.fill_gbuffer, cmdBuffer,
-                                                resources.pipeline_factory, resources.vis_buffer_pass_resources,
-                                                render_extent);
-
         record_depth_copy(frame_graph_helper, light_raster_record.tile_depth_copy, cmdBuffer,
                           resources.pipeline_factory, resources.tiled_raster_resources);
 

src/renderer/vulkan/renderpass/TiledLightingPass.cpp

@@ -183,7 +183,7 @@ TiledLightingFrameGraphRecord create_tiled_lighting_pass_record(FrameGraph::Buil
                                               VK_IMAGE_LAYOUT_READ_ONLY_OPTIMAL});
 
     tiled_lighting.depth =
-        builder.read_texture(tiled_lighting.pass_handle, vis_buffer_record.render.depth,
+        builder.read_texture(tiled_lighting.pass_handle, vis_buffer_record.depth,
                              GPUTextureAccess{VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_READ_BIT,
                                               VK_IMAGE_LAYOUT_DEPTH_READ_ONLY_OPTIMAL});