Feature/capture unit test (#4898)

* record/replay: Add tests

Modify the hello-world example to generate the hash code for the
entry point spirv code, so that we can compare it with replaying
the example.

Add the test script to run the example and compare the hash code
with replaying it.

* Check nullptr for out Diagnostics

We need to check whether the output Diagnostics is a nullptr,
because it's allowed.

* Fix the double free pointers

* Add triangle example as the new test for record-replay

Change the example base to add the offline rendering path
because we don't want to display anything when we're in the
test mode.

This change involves introducing a TestBase that will parse
the command line option. It will decide whether we are in
the test mode.

Disable all the swapchain and windows related creation, instead
we will only create one single framebuffer for the render target.

* Address comments

TODO:
In the follow up patches, I will add more tests and integrate the test flow into slang-unit-test.

examples/example-base/example-base.cpp

@@ -17,19 +17,6 @@ Slang::Result WindowedAppBase::initializeBase(
     int height,
     DeviceType deviceType)
 {
-    // Create a window for our application to render into.
-    //
-    platform::WindowDesc windowDesc;
-    windowDesc.title = title;
-    windowDesc.width = width;
-    windowDesc.height = height;
-    windowWidth = width;
-    windowHeight = height;
-    windowDesc.style = platform::WindowStyle::Default;
-    gWindow = platform::Application::createWindow(windowDesc);
-    gWindow->events.mainLoop = [this]() { mainLoop(); };
-    gWindow->events.sizeChanged = Slang::Action<>(this, &WindowedAppBase::windowSizeChanged);
-
     // Initialize the rendering layer.
 #ifdef _DEBUG
     // Enable debug layer in debug config.
@@ -41,26 +28,44 @@ Slang::Result WindowedAppBase::initializeBase(
     if (SLANG_FAILED(res))
         return res;
 
-    auto deviceInfo = gDevice->getDeviceInfo();
-    Slang::StringBuilder titleSb;
-    titleSb << title << " (" << deviceInfo.apiName << ": " << deviceInfo.adapterName << ")";
-    gWindow->setText(titleSb.getBuffer());
-
     ICommandQueue::Desc queueDesc = {};
     queueDesc.type = ICommandQueue::QueueType::Graphics;
     gQueue = gDevice->createCommandQueue(queueDesc);
 
-    // Create swapchain and framebuffers.
-    gfx::ISwapchain::Desc swapchainDesc = {};
-    swapchainDesc.format = gfx::Format::R8G8B8A8_UNORM;
-    swapchainDesc.width = width;
-    swapchainDesc.height = height;
-    swapchainDesc.imageCount = kSwapchainImageCount;
-    swapchainDesc.queue = gQueue;
-    gfx::WindowHandle windowHandle = gWindow->getNativeHandle().convert<gfx::WindowHandle>();
-    gSwapchain = gDevice->createSwapchain(swapchainDesc, windowHandle);
-
-    IFramebufferLayout::TargetLayout renderTargetLayout = {gSwapchain->getDesc().format, 1};
+    windowWidth = width;
+    windowHeight = height;
+    // Do not create swapchain and windows in test mode, because there won't be any display.
+    if (!isTestMode())
+    {
+        // Create a window for our application to render into.
+        //
+        platform::WindowDesc windowDesc;
+        windowDesc.title = title;
+        windowDesc.width = width;
+        windowDesc.height = height;
+        windowDesc.style = platform::WindowStyle::Default;
+        gWindow = platform::Application::createWindow(windowDesc);
+        gWindow->events.mainLoop = [this]() { mainLoop(); };
+        gWindow->events.sizeChanged = Slang::Action<>(this, &WindowedAppBase::windowSizeChanged);
+
+        auto deviceInfo = gDevice->getDeviceInfo();
+        Slang::StringBuilder titleSb;
+        titleSb << title << " (" << deviceInfo.apiName << ": " << deviceInfo.adapterName << ")";
+        gWindow->setText(titleSb.getBuffer());
+
+        // Create swapchain and framebuffers.
+        gfx::ISwapchain::Desc swapchainDesc = {};
+        swapchainDesc.format = gfx::Format::R8G8B8A8_UNORM;
+        swapchainDesc.width = width;
+        swapchainDesc.height = height;
+        swapchainDesc.imageCount = kSwapchainImageCount;
+        swapchainDesc.queue = gQueue;
+        gfx::WindowHandle windowHandle = gWindow->getNativeHandle().convert<gfx::WindowHandle>();
+        gSwapchain = gDevice->createSwapchain(swapchainDesc, windowHandle);
+        createSwapchainFramebuffers();
+    }
+
+    IFramebufferLayout::TargetLayout renderTargetLayout = {gfx::Format::R8G8B8A8_UNORM, 1};
     IFramebufferLayout::TargetLayout depthLayout = {gfx::Format::D32_FLOAT, 1};
     IFramebufferLayout::Desc framebufferLayoutDesc;
     framebufferLayoutDesc.renderTargetCount = 1;
@@ -69,7 +74,10 @@ Slang::Result WindowedAppBase::initializeBase(
     SLANG_RETURN_ON_FAIL(
         gDevice->createFramebufferLayout(framebufferLayoutDesc, gFramebufferLayout.writeRef()));
 
-    createSwapchainFramebuffers();
+    if (isTestMode())
+    {
+        createOfflineFramebuffers();
+    }
 
     for (uint32_t i = 0; i < kSwapchainImageCount; i++)
     {
@@ -102,23 +110,27 @@ Slang::Result WindowedAppBase::initializeBase(
 void WindowedAppBase::mainLoop()
 {
     int frameBufferIndex = gSwapchain->acquireNextImage();
-    if (frameBufferIndex == -1)
-        return;
 
     gTransientHeaps[frameBufferIndex]->synchronizeAndReset();
     renderFrame(frameBufferIndex);
     gTransientHeaps[frameBufferIndex]->finish();
 }
 
-void WindowedAppBase::createSwapchainFramebuffers()
+void WindowedAppBase::offlineRender()
 {
-    gFramebuffers.clear();
-    for (uint32_t i = 0; i < kSwapchainImageCount; i++)
+    gTransientHeaps[0]->synchronizeAndReset();
+    renderFrame(0);
+    gTransientHeaps[0]->finish();
+}
+
+void WindowedAppBase::createFramebuffers(uint32_t width, uint32_t height, gfx::Format colorFormat, uint32_t frameBufferCount)
+{
+    for (uint32_t i = 0; i < frameBufferCount; i++)
     {
         gfx::ITextureResource::Desc depthBufferDesc;
         depthBufferDesc.type = IResource::Type::Texture2D;
-        depthBufferDesc.size.width = gSwapchain->getDesc().width;
-        depthBufferDesc.size.height = gSwapchain->getDesc().height;
+        depthBufferDesc.size.width = width;
+        depthBufferDesc.size.height = height;
         depthBufferDesc.size.depth = 1;
         depthBufferDesc.format = gfx::Format::D32_FLOAT;
         depthBufferDesc.defaultState = ResourceState::DepthWrite;
@@ -127,12 +139,28 @@ void WindowedAppBase::createSwapchainFramebuffers()
         depthBufferDesc.optimalClearValue = &depthClearValue;
         ComPtr<gfx::ITextureResource> depthBufferResource =
             gDevice->createTextureResource(depthBufferDesc, nullptr);
+
         ComPtr<gfx::ITextureResource> colorBuffer;
-        gSwapchain->getImage(i, colorBuffer.writeRef());
+        if (isTestMode())
+        {
+            gfx::ITextureResource::Desc colorBufferDesc;
+            colorBufferDesc.type = IResource::Type::Texture2D;
+            colorBufferDesc.size.width = width;
+            colorBufferDesc.size.height = height;
+            colorBufferDesc.size.depth = 1;
+            colorBufferDesc.format = colorFormat;
+            colorBufferDesc.defaultState = ResourceState::RenderTarget;
+            colorBufferDesc.allowedStates = ResourceStateSet(ResourceState::RenderTarget, ResourceState::CopyDestination);
+            colorBuffer = gDevice->createTextureResource(colorBufferDesc, nullptr);
+        }
+        else
+        {
+            gSwapchain->getImage(i, colorBuffer.writeRef());
+        }
 
         gfx::IResourceView::Desc colorBufferViewDesc;
         memset(&colorBufferViewDesc, 0, sizeof(colorBufferViewDesc));
-        colorBufferViewDesc.format = gSwapchain->getDesc().format;
+        colorBufferViewDesc.format = colorFormat;
         colorBufferViewDesc.renderTarget.shape = gfx::IResource::Type::Texture2D;
         colorBufferViewDesc.type = gfx::IResourceView::Type::RenderTarget;
         ComPtr<gfx::IResourceView> rtv =
@@ -152,10 +180,24 @@ void WindowedAppBase::createSwapchainFramebuffers()
         framebufferDesc.renderTargetViews = rtv.readRef();
         framebufferDesc.layout = gFramebufferLayout;
         ComPtr<gfx::IFramebuffer> frameBuffer = gDevice->createFramebuffer(framebufferDesc);
+
         gFramebuffers.add(frameBuffer);
     }
 }
 
+void WindowedAppBase::createOfflineFramebuffers()
+{
+    gFramebuffers.clear();
+    createFramebuffers(windowWidth, windowHeight, gfx::Format::R8G8B8A8_UNORM, 1);
+}
+
+void WindowedAppBase::createSwapchainFramebuffers()
+{
+    gFramebuffers.clear();
+    createFramebuffers(gSwapchain->getDesc().width, gSwapchain->getDesc().height,
+                        gSwapchain->getDesc().format, kSwapchainImageCount);
+}
+
 ComPtr<gfx::IResourceView> WindowedAppBase::createTextureFromFile(String fileName, int& textureWidth, int& textureHeight)
 {
     int channelsInFile = 0;

examples/example-base/example-base.h

@@ -4,12 +4,13 @@
 #include "tools/platform/window.h"
 #include "source/core/slang-basic.h"
 #include "source/core/slang-io.h"
+#include "test-base.h"
 
 #ifdef _WIN32
 void _Win32OutputDebugString(const char* str);
 #endif
 
-struct WindowedAppBase
+struct WindowedAppBase : public TestBase
 {
 protected:
     static const int kSwapchainImageCount = 2;
@@ -32,8 +33,13 @@ struct WindowedAppBase
         int width,
         int height,
         gfx::DeviceType deviceType = gfx::DeviceType::Default);
+
+    void createFramebuffers(uint32_t width, uint32_t height, gfx::Format colorFormat, uint32_t frameBufferCount);
     void createSwapchainFramebuffers();
+    void createOfflineFramebuffers();
+
     void mainLoop();
+
     Slang::ComPtr<gfx::IResourceView> createTextureFromFile(Slang::String fileName, int& textureWidth, int& textureHeight);
     virtual void windowSizeChanged();
 
@@ -42,6 +48,7 @@ struct WindowedAppBase
 public:
     platform::Window* getWindow() { return gWindow.Ptr(); }
     virtual void finalize() { gQueue->waitOnHost(); }
+    void offlineRender();
 };
 
 struct ExampleResources {
@@ -103,18 +110,26 @@ inline void diagnoseIfNeeded(slang::IBlob* diagnosticsBlob)
 void initDebugCallback();
 
 template<typename TApp>
-int innerMain()
+int innerMain(int argc, char** argv)
 {
     initDebugCallback();
 
     TApp app;
 
+    app.parseOption(argc, argv);
     if (SLANG_FAILED(app.initialize()))
     {
         return -1;
     }
 
-    platform::Application::run(app.getWindow());
+    if (!app.isTestMode())
+    {
+        platform::Application::run(app.getWindow());
+    }
+    else
+    {
+        app.offlineRender();
+    }
 
     app.finalize();
     return 0;

examples/example-base/test-base.cpp

@@ -0,0 +1,50 @@
+#include "test-base.h"
+
+#ifdef _WIN32
+#include <windows.h>
+#include <shellapi.h>
+#endif
+
+int TestBase::parseOption(int argc, char** argv)
+{
+    // We only make the parse in a very loose way for only extracting the test option.
+#ifdef _WIN32
+    wchar_t** szArglist;
+    szArglist = CommandLineToArgvW(GetCommandLineW(), &argc);
+#endif
+
+    for (int i = 0; i < argc; i++)
+    {
+#ifdef _WIN32
+        if (wcscmp(szArglist[i], L"--test-mode") == 0)
+#else
+        if (strcmp(argv[i], "--test-mode") == 0)
+#endif
+        {
+            m_isTestMode = true;
+        }
+    }
+    return 0;
+}
+
+void TestBase::printEntrypointHashes(int entryPointCount, int targetCount, ComPtr<slang::IComponentType>& composedProgram)
+{
+    for (int targetIndex = 0; targetIndex < targetCount; targetIndex++)
+    {
+        for (int entryPointIndex = 0; entryPointIndex < entryPointCount; entryPointIndex++)
+        {
+            ComPtr<slang::IBlob> entryPointHashBlob;
+            composedProgram->getEntryPointHash(entryPointIndex, targetIndex, entryPointHashBlob.writeRef());
+
+            Slang::StringBuilder strBuilder;
+            strBuilder << "entrypoint: "<< entryPointIndex << ", target: " << targetIndex << ", hash: ";
+
+            uint8_t* buffer = (uint8_t*)entryPointHashBlob->getBufferPointer();
+            for (size_t i = 0; i < entryPointHashBlob->getBufferSize(); i++)
+            {
+                strBuilder<<Slang::StringUtil::makeStringWithFormat("%.2X", buffer[i]);
+            }
+            fprintf(stdout, "%s\n", strBuilder.begin());
+        }
+    }
+}

examples/example-base/test-base.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include "slang.h"
+#include "slang-com-ptr.h"
+#include "source/core/slang-string-util.h"
+
+using Slang::ComPtr;
+
+class TestBase {
+
+public:
+    // Parses command line options. This example only has one option for testing purpose.
+    int parseOption(int argc, char** argv);
+
+    void printEntrypointHashes(int entryPointCount, int targetCount, ComPtr<slang::IComponentType>& composedProgram);
+
+    bool isTestMode() const { return m_isTestMode; }
+
+private:
+    bool m_isTestMode = false;
+};

examples/hello-world/main.cpp

@@ -12,12 +12,14 @@
 
 #include "vulkan-api.h"
 #include "examples/example-base/example-base.h"
+#include "examples/example-base/test-base.h"
+#include "source/core/slang-string-util.h"
 
 using Slang::ComPtr;
 
 static const ExampleResources resourceBase("hello-world");
 
-struct HelloWorldExample
+struct HelloWorldExample : public TestBase
 {
     // The Vulkan functions pointers result from loading the vulkan library.
     VulkanAPI vkAPI;
@@ -66,10 +68,11 @@ struct HelloWorldExample
 
 };
 
-int main()
+int main(int argc, char* argv[])
 {
     initDebugCallback();
     HelloWorldExample example;
+    example.parseOption(argc, argv);
     return example.run();
 }
 
@@ -204,6 +207,11 @@ int HelloWorldExample::createComputePipelineFromShader()
             0, 0, spirvCode.writeRef(), diagnosticsBlob.writeRef());
         diagnoseIfNeeded(diagnosticsBlob);
         RETURN_ON_FAIL(result);
+
+        if (isTestMode())
+        {
+            printEntrypointHashes(1, 1, composedProgram);
+        }
     }
 
     // The following steps are all Vulkan API calls to create a pipeline.

examples/triangle/main.cpp

@@ -187,6 +187,11 @@ gfx::Result loadShaderProgram(
     programDesc.slangGlobalScope = linkedProgram;
     SLANG_RETURN_ON_FAIL(device->createProgram(programDesc, outProgram));
 
+    if (isTestMode())
+    {
+        printEntrypointHashes(entryPointCount, 1, linkedProgram);
+    }
+
     return SLANG_OK;
 }
 
@@ -387,9 +392,12 @@ virtual void renderFrame(int frameBufferIndex) override
     commandBuffer->close();
     gQueue->executeCommandBuffer(commandBuffer);
 
-    // With that, we are done drawing for one frame, and ready for the next.
-    //
-    gSwapchain->present();
+    if (!isTestMode())
+    {
+        // With that, we are done drawing for one frame, and ready for the next.
+        //
+        gSwapchain->present();
+    }
 }
 
 };