Skip to content

Commit

Permalink
Merge pull request #16038 from unknownbrackets/headless
Browse files Browse the repository at this point in the history 
Switch headless screenshot error to MSE, add benchmarking
  • Loading branch information
@hrydgard
hrydgard authored Sep 17, 2022
2 parents cd2c977 + 749268c commit c80cc1e
Show file tree
Hide file tree
Showing 4 changed files with 92 additions and 55 deletions.
23 changes: 15 additions & 8 deletions headless/Compare.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -305,11 +305,18 @@ bool CompareOutput(const Path &bootFilename, const std::string &output, bool ver
}
}

inline int ComparePixel(u32 pix1, u32 pix2) {
// For now, if they're different at all except alpha, it's an error.
if ((pix1 & 0xFFFFFF) != (pix2 & 0xFFFFFF))
return 1;
return 0;
static inline double CompareChannel(int pix1, int pix2) {
double diff = pix1 - pix2;
return diff * diff;
}

static inline double ComparePixel(u32 pix1, u32 pix2) {
// Ignore alpha.
double r = CompareChannel(pix1 & 0xFF, pix2 & 0xFF);
double g = CompareChannel((pix1 >> 8) & 0xFF, (pix2 >> 8) & 0xFF);
double b = CompareChannel((pix1 >> 16) & 0xFF, (pix2 >> 16) & 0xFF);

return r + g + b;
}

std::vector<u32> TranslateDebugBufferToCompare(const GPUDebugBuffer *buffer, u32 stride, u32 h) {
Expand Down Expand Up @@ -338,7 +345,6 @@ std::vector<u32> TranslateDebugBufferToCompare(const GPUDebugBuffer *buffer, u32
dst += (h - safeH) * stride;
}

u32 errors = 0;
for (u32 y = 0; y < safeH; ++y) {
switch (buffer->GetFormat()) {
case GPU_DBG_FORMAT_8888:
Expand Down Expand Up @@ -429,7 +435,7 @@ double ScreenshotComparer::Compare(const Path &screenshotFilename) {
return -1.0f;
}

u32 errors = 0;
double errors = 0;
if (asBitmap_) {
// The reference is flipped and BGRA by default for the common BMP compare case.
for (u32 y = 0; y < h_; ++y) {
Expand All @@ -447,7 +453,8 @@ double ScreenshotComparer::Compare(const Path &screenshotFilename) {
}
}

return (double) errors / (double) (w_ * h_);
// Convert to MSE, accounting for all three channels (RGB.)
return errors / (double)(w_ * h_ * 3);
}

bool ScreenshotComparer::SaveActualBitmap(const Path &resultFilename) {
Expand Down
110 changes: 67 additions & 43 deletions headless/Headless.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,20 +128,18 @@ int printUsage(const char *progname, const char *reason)
fprintf(stderr, " -l, --log full log output, not just emulated printfs\n");
fprintf(stderr, " --debugger=PORT enable websocket debugger and break at start\n");

#if defined(HEADLESSHOST_CLASS)
{
fprintf(stderr, " --graphics=BACKEND use the full gpu backend (slower)\n");
fprintf(stderr, " options: gles, software, directx9, etc.\n");
fprintf(stderr, " --screenshot=FILE compare against a screenshot\n");
}
#endif
fprintf(stderr, " --graphics=BACKEND use a different gpu backend\n");
fprintf(stderr, " options: gles, software, directx9, etc.\n");
fprintf(stderr, " --screenshot=FILE compare against a screenshot\n");
fprintf(stderr, " --max-mse=NUMBER maximum allowed MSE error for screenshot\n");
fprintf(stderr, " --timeout=SECONDS abort test it if takes longer than SECONDS\n");

fprintf(stderr, " -v, --verbose show the full passed/failed result\n");
fprintf(stderr, " -i use the interpreter\n");
fprintf(stderr, " --ir use ir interpreter\n");
fprintf(stderr, " -j use jit (default)\n");
fprintf(stderr, " -c, --compare compare with output in file.expected\n");
fprintf(stderr, " --bench run multiple times and output speed\n");
fprintf(stderr, "\nSee headless.txt for details.\n");

return 1;
Expand All @@ -161,13 +159,20 @@ static HeadlessHost *getHost(GPUCore gpuCore) {
}
}

bool RunAutoTest(HeadlessHost *headlessHost, CoreParameter &coreParameter, bool autoCompare, bool verbose, double timeout)
{
struct AutoTestOptions {
double timeout;
double maxScreenshotError;
bool compare : 1;
bool verbose : 1;
bool bench : 1;
};

bool RunAutoTest(HeadlessHost *headlessHost, CoreParameter &coreParameter, const AutoTestOptions &opt) {
// Kinda ugly, trying to guesstimate the test name from filename...
currentTestName = GetTestName(coreParameter.fileToStart);

std::string output;
if (autoCompare)
if (opt.compare || opt.bench)
coreParameter.collectEmuLog = &output;

std::string error_string;
Expand All @@ -181,30 +186,28 @@ bool RunAutoTest(HeadlessHost *headlessHost, CoreParameter &coreParameter, bool

TeamCityPrint("testStarted name='%s' captureStandardOutput='true'", currentTestName.c_str());

host->BootDone();

if (autoCompare)
headlessHost->SetComparisonScreenshot(ExpectedScreenshotFromFilename(coreParameter.fileToStart));
if (opt.compare)
headlessHost->SetComparisonScreenshot(ExpectedScreenshotFromFilename(coreParameter.fileToStart), opt.maxScreenshotError);

while (!PSP_InitUpdate(&error_string))
sleep_ms(1);
if (!PSP_IsInited()) {
TeamCityPrint("testFailed name='%s' message='Startup failed'", currentTestName.c_str());
TeamCityPrint("testFinished name='%s'", currentTestName.c_str());
GitHubActionsPrint("error", "Test timeout for %s", currentTestName.c_str());
GitHubActionsPrint("error", "Test init failed for %s", currentTestName.c_str());
return false;
}

bool passed = true;
double deadline;
deadline = time_now_d() + timeout;
host->BootDone();

Core_UpdateDebugStats(g_Config.bShowDebugStats || g_Config.bLogFrameDrops);

PSP_BeginHostFrame();
if (coreParameter.graphicsContext && coreParameter.graphicsContext->GetDrawContext())
coreParameter.graphicsContext->GetDrawContext()->BeginFrame();

bool passed = true;
double deadline = time_now_d() + opt.timeout;
coreState = coreParameter.startBreak ? CORE_STEPPING : CORE_RUNNING;
while (coreState == CORE_RUNNING || coreState == CORE_STEPPING)
{
Expand All @@ -221,12 +224,15 @@ bool RunAutoTest(HeadlessHost *headlessHost, CoreParameter &coreParameter, bool
}
if (time_now_d() > deadline) {
// Don't compare, print the output at least up to this point, and bail.
printf("%s", output.c_str());
passed = false;
if (!opt.bench) {
printf("%s", output.c_str());

host->SendDebugOutput("TIMEOUT\n");
TeamCityPrint("testFailed name='%s' message='Test timeout'", currentTestName.c_str());
GitHubActionsPrint("error", "Test timeout for %s", currentTestName.c_str());
}

host->SendDebugOutput("TIMEOUT\n");
TeamCityPrint("testFailed name='%s' message='Test timeout'", currentTestName.c_str());
GitHubActionsPrint("error", "Test timeout for %s", currentTestName.c_str());
passed = false;
Core_Stop();
}
}
Expand All @@ -237,10 +243,11 @@ bool RunAutoTest(HeadlessHost *headlessHost, CoreParameter &coreParameter, bool

PSP_Shutdown();

headlessHost->FlushDebugOutput();
if (!opt.bench)
headlessHost->FlushDebugOutput();

if (autoCompare && passed)
passed = CompareOutput(coreParameter.fileToStart, output, verbose);
if (opt.compare && passed)
passed = CompareOutput(coreParameter.fileToStart, output, opt.verbose);

TeamCityPrint("testFinished name='%s'", currentTestName.c_str());

Expand All @@ -263,9 +270,9 @@ int main(int argc, const char* argv[])
_CrtSetDbgFlag(_CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF);
#endif

AutoTestOptions testOptions{};
testOptions.timeout = std::numeric_limits<double>::infinity();
bool fullLog = false;
bool autoCompare = false;
bool verbose = false;
const char *stateToLoad = 0;
GPUCore gpuCore = GPUCORE_SOFTWARE;
CPUCore cpuCore = CPUCore::JIT;
Expand All @@ -275,7 +282,6 @@ int main(int argc, const char* argv[])
const char *mountIso = nullptr;
const char *mountRoot = nullptr;
const char *screenshotFilename = nullptr;
float timeout = std::numeric_limits<float>::infinity();

for (int i = 1; i < argc; i++)
{
Expand All @@ -300,9 +306,11 @@ int main(int argc, const char* argv[])
else if (!strcmp(argv[i], "--ir"))
cpuCore = CPUCore::IR_JIT;
else if (!strcmp(argv[i], "-c") || !strcmp(argv[i], "--compare"))
autoCompare = true;
testOptions.compare = true;
else if (!strcmp(argv[i], "--bench"))
testOptions.bench = true;
else if (!strcmp(argv[i], "-v") || !strcmp(argv[i], "--verbose"))
verbose = true;
testOptions.verbose = true;
else if (!strncmp(argv[i], "--graphics=", strlen("--graphics=")) && strlen(argv[i]) > strlen("--graphics="))
{
const char *gpuName = argv[i] + strlen("--graphics=");
Expand Down Expand Up @@ -330,7 +338,9 @@ int main(int argc, const char* argv[])
} else if (!strncmp(argv[i], "--screenshot=", strlen("--screenshot=")) && strlen(argv[i]) > strlen("--screenshot="))
screenshotFilename = argv[i] + strlen("--screenshot=");
else if (!strncmp(argv[i], "--timeout=", strlen("--timeout=")) && strlen(argv[i]) > strlen("--timeout="))
timeout = (float)strtod(argv[i] + strlen("--timeout="), NULL);
testOptions.timeout = strtod(argv[i] + strlen("--timeout="), nullptr);
else if (!strncmp(argv[i], "--max-mse=", strlen("--max-mse=")) && strlen(argv[i]) > strlen("--max-mse="))
testOptions.maxScreenshotError = strtod(argv[i] + strlen("--max-mse="), nullptr);
else if (!strncmp(argv[i], "--debugger=", strlen("--debugger=")) && strlen(argv[i]) > strlen("--debugger="))
debuggerPort = (int)strtoul(argv[i] + strlen("--debugger="), NULL, 10);
else if (!strcmp(argv[i], "--teamcity"))
Expand Down Expand Up @@ -388,7 +398,7 @@ int main(int argc, const char* argv[])
coreParameter.mountIso = mountIso ? Path(std::string(mountIso)) : Path();
coreParameter.mountRoot = mountRoot ? Path(std::string(mountRoot)) : Path();
coreParameter.startBreak = false;
coreParameter.printfEmuLog = !autoCompare;
coreParameter.printfEmuLog = !testOptions.compare;
coreParameter.headLess = true;
coreParameter.renderScaleFactor = 1;
coreParameter.renderWidth = 480;
Expand Down Expand Up @@ -456,8 +466,9 @@ int main(int argc, const char* argv[])
if (!File::Exists(g_Config.flash0Directory))
g_Config.flash0Directory = File::GetExeDirectory() / "assets/flash0";

if (screenshotFilename != 0)
headlessHost->SetComparisonScreenshot(Path(std::string(screenshotFilename)));
if (screenshotFilename)
headlessHost->SetComparisonScreenshot(Path(std::string(screenshotFilename)), testOptions.maxScreenshotError);
headlessHost->SetWriteFailureScreenshot(!teamCityMode && !getenv("GITHUB_ACTIONS") && !testOptions.bench);

#if PPSSPP_PLATFORM(ANDROID)
// For some reason the debugger installs it with this name?
Expand Down Expand Up @@ -487,14 +498,28 @@ int main(int argc, const char* argv[])
for (size_t i = 0; i < testFilenames.size(); ++i)
{
coreParameter.fileToStart = Path(testFilenames[i]);
if (autoCompare)
if (testOptions.compare)
printf("%s:\n", coreParameter.fileToStart.c_str());
bool passed = RunAutoTest(headlessHost, coreParameter, autoCompare, verbose, timeout);
if (autoCompare)
{
bool passed = RunAutoTest(headlessHost, coreParameter, testOptions);
if (testOptions.bench) {
double st = time_now_d();
double deadline = st + testOptions.timeout;
double runs = 0.0;
for (int i = 0; i < 100; ++i) {
RunAutoTest(headlessHost, coreParameter, testOptions);
runs++;

if (time_now_d() > deadline)
break;
}
double et = time_now_d();

std::string testName = GetTestName(coreParameter.fileToStart);
if (passed)
{
printf(" %s - %f seconds average\n", testName.c_str(), (et - st) / runs);
}
if (testOptions.compare) {
std::string testName = GetTestName(coreParameter.fileToStart);
if (passed) {
passedTests.push_back(testName);
printf(" %s - passed!\n", testName.c_str());
}
Expand All @@ -503,8 +528,7 @@ int main(int argc, const char* argv[])
}
}

if (autoCompare)
{
if (testOptions.compare) {
printf("%d tests passed, %d tests failed.\n", (int)passedTests.size(), (int)failedTests.size());
if (!failedTests.empty())
{
Expand Down
6 changes: 3 additions & 3 deletions headless/StubHost.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,10 @@ void HeadlessHost::SendDebugScreenshot(const u8 *pixbuf, u32 w, u32 h) {
if (errors < 0)
SendOrCollectDebugOutput(comparer.GetError() + "\n");

if (errors > 0)
SendOrCollectDebugOutput(StringFromFormat("Screenshot error: %f%%\n", errors * 100.0f));
if (errors > maxScreenshotError_)
SendOrCollectDebugOutput(StringFromFormat("Screenshot MSE: %f\n", errors));

if (errors > 0 && !teamCityMode && !getenv("GITHUB_ACTIONS")) {
if (errors > maxScreenshotError_ && writeFailureScreenshot_) {
if (comparer.SaveActualBitmap(Path("__testfailure.bmp")))
SendOrCollectDebugOutput("Actual output written to: __testfailure.bmp\n");
comparer.SaveVisualComparisonPNG(Path("__testcompare.png"));
Expand Down
8 changes: 7 additions & 1 deletion headless/StubHost.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,12 @@ class HeadlessHost : public Host {
}
}

virtual void SetComparisonScreenshot(const Path &filename) {
void SetComparisonScreenshot(const Path &filename, double maxError) {
comparisonScreenshot_ = filename;
maxScreenshotError_ = maxError;
}
void SetWriteFailureScreenshot(bool flag) {
writeFailureScreenshot_ = flag;
}

void SendDebugScreenshot(const u8 *pixbuf, u32 w, u32 h) override;
Expand All @@ -83,7 +87,9 @@ class HeadlessHost : public Host {
void SendOrCollectDebugOutput(const std::string &output);

Path comparisonScreenshot_;
double maxScreenshotError_ = 0.0;
std::string debugOutputBuffer_;
GPUCore gpuCore_;
GraphicsContext *gfx_ = nullptr;
bool writeFailureScreenshot_ = true;
};

0 comments on commit c80cc1e

Please sign in to comment.