From a60be6b81f22d3a18ee624e2414e91c37e2c607f Mon Sep 17 00:00:00 2001 From: Sergio Martins Date: Wed, 5 Jun 2024 23:25:18 +0100 Subject: [PATCH] feat: Add support for XPASS --- out/qttest.js | 27 +++++++++++++++++++-------- out/test.js | 14 ++++++++++++-- src/qttest.ts | 30 ++++++++++++++++++++++-------- src/test.ts | 18 ++++++++++++++++-- test/qt_test/test2.cpp | 4 ++++ 5 files changed, 73 insertions(+), 20 deletions(-) diff --git a/out/qttest.js b/out/qttest.js index 29fb9ba..13d523d 100644 --- a/out/qttest.js +++ b/out/qttest.js @@ -313,16 +313,27 @@ class QtTest { if (event.at(0) != "assert") continue; var obj = event.at(1); - if (obj["ok"] === false) { - if (obj["todo"] !== false) { - // This is a QEXPECT_FAIL test, all good. - // QtTest outputs it as "todo" - continue; - } + let pass = obj["ok"] === true; + let xfail = !pass && obj["todo"] !== false; + if (xfail) { + // This is a QEXPECT_FAIL test, all good. + // QtTest outputs it as "todo" + continue; + } + // There's an QEXPECT_FAIL but test passed, not good. + let xpass = pass && obj["todo"].includes("returned TRUE unexpectedly"); + if (!pass || xpass) { // We found a failure - var filename = obj["diag"]["file"]; - var lineNumber = obj["diag"]["line"]; var name = obj["name"].replace(/\(.*\)/, ""); + var filename = ""; + var lineNumber = -1; + if (obj["diag"]) { + filename = obj["diag"]["file"]; + lineNumber = obj["diag"]["line"]; + } + else { + // A XPASS for example misses file:line info. Nothing we can do, it's a Qt bug arguably. + } failedResults.push({ name: name, filePath: filename, diff --git a/out/test.js b/out/test.js index b6bf138..59dc270 100644 --- a/out/test.js +++ b/out/test.js @@ -58,7 +58,7 @@ function runTests(buildDirPath) { yield qt.dumpTestSlots(); let expected_slots = { "test/qt_test/build-dev/test1": ["testA", "testB", "testC", "testXFAIL"], - "test/qt_test/build-dev/test2": ["testD", "testE", "testF"], + "test/qt_test/build-dev/test2": ["testD", "testE", "testF", "testXPASS"], "test/qt_test/build-dev/test3": ["testAbortsEverythig", "testH", "testI"], }; for (var executable of qt.qtTestExecutables) { @@ -124,7 +124,6 @@ function runTests(buildDirPath) { } // 6. Run a slot that has XFAIL slot = qt.qtTestExecutables[0].slots[3]; - // assert it's called testXFAIL if (slot.name != "testXFAIL") { console.error("Expected slot name to be testXFAIL"); process.exit(1); @@ -134,6 +133,17 @@ function runTests(buildDirPath) { console.error("Expected test to pass: " + slot.name); process.exit(1); } + // 7. Run a slot that has XPASS + slot = qt.qtTestExecutables[1].slots[3]; + if (slot.name != "testXPASS") { + console.error("Expected slot name to be testXPASS"); + process.exit(1); + } + yield slot.runTest(); + if (!slot.lastTestFailure) { + console.error("Expected test to fail: " + slot.name); + process.exit(1); + } }); } function runCodeModelTests(codeModelFile) { diff --git a/src/qttest.ts b/src/qttest.ts index 34c8a87..7a8019e 100644 --- a/src/qttest.ts +++ b/src/qttest.ts @@ -315,18 +315,32 @@ export class QtTest { if (event.at(0) != "assert") continue; var obj = event.at(1); + let pass = obj["ok"] === true; - if (obj["ok"] === false) { - if (obj["todo"] !== false) { - // This is a QEXPECT_FAIL test, all good. - // QtTest outputs it as "todo" - continue; - } + let xfail = !pass && obj["todo"] !== false; + if (xfail) { + // This is a QEXPECT_FAIL test, all good. + // QtTest outputs it as "todo" + continue; + } + + // There's an QEXPECT_FAIL but test passed, not good. + let xpass = + pass && obj["todo"].includes("returned TRUE unexpectedly"); + if (!pass || xpass) { // We found a failure - var filename = obj["diag"]["file"]; - var lineNumber = obj["diag"]["line"]; + var name = obj["name"].replace(/\(.*\)/, ""); + var filename = ""; + var lineNumber = -1; + + if (obj["diag"]) { + filename = obj["diag"]["file"]; + lineNumber = obj["diag"]["line"]; + } else { + // A XPASS for example misses file:line info. Nothing we can do, it's a Qt bug arguably. + } failedResults.push({ name: name, diff --git a/src/test.ts b/src/test.ts index 6d02d62..31ba1d7 100644 --- a/src/test.ts +++ b/src/test.ts @@ -67,7 +67,7 @@ async function runTests(buildDirPath: string) { } let expected_slots: ExpectedSlots = { "test/qt_test/build-dev/test1": ["testA", "testB", "testC", "testXFAIL"], - "test/qt_test/build-dev/test2": ["testD", "testE", "testF"], + "test/qt_test/build-dev/test2": ["testD", "testE", "testF", "testXPASS"], "test/qt_test/build-dev/test3": ["testAbortsEverythig", "testH", "testI"], }; @@ -145,7 +145,7 @@ async function runTests(buildDirPath: string) { // 6. Run a slot that has XFAIL slot = qt.qtTestExecutables[0].slots![3]; - // assert it's called testXFAIL + if (slot.name != "testXFAIL") { console.error("Expected slot name to be testXFAIL"); process.exit(1); @@ -156,6 +156,20 @@ async function runTests(buildDirPath: string) { console.error("Expected test to pass: " + slot.name); process.exit(1); } + + // 7. Run a slot that has XPASS + slot = qt.qtTestExecutables[1].slots![3]; + + if (slot.name != "testXPASS") { + console.error("Expected slot name to be testXPASS"); + process.exit(1); + } + + await slot.runTest(); + if (!slot.lastTestFailure) { + console.error("Expected test to fail: " + slot.name); + process.exit(1); + } } async function runCodeModelTests(codeModelFile: string) { diff --git a/test/qt_test/test2.cpp b/test/qt_test/test2.cpp index 4e03b80..1bbe3bb 100644 --- a/test/qt_test/test2.cpp +++ b/test/qt_test/test2.cpp @@ -11,6 +11,10 @@ private Q_SLOTS: void testD() {} void testE() {} void testF() { QFAIL("failed"); } + void testXPASS() { + QEXPECT_FAIL("", "To be fixed", Continue); + QVERIFY(true); + } }; QTEST_MAIN(MyTest);