Skip to content

Commit

Permalink
feat: Add support for XPASS
Browse files Browse the repository at this point in the history
  • Loading branch information
@iamsergio
iamsergio committed Jun 5, 2024
1 parent 0037921 commit a60be6b
Show file tree
Hide file tree
Showing 5 changed files with 73 additions and 20 deletions.
27 changes: 19 additions & 8 deletions out/qttest.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -313,16 +313,27 @@ class QtTest {
if (event.at(0) != "assert")
continue;
var obj = event.at(1);
if (obj["ok"] === false) {
if (obj["todo"] !== false) {
// This is a QEXPECT_FAIL test, all good.
// QtTest outputs it as "todo"
continue;
}
let pass = obj["ok"] === true;
let xfail = !pass && obj["todo"] !== false;
if (xfail) {
// This is a QEXPECT_FAIL test, all good.
// QtTest outputs it as "todo"
continue;
}
// There's an QEXPECT_FAIL but test passed, not good.
let xpass = pass && obj["todo"].includes("returned TRUE unexpectedly");
if (!pass || xpass) {
// We found a failure
var filename = obj["diag"]["file"];
var lineNumber = obj["diag"]["line"];
var name = obj["name"].replace(/\(.*\)/, "");
var filename = "";
var lineNumber = -1;
if (obj["diag"]) {
filename = obj["diag"]["file"];
lineNumber = obj["diag"]["line"];
}
else {
// A XPASS for example misses file:line info. Nothing we can do, it's a Qt bug arguably.
}
failedResults.push({
name: name,
filePath: filename,
Expand Down
14 changes: 12 additions & 2 deletions out/test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ function runTests(buildDirPath) {
yield qt.dumpTestSlots();
let expected_slots = {
"test/qt_test/build-dev/test1": ["testA", "testB", "testC", "testXFAIL"],
"test/qt_test/build-dev/test2": ["testD", "testE", "testF"],
"test/qt_test/build-dev/test2": ["testD", "testE", "testF", "testXPASS"],
"test/qt_test/build-dev/test3": ["testAbortsEverythig", "testH", "testI"],
};
for (var executable of qt.qtTestExecutables) {
Expand Down Expand Up @@ -124,7 +124,6 @@ function runTests(buildDirPath) {
}
// 6. Run a slot that has XFAIL
slot = qt.qtTestExecutables[0].slots[3];
// assert it's called testXFAIL
if (slot.name != "testXFAIL") {
console.error("Expected slot name to be testXFAIL");
process.exit(1);
Expand All @@ -134,6 +133,17 @@ function runTests(buildDirPath) {
console.error("Expected test to pass: " + slot.name);
process.exit(1);
}
// 7. Run a slot that has XPASS
slot = qt.qtTestExecutables[1].slots[3];
if (slot.name != "testXPASS") {
console.error("Expected slot name to be testXPASS");
process.exit(1);
}
yield slot.runTest();
if (!slot.lastTestFailure) {
console.error("Expected test to fail: " + slot.name);
process.exit(1);
}
});
}
function runCodeModelTests(codeModelFile) {
Expand Down
30 changes: 22 additions & 8 deletions src/qttest.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -315,18 +315,32 @@ export class QtTest {
if (event.at(0) != "assert") continue;

var obj = event.at(1);
let pass = obj["ok"] === true;

if (obj["ok"] === false) {
if (obj["todo"] !== false) {
// This is a QEXPECT_FAIL test, all good.
// QtTest outputs it as "todo"
continue;
}
let xfail = !pass && obj["todo"] !== false;
if (xfail) {
// This is a QEXPECT_FAIL test, all good.
// QtTest outputs it as "todo"
continue;
}

// There's an QEXPECT_FAIL but test passed, not good.
let xpass =
pass && obj["todo"].includes("returned TRUE unexpectedly");

if (!pass || xpass) {
// We found a failure
var filename = obj["diag"]["file"];
var lineNumber = obj["diag"]["line"];

var name = obj["name"].replace(/\(.*\)/, "");
var filename = "";
var lineNumber = -1;

if (obj["diag"]) {
filename = obj["diag"]["file"];
lineNumber = obj["diag"]["line"];
} else {
// A XPASS for example misses file:line info. Nothing we can do, it's a Qt bug arguably.
}

failedResults.push({
name: name,
Expand Down
18 changes: 16 additions & 2 deletions src/test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ async function runTests(buildDirPath: string) {
}
let expected_slots: ExpectedSlots = {
"test/qt_test/build-dev/test1": ["testA", "testB", "testC", "testXFAIL"],
"test/qt_test/build-dev/test2": ["testD", "testE", "testF"],
"test/qt_test/build-dev/test2": ["testD", "testE", "testF", "testXPASS"],
"test/qt_test/build-dev/test3": ["testAbortsEverythig", "testH", "testI"],
};

Expand Down Expand Up @@ -145,7 +145,7 @@ async function runTests(buildDirPath: string) {

// 6. Run a slot that has XFAIL
slot = qt.qtTestExecutables[0].slots![3];
// assert it's called testXFAIL

if (slot.name != "testXFAIL") {
console.error("Expected slot name to be testXFAIL");
process.exit(1);
Expand All @@ -156,6 +156,20 @@ async function runTests(buildDirPath: string) {
console.error("Expected test to pass: " + slot.name);
process.exit(1);
}

// 7. Run a slot that has XPASS
slot = qt.qtTestExecutables[1].slots![3];

if (slot.name != "testXPASS") {
console.error("Expected slot name to be testXPASS");
process.exit(1);
}

await slot.runTest();
if (!slot.lastTestFailure) {
console.error("Expected test to fail: " + slot.name);
process.exit(1);
}
}

async function runCodeModelTests(codeModelFile: string) {
Expand Down
4 changes: 4 additions & 0 deletions test/qt_test/test2.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,10 @@ private Q_SLOTS:
void testD() {}
void testE() {}
void testF() { QFAIL("failed"); }
void testXPASS() {
QEXPECT_FAIL("", "To be fixed", Continue);
QVERIFY(true);
}
};

QTEST_MAIN(MyTest);
Expand Down

0 comments on commit a60be6b

Please sign in to comment.