Skip to content

Commit

Permalink
test: update WPT runner
Browse files Browse the repository at this point in the history
PR-URL: #43455
Reviewed-By: Rich Trott <[email protected]>
Reviewed-By: Joyee Cheung <[email protected]>
  • Loading branch information
panva authored Aug 10, 2022
1 parent 9b8ba25 commit c3ff198
Show file tree
Hide file tree
Showing 17 changed files with 3,606 additions and 136 deletions.
80 changes: 66 additions & 14 deletions test/common/wpt.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ const fixtures = require('../common/fixtures');
const fs = require('fs');
const fsPromises = fs.promises;
const path = require('path');
const events = require('events');
const { inspect } = require('util');
const { Worker } = require('worker_threads');

Expand Down Expand Up @@ -152,21 +153,30 @@ class WPTTestSpec {
this.filename = filename;

this.requires = new Set();
this.failReasons = [];
this.failedTests = [];
this.flakyTests = [];
this.skipReasons = [];
for (const item of rules) {
if (item.requires.length) {
for (const req of item.requires) {
this.requires.add(req);
}
}
if (item.fail) {
this.failReasons.push(item.fail);
if (Array.isArray(item.fail?.expected)) {
this.failedTests.push(...item.fail.expected);
}
if (Array.isArray(item.fail?.flaky)) {
this.failedTests.push(...item.fail.flaky);
this.flakyTests.push(...item.fail.flaky);
}
if (item.skip) {
this.skipReasons.push(item.skip);
}
}

this.failedTests = [...new Set(this.failedTests)];
this.flakyTests = [...new Set(this.flakyTests)];
this.skipReasons = [...new Set(this.skipReasons)];
}

getRelativePath() {
Expand Down Expand Up @@ -368,7 +378,7 @@ class WPTRunner {

// TODO(joyeecheung): work with the upstream to port more tests in .html
// to .js.
runJsTests() {
async runJsTests() {
let queue = [];

// If the tests are run as `node test/wpt/test-something.js subset.any.js`,
Expand Down Expand Up @@ -459,6 +469,8 @@ class WPTRunner {
);
this.inProgress.delete(testFileName);
});

await events.once(worker, 'exit').catch(() => {});
}

process.on('exit', () => {
Expand All @@ -469,34 +481,72 @@ class WPTRunner {
}
}
inspect.defaultOptions.depth = Infinity;
console.log(this.results);
// Sorts the rules to have consistent output
console.log(JSON.stringify(Object.keys(this.results).sort().reduce(
(obj, key) => {
obj[key] = this.results[key];
return obj;
},
{}
), null, 2));

const failures = [];
let expectedFailures = 0;
let skipped = 0;
for (const key of Object.keys(this.results)) {
const item = this.results[key];
if (item.fail && item.fail.unexpected) {
for (const [key, item] of Object.entries(this.results)) {
if (item.fail?.unexpected) {
failures.push(key);
}
if (item.fail && item.fail.expected) {
if (item.fail?.expected) {
expectedFailures++;
}
if (item.skip) {
skipped++;
}
}

const unexpectedPasses = [];
for (const [key, specMap] of this.specMap) {
// File has no expected failures
if (!specMap.failedTests.length) {
continue;
}

// File was (maybe even conditionally) skipped
if (this.results[key]?.skip) {
continue;
}

// Full check: every expected to fail test is present
if (specMap.failedTests.some((expectedToFail) => {
if (specMap.flakyTests.includes(expectedToFail)) {
return false;
}
return this.results[key]?.fail?.expected?.includes(expectedToFail) !== true;
})) {
unexpectedPasses.push(key);
continue;
}
}

const ran = total - skipped;
const passed = ran - expectedFailures - failures.length;
console.log(`Ran ${ran}/${total} tests, ${skipped} skipped,`,
`${passed} passed, ${expectedFailures} expected failures,`,
`${failures.length} unexpected failures`);
`${failures.length} unexpected failures,`,
`${unexpectedPasses.length} unexpected passes`);
if (failures.length > 0) {
const file = path.join('test', 'wpt', 'status', `${this.path}.json`);
throw new Error(
`Found ${failures.length} unexpected failures. ` +
`Consider updating ${file} for these files:\n${failures.join('\n')}`);
}
if (unexpectedPasses.length > 0) {
const file = path.join('test', 'wpt', 'status', `${this.path}.json`);
throw new Error(
`Found ${unexpectedPasses.length} unexpected passes. ` +
`Consider updating ${file} for these files:\n${unexpectedPasses.join('\n')}`);
}
});
}

Expand Down Expand Up @@ -577,8 +627,9 @@ class WPTRunner {
if (!result[item.status][key]) {
result[item.status][key] = [];
}
if (result[item.status][key].indexOf(item.reason) === -1) {
result[item.status][key].push(item.reason);
const hasName = result[item.status][key].includes(item.name);
if (!hasName) {
result[item.status][key].push(item.name);
}
}
}
Expand All @@ -589,10 +640,10 @@ class WPTRunner {

fail(filename, test, status) {
const spec = this.specMap.get(filename);
const expected = !!(spec.failReasons.length);
const expected = spec.failedTests.includes(test.name);
if (expected) {
console.log(`[EXPECTED_FAILURE][${status.toUpperCase()}] ${test.name}`);
console.log(spec.failReasons.join('; '));
console.log(test.message || status);
} else {
console.log(`[UNEXPECTED_FAILURE][${status.toUpperCase()}] ${test.name}`);
}
Expand All @@ -604,6 +655,7 @@ class WPTRunner {
` ${require.main.filename} ${filename}`;
console.log(`Command: ${command}\n`);
this.addTestResult(filename, {
name: test.name,
expected,
status: kFail,
reason: test.message || status
Expand Down
19 changes: 16 additions & 3 deletions test/wpt/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,11 @@ add this to `test/wpt/status/url.json`:

```json
"url-searchparams.any.js": {
"fail": "explain why the test fails, ideally with links"
"fail": {
"expected": [
"test name in the WPT test case, e.g. second argument passed to test()"
]
}
}
```

Expand Down Expand Up @@ -155,8 +159,17 @@ expected failures.
// Optional: the test will be skipped with the reason printed
"skip": "explain why we cannot run a test that's supposed to pass",
// Optional: the test will be skipped with the reason printed
"fail": "explain why we the test is expected to fail"
// Optional: failing tests
"fail": {
"note": "You may leave an optional arbitrary note e.g. with TODOs",
"expected": [
"test name in the WPT test case, e.g. second argument passed to test()",
"another test name"
],
"flaky": [
"flaky test name"
]
}
}
}
```
Expand Down
43 changes: 38 additions & 5 deletions test/wpt/status/FileAPI/blob.json
Original file line number Diff line number Diff line change
@@ -1,14 +1,47 @@
{
"Blob-constructor.any.js": {
"skip": "Depends on File API"
},
"Blob-constructor-dom.window.js": {
"skip": "Depends on DOM API"
},
"Blob-slice.any.js": {
"skip": "Depends on File API"
"Blob-constructor.any.js": {
"fail": {
"note": "Depends on File API",
"expected": [
"A plain object with @@iterator should be treated as a sequence for the blobParts argument.",
"A plain object with @@iterator and a length property should be treated as a sequence for the blobParts argument.",
"A String object should be treated as a sequence for the blobParts argument.",
"A Uint8Array object should be treated as a sequence for the blobParts argument.",
"Getters and value conversions should happen in order until an exception is thrown.",
"Changes to the blobParts array should be reflected in the returned Blob (pop).",
"Changes to the blobParts array should be reflected in the returned Blob (unshift).",
"ToString should be called on elements of the blobParts array.",
"ArrayBuffer elements of the blobParts array should be supported.",
"Passing typed arrays as elements of the blobParts array should work.",
"Passing a Float64Array as element of the blobParts array should work.",
"Array with two blobs",
"Array with two buffers",
"Array with two bufferviews",
"Array with mixed types",
"options properties should be accessed in lexicographic order.",
"Arguments should be evaluated from left to right.",
"Passing null (index 0) for options should use the defaults.",
"Passing null (index 0) for options should use the defaults (with newlines).",
"Passing undefined (index 1) for options should use the defaults.",
"Passing undefined (index 1) for options should use the defaults (with newlines).",
"Passing object \"[object Object]\" (index 2) for options should use the defaults.",
"Passing object \"[object Object]\" (index 2) for options should use the defaults (with newlines).",
"Passing object \"[object Object]\" (index 3) for options should use the defaults.",
"Passing object \"[object Object]\" (index 3) for options should use the defaults (with newlines).",
"Passing object \"/regex/\" (index 4) for options should use the defaults.",
"Passing object \"/regex/\" (index 4) for options should use the defaults (with newlines).",
"Passing function \"function() {}\" (index 5) for options should use the defaults.",
"Passing function \"function() {}\" (index 5) for options should use the defaults (with newlines)."
]
}
},
"Blob-in-worker.worker.js": {
"skip": "Depends on Web Workers API"
},
"Blob-slice.any.js": {
"skip": "Depends on File API"
}
}
Loading

0 comments on commit c3ff198

Please sign in to comment.