From 9056afd8594fb734336c9ff4ae3634e92b46ec80 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Tue, 4 Dec 2018 17:44:20 +0800 Subject: [PATCH 1/3] test: remove reference to whatwg in file names under test/wpt WPT covers standards in both W3C and WHATWG, as such it would be strange to make this disparity explicit in our file names (e.g. when testing standards that are solely in W3C, like performance-timeline). Remove the reference to WHATWG will also make the file names shorter. --- test/wpt/README.md | 6 +++--- test/wpt/{test-whatwg-console.js => test-console.js} | 0 test/wpt/{test-whatwg-url.js => test-url.js} | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename test/wpt/{test-whatwg-console.js => test-console.js} (100%) rename test/wpt/{test-whatwg-url.js => test-url.js} (100%) diff --git a/test/wpt/README.md b/test/wpt/README.md index 1810a98c8dc982..ff9092e6a1acd9 100644 --- a/test/wpt/README.md +++ b/test/wpt/README.md @@ -40,7 +40,7 @@ $ git node wpt url ### 3. Create the test driver -For example, for the URL tests, add a file `test/wpt/test-whatwg-url.js`: +For example, for the URL tests, add a file `test/wpt/test-url.js`: ```js 'use strict'; @@ -75,14 +75,14 @@ Run the test using `tools/test.py` and see if there are any failures. For example, to run all the URL tests under `test/fixtures/wpt/url`: ```text -$ tools/test.py wpt/test-whatwg-url +$ tools/test.py wpt/test-url ``` To run a specific test in WPT, for example, `url/url-searchparams.any.js`, pass the file name as argument to the corresponding test driver: ```text -node --expose-internals test/wpt/test-whatwg-url.js url-searchparams.any.js +node --expose-internals test/wpt/test-url.js url-searchparams.any.js ``` If there are any failures, update the corresponding status file diff --git a/test/wpt/test-whatwg-console.js b/test/wpt/test-console.js similarity index 100% rename from test/wpt/test-whatwg-console.js rename to test/wpt/test-console.js diff --git a/test/wpt/test-whatwg-url.js b/test/wpt/test-url.js similarity index 100% rename from test/wpt/test-whatwg-url.js rename to test/wpt/test-url.js From b7d41c1da33b2c192b0a0d703660a086491948e5 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Sun, 18 Nov 2018 13:15:15 +0800 Subject: [PATCH 2/3] test: improve WPT runner name matching This patch: - Support wildcards(*) in WPT runner name matching (needed by e.g. encoding where all the tests requires i18n support in the build) - Print failure reasons when encountering an expected failure - Fix a bug in copyGlobalsFromObject (previously it copies properties from `global` instead of the given `obj`) Previously an expected failure is printed as ``` [EXPECTED_FAILURE] response.formData() with input: %61+%4d%4D= ``` Now it is printed as ``` [EXPECTED_FAILURE] response.formData() with input: %61+%4d%4D= missing Request and Response ``` --- test/common/wpt.js | 137 ++++++++++++++++++++++++++++++++------------- 1 file changed, 98 insertions(+), 39 deletions(-) diff --git a/test/common/wpt.js b/test/common/wpt.js index 4f2f39c8e595d2..be6907e7cd5e8b 100644 --- a/test/common/wpt.js +++ b/test/common/wpt.js @@ -66,20 +66,92 @@ class ResourceLoader { } } +class StatusRule { + constructor(key, value, pattern = undefined) { + this.key = key; + this.requires = value.requires || []; + this.fail = value.fail; + this.skip = value.skip; + if (pattern) { + this.pattern = this.transformPattern(pattern); + } + // TODO(joyeecheung): implement this + this.scope = value.scope; + this.comment = value.comment; + } + + /** + * Transform a filename pattern into a RegExp + * @param {string} pattern + * @returns {RegExp} + */ + transformPattern(pattern) { + const result = pattern.replace(/[-/\\^$+?.()|[\]{}]/g, '\\$&'); + return new RegExp(result.replace('*', '.*')); + } +} + +class StatusRuleSet { + constructor() { + // We use two sets of rules to speed up matching + this.exactMatch = {}; + this.patternMatch = []; + } + + /** + * @param {object} rules + */ + addRules(rules) { + for (const key of Object.keys(rules)) { + if (key.includes('*')) { + this.patternMatch.push(new StatusRule(key, rules[key], key)); + } else { + this.exactMatch[key] = new StatusRule(key, rules[key]); + } + } + } + + match(file) { + const result = []; + const exact = this.exactMatch[file]; + if (exact) { + result.push(exact); + } + for (const item of this.patternMatch) { + if (item.pattern.test(file)) { + result.push(item); + } + } + return result; + } +} + class WPTTest { /** * @param {string} mod * @param {string} filename - * @param {string[]} requires - * @param {string | undefined} failReason - * @param {string | undefined} skipReason + * @param {StatusRule[]} rules */ - constructor(mod, filename, requires, failReason, skipReason) { + constructor(mod, filename, rules) { this.module = mod; // name of the WPT module, e.g. 'url' this.filename = filename; // name of the test file - this.requires = requires; - this.failReason = failReason; - this.skipReason = skipReason; + + this.requires = new Set(); + this.failReasons = []; + this.skipReasons = []; + for (const item of rules) { + if (item.requires.length) { + for (const req of item.requires) { + this.requires.add(req); + } + } + if (item.fail) { + this.failReasons.push(item.fail); + } + if (item.skip) { + this.skipReasons.push(item.skip); + } + } } getAbsolutePath() { @@ -90,12 +162,8 @@ class WPTTest { return fs.readFileSync(this.getAbsolutePath(), 'utf8'); } - shouldSkip() { - return this.failReason || this.skipReason; - } - requireIntl() { - return this.requires.includes('intl'); + return this.requires.has('intl'); } } @@ -103,41 +171,28 @@ class StatusLoader { constructor(path) { this.path = path; this.loaded = false; - this.status = null; + this.rules = new StatusRuleSet(); /** @type {WPTTest[]} */ this.tests = []; } - loadTest(file) { - let requires = []; - let failReason; - let skipReason; - if (this.status[file]) { - requires = this.status[file].requires || []; - failReason = this.status[file].fail; - skipReason = this.status[file].skip; - } - return new WPTTest(this.path, file, requires, - failReason, skipReason); - } - load() { const dir = path.join(__dirname, '..', 'wpt'); const statusFile = path.join(dir, 'status', `${this.path}.json`); const result = JSON.parse(fs.readFileSync(statusFile, 'utf8')); - this.status = result; + this.rules.addRules(result); const list = fs.readdirSync(fixtures.path('wpt', this.path)); for (const file of list) { - this.tests.push(this.loadTest(file)); + if (!(/\.\w+\.js$/.test(file))) { + continue; + } + const match = this.rules.match(file); + this.tests.push(new WPTTest(this.path, file, match)); } this.loaded = true; } - - get jsTests() { - return this.tests.filter((test) => test.filename.endsWith('.js')); - } } const PASSED = 1; @@ -156,7 +211,7 @@ class WPTRunner { this.status = new StatusLoader(path); this.status.load(); this.tests = new Map( - this.status.jsTests.map((item) => [item.filename, item]) + this.status.tests.map((item) => [item.filename, item]) ); this.results = new Map(); @@ -171,7 +226,10 @@ class WPTRunner { */ copyGlobalsFromObject(obj, names) { for (const name of names) { - const desc = Object.getOwnPropertyDescriptor(global, name); + const desc = Object.getOwnPropertyDescriptor(obj, name); + if (!desc) { + assert.fail(`${name} does not exist on the object`); + } this.globals.set(name, desc); } } @@ -328,8 +386,9 @@ class WPTRunner { for (const item of items) { switch (item.type) { case FAILED: { - if (test.failReason) { + if (test.failReasons.length) { console.log(`[EXPECTED_FAILURE] ${item.test.name}`); + console.log(test.failReasons.join('; ')); } else { console.log(`[UNEXPECTED_FAILURE] ${item.test.name}`); unexpectedFailures.push([title, filename, item]); @@ -386,10 +445,10 @@ class WPTRunner { }); } - skip(filename, reason) { + skip(filename, reasons) { this.addResult(filename, { type: SKIPPED, - reason + reason: reasons.join('; ') }); } @@ -435,8 +494,8 @@ class WPTRunner { const queue = []; for (const test of this.tests.values()) { const filename = test.filename; - if (test.skipReason) { - this.skip(filename, test.skipReason); + if (test.skipReasons.length > 0) { + this.skip(filename, test.skipReasons); continue; } From 8671556b91b1df6c820fa456197c754c2f36f704 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Sat, 15 Dec 2018 04:21:13 +0800 Subject: [PATCH 3/3] fixup! test: improve WPT runner name matching --- test/common/wpt.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/common/wpt.js b/test/common/wpt.js index be6907e7cd5e8b..2a25a22af53710 100644 --- a/test/common/wpt.js +++ b/test/common/wpt.js @@ -500,7 +500,7 @@ class WPTRunner { } if (!common.hasIntl && test.requireIntl()) { - this.skip(filename, 'missing Intl'); + this.skip(filename, [ 'missing Intl' ]); continue; }