From 175b10d8e2a103a0d3e864d9516616f2850b3f45 Mon Sep 17 00:00:00 2001 From: Khafra Date: Wed, 8 May 2024 18:10:14 -0400 Subject: [PATCH] add pipe support for wpt server --- test/wpt/runner/runner.mjs | 3 + test/wpt/server/constants.mjs | 3 + test/wpt/server/server.mjs | 57 ++++-- test/wpt/server/util.mjs | 290 ++++++++++++++++++++++++++++++ test/wpt/start-FileAPI.mjs | 2 + test/wpt/start-cacheStorage.mjs | 2 + test/wpt/start-eventsource.mjs | 2 + test/wpt/start-fetch.mjs | 2 + test/wpt/start-mimesniff.mjs | 2 + test/wpt/start-websockets.mjs | 2 + test/wpt/status/fetch.status.json | 2 + 11 files changed, 348 insertions(+), 19 deletions(-) create mode 100644 test/wpt/server/constants.mjs create mode 100644 test/wpt/server/util.mjs diff --git a/test/wpt/runner/runner.mjs b/test/wpt/runner/runner.mjs index a24576f7415..33c1b757bbf 100644 --- a/test/wpt/runner/runner.mjs +++ b/test/wpt/runner/runner.mjs @@ -191,6 +191,9 @@ export class WPTRunner extends EventEmitter { } }) + worker.stdout.pipe(process.stdout) + worker.stderr.pipe(process.stderr) + const fileUrl = new URL(`/${this.#folderName}${test.slice(this.#folderPath.length)}`, 'http://wpt') fileUrl.pathname = fileUrl.pathname.replace(/\.js$/, '.html') fileUrl.search = variant diff --git a/test/wpt/server/constants.mjs b/test/wpt/server/constants.mjs new file mode 100644 index 00000000000..48444c16dd1 --- /dev/null +++ b/test/wpt/server/constants.mjs @@ -0,0 +1,3 @@ +export const symbols = { + kContent: Symbol('content') +} diff --git a/test/wpt/server/server.mjs b/test/wpt/server/server.mjs index 7e4bbb0fb75..eb8a57a94f7 100644 --- a/test/wpt/server/server.mjs +++ b/test/wpt/server/server.mjs @@ -7,6 +7,8 @@ import { createReadStream, readFileSync, existsSync } from 'node:fs' import { setTimeout as sleep } from 'node:timers/promises' import { route as networkPartitionRoute } from './routes/network-partition-key.mjs' import { route as redirectRoute } from './routes/redirect.mjs' +import { Pipeline } from './util.mjs' +import { symbols } from './constants.mjs' const tests = fileURLToPath(join(import.meta.url, '../../tests')) @@ -31,6 +33,11 @@ const stash = new Stash() const server = createServer(async (req, res) => { const fullUrl = new URL(req.url, `http://localhost:${server.address().port}`) + if (fullUrl.searchParams.has('pipe')) { + const pipe = new Pipeline(fullUrl.searchParams.get('pipe')) + res = await pipe.call(req, res) + } + switch (fullUrl.pathname) { case '/service-workers/cache-storage/resources/blank.html': { res.setHeader('content-type', 'text/html') @@ -55,7 +62,8 @@ const server = createServer(async (req, res) => { case '/fetch/data-urls/resources/base64.json': case '/fetch/data-urls/resources/data-urls.json': case '/fetch/api/resources/empty.txt': - case '/fetch/api/resources/data.json': { + case '/fetch/api/resources/data.json': + case '/common/text-plain.txt': { // If this specific resources requires custom headers const customHeadersPath = join(tests, fullUrl.pathname + '.headers') if (existsSync(customHeadersPath)) { @@ -74,9 +82,11 @@ const server = createServer(async (req, res) => { } // https://github.com/web-platform-tests/wpt/blob/6ae3f702a332e8399fab778c831db6b7dca3f1c6/fetch/api/resources/data.json - return createReadStream(join(tests, fullUrl.pathname)) + createReadStream(join(tests, fullUrl.pathname)) .on('end', () => res.end()) .pipe(res) + + break } case '/fetch/api/resources/trickle.py': { // Note: python's time.sleep(...) takes seconds, while setTimeout @@ -133,7 +143,7 @@ const server = createServer(async (req, res) => { } res.end() - return + break } case '/fetch/api/resources/stash-take.py': { // https://github.com/web-platform-tests/wpt/blob/6ae3f702a332e8399fab778c831db6b7dca3f1c6/fetch/api/resources/stash-take.py @@ -144,7 +154,8 @@ const server = createServer(async (req, res) => { const took = stash.take(key, fullUrl.pathname) ?? null res.write(JSON.stringify(took)) - return res.end() + res.end() + break } case '/fetch/api/resources/echo-content.py': { res.setHeader('X-Request-Method', req.method) @@ -268,15 +279,19 @@ const server = createServer(async (req, res) => { break } case '/fetch/connection-pool/resources/network-partition-key.py': { - return networkPartitionRoute(req, res, fullUrl) + networkPartitionRoute(req, res, fullUrl) + break } case '/resources/top.txt': { - return createReadStream(join(tests, 'fetch/api/', fullUrl.pathname)) + createReadStream(join(tests, 'fetch/api/', fullUrl.pathname)) .on('end', () => res.end()) .pipe(res) + + break } case '/fetch/api/resources/redirect.py': { - return redirectRoute(req, res, fullUrl) + redirectRoute(req, res, fullUrl) + break } case '/fetch/api/resources/method.py': { if (fullUrl.searchParams.has('cors')) { @@ -299,7 +314,7 @@ const server = createServer(async (req, res) => { } res.end() - return + break } case '/fetch/api/resources/clean-stash.py': { const token = fullUrl.searchParams.get('token') @@ -334,7 +349,7 @@ const server = createServer(async (req, res) => { if (req.headers.authorization) { res.end(req.headers.authorization) - return + break } res.end('none') @@ -363,7 +378,7 @@ const server = createServer(async (req, res) => { if (user === 'user' && password === 'password') { res.end('Authentication done') - return + break } const realm = fullUrl.searchParams.get('realm') ?? 'test' @@ -371,32 +386,32 @@ const server = createServer(async (req, res) => { res.statusCode = 401 res.setHeader('WWW-Authenticate', `Basic realm="${realm}"`) res.end('Please login with credentials \'user\' and \'password\'') - return + break } case '/fetch/api/resources/redirect-empty-location.py': { res.setHeader('location', '') res.statusCode = 302 res.end('') - return + break } case '/service-workers/cache-storage/resources/fetch-status.py': { const status = Number(fullUrl.searchParams.get('status')) res.statusCode = status res.end() - return + break } case '/service-workers/cache-storage/this-resource-should-not-exist': case '/service-workers/cache-storage/this-does-not-exist-please-dont-create-it': { res.statusCode = 404 res.end() - return + break } case '/service-workers/cache-storage/resources/vary.py': { if (fullUrl.searchParams.has('clear-vary-value-override-cookie')) { res.setHeader('cookie', '') res.end('vary cookie cleared') - return + break } const setCookieVary = fullUrl.searchParams.get('set-vary-value-override-cookie') ?? '' @@ -404,7 +419,7 @@ const server = createServer(async (req, res) => { if (setCookieVary) { res.setHeader('set-cookie', `vary-value-override=${setCookieVary}`) res.end('vary cookie set') - return + break } const cookieVary = req.headers.cookie?.split(';').find((c) => c.includes('vary-value-override=')) @@ -420,7 +435,7 @@ const server = createServer(async (req, res) => { } res.end('vary response') - return + break } case '/eventsource/resources/message.py': { const mime = fullUrl.searchParams.get('mime') ?? 'text/event-stream' @@ -435,7 +450,7 @@ const server = createServer(async (req, res) => { res.end() }, sleep) - return + break } case '/eventsource/resources/last-event-id.py': { const lastEventId = req.headers['Last-Event-ID'] ?? '' @@ -451,13 +466,17 @@ const server = createServer(async (req, res) => { res.end() } - return + break } default: { res.statusCode = 200 res.end(fullUrl.toString()) } } + + if (res[symbols.kContent]) { + res.write(res[symbols.kContent]) + } }).listen(0) await once(server, 'listening') diff --git a/test/wpt/server/util.mjs b/test/wpt/server/util.mjs new file mode 100644 index 00000000000..bda9c520513 --- /dev/null +++ b/test/wpt/server/util.mjs @@ -0,0 +1,290 @@ +import { symbols } from './constants.mjs' + +// Adapted from +// https://github.com/web-platform-tests/wpt/blob/a9f92afbb621a71872388c843858553808bb9fa6/tools/wptserve/wptserve/pipes.py + +class ReplacementTokenizer { + constructor () { + this.tokenTypes = [ + [/\$\w+:/, this.var.bind(this)], + [/\$?\w+/, this.ident.bind(this)], + [/\[[^\]]*\]/, this.index.bind(this)], + [/\([^)]*\)/, this.arguments.bind(this)] + ] + } + + arguments (token) { + const unwrapped = token.slice(1, -1) + const args = unwrapped.toString('utf8').split(/,\s*/) + return ['arguments', args] + } + + ident (token) { + const value = token.toString('utf8') + return ['ident', value] + } + + index (token) { + let value = token.slice(1, -1).toString('utf8') + value = isNaN(value) ? value : parseInt(value) + return ['index', value] + } + + var (token) { + const value = token.slice(0, -1).toString('utf8') + return ['var', value] + } + + tokenize (string) { + const tokens = [] + while (string.length > 0) { + let matched = false + for (const [pattern, handler] of this.tokenTypes) { + const match = string.match(pattern) + if (match) { + tokens.push(handler(match[0])) + string = string.slice(match[0].length) + matched = true + break + } + } + if (!matched) { + throw new Error(`Invalid token at position ${string}`) + } + } + return tokens + } +} + +class PipeTokenizer { + constructor () { + this.state = null + this.string = '' + this.index = 0 + } + + * tokenize (string) { + this.string = string + this.state = this.funcNameState + this.index = 0 + while (this.state) { + yield this.state() + } + yield null + } + + getChar () { + if (this.index >= this.string.length) { + return null + } + const char = this.string[this.index] + this.index++ + return char + } + + funcNameState () { + let rv = '' + while (true) { + const char = this.getChar() + if (char === null) { + this.state = null + if (rv) { + return ['function', rv] + } else { + return null + } + } else if (char === '(') { + this.state = this.argumentState + return ['function', rv] + } else if (char === '|') { + if (rv) { + return ['function', rv] + } + } else { + rv += char + } + } + } + + argumentState () { + let rv = '' + while (true) { + const char = this.getChar() + if (char === null) { + // this.state = null; + return ['argument', rv] + } else if (char === '\\') { + rv += this.getEscape() + if (rv === null) { + // This should perhaps be an error instead + return ['argument', rv] + } + } else if (char === ',') { + return ['argument', rv] + } else if (char === ')') { + this.state = this.funcNameState + return ['argument', rv] + } else { + rv += char + } + } + } + + getEscape () { + const char = this.getChar() + const escapes = { + n: '\n', + r: '\r', + t: '\t' + } + return escapes[char] || char + } +} + +export class Pipeline { + static pipes = {} + + constructor (pipeString) { + this.pipeFunctions = this.parse(pipeString) + } + + parse (pipeString) { + const functions = [] + const tokenizer = new PipeTokenizer() + for (const item of tokenizer.tokenize(pipeString)) { + if (!item) { + break + } + if (item[0] === 'function') { + if (!Pipeline.pipes[item[1]]) { + throw new Error(`Pipe function ${item[1]} is not implemented.`) + } + + functions.push([Pipeline.pipes[item[1]], []]) + } else if (item[0] === 'argument') { + functions[functions.length - 1][1].push(item[1]) + } + } + return functions + } + + /** + * @param {import('node:http').IncomingMessage} request + * @param {import('node:http').ServerResponse} response + * @returns + */ + async call (request, response) { + let res = response + for (const [func, args] of this.pipeFunctions) { + res = await func(request, res, ...args) + } + return res + } +} + +/** + * @param {import('node:http').IncomingMessage} req + * @param {import('node:http').ServerResponse} res + */ +Pipeline.pipes.sub = async (req, res, escapeType = 'html') => { + let content = '' + + for await (const chunk of req) { + content += chunk + } + + content = template(req, content) + res[symbols.kContent] = content + + return res +} + +Pipeline.pipes.slice = async (req, res, start, end = null) => { + let content = '' + + for await (const chunk of req) { + content += chunk + } + + start ??= 0 + end ??= content.length + content = content.slice(start, end) + + res.setHeader('content-length', `${content.length}`) + res[symbols.kContent] = content + + return res +} + +Pipeline.pipes.status = (req, res, code) => { + res.statusCode = code + return res +} + +Pipeline.pipes.header = (req, res, name, value, append = false) => { + if (!append) { + res.setHeader(name, value) + } else { + res.appendHeader(name, value) + } + + return res +} + +function template (request, content, escapeType = 'html') { + const tokenizer = new ReplacementTokenizer() + const variables = {} + + function configReplacement (match) { + const content = match + const tokens = tokenizer.tokenize(content) + const variable = null + + let [tokenType, field] = tokens.shift().split(':') + let value + + if (!field) { + field = tokenType + tokenType = null + } + + if (field === 'headers') { + value = request.headers + } else { + throw new Error(`Undefined template variable: ${field}`) + } + + while (tokens.length > 0) { + const [ttype, tfield] = tokens.shift().split(':') + if (ttype === 'index') { + value = value[tfield] + } else if (ttype === 'arguments') { + const args = tfield.split(',') + value = value(request, ...args) + } else { + throw new Error(`Unexpected token type: ${ttype}`) + } + } + + if (variable !== null) { + variables[variable] = value + } + + const escapeFunc = { + html: (x) => escape(x), + none: (x) => x + }[escapeType] + + let result = value + if (typeof result === 'object') { + result = JSON.stringify(result) + } + + return escapeFunc(result) + } + + const templateRegexp = /{{([^}]*)}}/g + const newContent = content.replace(templateRegexp, configReplacement) + + return newContent +} diff --git a/test/wpt/start-FileAPI.mjs b/test/wpt/start-FileAPI.mjs index 5a92ab8b1f8..012acd28a7a 100644 --- a/test/wpt/start-FileAPI.mjs +++ b/test/wpt/start-FileAPI.mjs @@ -10,6 +10,8 @@ const child = fork(serverPath, [], { stdio: ['pipe', 'pipe', 'pipe', 'ipc'] }) +child.stdout.pipe(process.stdout) +child.stderr.pipe(process.stderr) child.on('exit', (code) => process.exit(code)) for await (const [message] of on(child, 'message')) { diff --git a/test/wpt/start-cacheStorage.mjs b/test/wpt/start-cacheStorage.mjs index a630e052285..fe818dddc14 100644 --- a/test/wpt/start-cacheStorage.mjs +++ b/test/wpt/start-cacheStorage.mjs @@ -10,6 +10,8 @@ const child = fork(serverPath, [], { stdio: ['pipe', 'pipe', 'pipe', 'ipc'] }) +child.stdout.pipe(process.stdout) +child.stderr.pipe(process.stderr) child.on('exit', (code) => process.exit(code)) for await (const [message] of on(child, 'message')) { diff --git a/test/wpt/start-eventsource.mjs b/test/wpt/start-eventsource.mjs index 44d7df30f83..f2a92562108 100644 --- a/test/wpt/start-eventsource.mjs +++ b/test/wpt/start-eventsource.mjs @@ -10,6 +10,8 @@ const child = fork(serverPath, [], { stdio: ['pipe', 'pipe', 'pipe', 'ipc'] }) +child.stdout.pipe(process.stdout) +child.stderr.pipe(process.stderr) child.on('exit', (code) => process.exit(code)) for await (const [message] of on(child, 'message')) { diff --git a/test/wpt/start-fetch.mjs b/test/wpt/start-fetch.mjs index 59c9f830916..0e2168e610c 100644 --- a/test/wpt/start-fetch.mjs +++ b/test/wpt/start-fetch.mjs @@ -12,6 +12,8 @@ const child = fork(serverPath, [], { stdio: ['pipe', 'pipe', 'pipe', 'ipc'] }) +child.stdout.pipe(process.stdout) +child.stderr.pipe(process.stderr) child.on('exit', (code) => process.exit(code)) for await (const [message] of on(child, 'message')) { diff --git a/test/wpt/start-mimesniff.mjs b/test/wpt/start-mimesniff.mjs index fbdb9bf0ece..ebe4e783e3b 100644 --- a/test/wpt/start-mimesniff.mjs +++ b/test/wpt/start-mimesniff.mjs @@ -12,6 +12,8 @@ const child = fork(serverPath, [], { stdio: ['pipe', 'pipe', 'pipe', 'ipc'] }) +child.stdout.pipe(process.stdout) +child.stderr.pipe(process.stderr) child.on('exit', (code) => process.exit(code)) for await (const [message] of on(child, 'message')) { diff --git a/test/wpt/start-websockets.mjs b/test/wpt/start-websockets.mjs index 79aa297b265..ec42af0eeb1 100644 --- a/test/wpt/start-websockets.mjs +++ b/test/wpt/start-websockets.mjs @@ -28,6 +28,8 @@ const child = fork(serverPath, [], { stdio: ['pipe', 'pipe', 'pipe', 'ipc'] }) +child.stdout.pipe(process.stdout) +child.stderr.pipe(process.stderr) child.on('exit', (code) => process.exit(code)) for await (const [message] of on(child, 'message')) { diff --git a/test/wpt/status/fetch.status.json b/test/wpt/status/fetch.status.json index 2b713611676..c94ef2f1c3b 100644 --- a/test/wpt/status/fetch.status.json +++ b/test/wpt/status/fetch.status.json @@ -446,6 +446,8 @@ }, "security": { "1xx-response.any.js": { + "note": "TODO(@KhafraDev): investigate timeout", + "skip": true, "fail": [ "Status(100) should be ignored.", "Status(101) should be accepted, with removing body.",