Skip to content

Commit

Permalink
Separate raw and parsed IDL in in-memory spec structure
Browse files Browse the repository at this point in the history 
The in-memory representation of a crawled spec now properly separates the raw
IDL, stored in an `idl` property, and the parsed IDL, stored in an `idlparsed`
property.

This is a breaking change for projects that use `expandCrawlResults` to load a
crawl report into memory, and a major bump will be needed.

See discussion in #814 (comment)
  • Loading branch information
@tidoust
tidoust committed Jan 7, 2022
1 parent b2d46f6 commit 747df3c
Show file tree
Hide file tree
Showing 6 changed files with 61 additions and 114 deletions.
2 changes: 1 addition & 1 deletion src/cli/check-missing-dfns.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -399,7 +399,7 @@ function checkSpecDefinitions(spec, options = {}) {
(spec.css || {});
const idl = (typeof spec.idlparsed === "string") ?
require(path.resolve(options.rootFolder, spec.idlparsed)).idlparsed :
spec.idl;
spec.idlparsed;

// Make sure that all expected CSS definitions exist in the dfns extract
const expectedCSSDfns = getExpectedDfnsFromCSS(css);
Expand Down
25 changes: 7 additions & 18 deletions src/cli/generate-idlnames.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ const { matchIdlDfn, getExpectedDfnFromIdlDesc } = require('./check-missing-dfns
const {
expandCrawlResult,
isLatestLevelThatPasses,
requireFromWorkingDirectory
requireFromWorkingDirectory,
createFolderIfNeeded
} = require('../lib/util');


Expand Down Expand Up @@ -118,7 +119,7 @@ function generateIdlNames(results, options = {}) {
const names = {};

function defineIDLContent(spec) {
return spec.idl && (spec.idl.idlNames || spec.idl.idlExtendedNames);
return spec.idlparsed?.idlNames || spec.idlparsed?.idlExtendedNames;
}

// Only keep latest version of specs and delta specs that define some IDL
Expand All @@ -129,10 +130,10 @@ function generateIdlNames(results, options = {}) {
// Add main definitions of all IDL names
// (using the latest version of a spec that defines some IDL)
results.forEach(spec => {
if (!spec.idl || !spec.idl.idlNames) {
if (!spec.idlparsed.idlNames) {
return;
}
Object.entries(spec.idl.idlNames).forEach(([name, idl]) => {
Object.entries(spec.idlparsed.idlNames).forEach(([name, idl]) => {
const desc = Object.assign(specInfo(spec), { fragment: idl.fragment });
fragments[idl.fragment] = idl;

Expand All @@ -157,10 +158,10 @@ function generateIdlNames(results, options = {}) {

// Add definitions that extend base definitions
results.forEach(spec => {
if (!spec.idl || !spec.idl.idlExtendedNames) {
if (!spec.idlparsed.idlExtendedNames) {
return;
}
Object.entries(spec.idl.idlExtendedNames).forEach(([name, extensions]) =>
Object.entries(spec.idlparsed.idlExtendedNames).forEach(([name, extensions]) =>
extensions.forEach(idl => {
const desc = Object.assign(specInfo(spec), { fragment: idl.fragment });
fragments[idl.fragment] = idl;
Expand Down Expand Up @@ -308,18 +309,6 @@ async function generateIdlNamesFromPath(crawlPath, options = {}) {
}


async function createFolderIfNeeded(name) {
try {
await fs.promises.mkdir(name);
}
catch (err) {
if (err.code !== 'EEXIST') {
throw err;
}
}
}


/**
* Save IDL names to individual JSON files in the given folder
*
Expand Down
30 changes: 7 additions & 23 deletions src/cli/generate-idlparsed.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ const path = require('path');
const webidlParser = require('../cli/parse-webidl');
const {
expandCrawlResult,
requireFromWorkingDirectory
requireFromWorkingDirectory,
createFolderIfNeeded
} = require('../lib/util');


Expand All @@ -47,19 +48,15 @@ async function generateIdlParsed(spec) {
if (!spec?.idl) {
return spec;
}
const rawIdl = spec.idl.idl ?? spec.idl;
try {
const parsedIdl = await webidlParser.parse(rawIdl);
parsedIdl.hasObsoleteIdl = webidlParser.hasObsoleteIdl(rawIdl);
parsedIdl.idl = rawIdl;
spec.idl = parsedIdl;
spec.idlparsed = await webidlParser.parse(spec.idl);
spec.idlparsed.hasObsoleteIdl = webidlParser.hasObsoleteIdl(spec.idl);
}
catch (err) {
// IDL content is invalid and cannot be parsed.
// Let's return the error, along with the raw IDL
// content so that it may be saved to a file.
err.idl = rawIdl;
spec.idl = err;
spec.idlparsed = err;
}
return spec;
}
Expand All @@ -73,18 +70,6 @@ async function generateIdlParsedFromPath(crawlPath) {
}


async function createFolderIfNeeded(name) {
try {
await fs.promises.mkdir(name);
}
catch (err) {
if (err.code !== 'EEXIST') {
throw err;
}
}
}


/**
* Generate the `idlparsed` export for the spec.
*
Expand All @@ -110,13 +95,12 @@ async function saveIdlParsed(spec, folder) {
const subfolder = path.join(folder, 'idlparsed');
await createFolderIfNeeded(subfolder);

if (!spec?.idl?.idl) {
if (!spec?.idlparsed) {
return;
}

delete spec.idl.idl;
const json = JSON.stringify(
Object.assign(specInfo(spec), { idlparsed: spec.idl }),
Object.assign(specInfo(spec), { idlparsed: spec.idlparsed }),
null, 2);
const filename = path.join(subfolder, spec.shortname + '.json');
await fs.promises.writeFile(filename, json);
Expand Down
32 changes: 14 additions & 18 deletions src/lib/specs-crawler.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@ const {
isLatestLevelThatPasses,
processSpecification,
setupBrowser,
teardownBrowser
teardownBrowser,
createFolderIfNeeded
} = require('./util');


Expand Down Expand Up @@ -138,6 +139,9 @@ async function crawlSpec(spec, crawlOptions) {
crawlOptions.modules.forEach(mod => {
if (result[mod.property]) {
spec[mod.property] = result[mod.property];
if (mod.property === 'idl') {
spec.idlparsed = result.idlparsed;
}
}
});
}
Expand Down Expand Up @@ -173,14 +177,7 @@ async function saveSpecResults(spec, settings) {

async function getSubfolder(name) {
let subfolder = path.join(settings.output, name);
try {
await fs.promises.mkdir(subfolder);
}
catch (err) {
if (err.code !== 'EEXIST') {
throw err;
}
}
await createFolderIfNeeded(subfolder);
return subfolder;
}

Expand Down Expand Up @@ -229,13 +226,14 @@ async function saveSpecResults(spec, settings) {
// (https://github.com/w3c/webref)
// Source: ${spec.title} (${spec.crawled})`;
idlHeader = idlHeader.replace(/^\s+/gm, '').trim() + '\n\n';
let idl = spec.idl.idl
let idl = spec.idl
.replace(/\s+$/gm, '\n')
.replace(/\t/g, ' ')
.trim();
idl = idlHeader + idl + '\n';
await fs.promises.writeFile(
path.join(folders.idl, spec.shortname + '.idl'), idl);
return `idl/${spec.shortname}.idl`;
};

async function saveCss(spec) {
Expand All @@ -252,17 +250,15 @@ async function saveSpecResults(spec, settings) {
}, 2) + '\n';
const pathname = path.join(folders.css, spec.shortname + '.json')
await fs.promises.writeFile(pathname, json);
spec.css = `css/${spec.shortname}.json`;
return `css/${spec.shortname}.json`;
};

// Save IDL dumps
if (spec.idl && spec.idl.idl) {
await saveIdl(spec);
spec.idlparsed = await saveIdlParsed(spec, settings.output);
spec.idl = `idl/${spec.shortname}.idl`;
if (spec.idl) {
spec.idl = await saveIdl(spec);
}
else if (spec.idl) {
delete spec.idl;
if (spec.idlparsed) {
spec.idlparsed = await saveIdlParsed(spec, settings.output);
}

// Save CSS dumps
Expand All @@ -273,7 +269,7 @@ async function saveSpecResults(spec, settings) {
(Object.keys(spec.css.valuespaces || {}).length > 0));
}
if (defineCSSContent(spec)) {
await saveCss(spec);
spec.css = await saveCss(spec);
}

// Specs that define CSS now have a "css" key that point to the CSS extract.
Expand Down
62 changes: 30 additions & 32 deletions src/lib/util.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -717,30 +717,6 @@ async function expandCrawlResult(crawl, baseFolder, properties) {
baseFolder = baseFolder || '';

async function expandSpec(spec) {
// Special case for "idl" that must be processed first
if (spec.idl && (typeof spec.idl === 'string') &&
(!properties || properties.includes('idl') || properties.includes('idlparsed'))) {
if (baseFolder.startsWith('https:')) {
const url = (new URL(spec.idl, baseFolder)).toString();
let response = await fetch(url, { nolog: true });
spec.idl = {
idl: await response.text()
};
}
else {
spec.idl = {
idl: await fs.readFile(path.join(baseFolder, spec.idl), 'utf8')
};
}

// Drop IDL header comment that got added when IDL content was
// serialized to a file.
if (spec.idl.idl.startsWith('// GENERATED CONTENT - DO NOT EDIT')) {
const endOfHeader = spec.idl.idl.indexOf('\n\n');
spec.idl.idl = spec.idl.idl.substring(endOfHeader + 2);
}
}

await Promise.all(Object.keys(spec).map(async property => {
// Only consider properties explicitly requested
if (properties && !properties.includes(property)) {
Expand Down Expand Up @@ -774,14 +750,15 @@ async function expandCrawlResult(crawl, baseFolder, properties) {
delete css.spec;
spec[property] = css;
}
else if (property === 'idlparsed') {
// Special case for parsed IDL extracts, as result needs to be
// attached to "idl"
if (!spec.idl) {
spec.idl = {};
else if (property === 'idl') {
// Special case for raw IDL extracts, which are text extracts.
// Also drop header that may have been added when extract was
// serialized.
if (contents.startsWith('// GENERATED CONTENT - DO NOT EDIT')) {
const endOfHeader = contents.indexOf('\n\n');
contents = contents.substring(endOfHeader + 2);
}
Object.assign(spec.idl, contents[property]);
delete spec.idlparsed;
spec.idl = contents;
}
else {
spec[property] = contents[property];
Expand Down Expand Up @@ -846,6 +823,26 @@ function getGeneratedIDLNamesByCSSProperty(property) {
};


/**
* Creates the given folder if it does not exist yet.
*
* @function
* @public
* @param {String} folder Path to folder to create
* (from current working directory)
*/
async function createFolderIfNeeded(folder) {
try {
await fs.mkdir(folder);
}
catch (err) {
if (err.code !== 'EEXIST') {
throw err;
}
}
}


module.exports = {
fetch,
requireFromWorkingDirectory,
Expand All @@ -856,5 +853,6 @@ module.exports = {
completeWithAlternativeUrls,
isLatestLevelThatPasses,
expandCrawlResult,
getGeneratedIDLNamesByCSSProperty
getGeneratedIDLNamesByCSSProperty,
createFolderIfNeeded
};
24 changes: 2 additions & 22 deletions tests/generate-idlparsed.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,26 +12,7 @@ describe('The parsed IDL generator', function () {
const idl = 'interface foo {};';
const spec = { idl };
const result = await generateIdlParsed(spec);
assert.equal(result?.idl?.idl, idl);
assert.deepEqual(result?.idl?.idlNames, {
foo: {
extAttrs: [],
fragment: 'interface foo {};',
inheritance: null,
members: [],
name: 'foo',
partial: false,
type: 'interface'
}
});
});

it('parses raw IDL defined in the `idl.idl` property', async () => {
const idl = 'interface foo {};';
const spec = { idl: { idl } };
const result = await generateIdlParsed(spec);
assert.equal(result?.idl?.idl, idl);
assert.deepEqual(result?.idl?.idlNames, {
assert.deepEqual(result?.idlparsed?.idlNames, {
foo: {
extAttrs: [],
fragment: 'interface foo {};',
Expand All @@ -48,8 +29,7 @@ describe('The parsed IDL generator', function () {
const idl = 'intraface foo {};';
const spec = { idl };
const result = await generateIdlParsed(spec);
assert.equal(result?.idl?.idl, idl);
assert.equal(result.idl, `WebIDLParseError: Syntax error at line 1:
assert.equal(result.idlparsed, `WebIDLParseError: Syntax error at line 1:
intraface foo {};
^ Unrecognised tokens`);
});
Expand Down

0 comments on commit 747df3c

Please sign in to comment.