diff --git a/package-lock.json b/package-lock.json index 618345c5..2dccc641 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2133,8 +2133,7 @@ "node-fetch": { "version": "2.6.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", - "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==", - "dev": true + "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==" }, "normalize-path": { "version": "3.0.0", diff --git a/package.json b/package.json index 8eb606ca..28babcc5 100644 --- a/package.json +++ b/package.json @@ -23,5 +23,8 @@ "auto-changelog": "^1.16.4", "mocha": "^7.0.0", "release-it": "^13.5.1" + }, + "dependencies": { + "node-fetch": "^2.6.0" } } diff --git a/src/data/ignore.json b/src/data/ignore.json new file mode 100644 index 00000000..a183f677 --- /dev/null +++ b/src/data/ignore.json @@ -0,0 +1,45 @@ +{ + "repos": { + "w3c/adpt": { + "comment": "not targeted at browsers" + }, + "w3c/imsc": { + "comment": "not targeted at browsers" + }, + "w3c/tt-module-karaoke": { + "comment": "not targeted at browsers" + }, + "w3c/tt-module-live":{ + "comment": "not targeted at browsers" + }, + "w3c/ttml1": { + "comment": "not targeted at browsers" + }, + "w3c/ttml2": { + "comment": "not targeted at browsers" + }, + "w3c/ttml3": { + "comment": "not targeted at browsers" + }, + "WICG/open-ui": { + "comment": "not targeted at browsers" + } + }, + "specs": { + "https://www.w3.org/TR/ttml-imsc1.2/": { + "comment": "not targeted at browsers" + }, + "https://www.w3.org/TR/ttml2/": { + "comment": "not targeted at browsers" + }, + "https://www.w3.org/TR/html53/": { + "comment": "about to be deprecated" + }, + "https://www.w3.org/TR/microdata/": { + "comment": "about to be deprecated" + }, + "https://www.w3.org/TR/dom/": { + "comment": "about to be deprecated" + } + } +} diff --git a/src/data/monitor-repos.json b/src/data/monitor-repos.json new file mode 100644 index 00000000..4d642331 --- /dev/null +++ b/src/data/monitor-repos.json @@ -0,0 +1,42 @@ +{ + "WICG/datacue": { + "comment": "mostly TODOs", + "lastreviewed": "2020-06-11" + }, + "privacycg/storage-partitioning/": { + "comment": "mostly TODOs", + "lastreviewed": "2020-06-11" + }, + "w3c/system-wake-lock": { + "comment": "no spec yet", + "lastreviewed": "2020-06-11" + }, + "WICG/audio-focus": { + "comment": "only an explainer", + "lastreviewed": "2020-06-11" + }, + "WICG/file-handling": { + "comment": "only points to explainer", + "lastreviewed": "2020-06-11" + }, + "WICG/first-party-sets": { + "comment": "no spec yet", + "lastreviewed": "2020-06-11" + }, + "WICG/indexed-db-observers": { + "comment": "no spec yet (but 3 years old?)", + "lastreviewed": "2020-06-11" + }, + "WICG/media-latency-hint": { + "comment": "empty spec", + "lastreviewed": "2020-06-11" + }, + "WICG/sw-launch": { + "comment": "no spec yet", + "lastreviewed": "2020-06-11" + }, + "WICG/virtual-scroller": { + "comment": "no spec yet", + "lastreviewed": "2020-06-11" + } +} diff --git a/src/find-specs.js b/src/find-specs.js new file mode 100644 index 00000000..02a32dae --- /dev/null +++ b/src/find-specs.js @@ -0,0 +1,134 @@ +'use strict'; + +const fetch = require("node-fetch"); + +const specs = require("../index.json"); +const ignorable = require("./data/ignore.json"); +const temporarilyIgnorableRepos = require("./data/monitor-repos.json"); + +const nonBrowserSpecWgs = [ + "Accessibility Guidelines Working Group", + "Accessible Platform Architectures Working Group", + "Automotive Working Group", + "Dataset Exchange Working Group", + "Decentralized Identifier Working Group", + "Distributed Tracing Working Group", + "Education and Outreach Working Group", + "JSON-LD Working Group", + "Publishing Working Group", + "Verifiable Credentials Working Group", + "Web of Things Working Group" +]; +const watchedBrowserCgs = [ + "Web Platform Incubator Community Group", + "Web Assembly Community Group", + "Immersive Web Community Group", + "Audio Community Group", + "Privacy Community Group", + "GPU for the Web Community Group" +]; + +function canonicalizeGhUrl(r) { + const url = new URL(r.homepageUrl); + url.protocol = 'https:'; + if (url.pathname.lastIndexOf('/') === 0 && url.pathname.length > 1) { + url.pathname += '/'; + } + return url.toString(); +} + +function canonicalizeTRUrl(url) { + url = new URL(url); + url.protocol = 'https:'; + return url.toString(); +} + +const toGhUrl = repo => `https://${repo.owner.login.toLowerCase()}.github.io/${repo.name}/` +const matchRepoName = fullName => r => fullName === r.owner.login + '/' + r.name; +const isRelevantRepo = fullName => !Object.keys(ignorable.repos).includes(fullName) && !Object.keys(temporarilyIgnorableRepos).includes(fullName); +const isRelevantSpec = url => !Object.keys(ignorable.specs).includes(url); +const isUnknownSpec = url => !specs.find(s => s.nightly.url.startsWith(url) + || (s.release && s.release.url === url)) +const hasRepoType = type => r => r.w3c && r.w3c["repo-type"] + && (r.w3c["repo-type"] === type || r.w3c["repo-type"].includes(type)); +const urlIfExists = u => fetch(u).then(({ok, url}) => { + if (ok) return url; +}); + +(async function() { + const {groups, repos} = await fetch("https://w3c.github.io/validate-repos/report.json").then(r => r.json()); + const specRepos = await fetch("https://w3c.github.io/spec-dashboard/repo-map.json").then(r => r.json()); + const whatwgSpecs = await fetch("https://raw.githubusercontent.com/whatwg/sg/master/db.json").then(r => r.json()) + .then(d => d.workstreams.map(w => w.standards).flat()); + + const wgs = Object.values(groups).filter(g => g.type === "working group" && !nonBrowserSpecWgs.includes(g.name)); + const cgs = Object.values(groups).filter(g => g.type === "community group" && watchedBrowserCgs.includes(g.name)); + + // WGs + // * check repos with w3c.json/repo-type including rec-track + const wgRepos = wgs.map(g => g.repos.map(r => r.fullName)).flat() + .filter(isRelevantRepo) + .map(fullName => repos.find(matchRepoName(fullName))); + const recTrackRepos = wgRepos.filter(hasRepoType('rec-track')); + + // * look if those with homepage URLs have a match in the list of specs + console.log("URLs from a repo of a browser-spec producing WG with no matching URL in spec list") + console.log(recTrackRepos.filter(r => r.homepageUrl) + .map(canonicalizeGhUrl) + .filter(isUnknownSpec) + .filter(isRelevantSpec) + ); + + // * look if those without a homepage URL have a match with their generated URL + const wgUrls = (await Promise.all(recTrackRepos.filter(r => !r.homepageUrl) + .map(toGhUrl) + .filter(isUnknownSpec) + .filter(isRelevantSpec) + .map(urlIfExists))).filter(x => x); + console.log("Unadvertized URLs from a repo of a browser-spec producing WG with no matching URL in spec list") + console.log(wgUrls); + + // Look which of the specRepos on recTrack from a browser-producing WG have no match + console.log("TR specs from browser-producing WGs") + console.log( + Object.keys(specRepos).map( + r => specRepos[r].filter(s => s.recTrack && wgs.find(g => g.id === s.group)).map(s => canonicalizeTRUrl(s.url))) + .flat() + .filter(isUnknownSpec) + .filter(isRelevantSpec) + ); + + // CGs + //check repos with w3c.json/repo-type includes cg-report or with no w3c.json + const cgRepos = cgs.map(g => g.repos.map(r => r.fullName)).flat() + .filter(isRelevantRepo) + .map(fullName => repos.find(matchRepoName(fullName))); + + const cgSpecRepos = cgRepos.filter(r => !r.w3c + || hasRepoType('cg-report')(r)); + // * look if those with homepage URLs have a match in the list of specs + console.log("URLs from a repo of a browser-spec producing CG with no matching URL in spec list") + console.log(cgSpecRepos.filter(r => r.homepageUrl) + .map(canonicalizeGhUrl) + .filter(isUnknownSpec) + .filter(isRelevantSpec) + ); + // * look if those without a homepage URL have a match with their generated URL + const cgUrls = (await Promise.all(cgSpecRepos.filter(r => !r.homepageUrl) + .map(toGhUrl) + .filter(isUnknownSpec) + .filter(isRelevantSpec) + .map(urlIfExists))).filter(x => x); + console.log("Unadvertized URLs from a repo of a browser-spec producing CG with no matching URL in spec list") + console.log(cgUrls); + + + const whatwgUrls = whatwgSpecs.map(s => s.href) + .filter(isUnknownSpec) + .filter(isRelevantSpec); + console.log("URLs from WHATWG with no matching URL in spec list") + console.log(whatwgUrls); +})().catch(e => { + console.error(e); + process.exit(1); +});