Add script to find list of specs potentially in scope

package.json

@@ -23,5 +23,8 @@
     "auto-changelog": "^1.16.4",
     "mocha": "^7.0.0",
     "release-it": "^13.5.1"
+  },
+  "dependencies": {
+    "node-fetch": "^2.6.0"
   }
 }

src/data/ignore.json

@@ -0,0 +1,45 @@
+{
+  "repos": {
+    "w3c/adpt": {
+      "comment": "not targeted at browsers"
+    },
+    "w3c/imsc": {
+      "comment": "not targeted at browsers"
+    },
+    "w3c/tt-module-karaoke": {
+      "comment": "not targeted at browsers"
+    },
+    "w3c/tt-module-live":{
+      "comment": "not targeted at browsers"
+    },
+    "w3c/ttml1": {
+      "comment": "not targeted at browsers"
+    },
+    "w3c/ttml2": {
+      "comment": "not targeted at browsers"
+    },
+    "w3c/ttml3": {
+      "comment": "not targeted at browsers"
+    },
+    "WICG/open-ui": {
+      "comment": "not targeted at browsers"
+    }
+  },
+  "specs": {
+    "https://www.w3.org/TR/ttml-imsc1.2/": {
+      "comment": "not targeted at browsers"
+    },
+    "https://www.w3.org/TR/ttml2/": {
+      "comment": "not targeted at browsers"
+    },
+    "https://www.w3.org/TR/html53/": {
+      "comment": "about to be deprecated"
+    },
+    "https://www.w3.org/TR/microdata/": {
+      "comment": "about to be deprecated"
+    },
+    "https://www.w3.org/TR/dom/": {
+      "comment": "about to be deprecated"
+    }
+  }
+}

src/data/monitor-repos.json

@@ -0,0 +1,42 @@
+{
+  "WICG/datacue": {
+    "comment": "mostly TODOs",
+    "lastreviewed": "2020-06-11"
+  },
+  "privacycg/storage-partitioning/": {
+    "comment": "mostly TODOs",
+    "lastreviewed": "2020-06-11"
+  },
+  "w3c/system-wake-lock": {
+    "comment": "no spec yet",
+    "lastreviewed": "2020-06-11"
+  },
+  "WICG/audio-focus": {
+    "comment": "only an explainer",
+    "lastreviewed": "2020-06-11"
+  },
+  "WICG/file-handling": {
+    "comment": "only points to explainer",
+    "lastreviewed": "2020-06-11"
+  },
+  "WICG/first-party-sets": {
+    "comment": "no spec yet",
+    "lastreviewed": "2020-06-11"
+  },
+  "WICG/indexed-db-observers": {
+    "comment": "no spec yet (but 3 years old?)",
+    "lastreviewed": "2020-06-11"
+  },
+  "WICG/media-latency-hint": {
+    "comment": "empty spec",
+    "lastreviewed": "2020-06-11"
+  },
+  "WICG/sw-launch": {
+    "comment": "no spec yet",
+    "lastreviewed": "2020-06-11"
+  },
+  "WICG/virtual-scroller": {
+    "comment": "no spec yet",
+    "lastreviewed": "2020-06-11"
+  }
+}

src/find-specs.js

@@ -0,0 +1,134 @@
+'use strict';
+
+const fetch = require("node-fetch");
+
+const specs = require("../index.json");
+const ignorable = require("./data/ignore.json");
+const temporarilyIgnorableRepos = require("./data/monitor-repos.json");
+
+const nonBrowserSpecWgs = [
+  "Accessibility Guidelines Working Group",
+  "Accessible Platform Architectures Working Group",
+  "Automotive Working Group",
+  "Dataset Exchange Working Group",
+  "Decentralized Identifier Working Group",
+  "Distributed Tracing Working Group",
+  "Education and Outreach Working Group",
+  "JSON-LD Working Group",
+  "Publishing Working Group",
+  "Verifiable Credentials Working Group",
+  "Web of Things Working Group"
+];
+const watchedBrowserCgs = [
+  "Web Platform Incubator Community Group",
+  "Web Assembly Community Group",
+  "Immersive Web Community Group",
+  "Audio Community Group",
+  "Privacy Community Group",
+  "GPU for the Web Community Group"
+];
+
+function canonicalizeGhUrl(r) {
+  const url = new URL(r.homepageUrl);
+  url.protocol = 'https:';
+  if (url.pathname.lastIndexOf('/') === 0 && url.pathname.length > 1) {
+      url.pathname += '/';
+  }
+  return url.toString();
+}
+
+function canonicalizeTRUrl(url) {
+  url = new URL(url);
+  url.protocol = 'https:';
+  return url.toString();
+}
+
+const toGhUrl = repo => `https://${repo.owner.login.toLowerCase()}.github.io/${repo.name}/`
+const matchRepoName = fullName => r => fullName === r.owner.login + '/' + r.name;
+const isRelevantRepo = fullName => !Object.keys(ignorable.repos).includes(fullName) && !Object.keys(temporarilyIgnorableRepos).includes(fullName);
+const isRelevantSpec = url => !Object.keys(ignorable.specs).includes(url);
+const isUnknownSpec = url => !specs.find(s => s.nightly.url.startsWith(url)
+                                         || (s.release && s.release.url === url))
+const hasRepoType = type => r => r.w3c && r.w3c["repo-type"]
+      && (r.w3c["repo-type"] === type || r.w3c["repo-type"].includes(type));
+const urlIfExists = u => fetch(u).then(({ok, url}) => {
+  if (ok) return url;
+});
+
+(async function() {
+  const {groups, repos} = await fetch("https://w3c.github.io/validate-repos/report.json").then(r => r.json());
+  const specRepos = await fetch("https://w3c.github.io/spec-dashboard/repo-map.json").then(r => r.json());
+  const whatwgSpecs = await fetch("https://raw.githubusercontent.com/whatwg/sg/master/db.json").then(r => r.json())
+        .then(d => d.workstreams.map(w => w.standards).flat());
+
+  const wgs = Object.values(groups).filter(g => g.type === "working group" && !nonBrowserSpecWgs.includes(g.name));
+  const cgs = Object.values(groups).filter(g => g.type === "community group" && watchedBrowserCgs.includes(g.name));
+
+  // WGs
+  // * check repos with w3c.json/repo-type including rec-track
+  const wgRepos = wgs.map(g => g.repos.map(r => r.fullName)).flat()
+        .filter(isRelevantRepo)
+        .map(fullName => repos.find(matchRepoName(fullName)));
+  const recTrackRepos = wgRepos.filter(hasRepoType('rec-track'));
+
+  // * look if those with homepage URLs have a match in the list of specs
+  console.log("URLs from a repo of a browser-spec producing WG with no matching URL in spec list")
+  console.log(recTrackRepos.filter(r => r.homepageUrl)
+              .map(canonicalizeGhUrl)
+              .filter(isUnknownSpec)
+              .filter(isRelevantSpec)
+             );
+
+  // * look if those without a homepage URL have a match with their generated URL
+  const wgUrls = (await Promise.all(recTrackRepos.filter(r => !r.homepageUrl)
+                                    .map(toGhUrl)
+                                    .filter(isUnknownSpec)
+                                    .filter(isRelevantSpec)
+                                    .map(urlIfExists))).filter(x => x);
+  console.log("Unadvertized URLs from a repo of a browser-spec producing WG with no matching URL in spec list")
+  console.log(wgUrls);
+
+  // Look which of the specRepos on recTrack from a browser-producing WG have no match
+  console.log("TR specs from browser-producing WGs")
+  console.log(
+    Object.keys(specRepos).map(
+      r => specRepos[r].filter(s => s.recTrack && wgs.find(g => g.id === s.group)).map(s => canonicalizeTRUrl(s.url)))
+      .flat()
+      .filter(isUnknownSpec)
+      .filter(isRelevantSpec)
+  );
+
+  // CGs
+  //check repos with w3c.json/repo-type includes cg-report or with no w3c.json
+  const cgRepos = cgs.map(g => g.repos.map(r => r.fullName)).flat()
+        .filter(isRelevantRepo)
+        .map(fullName => repos.find(matchRepoName(fullName)));
+
+  const cgSpecRepos = cgRepos.filter(r => !r.w3c
+                                     || hasRepoType('cg-report')(r));
+  // * look if those with homepage URLs have a match in the list of specs
+  console.log("URLs from a repo of a browser-spec producing CG with no matching URL in spec list")
+  console.log(cgSpecRepos.filter(r => r.homepageUrl)
+              .map(canonicalizeGhUrl)
+              .filter(isUnknownSpec)
+              .filter(isRelevantSpec)
+             );
+  // * look if those without a homepage URL have a match with their generated URL
+  const cgUrls = (await Promise.all(cgSpecRepos.filter(r => !r.homepageUrl)
+                                    .map(toGhUrl)
+                                    .filter(isUnknownSpec)
+                                    .filter(isRelevantSpec)
+                                    .map(urlIfExists))).filter(x => x);
+  console.log("Unadvertized URLs from a repo of a browser-spec producing CG with no matching URL in spec list")
+  console.log(cgUrls);
+
+
+  const whatwgUrls = whatwgSpecs.map(s => s.href)
+        .filter(isUnknownSpec)
+        .filter(isRelevantSpec);
+  console.log("URLs from WHATWG with no matching URL in spec list")
+  console.log(whatwgUrls);
+})().catch(e => {
+  console.error(e);
+  process.exit(1);
+});