Merge remote-tracking branch 'upstream/main' into main

* upstream/main: (288 commits)
  remove tests for obsolete search endpoints (mdn#3688)
  Limit non-English locales (mdn#3661)
  move footer icons to CSS bg images (mdn#3655)
  not all http:// links can be https:// (mdn#3657)
  add the lost part and update the translation for Express (mdn#3664)
  update the translation of the zh-TW (mdn#3663)
  Add a Specifications macro and extract a special specification section to the Document (mdn#3518)
  build(deps-dev): bump & migrate use-debounce to 6.0.0 (mdn#3684)
  build(deps-dev): bump puppeteer from 9.0.0 to 9.1.0 (mdn#3686)
  build(deps-dev): bump & migrate husky to 6.0.0 (mdn#3681)
  build(deps-dev): bump jest-environment-jsdom-sixteen from 1.0.3 to 2.0.0 (mdn#3615)
  More concurrent open npm PRs (mdn#3685)
  build(deps-dev): bump black from 21.4b0 to 21.4b2 in /deployer (mdn#3668)
  build(deps-dev): bump black in /testing/integration (mdn#3666)
  build(deps-dev): bump @babel/core from 7.13.16 to 7.14.0 (mdn#3683)
  build(deps-dev): bump jest-puppeteer from 5.0.2 to 5.0.3 (mdn#3680)
  build(deps): bump @mdn/browser-compat-data from 3.3.0 to 3.3.2 (mdn#3679)
  build(deps-dev): bump sass from 1.32.11 to 1.32.12 (mdn#3678)
  build(deps-dev): bump eslint-plugin-jest from 24.3.5 to 24.3.6 (mdn#3677)
  build(deps): bump http-proxy-middleware from 1.3.0 to 1.3.1 (mdn#3675)
  ...

.github/dependabot.yml

@@ -5,7 +5,7 @@ updates:
     directory: "/"
     schedule:
       interval: daily
-    open-pull-requests-limit: 10
+    open-pull-requests-limit: 15
     ignore:
       # Need manual updates with "resolutions" in package.json
       # to property hoist versions from react-scripts
@@ -20,12 +20,12 @@ updates:
   - package-ecosystem: "pip"
     directory: "/deployer"
     schedule:
-      interval: "daily"
+      interval: "weekly"
 
   - package-ecosystem: "pip"
     directory: "/testing/integration"
     schedule:
-      interval: "daily"
+      interval: "weekly"
 
   - package-ecosystem: npm
     directory: "/deployer/aws-lambda/content-origin-request"

.github/workflows/content-origin-request.yml

@@ -24,7 +24,7 @@ jobs:
           node-version: "12"
 
       - name: Cache node_modules
-        uses: actions/[email protected].4
+        uses: actions/[email protected].5
         id: cached-node_modules
         with:
           path: |

.github/workflows/dev-build.yml

@@ -72,7 +72,7 @@ jobs:
           node-version: "12"
 
       - name: Cache node_modules
-        uses: actions/[email protected].4
+        uses: actions/[email protected].5
         id: cached-node_modules
         with:
           path: |
@@ -91,7 +91,7 @@ jobs:
           python-version: "3.8"
 
       - name: Install Python poetry
-        uses: snok/[email protected].2
+        uses: snok/[email protected].4
 
       - name: Install deployer
         run: |
@@ -204,7 +204,9 @@ jobs:
           # XXX would be nice to validate here that $DEPLOYER_BUCKET_PREFIX is truthy
           echo "DEPLOYER_BUCKET_PREFIX=$DEPLOYER_BUCKET_PREFIX"
 
-          poetry run deployer upload --prune ../client/build
+          # Prune again once https://github.com/mdn/yari/issues/3323 is resolved.
+          # poetry run deployer upload --prune --archived-files ../content/archived.txt ../client/build
+          poetry run deployer upload ../client/build
           poetry run deployer update-lambda-functions ./aws-lambda
           # TODO
           # Execute command to tell the Dev CloudFront distribution to use the

.github/workflows/developing.yml

@@ -21,7 +21,7 @@ jobs:
           node-version: "12"
 
       - name: Cache node_modules
-        uses: actions/[email protected].4
+        uses: actions/[email protected].5
         id: cached-node_modules
         with:
           path: |

.github/workflows/npm-publish.yml

@@ -26,7 +26,7 @@ jobs:
           node-version: "12"
 
       - name: Cache node_modules
-        uses: actions/[email protected].4
+        uses: actions/[email protected].5
         id: cached-node_modules
         with:
           path: |

.github/workflows/npm-published-simulation.yml

@@ -26,7 +26,7 @@ jobs:
           node-version: "12"
 
       - name: Cache node_modules
-        uses: actions/[email protected].4
+        uses: actions/[email protected].5
         id: cached-node_modules
         with:
           path: |

.github/workflows/performance.yml

@@ -30,7 +30,7 @@ jobs:
           node-version: "12"
 
       - name: Cache node_modules
-        uses: actions/[email protected].4
+        uses: actions/[email protected].5
         id: cached-node_modules
         with:
           path: |

.github/workflows/pr-deployer.yml

@@ -19,14 +19,14 @@ jobs:
           python-version: "3.8"
 
       - name: Install Python poetry
-        uses: snok/[email protected].2
+        uses: snok/[email protected].4
         with:
           virtualenvs-create: true
           virtualenvs-in-project: true
 
       - name: Load cached venv
         id: cached-poetry-dependencies
-        uses: actions/[email protected].4
+        uses: actions/[email protected].5
         with:
           path: deployer/.venv
           key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}

.github/workflows/pr-kumascript.yml

@@ -29,7 +29,7 @@ jobs:
           node-version: "12"
 
       - name: Cache node_modules
-        uses: actions/[email protected].4
+        uses: actions/[email protected].5
         id: cached-node_modules
         with:
           path: |

.github/workflows/prod-build.yml

@@ -51,6 +51,9 @@ jobs:
   build:
     runs-on: ubuntu-latest
 
+    # Only run the scheduled workflows on the main repo.
+    if: github.repository == 'mdn/yari'
+
     steps:
       - uses: actions/checkout@v2
 
@@ -94,7 +97,7 @@ jobs:
           node-version: "12"
 
       - name: Cache node_modules
-        uses: actions/[email protected].4
+        uses: actions/[email protected].5
         id: cached-node_modules
         with:
           path: |
@@ -113,7 +116,7 @@ jobs:
           python-version: "3.8"
 
       - name: Install Python poetry
-        uses: snok/[email protected].2
+        uses: snok/[email protected].4
 
       - name: Install deployer
         run: |
@@ -224,16 +227,11 @@ jobs:
           # XXX would be nice to validate here that $DEPLOYER_BUCKET_PREFIX is truthy
           echo "DEPLOYER_BUCKET_PREFIX=$DEPLOYER_BUCKET_PREFIX"
 
+          # Prune again once https://github.com/mdn/yari/issues/3323 is resolved.
+          # poetry run deployer upload --prune --archived-files ../content/archived.txt ../client/build
           poetry run deployer upload ../client/build
           poetry run deployer update-lambda-functions ./aws-lambda
-
-          # TODO: Depending on how long the upload takes, consider switching to
-          # use `--update` in this command. It will skip the index re-creation
-          # and "cake on". This is slightly faster and means no window of time
-          # where the Elasticsearch server is nearly empty.
-          # Consider doing something like a clean indexing on Sundays and
-          # use `--update` on all the other week days.
-          poetry run deployer search-index ../client/build --priority-prefix=en-us/docs/web
+          poetry run deployer search-index ../client/build
 
           # Record the deployment in our Speedcurve account. This should always
           # be done after the upload of the documents and Lambda functions, as well

.github/workflows/stage-build.yml

@@ -51,6 +51,9 @@ jobs:
   build:
     runs-on: ubuntu-latest
 
+    # Only run the scheduled workflows on the main repo.
+    if: github.repository == 'mdn/yari'
+
     steps:
       - uses: actions/checkout@v2
 
@@ -94,7 +97,7 @@ jobs:
           node-version: "12"
 
       - name: Cache node_modules
-        uses: actions/[email protected].4
+        uses: actions/[email protected].5
         id: cached-node_modules
         with:
           path: |
@@ -113,7 +116,7 @@ jobs:
           python-version: "3.8"
 
       - name: Install Python poetry
-        uses: snok/[email protected].2
+        uses: snok/[email protected].4
 
       - name: Install deployer
         run: |
@@ -165,13 +168,6 @@ jobs:
           # kind of document it is.
           BUILD_ALWAYS_NO_ROBOTS: true
 
-          # (peterbe, Mar 2021) This is unique to Stage and temporary. We want to
-          # test the new Yari-based sign-in and sign-up. Setting this environment
-          # variable changes the behavior of the "Sign in" link in the top navbar.
-          # Once we know for sure that this new Yari sign in/up is working in Stage
-          # we'll make this the new default can we can remove these lines.
-          REACT_APP_USE_YARI_SIGNIN: true
-
         run: |
           if [ ${{ env.BUILD_ARCHIVED_CONTENT }} == "true" ]; then
             echo "Will build mdn/archived-content too"
@@ -237,16 +233,11 @@ jobs:
           # XXX would be nice to validate here that $DEPLOYER_BUCKET_PREFIX is truthy
           echo "DEPLOYER_BUCKET_PREFIX=$DEPLOYER_BUCKET_PREFIX"
 
-          poetry run deployer upload  --prune ../client/build
+          # Prune again once https://github.com/mdn/yari/issues/3323 is resolved.
+          # poetry run deployer upload --prune --archived-files ../content/archived.txt ../client/build
+          poetry run deployer upload ../client/build
           poetry run deployer update-lambda-functions ./aws-lambda
-
-          # TODO: Depending on how long the upload takes, consider switching to
-          # use `--update` in this command. It will skip the index re-creation
-          # and "cake on". This is slightly faster and means no window of time
-          # where the Elasticsearch server is nearly empty.
-          # Consider doing something like a clean indexing on Sundays and
-          # use `--update` on all the other week days.
-          poetry run deployer search-index ../client/build --priority-prefix=en-us/docs/web
+          poetry run deployer search-index ../client/build
 
           # Record the deployment in our Speedcurve account. This should always
           # be done after the upload of the documents and Lambda functions, as well

.github/workflows/testing.yml

@@ -22,7 +22,7 @@ jobs:
           node-version: "12"
 
       - name: Cache node_modules
-        uses: actions/[email protected].4
+        uses: actions/[email protected].5
         id: cached-node_modules
         with:
           path: |

.husky/.gitignore

@@ -0,0 +1 @@
+_

.husky/pre-commit

@@ -0,0 +1,4 @@
+#!/bin/sh
+. "$(dirname "$0")/_/husky.sh"
+
+yarn pretty-quick --staged && yarn install --ignore-scripts

build/bcd-urls.js

@@ -56,6 +56,11 @@ function normalizeBCDURLs(doc, options) {
 
   for (const section of doc.body) {
     if (section.type !== "browser_compatibility") continue;
+
+    // This happens if a query is "broken".
+    // E.g. <div class="bc-data" id="bcd:apii.TypoCatching">
+    if (!section.value.data) continue;
+
     for (const [key, data] of Object.entries(section.value.data)) {
       // First block from the BCD data does not have its name as the root key
       // so mdn_url is accessible at the root. If the block has a key for
@@ -119,6 +124,12 @@ function extractBCDData(doc) {
   for (const section of doc.body) {
     if (section.type === "browser_compatibility") {
       // Most pages only have exactly 1 so no need to put the prefix on them.
+
+      if (!section.value.data) {
+        // This happens if a query is "broken".
+        // E.g. <div class="bc-data" id="bcd:apii.TypoCatching">
+        continue;
+      }
       const fileName = ++nextId > 1 ? `bcd-${nextId}.json` : "bcd.json";
       const dataURL = `${doc.mdn_url}/${fileName}`;
       data.push({

build/cli.js

@@ -2,6 +2,7 @@ const fs = require("fs");
 const path = require("path");
 const zlib = require("zlib");
 
+const chalk = require("chalk");
 const cliProgress = require("cli-progress");
 const program = require("@caporal/core").default;
 const { prompt } = require("inquirer");
@@ -10,12 +11,13 @@ const {
   Document,
   slugToFolder,
   translationsOf,
+  CONTENT_ROOT,
   CONTENT_TRANSLATED_ROOT,
+  CONTENT_ARCHIVED_ROOT,
 } = require("../content");
-
+const { VALID_LOCALES } = require("../libs/constants");
 // eslint-disable-next-line node/no-missing-require
 const { renderDocHTML } = require("../ssr/dist/main");
-
 const options = require("./build-options");
 const { buildDocument, renderContributorsTxt } = require("./index");
 const SearchIndex = require("./search-index");
@@ -33,6 +35,10 @@ async function buildDocumentInteractive(
       ? Document.read(documentPath, Document.MEMOIZE_INVALIDATE)
       : Document.read(documentPath);
 
+    if (!document) {
+      throw new Error(`${documentPath} could not be read`);
+    }
+
     if (!interactive) {
       const translations = translationsOf(document.metadata);
       if (translations && translations.length > 0) {
@@ -74,13 +80,15 @@ async function buildDocumentInteractive(
 async function buildDocuments(
   files = null,
   quiet = false,
-  interactive = false
+  interactive = false,
+  locales = new Map()
 ) {
   // If a list of files was set, it came from the CLI.
   // Override whatever was in the build options.
-  const findAllOptions = files
-    ? Object.assign({}, options, { files: new Set(files) })
-    : options;
+  const findAllOptions = Object.assign({}, options, { locales });
+  if (files) {
+    findAllOptions.files = new Set(files);
+  }
 
   const documents = Document.findAll(findAllOptions);
   const progressBar = new cliProgress.SingleBar(
@@ -275,18 +283,47 @@ program
   .option("-i, --interactive", "Ask what to do when encountering flaws", {
     default: false,
   })
+  .option("-l, --locale <locale...>", "Filtered specific locales", {
+    default: [],
+    validator: [...VALID_LOCALES.keys()],
+  })
   .argument("[files...]", "specific files to build")
   .action(async ({ args, options }) => {
     try {
       if (!options.quiet) {
-        console.log("\nBuilding Documents...");
+        const roots = [
+          ["CONTENT_ROOT", CONTENT_ROOT],
+          ["CONTENT_TRANSLATED_ROOT", CONTENT_TRANSLATED_ROOT],
+          ["CONTENT_ARCHIVED_ROOT", CONTENT_ARCHIVED_ROOT],
+        ];
+        for (const [key, value] of roots) {
+          console.log(
+            `${chalk.grey((key + ":").padEnd(25, " "))}${
+              value ? chalk.white(value) : chalk.grey("not set")
+            }`
+          );
+        }
       }
       const { files } = args;
+
+      // 'true' means we include this locale and all others get excluded.
+      // Some day we might make it an option to set `--not-locale` to
+      // filter out specific locales.
+      const locales = new Map(
+        // The `options.locale` is either an empty array (e.g. no --locale used),
+        // a string (e.g. one single --locale) or an array of strings
+        // (e.g. multiple --locale options).
+        (Array.isArray(options.locale)
+          ? options.locale
+          : [options.locale]
+        ).map((locale) => [locale, true])
+      );
       const t0 = new Date();
       const { slugPerLocale, peakHeapBytes, totalFlaws } = await buildDocuments(
         files,
         Boolean(options.quiet),
-        Boolean(options.interactive)
+        Boolean(options.interactive),
+        locales
       );
       const t1 = new Date();
       const count = Object.values(slugPerLocale).reduce(

build/constants.js

@@ -31,6 +31,7 @@ const VALID_FLAW_CHECKS = new Set([
   "bad_pre_tags",
   "sectioning",
   "heading_links",
+  "translation_differences",
   "unsafe_html",
 ]);