Merge remote-tracking branch 'upstream/main' into main

* upstream/main: (164 commits)
  skip empty srcs for safe iframe srcs (mdn#3216)
  correct code comment (mdn#3223)
  build(deps): bump ahmadnassri/action-dependabot-auto-merge (mdn#3197)
  build(deps-dev): bump @types/react-dom from 17.0.1 to 17.0.2 (mdn#3164)
  create a whatsdeployed.json for translated-content too (mdn#3221)
  avoid double-slash redirects (mdn#3222)
  build(deps): bump image-size from 0.9.4 to 0.9.5 (mdn#3214)
  build(deps): bump boto3 from 1.17.22 to 1.17.26 in /deployer (mdn#3212)
  Fix our auto-merge workflow (mdn#3218)
  build(deps-dev): bump ts-loader from 8.0.17 to 8.0.18 (mdn#3208)
  disable lighthouse PR check unless relevant changes (mdn#3203)
  hide toolbar for frozen locales (mdn#3213)
  build(deps): bump is-svg from 4.2.1 to 4.2.2 (mdn#3209)
  build(deps): bump @mdn/browser-compat-data from 3.1.3 to 3.2.0 (mdn#3210)
  downloading external images for translated-content (mdn#3207)
  add active locales (mdn#3201)
  add tool command for rendering/removing macros (mdn#2955)
  unsafe html should be a breaking flaw (mdn#3192)
  open editor for translated content (mdn#3196)
  add fundamental redirects for /en-US/Security/CSP (mdn#3200)
  ...

.github/workflows/auto-merge.yml

@@ -1,7 +1,7 @@
 name: auto-merge
 
 on:
-  pull_request:
+  pull_request_target:
     branches:
       - main
 
@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     if: github.actor == 'dependabot[bot]'
     steps:
-      - uses: ahmadnassri/action-dependabot-auto-merge@v2.3
+      - uses: ahmadnassri/action-dependabot-auto-merge@v2.4
         with:
           github-token: ${{ secrets.AUTOMERGE_TOKEN }}
           command: "squash and merge"

.github/workflows/content-origin-request.yml

@@ -0,0 +1,62 @@
+# This starts up a simulator that tries to do what our Lambda@Edge does.
+
+name: content-origin-request
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - deployer/aws-lambda/**
+      - libs/**
+      - .github/workflows/content-origin-request.yml
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup Node.js environment
+        uses: actions/[email protected]
+        with:
+          node-version: "12"
+
+      - name: Cache node_modules
+        uses: actions/[email protected]
+        id: cached-node_modules
+        with:
+          path: |
+            deployer/aws-lambda/content-origin-request/node_modules
+          key: ${{ runner.os }}-${{ hashFiles('deployer/aws-lambda/content-origin-request/yarn.lock') }}-${{ hashFiles('libs/**/*.js') }}
+
+      - name: Install all yarn packages
+        if: steps.cached-node_modules.outputs.cache-hit != 'true'
+        working-directory: deployer/aws-lambda/content-origin-request
+        run: yarn --frozen-lockfile
+
+      - name: Run test server
+        working-directory: deployer/aws-lambda/content-origin-request
+        run: |
+          yarn serve > /tmp/stdout.log 2> /tmp/stderr.log &
+
+      - name: Check that the server started
+        run: curl --retry-connrefused --retry 5 -I http://localhost:7000/ping
+
+      - name: Preflight the integration tests
+        run: |
+          curl -I http://localhost:7000/docs/Web
+          curl -I http://localhost:7000/en-US/docs/Web/
+
+      - name: Unit test
+        working-directory: deployer/aws-lambda/content-origin-request
+        run: |
+          yarn test-server
+
+      - name: Debug any server outputs
+        run: |
+          echo "____STDOUT____"
+          cat /tmp/stdout.log
+          echo "____STDERR____"
+          cat /tmp/stderr.log

.github/workflows/dev-build.yml

@@ -21,10 +21,6 @@ on:
         description: "Build archived content"
         required: false
         default: "false"
-      translated_content:
-        description: "Build translated content"
-        required: false
-        default: "true"
 
       # This is very useful when combined with the "Use workflow from"
       # feature that is built into the "Run workflow" button on
@@ -66,13 +62,12 @@ jobs:
           path: mdn/archived-content
 
       - uses: actions/checkout@v2
-        if: "contains(github.event.inputs.translated_content, 'true')"
         with:
-          repository: mdn/translated-content-rendered
+          repository: mdn/translated-content
           path: mdn/translated-content
 
       - name: Setup Node.js environment
-        uses: actions/[email protected].4
+        uses: actions/[email protected].5
         with:
           node-version: "12"
 
@@ -96,7 +91,7 @@ jobs:
           python-version: "3.8"
 
       - name: Install Python poetry
-        uses: snok/[email protected].1
+        uses: snok/[email protected].2
 
       - name: Install deployer
         run: |
@@ -112,7 +107,6 @@ jobs:
         run: |
           echo "notes: ${{ github.event.inputs.notes }}"
           echo "archived_content: ${{ github.event.inputs.archived_content }}"
-          echo "translated_content: ${{ github.event.inputs.translated_content }}"
           echo "log_each_successful_upload: ${{ github.event.inputs.log_each_successful_upload }}"
           echo "deployment_prefix: ${{ github.event.inputs.deployment_prefix }}"
 
@@ -121,6 +115,7 @@ jobs:
           # Remember, the mdn/content repo got cloned into `pwd` into a
           # sub-folder called "mdn/content"
           CONTENT_ROOT: ${{ github.workspace }}/mdn/content/files
+          CONTENT_TRANSLATED_ROOT: ${{ github.workspace }}/mdn/translated-content/files
 
           # This basically means that all live-sample iframes run on the same
           # host as the page that includes the iframe. Not great security but the
@@ -152,19 +147,14 @@ jobs:
           else
             echo "Will NOT build mdn/archived-content too"
           fi
-          if [ ${{ github.event.inputs.translated_content }} == "true" ]; then
-            echo "Will build mdn/translated-content too"
-            export CONTENT_TRANSLATED_ROOT=${{ github.workspace }}/mdn/translated-content/files
-          else
-            echo "Will NOT build mdn/translated-content too"
-          fi
 
           # Info about which CONTENT_* environment variables were set and to what.
           echo "CONTENT_ROOT=$CONTENT_ROOT"
           echo "CONTENT_ARCHIVED_ROOT=$CONTENT_ARCHIVED_ROOT"
           echo "CONTENT_TRANSLATED_ROOT=$CONTENT_TRANSLATED_ROOT"
           yarn prepare-build
 
+          yarn tool sync-translated-content
           yarn build
 
           # TODO: When the deployer is available this is where we
@@ -175,6 +165,7 @@ jobs:
         env:
           # Set the CONTENT_ROOT first
           CONTENT_ROOT: ${{ github.workspace }}/mdn/content/files
+          CONTENT_TRANSLATED_ROOT: ${{ github.workspace }}/mdn/translated-content/files
 
           DEPLOYER_BUCKET_NAME: mdn-content-dev
           DEPLOYER_BUCKET_PREFIX: ${{ github.event.inputs.deployment_prefix }}
@@ -209,10 +200,11 @@ jobs:
 
           poetry run deployer whatsdeployed --output ../client/build/_whatsdeployed/code.json
           poetry run deployer whatsdeployed --output ../client/build/_whatsdeployed/content.json $CONTENT_ROOT
+          poetry run deployer whatsdeployed --output ../client/build/_whatsdeployed/translated-content.json $CONTENT_TRANSLATED_ROOT
           # XXX would be nice to validate here that $DEPLOYER_BUCKET_PREFIX is truthy
           echo "DEPLOYER_BUCKET_PREFIX=$DEPLOYER_BUCKET_PREFIX"
 
-          poetry run deployer upload ../client/build
+          poetry run deployer upload --prune ../client/build
           poetry run deployer update-lambda-functions ./aws-lambda
           # TODO
           # Execute command to tell the Dev CloudFront distribution to use the

.github/workflows/developing.yml

@@ -16,7 +16,7 @@ jobs:
           path: mdn/content
 
       - name: Setup Node.js environment
-        uses: actions/[email protected].4
+        uses: actions/[email protected].5
         with:
           node-version: "12"
 
@@ -62,7 +62,7 @@ jobs:
       - name: Wait for servers
         run: |
           # Just a slight delay to wait until the dev server is ready.
-          sleep 5
+          sleep 3
 
           curl --retry-connrefused --retry 5 http://localhost:5000 > /dev/null
           curl --retry-connrefused --retry 5 --silent http://localhost:3000 > /dev/null
@@ -76,15 +76,13 @@ jobs:
           # of the yarn installs above
           PUPPETEER_EXECUTABLE_PATH: /usr/bin/google-chrome
         run: |
-          status=0
-          yarn test:testing developing || (
-            status=$?
-            echo "Testing failed! Going to dump stdout and stderr"
-            echo "STDOUT..................................................."
-            cat /tmp/stdout.log
-            echo "STDERR..................................................."
-            cat /tmp/stderr.log
-            echo $status
-            exit $status
-          )
-          exit $status
+          yarn test:testing developing
+
+      - name: Debug server's stdout and stderr if tests failed
+        if: failure()
+        run: |
+          echo "STDOUT..................................................."
+          cat /tmp/stdout.log
+          echo ""
+          echo "STDERR..................................................."
+          cat /tmp/stderr.log

.github/workflows/npm-publish.yml

@@ -21,7 +21,7 @@ jobs:
           fetch-depth: 10
 
       - name: Setup Node.js environment
-        uses: actions/[email protected].4
+        uses: actions/[email protected].5
         with:
           node-version: "12"
 

.github/workflows/npm-published-simulation.yml

@@ -0,0 +1,113 @@
+# Instead of waiting for Yari to be published to npmjs.com and be upgraded
+# inside mdn/content by Dependabot, we do all those steps here using `npm pack`.
+
+name: NPM Publish simulation
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/checkout@v2
+        with:
+          repository: mdn/content
+          path: mdn/content
+
+      - name: Setup Node.js environment
+        uses: actions/[email protected]
+        with:
+          node-version: "12"
+
+      - name: Cache node_modules
+        uses: actions/[email protected]
+        id: cached-node_modules
+        with:
+          path: |
+            node_modules
+          key: ${{ runner.os }}-${{ hashFiles('yarn.lock') }}
+
+      - name: Install all yarn packages
+        if: steps.cached-node_modules.outputs.cache-hit != 'true'
+        env:
+          PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: 1
+        run: yarn --frozen-lockfile
+
+      - name: Setup kernel for react native, increase watchers
+        run: |
+          # When running Yari on Linux, you might get the
+          # "Error: ENOSPC: System limit for number of file watchers reached" error.
+          # This, resolves that.
+          # Source https://github.com/expo/expo-github-action/issues/20#issuecomment-541676895
+          echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p
+
+      - name: Prepare to build
+        env:
+          # The following env vars is what we do in npm-publish.yml
+          # Each variable set is documented there.
+
+          REACT_APP_CRUD_MODE: true
+          REACT_APP_DISABLE_AUTH: true
+          CONTENT_ROOT: testing/content/files
+        run: |
+          yarn prepare-build
+
+      - name: Build and install tarball
+        run: |
+          echo mdn/content/ >> .npmignore
+          npm pack
+          TARBALL=`ls mdn-yari-*.tgz`
+          echo $TARBALL
+          ls -lh $TARBALL
+          mv $TARBALL mdn/content/
+          cd mdn/content
+          yarn add file:$TARBALL
+
+      - name: Start Yari from mock content repo
+        working-directory: mdn/content
+        run: |
+          yarn start > /tmp/stdout.log 2> /tmp/stderr.log &
+
+      - name: View some URLs on localhost:5000
+        run: |
+          curl --retry-connrefused --retry 5 -I http://localhost:5000
+
+          # Basically, test if it 200 OKs. If not, this'll exit non-zero.
+          curl http://localhost:5000/en-US/ > /dev/null
+          curl http://localhost:5000/en-US/docs/MDN/Kitchensink > /dev/null
+
+      - name: Test viewing the dev server
+        env:
+          # This will make sure the tests in `testing/tests/*.test.js` only run
+          # if the development server is up and ready to be tested.
+          TESTING_DEVELOPING: true
+          # Use local chrome installs since we skip downloading it as part
+          # of the yarn installs above
+          PUPPETEER_EXECUTABLE_PATH: /usr/bin/google-chrome
+          # When running Yari from within mdn/content it only starts 1 server;
+          # the one on localhost:5000. No React dev server; the one
+          # on localhost:3000.
+          # Testing that dev server is not relevant or important in this context.
+          DEVELOPING_SKIP_DEV_URL: true
+        run: |
+          yarn test:testing developing
+
+      - name: Debug server's stdout and stderr if tests failed
+        if: failure()
+        run: |
+          echo "STDOUT..................................................."
+          cat /tmp/stdout.log
+          echo ""
+          echo "STDERR..................................................."
+          cat /tmp/stderr.log
+
+      - name: SSR build a page
+        working-directory: mdn/content
+        run: |
+          yarn build files/en-us/mdn/kitchensink/index.html

.github/workflows/performance.yml

@@ -2,6 +2,11 @@ name: Performance
 
 on:
   pull_request:
+    paths:
+      - client/src/**
+      - .github/workflows/performance.yml
+      - package.json
+      - yarn.lock
 
 jobs:
   lighthouse:
@@ -20,7 +25,7 @@ jobs:
           path: mdn/content
 
       - name: Setup Node.js environment
-        uses: actions/[email protected].4
+        uses: actions/[email protected].5
         with:
           node-version: "12"
 

.github/workflows/pr-deployer.yml

@@ -19,7 +19,7 @@ jobs:
           python-version: "3.8"
 
       - name: Install Python poetry
-        uses: snok/[email protected].1
+        uses: snok/[email protected].2
         with:
           virtualenvs-create: true
           virtualenvs-in-project: true

.github/workflows/pr-docs.yml

@@ -14,7 +14,7 @@ jobs:
       - uses: actions/checkout@v2
 
       - name: Setup Node.js environment
-        uses: actions/[email protected].4
+        uses: actions/[email protected].5
         with:
           node-version: "12"
 

.github/workflows/pr-kumascript.yml

@@ -24,7 +24,7 @@ jobs:
       - uses: actions/checkout@v2
 
       - name: Setup Node.js environment
-        uses: actions/[email protected].4
+        uses: actions/[email protected].5
         with:
           node-version: "12"