diff --git a/.github/scripts/generate-devtools-hash.sh b/.github/scripts/generate-devtools-hash.sh index a78aad06570e..a44ca58d13e7 100644 --- a/.github/scripts/generate-devtools-hash.sh +++ b/.github/scripts/generate-devtools-hash.sh @@ -7,7 +7,7 @@ ## # Prints to stdout text that, when it changes, indicates that the devtools tests -# should update its dependencies (devtools frontend, content shell, blink tools). +# should rebuild the devtools frontend. SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" LH_ROOT="$SCRIPT_DIR/../.." @@ -15,6 +15,5 @@ LH_ROOT="$SCRIPT_DIR/../.." cd "$LH_ROOT" bash .github/scripts/print-devtools-relevant-commits.sh md5 \ - lighthouse-core/test/chromium-web-tests/* \ - third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/**/*.* \ + lighthouse-core/test/devtools-tests/* \ third-party/devtools-tests/e2e/**/*.* diff --git a/.github/workflows/devtools.yml b/.github/workflows/devtools.yml index 4fae5b028e9f..177a9fad1a88 100644 --- a/.github/workflows/devtools.yml +++ b/.github/workflows/devtools.yml @@ -8,7 +8,6 @@ on: env: DEPOT_TOOLS_PATH: ${{ github.workspace }}/depot-tools DEVTOOLS_PATH: ${{ github.workspace }}/devtools-frontend - BLINK_TOOLS_PATH: ${{ github.workspace }}/blink_tools jobs: build: @@ -35,15 +34,13 @@ jobs: # Caches are invalidated at least once a week, so that's the max time between testing # with the latest dependencies. Any commit to the DevTools repo touching Lighthouse # code will invalidate the cache sooner. - - name: Cache depot tools, devtools, blink tools and content shell + - name: Cache depot tools and devtools id: devtools-cache uses: actions/cache@v3 with: path: | ${{ env.DEPOT_TOOLS_PATH }} ${{ env.DEVTOOLS_PATH }} - ${{ env.BLINK_TOOLS_PATH }} - ${{ github.workspace }}/lighthouse/.tmp/chromium-web-tests/content-shells ${{ github.workspace }}/.gclient ${{ github.workspace }}/.gclient_entries # This hash key changes: @@ -52,8 +49,8 @@ jobs: # 3) every change to file in Lighthouse repo important to running these tests. # # The number is how many times this hash key was manually updated to break the cache. - key: ${{ runner.os }}-7-${{ env.WEEK_OF_THE_YEAR }}-${{ hashFiles('cdt-test-hash.txt') }} - restore-keys: ${{ runner.os }}-7 + key: ${{ runner.os }}-8-${{ env.WEEK_OF_THE_YEAR }}-${{ hashFiles('cdt-test-hash.txt') }} + restore-keys: ${{ runner.os }}-8 - name: Set GHA_DEVTOOLS_CACHE_HIT if: steps.devtools-cache.outputs.cache-hit == 'true' run: echo "GHA_DEVTOOLS_CACHE_HIT=1" >> $GITHUB_ENV @@ -62,12 +59,12 @@ jobs: working-directory: ${{ github.workspace }}/lighthouse - name: Download depot tools - run: bash $GITHUB_WORKSPACE/lighthouse/lighthouse-core/test/chromium-web-tests/download-depot-tools.sh + run: bash $GITHUB_WORKSPACE/lighthouse/lighthouse-core/test/devtools-tests/download-depot-tools.sh - name: Download DevTools Frontend - run: bash $GITHUB_WORKSPACE/lighthouse/lighthouse-core/test/chromium-web-tests/download-devtools.sh + run: bash $GITHUB_WORKSPACE/lighthouse/lighthouse-core/test/devtools-tests/download-devtools.sh - name: Roll Lighthouse + build DevTools - run: bash $GITHUB_WORKSPACE/lighthouse/lighthouse-core/test/chromium-web-tests/roll-devtools.sh + run: bash $GITHUB_WORKSPACE/lighthouse/lighthouse-core/test/devtools-tests/roll-devtools.sh - name: Cache build artifacts id: devtools-build-artifacts @@ -101,7 +98,7 @@ jobs: run: echo "$DEPOT_TOOLS_PATH" >> $GITHUB_PATH # For vpython. - name: Download depot tools - run: bash $GITHUB_WORKSPACE/lighthouse/lighthouse-core/test/chromium-web-tests/download-depot-tools.sh + run: bash $GITHUB_WORKSPACE/lighthouse/lighthouse-core/test/devtools-tests/download-depot-tools.sh - name: Load build artifacts id: devtools-build-artifacts @@ -116,49 +113,6 @@ jobs: - name: Run e2e tests run: bash $GITHUB_WORKSPACE/lighthouse/lighthouse-core/test/devtools-tests/run-e2e-tests.sh - web-tests: - needs: [build] - runs-on: macos-latest - - steps: - - name: git clone - uses: actions/checkout@v2 - with: - path: lighthouse - - - name: Use Node.js 14.x - uses: actions/setup-node@v1 - with: - node-version: 14.x - - - name: Load build artifacts - id: devtools-build-artifacts - uses: actions/cache@v3 - with: - path: | - ${{ env.DEVTOOLS_PATH }} - ${{ github.workspace }}/.gclient - ${{ github.workspace }}/.gclient_entries - key: devtools-build-artifacts-${{ github.run_id }} - - - name: Download Blink Tools - run: bash $GITHUB_WORKSPACE/lighthouse/lighthouse-core/test/chromium-web-tests/download-blink-tools.sh - - name: Download Content Shell - run: bash $GITHUB_WORKSPACE/lighthouse/lighthouse-core/test/chromium-web-tests/download-content-shell.sh - - - name: Install python deps - run: python3 -m pip install six requests - - - name: Run Web Tests - run: bash $GITHUB_WORKSPACE/lighthouse/lighthouse-core/test/chromium-web-tests/run-web-tests.sh - - - name: Upload results - uses: actions/upload-artifact@v2 - if: failure() - with: - name: results - path: ${{ github.workspace }}/lighthouse/.tmp/layout-test-results - smoke: needs: [build] strategy: @@ -190,11 +144,6 @@ jobs: ${{ github.workspace }}/.gclient_entries key: devtools-build-artifacts-${{ github.run_id }} - - name: Download Blink Tools - run: bash $GITHUB_WORKSPACE/lighthouse/lighthouse-core/test/chromium-web-tests/download-blink-tools.sh - - name: Download Content Shell - run: bash $GITHUB_WORKSPACE/lighthouse/lighthouse-core/test/chromium-web-tests/download-content-shell.sh - - run: yarn --frozen-lockfile --network-timeout 1000000 working-directory: ${{ github.workspace }}/lighthouse diff --git a/.npmignore b/.npmignore index d629c78ff6d4..6b9f4511e0aa 100644 --- a/.npmignore +++ b/.npmignore @@ -35,7 +35,7 @@ timings-data/ # ignore all folders named as such node_modules latest-run -chromium-webtests +devtools-tests .tmp # generated files diff --git a/lighthouse-cli/test/smokehouse/lighthouse-runners/devtools.js b/lighthouse-cli/test/smokehouse/lighthouse-runners/devtools.js index c9e5dd51e73e..c4fd7998583c 100644 --- a/lighthouse-cli/test/smokehouse/lighthouse-runners/devtools.js +++ b/lighthouse-cli/test/smokehouse/lighthouse-runners/devtools.js @@ -21,10 +21,10 @@ const devtoolsDir = * @param {string[]} logs * @param {string} command * @param {string[]} args + * @return {Promise} true if command exited successfully. */ -async function spawnAndLog(logs, command, args) { - /** @type {Promise} */ - const promise = new Promise((resolve) => { +function spawnAndLog(logs, command, args) { + return new Promise((resolve) => { const spawnHandle = spawn(command, args); spawnHandle.on('close', code => { if (code) { @@ -32,7 +32,7 @@ async function spawnAndLog(logs, command, args) { } else { logs.push('[SUCCESS] Command exited with code: 0\n'); } - resolve(); + resolve(code === 0); }); spawnHandle.on('error', (error) => { logs.push(`ERROR: ${error.toString()}`); @@ -48,21 +48,22 @@ async function spawnAndLog(logs, command, args) { logs.push(`STDERR: ${data}`); }); }); - await promise; } -/** @type {Promise} */ +/** @type {Promise} */ let buildDevtoolsPromise; /** - * @param {string[]} logs * Download/pull latest DevTools, build Lighthouse for DevTools, roll to DevTools, and build DevTools. + * @param {string[]} logs */ async function buildDevtools(logs) { - if (process.env.CI) return; + if (process.env.CI) return true; process.env.DEVTOOLS_PATH = devtoolsDir; - await spawnAndLog(logs, 'bash', ['lighthouse-core/test/chromium-web-tests/download-devtools.sh']); - await spawnAndLog(logs, 'bash', ['lighthouse-core/test/chromium-web-tests/roll-devtools.sh']); + const success = + await spawnAndLog(logs, 'bash', ['lighthouse-core/test/devtools-tests/download-devtools.sh']) && + await spawnAndLog(logs, 'bash', ['lighthouse-core/test/devtools-tests/roll-devtools.sh']); + return success; } /** @@ -80,7 +81,10 @@ async function runLighthouse(url, configJson, testRunnerOptions = {}) { const logs = []; if (!buildDevtoolsPromise) buildDevtoolsPromise = buildDevtools(logs); - await buildDevtoolsPromise; + if (!await buildDevtoolsPromise) { + const log = logs.join('') + '\n'; + throw new Error(`failed to build devtools:\n${log}`); + } const outputDir = fs.mkdtempSync(os.tmpdir() + '/lh-smoke-cdt-runner-'); const chromeFlags = [ diff --git a/lighthouse-core/scripts/build-devtools.sh b/lighthouse-core/scripts/build-devtools.sh index d0adfcf6632c..66569f5a26f9 100644 --- a/lighthouse-core/scripts/build-devtools.sh +++ b/lighthouse-core/scripts/build-devtools.sh @@ -12,7 +12,7 @@ set -euo pipefail # 2) Rolls to local devtools repo. By default, this is the temporary checkout in .tmp # 3) Builds devtools frontend with new Lighthouse roll # -# Run `bash lighthouse-core/test/chromium-web-tests/setup.sh` first to update the temporary devtools checkout. +# Run `bash lighthouse-core/test/devtools-tests/setup.sh` first to update the temporary devtools checkout. # Specify `$DEVTOOLS_PATH` to use a different devtools repo. SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" @@ -35,15 +35,12 @@ fi if ! which gn ; then # If the contributor doesn't have a separate depot tools in their path, use the tmp copy. DEPOT_TOOLS_PATH="$TEST_DIR/depot-tools" - BLINK_TOOLS_PATH="$TEST_DIR/blink_tools" export PATH=$DEPOT_TOOLS_PATH:$PATH - # Add typ to python path. The regular method assumes there is a Chromium checkout. - export PYTHONPATH="${PYTHONPATH:-}:$BLINK_TOOLS_PATH/latest/third_party/typ" fi yarn devtools "$DEVTOOLS_PATH" cd "$DEVTOOLS_PATH" -gn gen out/Default +gn gen out/Default --args='devtools_dcheck_always_on=true is_debug=false' gclient sync autoninja -C out/Default diff --git a/lighthouse-core/scripts/open-devtools.sh b/lighthouse-core/scripts/open-devtools.sh index 74f28363e918..705226f61f37 100644 --- a/lighthouse-core/scripts/open-devtools.sh +++ b/lighthouse-core/scripts/open-devtools.sh @@ -23,7 +23,7 @@ fi # If using the default .tmp devtools checkout, make sure it's up to date first. if [ -z "${DEVTOOLS_PATH:-}" ]; then - source "$LH_ROOT/lighthouse-core/test/chromium-web-tests/setup.sh" + source "$LH_ROOT/lighthouse-core/test/devtools-tests/setup.sh" fi echo "CHROME_PATH: $CHROME_PATH" diff --git a/lighthouse-core/scripts/pptr-run-devtools.js b/lighthouse-core/scripts/pptr-run-devtools.js index b2fd7961c7c6..2d6bf9beb8bd 100644 --- a/lighthouse-core/scripts/pptr-run-devtools.js +++ b/lighthouse-core/scripts/pptr-run-devtools.js @@ -14,7 +14,7 @@ * yarn run-devtools --chrome-flags=--custom-devtools-frontend=file://$HOME/src/devtools/devtools-frontend/out/Default/gen/front_end * * Or with the DevTools in .tmp: - * bash lighthouse-core/test/chromium-web-tests/setup.sh + * bash lighthouse-core/test/devtools-tests/setup.sh * yarn run-devtools --chrome-flags=--custom-devtools-frontend=file://$PWD/.tmp/chromium-web-tests/devtools/devtools-frontend/out/Default/gen/front_end * * URL list file: yarn run-devtools < path/to/urls.txt diff --git a/lighthouse-core/scripts/roll-to-devtools.sh b/lighthouse-core/scripts/roll-to-devtools.sh index c51f5bfa9a69..0a637bdd5504 100755 --- a/lighthouse-core/scripts/roll-to-devtools.sh +++ b/lighthouse-core/scripts/roll-to-devtools.sh @@ -73,14 +73,6 @@ fe_locales_dir="$fe_lh_dir/locales" rsync -avh "$lh_locales_dir" "$fe_locales_dir" --exclude="*.ctc.json" --delete echo -e "$check Locale JSON files copied." -# copy webtests -lh_webtests_dir="third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/" -fe_webtests_dir="$dt_dir/test/webtests/http/tests/devtools/lighthouse" -rsync -avh "$lh_webtests_dir" "$fe_webtests_dir" --exclude="OWNERS" --delete -lh_webtests_exp_dir="third-party/chromium-webtests/webtests/platform/generic/http/tests/devtools/lighthouse/" -fe_webtests_exp_dir="$dt_dir/test/webtests/platform/generic/http/tests/devtools/lighthouse" -rsync -avh "$lh_webtests_exp_dir" "$fe_webtests_exp_dir" --exclude="OWNERS" --delete - # copy e2e tests lh_e2e_dir="third-party/devtools-tests/e2e/lighthouse/" fe_e2e_dir="$dt_dir/test/e2e/lighthouse" @@ -90,5 +82,5 @@ fe_e2e_res_dir="$dt_dir/test/e2e/resources/lighthouse" rsync -avh "$lh_e2e_res_dir" "$fe_e2e_res_dir" --exclude="OWNERS" --delete echo "" -echo "Done. To run the webtests: " +echo "Done. To run the e2e tests: " echo " DEVTOOLS_PATH=\"$dt_dir\" yarn test-devtools" diff --git a/lighthouse-core/test/chromium-web-tests/README.md b/lighthouse-core/test/chromium-web-tests/README.md deleted file mode 100644 index a131bcd86e82..000000000000 --- a/lighthouse-core/test/chromium-web-tests/README.md +++ /dev/null @@ -1,112 +0,0 @@ -# Chromium Web Tests - -This runs the Chromium webtests using the devtools integration tester. - -`third-party/chromium-web-tests/webtests` contains all of our Lighthouse webtests. - -## Run - -```sh -yarn test-devtools - -# Reset the results. -yarn update:test-devtools - -# Run the test runner, without updating content_shell or getting the latest -# DevTools commits like `yarn test-devtools` does. -# This still bundles Lighthouse + rolls to DevTools before running the tests. -SKIP_DOWNLOADS=1 yarn test-devtools -``` - -### Prerequistes - -
- Install python dependencies - - Ensure you have these modules available globally on your python 3.x install: - - ```sh - python3 -m pip install requests six - ``` - -
- -### Debugging - -* Want logs from Lighthouse? Add `log.log('status', '**** hello test output ' + JSON.stringify({obj}));` which will be visible in the `lighthouse-successful-run.js` output thanks to `LighthouseTestRunner.addStatusListener`. -* Want logs from test files? Adding these flags to the invocation `yarn test-devtools --driver-logging --no-retry-failures` will print to terminal. -* Want logs from the inspected page? Add `testRunner.setDumpConsoleMessages(true);` to a test file. (also, [beware](https://source.chromium.org/chromium/chromium/src/+/main:content/web_test/renderer/web_view_test_proxy.cc;l=125-129;drc=437e5d9a05535b9e2cd7b983f78b23ebc3d92b3f) w/e this is about) - - -## How it works - -Normally, running these webtests requires a full Chromium checkout. However, that takes much too long, so it wouldn't be feasible for daily development or CI. Instead, we: - -1) **Grab just the source files we need from the Chromium repos** - -`download-blink-tools.sh` downloads just the code needed to run `third_party/blink/tools/run_web_tests.py`. This includes - -* `third_party/blink/tools`. -* `third_party/blink/web_tests/fast/harness` -* [`third_party/catapult/third_party/typ`](https://source.chromium.org/chromium/chromium/src/+/main:third_party/catapult/third_party/typ/)–necessary third party Python library - -2) **Apply a few custom patches** - -`run_web_tests.py` normally uses Apache to host the webtest resources while the tests run. However, Apache is a very heavy dependency, and doesn't work well in GitHub actions. The majority of how Apache is configured is unnecessary for our layout tests, so instead `npx http-server` is used. How the web server is launched is not configurable in `run_web_tests.py`, so instead we apply a small patch to skip its usage. - -`run_web_tests.py` also verifies that `image_diff` has been built from the Chromium checkout. We don't need that in our tests, so we skip that check. It's only for screenshot tests. - -`run_web_tests.py` requires a content shell binary. Instead of building it, we download it. - -3) **Download a prebuilt content shell** (barebones Chromium that is used by webtests) - -DevTools had a script for this, but it was removed. `download-content-shell.js` is from the old script, but with only the content shell downloading code and with a couple of minor changes. - -4) **Check out DevTools frontend, roll Lighthouse, build, and extract inspector resources** - -To run the devtools webtests, `run_web_tests.py` requires the inspector resources (the output of the frontend build process). These files are actually included in the content shell download from before, but instead we want to build the currently checked out Lighthouse, roll to DevTools, build DevTools, and use the newly created inspector files. - -`run_web_tests.py` normally serves these files by mounting the Chromium build folder for the DevTools output to the path `/inspector-sources` [1](https://source.chromium.org/chromium/chromium/src/+/main:third_party/blink/tools/blinkpy/web_tests/port/base.py;l=1280;drc=e8e4dcd1d1684251c33cda9b9fc93d7ea808e4bd) [2](https://source.chromium.org/chromium/chromium/src/+/main:third_party/blink/tools/blinkpy/web_tests/servers/apache_http.py;l=118;drc=32408e19204a7ffceebfe774d7e99f2041cf4338). Instead, we fetch the DevTools frontend, roll Lighthouse, build it, then copy the build output to the `inspector-sources` at the root of our `npx http-server` web server. - -## Testing Lighthouse from DevTools - -`run_web_tests.py` is used to automatically test Lighthouse from DevTools. - -```sh -# Runs Lighthouse from DevTools. Outputs results to ./latest-run/devtools-lhr.json. -yarn run-devtools http://example.com -``` - -## TODO - -* Keep LighthouseTestRunner here too. -* Implement `findMostRecentChromiumCommit` in `download-content-shell.js` -* Run smoke tests (started awhile ago [here](https://chromium-review.googlesource.com/c/chromium/src/+/1739566/3/third_party/blink/web_tests/http/tests/devtools/audits/audits-smoke-run.js)). -* auto commit rebaseline in CI [1] - -[1] The following would work for the core members, but not external contributors. -``` -# TODO: Must create a new token so external contributors can use. -# - name: Reset Results -# if: failure() && github.actor != 'patrickhulce' && github.actor != 'brendankenny' -# run: bash $GITHUB_WORKSPACE/lighthouse/lighthouse-core/test/chromium-web-tests/run-web-tests.sh --reset-results -# - name: Commit new expectations -# if: failure() && github.actor != 'patrickhulce' && github.actor != 'brendankenny' -# uses: EndBug/add-and-commit@v4 -# with: -# cwd: ${{ github.workspace }}/lighthouse -# add: third-party/chromium-webtests/webtests -# message: update webtest expectations -# env: -# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -``` - -### FAQ - -#### How to modify blink-tools.patch? - -Simply make your changes in `.tmp/chromium-web-tests/blink_tools/latest`, and run: `git -C .tmp/chromium-web-tests/blink_tools/latest diff > lighthouse-core/test/chromium-web-tests/blink-tools.patch` - -#### How does the python module `typ` get added to python sys path? - -Via a [hack](https://source.chromium.org/chromium/chromium/src/+/main:third_party/blink/tools/blinkpy/web_tests/models/typ_types.py;l=7?q=add_typ_dir_to_sys_path). diff --git a/lighthouse-core/test/chromium-web-tests/blink-tools.patch b/lighthouse-core/test/chromium-web-tests/blink-tools.patch deleted file mode 100644 index 5ea0ac348771..000000000000 --- a/lighthouse-core/test/chromium-web-tests/blink-tools.patch +++ /dev/null @@ -1,53 +0,0 @@ -diff --git a/third_party/blink/tools/blinkpy/web_tests/controllers/manager.py b/third_party/blink/tools/blinkpy/web_tests/controllers/manager.py -index 4323c8c..23ae146 100644 ---- a/third_party/blink/tools/blinkpy/web_tests/controllers/manager.py -+++ b/third_party/blink/tools/blinkpy/web_tests/controllers/manager.py -@@ -485,9 +485,9 @@ class Manager(object): - if (self._port.requires_http_server() - or any(self._is_http_test(test) for test in tests_to_run)): - self._printer.write_update('Starting HTTP server ...') -- self._port.start_http_server( -- additional_dirs={}, -- number_of_drivers=self._options.max_locked_shards) -+ # self._port.start_http_server( -+ # additional_dirs={}, -+ # number_of_drivers=self._options.max_locked_shards) - self._http_server_started = True - - if any(self._is_websocket_test(test) for test in tests_to_run): -diff --git a/third_party/blink/tools/blinkpy/web_tests/port/base.py b/third_party/blink/tools/blinkpy/web_tests/port/base.py -index 8f5ff1a..b7f0ae5 100644 ---- a/third_party/blink/tools/blinkpy/web_tests/port/base.py -+++ b/third_party/blink/tools/blinkpy/web_tests/port/base.py -@@ -63,8 +63,8 @@ from blinkpy.web_tests.port.factory import PortFactory - from blinkpy.web_tests.servers import apache_http - from blinkpy.web_tests.servers import pywebsocket - from blinkpy.web_tests.servers import wptserve --from blinkpy.web_tests.skia_gold import blink_skia_gold_properties as sgp --from blinkpy.web_tests.skia_gold import blink_skia_gold_session_manager as sgsm -+# from blinkpy.web_tests.skia_gold import blink_skia_gold_properties as sgp -+# from blinkpy.web_tests.skia_gold import blink_skia_gold_session_manager as sgsm - - _log = logging.getLogger(__name__) - -@@ -541,15 +541,15 @@ class Port(object): - self.get_option('configuration')): - return exit_codes.UNEXPECTED_ERROR_EXIT_STATUS - -- if not self._check_file_exists(self._path_to_image_diff(), -- 'image_diff'): -- return exit_codes.UNEXPECTED_ERROR_EXIT_STATUS -+ # if not self._check_file_exists(self._path_to_image_diff(), -+ # 'image_diff'): -+ # return exit_codes.UNEXPECTED_ERROR_EXIT_STATUS - - if self._dump_reader and not self._dump_reader.check_is_functional(): - return exit_codes.UNEXPECTED_ERROR_EXIT_STATUS - -- if needs_http and not self.check_httpd(): -- return exit_codes.UNEXPECTED_ERROR_EXIT_STATUS -+ # if needs_http and not self.check_httpd(): -+ # return exit_codes.UNEXPECTED_ERROR_EXIT_STATUS - - return exit_codes.OK_EXIT_STATUS - diff --git a/lighthouse-core/test/chromium-web-tests/download-blink-tools.sh b/lighthouse-core/test/chromium-web-tests/download-blink-tools.sh deleted file mode 100644 index 6271c2ecca25..000000000000 --- a/lighthouse-core/test/chromium-web-tests/download-blink-tools.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -## -# @license Copyright 2020 The Lighthouse Authors. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -## - -# Download blink tools for run_web_tests.py. - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -# https://chromium.googlesource.com/chromium/src/ -commit_chromium="e1c3b88f73fcbff831485c0ca37f877bd22b7a02" -# https://chromium.googlesource.com/catapult/ -commit_catapult="c2b4e773a8d77586ddea18b1962a3e02d19e682f" - -VERSIONED_DIR="$BLINK_TOOLS_PATH/$commit_chromium$commit_catapult" - -if ! type -P wget; then - echo "wget could not be found" - exit 1 -fi -if ! type -P tar; then - echo "tar could not be found" - exit 1 -fi -if ! type -P git; then - echo "git could not be found" - exit 1 -fi - -if [ -e "$VERSIONED_DIR" ]; then - echo "cached blink tools found" -else - mkdir -p "$VERSIONED_DIR/third_party/blink/tools" - rm "$BLINK_TOOLS_PATH/latest" || true - ln -s "$VERSIONED_DIR" "$BLINK_TOOLS_PATH/latest" - - wget "https://chromium.googlesource.com/chromium/src/+archive/$commit_chromium/third_party/blink/tools.tar.gz" --no-check-certificate -q -O blinktools.tar.gz - tar -xf blinktools.tar.gz -C "$VERSIONED_DIR/third_party/blink/tools" - rm blinktools.tar.gz - - # Just need this for the results.html template. - mkdir -p "$VERSIONED_DIR/third_party/blink/web_tests/fast/harness" - wget "https://chromium.googlesource.com/chromium/src/+archive/$commit_chromium/third_party/blink/web_tests/fast/harness.tar.gz" --no-check-certificate -q -O harness.tar.gz - tar -xf harness.tar.gz -C "$VERSIONED_DIR/third_party/blink/web_tests/fast/harness" - rm harness.tar.gz - - mkdir -p "$VERSIONED_DIR/third_party/typ" - wget "https://chromium.googlesource.com/catapult/+archive/$commit_catapult/third_party/typ.tar.gz" --no-check-certificate -q -O typ.tar.gz - tar -xf typ.tar.gz -C "$VERSIONED_DIR/third_party/typ" - rm typ.tar.gz - - cd "$VERSIONED_DIR" - git init - echo "*.pyc" > .gitignore - git add . - git commit -m baseline - patch -p1 < "$SCRIPT_DIR/blink-tools.patch" -fi diff --git a/lighthouse-core/test/chromium-web-tests/download-content-shell.sh b/lighthouse-core/test/chromium-web-tests/download-content-shell.sh deleted file mode 100644 index 3f61ba799e67..000000000000 --- a/lighthouse-core/test/chromium-web-tests/download-content-shell.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -## -# @license Copyright 2020 The Lighthouse Authors. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -## - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -LH_ROOT="$SCRIPT_DIR/../../.." - -node "$LH_ROOT/third-party/download-content-shell/download-content-shell.js" diff --git a/lighthouse-core/test/chromium-web-tests/run-web-tests.sh b/lighthouse-core/test/chromium-web-tests/run-web-tests.sh deleted file mode 100644 index d46ad75157e7..000000000000 --- a/lighthouse-core/test/chromium-web-tests/run-web-tests.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -## -# @license Copyright 2020 The Lighthouse Authors. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -## - -# Do not use directly. Requires setting up multiple environment variables first, -# see test-locally.sh for example. - -set -u - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -export LH_ROOT="$SCRIPT_DIR/../../.." - -bash "$SCRIPT_DIR/web-test-server.sh" http/tests/devtools/lighthouse $* -status=$? - -if [ ! $status -eq 0 ]; then - # Print failure diffs to stdout. - find "$LH_ROOT/.tmp/layout-test-results/retry_3" -name '*-diff.txt' -exec cat {} \; - echo "❌❌❌ webtests failed. to rebaseline run: yarn update:test-devtools ❌❌❌" -fi - -exit $status diff --git a/lighthouse-core/test/chromium-web-tests/setup.sh b/lighthouse-core/test/chromium-web-tests/setup.sh deleted file mode 100644 index 86821f849d77..000000000000 --- a/lighthouse-core/test/chromium-web-tests/setup.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash - -## -# @license Copyright 2021 The Lighthouse Authors. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -## - -# Setup dependencies for local layout test runner. -# Set SKIP_DOWNLOADS to skip all the downloading and just export variables. - -set -euo pipefail - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -LH_ROOT="$SCRIPT_DIR/../../.." -TEST_DIR="$LH_ROOT/.tmp/chromium-web-tests" - -export DEPOT_TOOLS_PATH="$TEST_DIR/depot-tools" -export DEVTOOLS_PATH=${DEVTOOLS_PATH:-"$TEST_DIR/devtools/devtools-frontend"} -export BLINK_TOOLS_PATH="$TEST_DIR/blink_tools" -export PATH=$DEPOT_TOOLS_PATH:$PATH - -if [ -z ${SKIP_DOWNLOADS+x} ] -then - echo "========================================" - echo "Downloading dependencies, including latest content_shell and DevTools" - echo "To skip this step, set SKIP_DOWNLOADS=1" - echo "========================================" - echo - - bash "$SCRIPT_DIR/download-depot-tools.sh" - bash "$SCRIPT_DIR/download-devtools.sh" - bash "$SCRIPT_DIR/download-blink-tools.sh" - bash "$SCRIPT_DIR/download-content-shell.sh" -fi diff --git a/lighthouse-core/test/chromium-web-tests/test-locally.sh b/lighthouse-core/test/chromium-web-tests/test-locally.sh deleted file mode 100644 index 10ad8b456fde..000000000000 --- a/lighthouse-core/test/chromium-web-tests/test-locally.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -## -# @license Copyright 2020 The Lighthouse Authors. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -## - -# Runs the layout tests in third-party/chromium-webtests using the latest -# Lighthouse, DevTools, and Chrome content_shell* -# -# * Not exactly the latest, but close. See download-content-shell.js - -set -euo pipefail - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -source "$SCRIPT_DIR/setup.sh" -bash "$SCRIPT_DIR/roll-devtools.sh" -bash "$SCRIPT_DIR/run-web-tests.sh" $* diff --git a/lighthouse-core/test/chromium-web-tests/web-test-server.sh b/lighthouse-core/test/chromium-web-tests/web-test-server.sh deleted file mode 100644 index c5d73ece5d80..000000000000 --- a/lighthouse-core/test/chromium-web-tests/web-test-server.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash - -## -# @license Copyright 2020 The Lighthouse Authors. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -## - -set -euo pipefail - -if ! type -P python3; then - echo "python3 could not be found" - exit 1 -fi - -# Get newest folder -latest_content_shell_dir=$(ls -t "$LH_ROOT/.tmp/chromium-web-tests/content-shells/" | head -n1) -export latest_content_shell="$LH_ROOT/.tmp/chromium-web-tests/content-shells/$latest_content_shell_dir" - -# Run a very basic server on port 8000. Only thing we need is: -# - /devtools -> the layout tests for devtools frontend -# - /inspector-sources -> the generated resources from devtools -# - CORS (Access-Control-Allow-Origin header) - -ln -s "$DEVTOOLS_PATH/out/Default/gen/front_end" "$DEVTOOLS_PATH/test/webtests/http/tests/inspector-sources" - -# Kill background jobs when script ends. -cleanup() { - rm "$DEVTOOLS_PATH/test/webtests/http/tests/inspector-sources" - kill ${SERVER_PID} -} -trap 'cleanup' EXIT - -# Serve from devtools frontend webtests folder. -(npx http-server@0.12.3 "$DEVTOOLS_PATH/test/webtests/http/tests" -p 8000 --cors > /dev/null 2>&1) & -SERVER_PID=$! - -echo "Waiting for server" -health_check_url='http://localhost:8000/inspector-sources/integration_test_runner.html?experiments=true&test=http://127.0.0.1:8000/devtools/lighthouse/lighthouse-view-trace-run.js' -until $(curl --output /dev/null --silent --head --fail $health_check_url); do - printf '.' - sleep 1 -done -echo "Server is up" - -# webtests sometimes error if results are already present. -rm -rf "$latest_content_shell/out/Release/layout-test-results" - -# Add typ to python path. The regular method assumes there is a Chromium checkout. -export PYTHONPATH="${PYTHONPATH:-}:$BLINK_TOOLS_PATH/latest/third_party/typ" - -# Don't quit if the python command fails. -set +e -# Print the python command. -set -x - -python3 \ - "$BLINK_TOOLS_PATH/latest/third_party/blink/tools/run_web_tests.py" \ - --layout-tests-directory="$DEVTOOLS_PATH/test/webtests" \ - --additional-platform-directory="$DEVTOOLS_PATH/test/webtests/platform/generic" \ - --build-directory="$latest_content_shell/out" \ - $* -status=$? - -set +x -set -e - -rm -rf "$LH_ROOT/.tmp/layout-test-results" -cp -r "$latest_content_shell/out/Release/layout-test-results" "$LH_ROOT/.tmp/layout-test-results" -cp "$DEVTOOLS_PATH/test/webtests/platform/generic/http/tests/devtools/lighthouse/"*-expected.txt "$LH_ROOT/third-party/chromium-webtests/webtests/platform/generic/http/tests/devtools/lighthouse" - -exit $status diff --git a/lighthouse-core/test/devtools-tests/README.md b/lighthouse-core/test/devtools-tests/README.md new file mode 100644 index 000000000000..2bb767721896 --- /dev/null +++ b/lighthouse-core/test/devtools-tests/README.md @@ -0,0 +1,15 @@ +# DevTools Tests + +This runs the DevTools e2e tests with latest Lighthouse. + +`third-party/devtools-tests` contains all of our e2e tests. + +## Run + +```sh +yarn test-devtools + +# Run the test runner, without getting the latest DevTools commits like `yarn test-devtools` does. +# This still bundles Lighthouse + rolls to DevTools before running the tests. +SKIP_DOWNLOADS=1 yarn test-devtools +``` diff --git a/lighthouse-core/test/chromium-web-tests/download-depot-tools.sh b/lighthouse-core/test/devtools-tests/download-depot-tools.sh similarity index 100% rename from lighthouse-core/test/chromium-web-tests/download-depot-tools.sh rename to lighthouse-core/test/devtools-tests/download-depot-tools.sh diff --git a/lighthouse-core/test/chromium-web-tests/download-devtools.sh b/lighthouse-core/test/devtools-tests/download-devtools.sh similarity index 93% rename from lighthouse-core/test/chromium-web-tests/download-devtools.sh rename to lighthouse-core/test/devtools-tests/download-devtools.sh index 3b37303d5f52..4fa3f7cff8fa 100644 --- a/lighthouse-core/test/chromium-web-tests/download-devtools.sh +++ b/lighthouse-core/test/devtools-tests/download-devtools.sh @@ -29,12 +29,7 @@ echo "Downloading DevTools frontend at $DEVTOOLS_PATH" mkdir -p `dirname $DEVTOOLS_PATH` cd `dirname $DEVTOOLS_PATH` -set -x - fetch --nohooks --no-history devtools-frontend cd devtools-frontend - gclient sync -gn gen out/Default - -set +x +gn gen out/Default --args='devtools_dcheck_always_on=true is_debug=false' diff --git a/lighthouse-core/test/chromium-web-tests/roll-devtools.sh b/lighthouse-core/test/devtools-tests/roll-devtools.sh similarity index 74% rename from lighthouse-core/test/chromium-web-tests/roll-devtools.sh rename to lighthouse-core/test/devtools-tests/roll-devtools.sh index 4f3202319411..aa052ad60cdc 100644 --- a/lighthouse-core/test/chromium-web-tests/roll-devtools.sh +++ b/lighthouse-core/test/devtools-tests/roll-devtools.sh @@ -13,7 +13,7 @@ LH_ROOT="$SCRIPT_DIR/../../.." roll_devtools() { # Roll devtools. Besides giving DevTools the latest lighthouse source files, - # this also copies over the webtests. + # this also copies over the e2e tests. cd "$LH_ROOT" yarn devtools "$DEVTOOLS_PATH" cd - @@ -22,7 +22,12 @@ roll_devtools() { cd "$DEVTOOLS_PATH" git --no-pager log -1 roll_devtools -# Build devtools. The creates `out/Default/gen/front_end`, -# which is served as `inspector-sources` by the webtests server. + +# Needed to re-generate ninja rules, because there may possibly be e2e test files +# referenced in the rules initially generated by `download-devtools.sh` that +# `yarn devtools` deleted. +gclient sync --delete_unversioned_trees --reset + +# Build devtools. This creates `out/Default/gen/front_end`. autoninja -C out/Default cd - diff --git a/lighthouse-core/test/devtools-tests/setup.sh b/lighthouse-core/test/devtools-tests/setup.sh index 756cfacf884e..b0658fd05363 100644 --- a/lighthouse-core/test/devtools-tests/setup.sh +++ b/lighthouse-core/test/devtools-tests/setup.sh @@ -15,16 +15,17 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" LH_ROOT="$SCRIPT_DIR/../../.." TEST_DIR="$LH_ROOT/.tmp/chromium-web-tests" +export DEPOT_TOOLS_PATH="$TEST_DIR/depot-tools" export DEVTOOLS_PATH=${DEVTOOLS_PATH:-"$TEST_DIR/devtools/devtools-frontend"} if [ -z ${SKIP_DOWNLOADS+x} ] then echo "========================================" - echo "Downloading latest content_shell and DevTools" + echo "Downloading latest DevTools" echo "To skip this step, set SKIP_DOWNLOADS=1" echo "========================================" echo - bash "$SCRIPT_DIR/../chromium-web-tests/download-devtools.sh" - bash "$SCRIPT_DIR/../chromium-web-tests/download-content-shell.sh" + bash "$SCRIPT_DIR/download-depot-tools.sh" + bash "$SCRIPT_DIR/download-devtools.sh" fi diff --git a/lighthouse-core/test/devtools-tests/test-locally.sh b/lighthouse-core/test/devtools-tests/test-locally.sh index 444dfd1b0bd9..119bda9e36b5 100644 --- a/lighthouse-core/test/devtools-tests/test-locally.sh +++ b/lighthouse-core/test/devtools-tests/test-locally.sh @@ -7,14 +7,12 @@ ## # Runs the devtools e2e tests in third-party/devtools-tests using the latest -# Lighthouse, DevTools, and Chrome content_shell* -# -# * Not exactly the latest, but close. See download-content-shell.js +# Lighthouse and DevTools set -euo pipefail SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" source "$SCRIPT_DIR/setup.sh" -bash "$SCRIPT_DIR/../chromium-web-tests/roll-devtools.sh" +bash "$SCRIPT_DIR/roll-devtools.sh" bash "$SCRIPT_DIR/run-e2e-tests.sh" $* diff --git a/package.json b/package.json index dfe4bfe2ea6d..39af2022400d 100644 --- a/package.json +++ b/package.json @@ -76,8 +76,7 @@ "update:sample-artifacts": "node lighthouse-core/scripts/update-report-fixtures.js", "update:sample-json": "yarn i18n:collect-strings && node ./lighthouse-cli -A=./lighthouse-core/test/results/artifacts --config-path=./lighthouse-core/test/results/sample-config.js --output=json --output-path=./lighthouse-core/test/results/sample_v2.json && node lighthouse-core/scripts/cleanup-LHR-for-diff.js ./lighthouse-core/test/results/sample_v2.json --only-remove-timing && node ./lighthouse-core/scripts/update-flow-fixtures.js", "update:flow-sample-json": "yarn i18n:collect-strings && node ./lighthouse-core/scripts/update-flow-fixtures.js", - "update:test-devtools": "bash lighthouse-core/test/chromium-web-tests/test-locally.sh --reset-results", - "test-devtools": "bash lighthouse-core/test/chromium-web-tests/test-locally.sh", + "test-devtools": "bash lighthouse-core/test/devtools-tests/test-locally.sh", "open-devtools": "bash lighthouse-core/scripts/open-devtools.sh", "run-devtools": "node lighthouse-core/scripts/pptr-run-devtools.js", "diff:sample-json": "yarn i18n:checks && bash lighthouse-core/scripts/assert-golden-lhr-unchanged.sh", diff --git a/third-party/chromium-webtests/README.md b/third-party/chromium-webtests/README.md deleted file mode 100644 index 917775a6ef6c..000000000000 --- a/third-party/chromium-webtests/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Chromium Web Tests - -These tests are rolled into the Chromium DevTools Frontend codebase. They "belong" to the devtools frontend, but are truly defined in this Lighthouse repo. - -See `lighthouse-core/test/chromium-web-tests/README.md` for more. - -## Sync - -```sh -rsync -ahvz --exclude='OWNERS' ~/src/devtools/devtools-frontend/test/webtests/http/tests/devtools/lighthouse/ third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/ -yarn && yarn build-devtools && yarn update:test-devtools -``` diff --git a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-block-url.js b/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-block-url.js deleted file mode 100644 index fdb8165acfaa..000000000000 --- a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-block-url.js +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2020 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -(async function() { - TestRunner.addResult('Tests that Lighthouse block requests specified in DevTools BlockedURLsPane.\n'); - await TestRunner.navigatePromise('resources/lighthouse-basic.html'); - - await TestRunner.loadTestModule('lighthouse_test_runner'); - await TestRunner.showPanel('lighthouse'); - - const containerElement = LighthouseTestRunner.getContainerElement(); - const ensureDisabledNames = ['Accessibility', 'Best practices', 'SEO', 'Progressive Web App']; - const checkboxes = Array.from(containerElement.querySelectorAll('.checkbox')); - for (const checkbox of checkboxes) { - if (!ensureDisabledNames.includes(checkbox.textElement.textContent)) { - continue; - } - - if (checkbox.checkboxElement.checked) { - checkbox.checkboxElement.click(); - } - } - - const networkManager = self.SDK.MultitargetNetworkManager.instance(); - networkManager.setBlockingEnabled(true); - networkManager.setBlockedPatterns([{enabled: true, url: '*.css'}]); - - LighthouseTestRunner.dumpStartAuditState(); - LighthouseTestRunner.getRunButton().click(); - const {lhr} = await LighthouseTestRunner.waitForResults(); - - const requestsMade = []; - const requestsBlocked = []; - for (const item of lhr.audits['network-requests'].details.items) { - if (item.statusCode === -1) { - requestsBlocked.push(item.url); - } else { - requestsMade.push(item.url); - } - } - - TestRunner.addResult(`\nRequests made: ${requestsMade.join(', ')}`); - TestRunner.addResult(`Requests blocked: ${requestsBlocked.join(', ')}`); - - networkManager.setBlockingEnabled(false); - networkManager.setBlockedPatterns([]); - - TestRunner.completeTest(); -})(); diff --git a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-clear-data-warning.js b/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-clear-data-warning.js deleted file mode 100644 index 09ea713ba42f..000000000000 --- a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-clear-data-warning.js +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2020 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -(async function() { - TestRunner.addResult('Tests that Lighthouse panel displays a warning when important data may affect performance.\n'); - await TestRunner.navigatePromise('resources/lighthouse-storage.html'); - - await TestRunner.loadTestModule('lighthouse_test_runner'); - await TestRunner.showPanel('lighthouse'); - await TestRunner.RuntimeAgent.invoke_evaluate({ - expression: 'webSqlPromise', - awaitPromise: true, - }); - - const containerElement = LighthouseTestRunner.getContainerElement(); - const ensureDisabledNames = ['Accessibility', 'Best practices', 'SEO', 'Progressive Web App']; - const checkboxes = Array.from(containerElement.querySelectorAll('.checkbox')); - for (const checkbox of checkboxes) { - if (!ensureDisabledNames.includes(checkbox.textElement.textContent)) { - continue; - } - - if (checkbox.checkboxElement.checked) { - checkbox.checkboxElement.click(); - } - } - - LighthouseTestRunner.dumpStartAuditState(); - - const Events = Lighthouse.LighthousePanel.getEvents(); - - // Wait for warning event to be handled - LighthouseTestRunner._panel().controller.addEventListener(Events.PageWarningsChanged, () => { - const warningText = containerElement.querySelector('.lighthouse-warning-text'); - TestRunner.addResult(`Warning Text: ${warningText.textContent}`); - TestRunner.completeTest(); - }); - LighthouseTestRunner.forcePageAuditabilityCheck(); -})(); diff --git a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-emulate-run.js b/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-emulate-run.js deleted file mode 100644 index 16c095bc57d2..000000000000 --- a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-emulate-run.js +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2019 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -(async function() { - TestRunner.addResult('Tests that mobile emulation works.\n'); - await TestRunner.navigatePromise('resources/lighthouse-emulate-pass.html'); - - await TestRunner.loadTestModule('lighthouse_test_runner'); - await TestRunner.showPanel('lighthouse'); - - LighthouseTestRunner.dumpStartAuditState(); - LighthouseTestRunner.getRunButton().click(); - const {lhr} = await LighthouseTestRunner.waitForResults(); - - TestRunner.addResult('\n=============== Lighthouse Results ==============='); - TestRunner.addResult(`URL: ${lhr.finalUrl}`); - TestRunner.addResult(`Version: ${lhr.lighthouseVersion}`); - TestRunner.addResult(`formFactor: ${lhr.configSettings.formFactor}`); - TestRunner.addResult(`screenEmulation: ${JSON.stringify(lhr.configSettings.screenEmulation, null, 2)}`); - TestRunner.addResult(`Mobile network UA?: ${lhr.environment.networkUserAgent.includes('Mobile')}`); - TestRunner.addResult(`Mobile configured UA?: ${lhr.configSettings.emulatedUserAgent.includes('Mobile')}`); - TestRunner.addResult(`throttlingMethod: ${lhr.configSettings.throttlingMethod}`); - TestRunner.addResult(`throttling.rttMs: ${lhr.configSettings.throttling.rttMs}`); - TestRunner.addResult('\n'); - - const auditName = 'content-width'; - const audit = lhr.audits[auditName]; - if (audit.scoreDisplayMode === 'error') { - TestRunner.addResult(`${auditName}: ERROR ${audit.errorMessage}`); - } else if (audit.scoreDisplayMode === 'binary') { - TestRunner.addResult(`${auditName}: ${audit.score ? 'pass' : 'fail'} ${audit.explanation}`); - } else { - TestRunner.addResult(`${auditName}: ${audit.scoreDisplayMode}`); - } - - TestRunner.completeTest(); -})(); diff --git a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-export-run.js b/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-export-run.js deleted file mode 100644 index b1f119779a89..000000000000 --- a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-export-run.js +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2019 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -(async function() { - TestRunner.addResult('Tests that exporting works.\n'); - await TestRunner.navigatePromise('resources/lighthouse-basic.html'); - - await TestRunner.loadTestModule('lighthouse_test_runner'); - await TestRunner.showPanel('lighthouse'); - - LighthouseTestRunner.getRunButton().click(); - await LighthouseTestRunner.waitForResults(); - - const resultsElement = LighthouseTestRunner.getResultsElement(); - const toolsMenu = resultsElement.querySelector('.lh-tools__dropdown'); - - function waitForSave() { - return new Promise(resolve => { - TestRunner.addSniffer(Workspace.FileManager.prototype, 'save', - (filename, content) => resolve(content)); - }); - } - - async function testExportHtml() { - const reportHtmlPromise = waitForSave(); - toolsMenu.querySelector('a[data-action="save-html"').click(); - const reportHtml = await reportHtmlPromise; - - let auditElements = resultsElement.querySelectorAll('.lh-audit'); - TestRunner.addResult(`\n# of .lh-audit divs (original): ${auditElements.length}`); - - const exportedReportIframe = resultsElement.ownerDocument.createElement('iframe'); - exportedReportIframe.srcdoc = reportHtml; - resultsElement.parentElement.append(exportedReportIframe); - await new Promise(resolve => exportedReportIframe.addEventListener('load', resolve)); - - auditElements = exportedReportIframe.contentDocument.querySelectorAll('.lh-audit'); - TestRunner.addResult(`\n# of .lh-audit divs (exported html): ${auditElements.length}`); - } - - async function testExportJson() { - const reportJsonPromise = waitForSave(); - toolsMenu.querySelector('a[data-action="save-json"').click(); - const reportJson = await reportJsonPromise; - const lhr = JSON.parse(reportJson); - TestRunner.addResult(`\n# of audits (json): ${Object.keys(lhr.audits).length}`); - } - - TestRunner.addResult('++++++++ testExportHtml'); - await testExportHtml(); - - TestRunner.addResult('\n++++++++ testExportJson'); - await testExportJson(); - - TestRunner.completeTest(); -})(); diff --git a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-flags-run.js b/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-flags-run.js deleted file mode 100644 index 9806e779c7de..000000000000 --- a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-flags-run.js +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2018 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -(async function() { - TestRunner.addResult('Tests that lighthouse panel passes flags.\n'); - await TestRunner.navigatePromise('resources/lighthouse-basic.html'); - - await TestRunner.loadTestModule('lighthouse_test_runner'); - await TestRunner.showPanel('lighthouse'); - - const dialogElement = LighthouseTestRunner.getContainerElement(); - - // Turn off simulated throttling. - const select = dialogElement.querySelector('.lighthouse-settings-pane .toolbar') - .shadowRoot.querySelector('select') - select.querySelector('option[value="devtools"]').selected = true; - - // Change event is not emitted automatically when updating select element programatically. - select.dispatchEvent(new Event('change')); - - // Use desktop environment - dialogElement.querySelector('input[name="lighthouse.device_type"][value="desktop"]').click(); - - LighthouseTestRunner.dumpStartAuditState(); - LighthouseTestRunner.getRunButton().click(); - - const {lhr} = await LighthouseTestRunner.waitForResults(); - TestRunner.addResult('\n=============== Lighthouse Results ==============='); - TestRunner.addResult(`formFactor: ${lhr.configSettings.formFactor}`); - TestRunner.addResult(`disableStorageReset: ${lhr.configSettings.disableStorageReset}`); - TestRunner.addResult(`throttlingMethod: ${lhr.configSettings.throttlingMethod}`); - - const viewTraceButton = LighthouseTestRunner.getResultsElement().querySelector('.lh-button--trace'); - TestRunner.addResult(`\nView Trace Button Text: "${viewTraceButton.textContent}"`); - TestRunner.addResult(`View Trace Button Title: "${viewTraceButton.title}"`); - - TestRunner.completeTest(); -})(); diff --git a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-i18n-run.js b/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-i18n-run.js deleted file mode 100644 index 21a91f253937..000000000000 --- a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-i18n-run.js +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2020 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -(async function() { - TestRunner.addResult('Tests that Lighthouse report is translated.\n'); - await TestRunner.navigatePromise('resources/lighthouse-basic.html'); - - await TestRunner.loadTestModule('lighthouse_test_runner'); - await TestRunner.showPanel('lighthouse'); - - const containerElement = LighthouseTestRunner.getContainerElement(); - const ensureDisabledNames = ['Accessibility', 'Best practices', 'SEO', 'Progressive Web App']; - const checkboxes = Array.from(containerElement.querySelectorAll('.checkbox')); - for (const checkbox of checkboxes) { - if (!ensureDisabledNames.includes(checkbox.textElement.textContent)) { - continue; - } - - if (checkbox.checkboxElement.checked) { - checkbox.checkboxElement.click(); - } - } - - TestRunner.override(LighthouseTestRunner._panel().protocolService, 'getLocales', overrideLookupLocale, true); - - const locales = ['invalid-locale', 'es']; - function overrideLookupLocale() { - return locales; - } - - LighthouseTestRunner.dumpStartAuditState(); - LighthouseTestRunner.getRunButton().click(); - - const {lhr} = await LighthouseTestRunner.waitForResults(); - - TestRunner.addResult(`resolved to locale ${lhr.configSettings.locale}`); - TestRunner.addResult(`\ni18n footerIssue: "${lhr.i18n.rendererFormattedStrings.footerIssue}"`); - - const footerIssueLink = LighthouseTestRunner.getResultsElement().querySelector('.lh-footer__version_issue'); - TestRunner.addResult(`\nFooter Issue Link Text: "${footerIssueLink.textContent}"`); - - TestRunner.completeTest(); -})(); diff --git a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-limited-run.js b/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-limited-run.js deleted file mode 100644 index 26c5cdaa4905..000000000000 --- a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-limited-run.js +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2017 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -(async function() { - TestRunner.addResult('Tests that audits panel works when only the pwa category is selected.\n'); - await TestRunner.navigatePromise('resources/lighthouse-basic.html'); - - await TestRunner.loadTestModule('lighthouse_test_runner'); - await TestRunner.showPanel('lighthouse'); - - const containerElement = LighthouseTestRunner.getContainerElement(); - const ensureDisabledNames = ['Performance', 'Accessibility', 'Best practices', 'SEO']; - const checkboxes = Array.from(containerElement.querySelectorAll('.checkbox')); - for (const checkbox of checkboxes) { - if (!ensureDisabledNames.includes(checkbox.textElement.textContent)) { - continue; - } - - if (checkbox.checkboxElement.checked) { - checkbox.checkboxElement.click(); - } - } - - LighthouseTestRunner.dumpStartAuditState(); - LighthouseTestRunner.getRunButton().click(); - - const {lhr} = await LighthouseTestRunner.waitForResults(); - TestRunner.addResult('\n=============== Audits run ==============='); - TestRunner.addResult(Object.keys(lhr.audits).sort().join('\n')); - - TestRunner.completeTest(); -})(); diff --git a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-prevent-run.js b/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-prevent-run.js deleted file mode 100644 index 74fbc6273c1d..000000000000 --- a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-prevent-run.js +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2017 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -(async function() { - // about:blank never fires a load event so just wait until we see the URL change - function navigateToAboutBlankAndWait() { - const listenerPromise = new Promise(resolve => { - self.SDK.targetManager.addEventListener(SDK.TargetManager.Events.InspectedURLChanged, resolve); - }); - - TestRunner.navigate('about:blank'); - return listenerPromise; - } - - TestRunner.addResult('Tests that audits panel prevents run of unauditable pages.\n'); - await TestRunner.navigatePromise('resources/lighthouse-basic.html'); - - await TestRunner.loadTestModule('lighthouse_test_runner'); - await TestRunner.showPanel('lighthouse'); - - TestRunner.addResult('\n\n**Prevents audit with no categories**'); - LighthouseTestRunner.openStartAudit(); - const containerElement = LighthouseTestRunner.getContainerElement(); - const ensureDisabledNames = ['Performance', 'Accessibility', 'Best practices', 'SEO', 'Progressive Web App']; - const checkboxes = Array.from(containerElement.querySelectorAll('.checkbox')); - for (const checkbox of checkboxes) { - if (!ensureDisabledNames.includes(checkbox.textElement.textContent)) { - continue; - } - - if (checkbox.checkboxElement.checked) { - checkbox.checkboxElement.click(); - } - } - LighthouseTestRunner.dumpStartAuditState(); - - TestRunner.addResult('\n\n**Allows audit with a single category**'); - for (const checkbox of checkboxes) { - if (checkbox.textElement.textContent !== 'Performance') { - continue; - } - - checkbox.checkboxElement.click(); - } - LighthouseTestRunner.dumpStartAuditState(); - - TestRunner.addResult('\n\n**Allows audit on undockable page**'); - // Extension page and remote debugging previously caused crashes (crbug.com/734532) - // However, the crashes have been resolved, so these should now pass. - LighthouseTestRunner.forcePageAuditabilityCheck(); - LighthouseTestRunner.dumpStartAuditState(); - - TestRunner.addResult('\n\n**Prevents audit on internal page**'); - await navigateToAboutBlankAndWait(); - TestRunner.addResult(`URL: ${TestRunner.mainTarget.inspectedURL()}`); - LighthouseTestRunner.dumpStartAuditState(); - - TestRunner.completeTest(); -})(); diff --git a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-successful-navigation.js b/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-successful-navigation.js deleted file mode 100644 index 72e52e28088f..000000000000 --- a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-successful-navigation.js +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2022 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -(async function() { - // screenshots in content shell are flaky and NO_NAVSTART occurs on bots way more frequently than local - // ignore the results of the trace-dependent audits, just make sure they ran - const FLAKY_AUDITS = [ - // metrics - 'first-contentful-paint', - 'first-meaningful-paint', - 'interactive', - 'speed-index', - 'metrics', - 'screenshot-thumbnails', - // misc trace-based audits - 'long-tasks', - 'user-timings', - 'bootup-time', - // opportunities - 'efficient-animated-content', - 'offscreen-images', - 'redirects', - 'render-blocking-resources', - 'unminified-css', - 'unminified-javascript', - 'unused-css-rules', - 'uses-optimized-images', - 'uses-rel-preload', - 'uses-responsive-images', - 'uses-text-compression', - 'modern-image-formats', - ]; - - TestRunner.addResult('Tests that navigation mode works correctly.\n'); - await TestRunner.navigatePromise('resources/lighthouse-basic.html'); - - await TestRunner.loadTestModule('lighthouse_test_runner'); - await TestRunner.showPanel('lighthouse'); - - // Enable Publisher Ads. - const containerElement = LighthouseTestRunner.getContainerElement(); - const checkboxes = containerElement.querySelectorAll('.checkbox'); - for (const checkbox of checkboxes) { - if (checkbox.textElement.textContent === 'Publisher Ads') { - checkbox.checkboxElement.click(); - } - } - - // Enable FR navigation. - containerElement.querySelector('.lighthouse-settings-pane > div').shadowRoot - .querySelectorAll('span')[0].shadowRoot - .querySelector('input').click(); - - LighthouseTestRunner.dumpStartAuditState(); - - TestRunner.addResult('\n=============== Lighthouse Status Updates ==============='); - LighthouseTestRunner.addStatusListener(msg => TestRunner.addResult(msg)); - LighthouseTestRunner.getRunButton().click(); - - const {artifacts, lhr} = await LighthouseTestRunner.waitForResults(); - TestRunner.addResult('\n=============== Lighthouse Results ==============='); - TestRunner.addResult(`URL: ${lhr.finalUrl}`); - TestRunner.addResult(`Version: ${lhr.lighthouseVersion}`); - TestRunner.addResult(`ViewportDimensions: ${JSON.stringify(artifacts.ViewportDimensions, null, 2)}`); - TestRunner.addResult('\n'); - - Object.keys(lhr.audits).sort().forEach(auditName => { - const audit = lhr.audits[auditName]; - - if (FLAKY_AUDITS.includes(auditName)) { - TestRunner.addResult(`${auditName}: flaky`); - } else if (audit.scoreDisplayMode === 'error') { - TestRunner.addResult(`${auditName}: ERROR ${audit.errorMessage}`); - } else if (audit.scoreDisplayMode === 'binary') { - TestRunner.addResult(`${auditName}: ${audit.score ? 'pass' : 'fail'}`); - } else { - TestRunner.addResult(`${auditName}: ${audit.scoreDisplayMode}`); - } - }); - - const resultsElement = LighthouseTestRunner.getResultsElement(); - const auditElements = [...resultsElement.querySelectorAll('.lh-audit')]; - const auditElementNames = auditElements.map(e => e.id).sort((a, b) => a.localeCompare(b)); - TestRunner.addResult(`\n# of .lh-audit divs: ${auditElements.length}`); - TestRunner.addResult(`\n.lh-audit divs:\n${auditElementNames.join('\n')}`); - - // Ensure duplicate events are not recieved. - // See https://github.com/GoogleChrome/lighthouse/issues/11415 - const devtoolsLog = artifacts.devtoolsLogs.defaultPass; - const networkResponseRecievedEvents = devtoolsLog.filter( - log => log.method === 'Network.responseReceived' && log.params.response.url.endsWith('lighthouse-basic.html')); - if (networkResponseRecievedEvents.length !== 1) { - TestRunner.addResult(`ERROR: Network.responseReceived events for main resource; expected 1, got ${ - networkResponseRecievedEvents.length}`); - } - - TestRunner.completeTest(); -})(); diff --git a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-successful-run.js b/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-successful-run.js deleted file mode 100644 index 10cbfd9287b0..000000000000 --- a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-successful-run.js +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2017 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -(async function() { - // screenshots in content shell are flaky and NO_NAVSTART occurs on bots way more frequently than local - // ignore the results of the trace-dependent audits, just make sure they ran - const FLAKY_AUDITS = [ - // metrics - 'first-contentful-paint', - 'first-meaningful-paint', - 'interactive', - 'speed-index', - 'metrics', - 'screenshot-thumbnails', - // misc trace-based audits - 'long-tasks', - 'user-timings', - 'bootup-time', - // opportunities - 'efficient-animated-content', - 'offscreen-images', - 'redirects', - 'render-blocking-resources', - 'unminified-css', - 'unminified-javascript', - 'unused-css-rules', - 'uses-optimized-images', - 'uses-rel-preload', - 'uses-responsive-images', - 'uses-text-compression', - 'modern-image-formats', - ]; - - TestRunner.addResult('Tests that audits panel works.\n'); - await TestRunner.navigatePromise('resources/lighthouse-basic.html'); - - await TestRunner.loadTestModule('lighthouse_test_runner'); - await TestRunner.showPanel('lighthouse'); - - // Use all the default settings, but also enable a plugin. - const containerElement = LighthouseTestRunner.getContainerElement(); - const checkboxes = containerElement.querySelectorAll('.checkbox'); - for (const checkbox of checkboxes) { - if (checkbox.textElement.textContent === 'Publisher Ads') { - checkbox.checkboxElement.click(); - } - } - - LighthouseTestRunner.dumpStartAuditState(); - - TestRunner.addResult('\n=============== Lighthouse Status Updates ==============='); - LighthouseTestRunner.addStatusListener(msg => TestRunner.addResult(msg)); - LighthouseTestRunner.getRunButton().click(); - - const {artifacts, lhr} = await LighthouseTestRunner.waitForResults(); - TestRunner.addResult('\n=============== Lighthouse Results ==============='); - TestRunner.addResult(`URL: ${lhr.finalUrl}`); - TestRunner.addResult(`Version: ${lhr.lighthouseVersion}`); - TestRunner.addResult(`ViewportDimensions: ${JSON.stringify(artifacts.ViewportDimensions, null, 2)}`); - TestRunner.addResult('\n'); - - Object.keys(lhr.audits).sort().forEach(auditName => { - const audit = lhr.audits[auditName]; - - if (FLAKY_AUDITS.includes(auditName)) { - TestRunner.addResult(`${auditName}: flaky`); - } else if (audit.scoreDisplayMode === 'error') { - TestRunner.addResult(`${auditName}: ERROR ${audit.errorMessage}`); - } else if (audit.scoreDisplayMode === 'binary') { - TestRunner.addResult(`${auditName}: ${audit.score ? 'pass' : 'fail'}`); - } else { - TestRunner.addResult(`${auditName}: ${audit.scoreDisplayMode}`); - } - }); - - const resultsElement = LighthouseTestRunner.getResultsElement(); - const auditElements = [...resultsElement.querySelectorAll('.lh-audit')]; - const auditElementNames = auditElements.map(e => e.id).sort((a, b) => a.localeCompare(b)); - TestRunner.addResult(`\n# of .lh-audit divs: ${auditElements.length}`); - TestRunner.addResult(`\n.lh-audit divs:\n${auditElementNames.join('\n')}`); - - // Ensure duplicate events are not recieved. - // See https://github.com/GoogleChrome/lighthouse/issues/11415 - const devtoolsLog = artifacts.devtoolsLogs.defaultPass; - const networkResponseRecievedEvents = devtoolsLog.filter( - log => log.method === 'Network.responseReceived' && log.params.response.url.endsWith('lighthouse-basic.html')); - if (networkResponseRecievedEvents.length !== 1) { - TestRunner.addResult(`ERROR: Network.responseReceived events for main resource; expected 1, got ${ - networkResponseRecievedEvents.length}`); - } - - TestRunner.completeTest(); -})(); diff --git a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-view-trace-run.js b/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-view-trace-run.js deleted file mode 100644 index bd93b05743f6..000000000000 --- a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-view-trace-run.js +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2019 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -(async function() { - TestRunner.addResult('Tests that audits panel renders View Trace button.\n'); - await TestRunner.navigatePromise('resources/lighthouse-basic.html'); - - await TestRunner.loadTestModule('lighthouse_test_runner'); - await TestRunner.showPanel('lighthouse'); - - const containerElement = LighthouseTestRunner.getContainerElement(); - const ensureDisabledNames = ['Accessibility', 'Best practices', 'SEO', 'Progressive Web App']; - const checkboxes = Array.from(containerElement.querySelectorAll('.checkbox')); - for (const checkbox of checkboxes) { - if (!ensureDisabledNames.includes(checkbox.textElement.textContent)) { - continue; - } - - if (checkbox.checkboxElement.checked) { - checkbox.checkboxElement.click(); - } - } - - LighthouseTestRunner.dumpStartAuditState(); - LighthouseTestRunner.getRunButton().click(); - - const {lhr} = await LighthouseTestRunner.waitForResults(); - TestRunner.addResult('\n=============== Audits run ==============='); - TestRunner.addResult(Object.keys(lhr.audits).sort().join('\n')); - - const waitForShowView = new Promise(resolve => { - TestRunner.addSniffer(UI.ViewManager.prototype, 'showView', resolve); - }); - const viewTraceButton = LighthouseTestRunner.getResultsElement().querySelector('.lh-button--trace'); - TestRunner.addResult(`\nView Trace Button Text: "${viewTraceButton.textContent}"`); - TestRunner.addResult(`View Trace Button Title: "${viewTraceButton.title}"`); - viewTraceButton.click(); - const viewShown = await waitForShowView; - TestRunner.addResult(`\nShowing view: ${viewShown}`); - - TestRunner.completeTest(); -})(); diff --git a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/resources/lighthouse-emulate-pass.html b/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/resources/lighthouse-emulate-pass.html deleted file mode 100644 index 2c74c1daee78..000000000000 --- a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/resources/lighthouse-emulate-pass.html +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - hi - diff --git a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/resources/lighthouse-storage.html b/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/resources/lighthouse-storage.html deleted file mode 100644 index 6155a567e21c..000000000000 --- a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/resources/lighthouse-storage.html +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - hi - diff --git a/third-party/chromium-webtests/webtests/platform/generic/http/tests/devtools/lighthouse/lighthouse-block-url-expected.txt b/third-party/chromium-webtests/webtests/platform/generic/http/tests/devtools/lighthouse/lighthouse-block-url-expected.txt deleted file mode 100644 index 49df53107e7f..000000000000 --- a/third-party/chromium-webtests/webtests/platform/generic/http/tests/devtools/lighthouse/lighthouse-block-url-expected.txt +++ /dev/null @@ -1,18 +0,0 @@ -Tests that Lighthouse block requests specified in DevTools BlockedURLsPane. - - -========== Lighthouse Start Audit State ========== -[x] Performance -[ ] Accessibility -[ ] Best practices -[ ] SEO -[ ] Progressive Web App -[ ] Publisher Ads -[x] Legacy navigation -[x] Clear storage -Throttling method: simulate -Analyze page load: enabled visible - -Requests made: http://127.0.0.1:8000/devtools/lighthouse/resources/lighthouse-basic.html -Requests blocked: http://127.0.0.1:8000/devtools/lighthouse/resources/lighthouse-basic.css - diff --git a/third-party/chromium-webtests/webtests/platform/generic/http/tests/devtools/lighthouse/lighthouse-clear-data-warning-expected.txt b/third-party/chromium-webtests/webtests/platform/generic/http/tests/devtools/lighthouse/lighthouse-clear-data-warning-expected.txt deleted file mode 100644 index b74d732da158..000000000000 --- a/third-party/chromium-webtests/webtests/platform/generic/http/tests/devtools/lighthouse/lighthouse-clear-data-warning-expected.txt +++ /dev/null @@ -1,16 +0,0 @@ -Tests that Lighthouse panel displays a warning when important data may affect performance. - - -========== Lighthouse Start Audit State ========== -[x] Performance -[ ] Accessibility -[ ] Best practices -[ ] SEO -[ ] Progressive Web App -[ ] Publisher Ads -[x] Legacy navigation -[x] Clear storage -Throttling method: simulate -Analyze page load: enabled visible -Warning Text: There may be stored data affecting loading performance in these locations: Web SQL, IndexedDB. Audit this page in an incognito window to prevent those resources from affecting your scores. - diff --git a/third-party/chromium-webtests/webtests/platform/generic/http/tests/devtools/lighthouse/lighthouse-emulate-run-expected.txt b/third-party/chromium-webtests/webtests/platform/generic/http/tests/devtools/lighthouse/lighthouse-emulate-run-expected.txt deleted file mode 100644 index ca9ff8b5c4b0..000000000000 --- a/third-party/chromium-webtests/webtests/platform/generic/http/tests/devtools/lighthouse/lighthouse-emulate-run-expected.txt +++ /dev/null @@ -1,34 +0,0 @@ -Tests that mobile emulation works. - - -========== Lighthouse Start Audit State ========== -[x] Performance -[x] Accessibility -[x] Best practices -[x] SEO -[x] Progressive Web App -[ ] Publisher Ads -[x] Legacy navigation -[x] Clear storage -Throttling method: simulate -Analyze page load: enabled visible - -=============== Lighthouse Results =============== -URL: http://127.0.0.1:8000/devtools/lighthouse/resources/lighthouse-emulate-pass.html -Version: 9.5.0 -formFactor: mobile -screenEmulation: { - "mobile": true, - "width": 360, - "height": 640, - "deviceScaleFactor": 2.625, - "disabled": true -} -Mobile network UA?: true -Mobile configured UA?: true -throttlingMethod: simulate -throttling.rttMs: 150 - - -content-width: pass undefined - diff --git a/third-party/chromium-webtests/webtests/platform/generic/http/tests/devtools/lighthouse/lighthouse-export-run-expected.txt b/third-party/chromium-webtests/webtests/platform/generic/http/tests/devtools/lighthouse/lighthouse-export-run-expected.txt deleted file mode 100644 index 57304d5d24dc..000000000000 --- a/third-party/chromium-webtests/webtests/platform/generic/http/tests/devtools/lighthouse/lighthouse-export-run-expected.txt +++ /dev/null @@ -1,12 +0,0 @@ -Tests that exporting works. - -++++++++ testExportHtml - -# of .lh-audit divs (original): 136 - -# of .lh-audit divs (exported html): 136 - -++++++++ testExportJson - -# of audits (json): 152 - diff --git a/third-party/chromium-webtests/webtests/platform/generic/http/tests/devtools/lighthouse/lighthouse-flags-run-expected.txt b/third-party/chromium-webtests/webtests/platform/generic/http/tests/devtools/lighthouse/lighthouse-flags-run-expected.txt deleted file mode 100644 index 06479d291833..000000000000 --- a/third-party/chromium-webtests/webtests/platform/generic/http/tests/devtools/lighthouse/lighthouse-flags-run-expected.txt +++ /dev/null @@ -1,23 +0,0 @@ -Tests that lighthouse panel passes flags. - - -========== Lighthouse Start Audit State ========== -[x] Performance -[x] Accessibility -[x] Best practices -[x] SEO -[x] Progressive Web App -[ ] Publisher Ads -[x] Legacy navigation -[x] Clear storage -Throttling method: devtools -Analyze page load: enabled visible - -=============== Lighthouse Results =============== -formFactor: desktop -disableStorageReset: false -throttlingMethod: devtools - -View Trace Button Text: "View Trace" -View Trace Button Title: "" - diff --git a/third-party/chromium-webtests/webtests/platform/generic/http/tests/devtools/lighthouse/lighthouse-i18n-run-expected.txt b/third-party/chromium-webtests/webtests/platform/generic/http/tests/devtools/lighthouse/lighthouse-i18n-run-expected.txt deleted file mode 100644 index b8cc8b30f7fc..000000000000 --- a/third-party/chromium-webtests/webtests/platform/generic/http/tests/devtools/lighthouse/lighthouse-i18n-run-expected.txt +++ /dev/null @@ -1,20 +0,0 @@ -Tests that Lighthouse report is translated. - - -========== Lighthouse Start Audit State ========== -[x] Performance -[ ] Accessibility -[ ] Best practices -[ ] SEO -[ ] Progressive Web App -[ ] Publisher Ads -[x] Legacy navigation -[x] Clear storage -Throttling method: simulate -Analyze page load: enabled visible -resolved to locale es - -i18n footerIssue: "Notificar un problema" - -Footer Issue Link Text: "Notificar un problema" - diff --git a/third-party/chromium-webtests/webtests/platform/generic/http/tests/devtools/lighthouse/lighthouse-limited-run-expected.txt b/third-party/chromium-webtests/webtests/platform/generic/http/tests/devtools/lighthouse/lighthouse-limited-run-expected.txt deleted file mode 100644 index 180af44d59e4..000000000000 --- a/third-party/chromium-webtests/webtests/platform/generic/http/tests/devtools/lighthouse/lighthouse-limited-run-expected.txt +++ /dev/null @@ -1,29 +0,0 @@ -Tests that audits panel works when only the pwa category is selected. - - -========== Lighthouse Start Audit State ========== -[ ] Performance -[ ] Accessibility -[ ] Best practices -[ ] SEO -[x] Progressive Web App -[ ] Publisher Ads -[x] Legacy navigation -[x] Clear storage -Throttling method: simulate -Analyze page load: enabled visible - -=============== Audits run =============== -apple-touch-icon -content-width -full-page-screenshot -installable-manifest -maskable-icon -pwa-cross-browser -pwa-each-page-has-url -pwa-page-transitions -service-worker -splash-screen -themed-omnibox -viewport - diff --git a/third-party/chromium-webtests/webtests/platform/generic/http/tests/devtools/lighthouse/lighthouse-prevent-run-expected.txt b/third-party/chromium-webtests/webtests/platform/generic/http/tests/devtools/lighthouse/lighthouse-prevent-run-expected.txt deleted file mode 100644 index ce53bfa9200d..000000000000 --- a/third-party/chromium-webtests/webtests/platform/generic/http/tests/devtools/lighthouse/lighthouse-prevent-run-expected.txt +++ /dev/null @@ -1,66 +0,0 @@ -Tests that audits panel prevents run of unauditable pages. - - - -**Prevents audit with no categories** - -========== Lighthouse Start Audit State ========== -[ ] Performance -[ ] Accessibility -[ ] Best practices -[ ] SEO -[ ] Progressive Web App -[ ] Publisher Ads -[x] Legacy navigation -[x] Clear storage -Throttling method: simulate -Help text: At least one category must be selected. -Analyze page load: disabled visible - - -**Allows audit with a single category** - -========== Lighthouse Start Audit State ========== -[x] Performance -[ ] Accessibility -[ ] Best practices -[ ] SEO -[ ] Progressive Web App -[ ] Publisher Ads -[x] Legacy navigation -[x] Clear storage -Throttling method: simulate -Analyze page load: enabled visible - - -**Allows audit on undockable page** - -========== Lighthouse Start Audit State ========== -[x] Performance -[ ] Accessibility -[ ] Best practices -[ ] SEO -[ ] Progressive Web App -[ ] Publisher Ads -[x] Legacy navigation -[x] Clear storage -Throttling method: simulate -Analyze page load: enabled visible - - -**Prevents audit on internal page** -URL: about:blank - -========== Lighthouse Start Audit State ========== -[x] Performance -[ ] Accessibility -[ ] Best practices -[ ] SEO -[ ] Progressive Web App -[ ] Publisher Ads -[x] Legacy navigation -[x] Clear storage -Throttling method: simulate -Help text: Can only audit HTTP/HTTPS pages and Chrome extensions. Navigate to a different page to start an audit. -Analyze page load: disabled visible - diff --git a/third-party/chromium-webtests/webtests/platform/generic/http/tests/devtools/lighthouse/lighthouse-successful-navigation-expected.txt b/third-party/chromium-webtests/webtests/platform/generic/http/tests/devtools/lighthouse/lighthouse-successful-navigation-expected.txt deleted file mode 100644 index d6c0336bdc6a..000000000000 --- a/third-party/chromium-webtests/webtests/platform/generic/http/tests/devtools/lighthouse/lighthouse-successful-navigation-expected.txt +++ /dev/null @@ -1,613 +0,0 @@ -Tests that navigation mode works correctly. - - -========== Lighthouse Start Audit State ========== -[x] Performance -[x] Accessibility -[x] Best practices -[x] SEO -[x] Progressive Web App -[x] Publisher Ads -[ ] Legacy navigation -[x] Clear storage -Throttling method: simulate -Analyze page load: enabled visible - -=============== Lighthouse Status Updates =============== -Loading… -Initialize config -Resolve artifact definitions -Resolve navigation definitions -Gather phase -Connecting to browser -Navigating to about:blank -Benchmarking machine -Getting browser version -Preparing target for navigation mode -Navigating to about:blank -Preparing target for navigation -Cleaning origin data -Cleaning browser cache -Preparing network conditions -Navigating to http://127.0.0.1:8000/devtools/lighthouse/resources/lighthouse-basic.html -Computing artifact: NetworkRecords$J -Get webapp installability errors -Computing artifact: MainResource$k -Collect stacks -Computing artifact: ProcessedTrace$h -Computing artifact: ProcessedNavigation$7 -Computing artifact: Responsiveness -Get webapp manifest -Audit phase -Analyzing and running audits... -Auditing: Uses HTTPS -Auditing: Registers a service worker that controls page and `start_url` -Auditing: Has a `` tag with `width` or `initial-scale` -Computing artifact: ViewportMeta -Auditing: First Contentful Paint -Computing artifact: FirstContentfulPaint$3 -Computing artifact: LanternFirstContentfulPaint$6 -Computing artifact: PageDependencyGraph$a -Computing artifact: LoadSimulator$a -Computing artifact: NetworkAnalysis$1 -Auditing: Largest Contentful Paint -Computing artifact: LargestContentfulPaint$2 -Computing artifact: LanternLargestContentfulPaint$1 -Auditing: First Meaningful Paint -Computing artifact: FirstMeaningfulPaint$2 -Computing artifact: LanternFirstMeaningfulPaint$2 -Auditing: Speed Index -Computing artifact: SpeedIndex$2 -Computing artifact: LanternSpeedIndex$2 -Computing artifact: Speedline$4 -Auditing: Screenshot Thumbnails -Auditing: Final Screenshot -Computing artifact: Screenshots$1 -Auditing: Total Blocking Time -Computing artifact: TotalBlockingTime$2 -Computing artifact: LanternTotalBlockingTime$1 -Computing artifact: LanternInteractive$4 -Auditing: Max Potential First Input Delay -Computing artifact: MaxPotentialFID$2 -Computing artifact: LanternMaxPotentialFID$1 -Auditing: Cumulative Layout Shift -Computing artifact: CumulativeLayoutShift$2 -Auditing: No browser errors logged to the console -Computing artifact: JSBundles -Auditing: Initial server response time was short -Auditing: Time to Interactive -Computing artifact: Interactive$4 -Auditing: User Timing marks and measures -Computing artifact: UserTimings$1 -Auditing: Avoid chaining critical requests -Computing artifact: CriticalRequestChains$2 -Auditing: Avoid multiple page redirects -Auditing: Web app manifest and service worker meet the installability requirements -Computing artifact: ManifestValues$4 -Auditing: Provides a valid `apple-touch-icon` -Auditing: Configured for a custom splash screen -Auditing: Sets a theme color for the address bar. -Auditing: Manifest has a maskable icon -Auditing: Content is sized correctly for the viewport -Auditing: Displays images with correct aspect ratio -Auditing: Serves images with appropriate resolution -Auditing: Fonts with `font-display: optional` are preloaded -Auditing: Avoids deprecated APIs -Auditing: Minimizes main-thread work -Computing artifact: MainThreadTasks$8 -Auditing: JavaScript execution time -Auditing: Preload key requests -Auditing: Preconnect to required origins -Auditing: All text remains visible during webfont loads -Auditing: Diagnostics -Auditing: Network Requests -Auditing: Network Round Trip Times -Auditing: Server Backend Latencies -Auditing: Tasks -Auditing: Metrics -Computing artifact: TimingSummary$1 -Auditing: Performance budget -Computing artifact: ResourceSummary$2 -Auditing: Timing budget -Auditing: Keep request counts low and transfer sizes small -Auditing: Minimize third-party usage -Auditing: Lazy load third-party resources with facades -Auditing: Largest Contentful Paint element -Auditing: Largest Contentful Paint image was not lazily loaded -Auditing: Avoid large layout shifts -Auditing: Avoid long main-thread tasks -Auditing: Avoids `unload` event listeners -Auditing: Avoid non-composited animations -Auditing: Image elements have explicit `width` and `height` -Auditing: Page has valid source maps -Auditing: Preload Largest Contentful Paint image -Auditing: Ensure CSP is effective against XSS attacks -Auditing: Full-page screenshot -Auditing: Script Treemap Data -Computing artifact: ModuleDuplication$2 -Auditing: Site works cross-browser -Auditing: Page transitions don't feel like they block on the network -Auditing: Each page has a URL -Auditing: `[accesskey]` values are unique -Auditing: `[aria-*]` attributes match their roles -Auditing: `button`, `link`, and `menuitem` elements have accessible names -Auditing: `[aria-hidden="true"]` is not present on the document `` -Auditing: `[aria-hidden="true"]` elements do not contain focusable descendents -Auditing: ARIA input fields have accessible names -Auditing: ARIA `meter` elements have accessible names -Auditing: ARIA `progressbar` elements have accessible names -Auditing: `[role]`s have all required `[aria-*]` attributes -Auditing: Elements with an ARIA `[role]` that require children to contain a specific `[role]` have all required children. -Auditing: `[role]`s are contained by their required parent element -Auditing: `[role]` values are valid -Auditing: ARIA toggle fields have accessible names -Auditing: ARIA `tooltip` elements have accessible names -Auditing: ARIA `treeitem` elements have accessible names -Auditing: `[aria-*]` attributes have valid values -Auditing: `[aria-*]` attributes are valid and not misspelled -Auditing: Buttons have an accessible name -Auditing: The page contains a heading, skip link, or landmark region -Auditing: Background and foreground colors have a sufficient contrast ratio -Auditing: `
`'s contain only properly-ordered `
` and `
` groups, `