tests(lantern): add lantern regression test scripts

.travis.yml

@@ -13,6 +13,7 @@ cache:
   yarn: true
   directories:
     - node_modules
+    - lantern-data
     - lighthouse-extension/node_modules
     - lighthouse-viewer/node_modules
     - /home/travis/.rvm/gems/
@@ -39,6 +40,7 @@ script:
   - yarn smoke:silentcoverage
   - yarn test-extension
   - yarn test-viewer
+  - yarn test-lantern
   # _JAVA_OPTIONS is breaking parsing of compiler output. See #3338.
   - unset _JAVA_OPTIONS
   # FIXME(paulirish): re-enable after a roll of LH->CDT

lighthouse-core/scripts/lantern/diff-expectations.js

@@ -0,0 +1,53 @@
+#!/usr/bin/env node
+/**
+ * @license Copyright 2018 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/* eslint-disable no-console */
+
+const fs = require('fs');
+const path = require('path');
+const execSync = require('child_process').execSync;
+
+const COMPUTED_INPUT_ARG = process.argv[2] || './lantern-data/lantern-computed.json';
+const COMPUTED_PATH = path.join(process.cwd(), COMPUTED_INPUT_ARG);
+const EXPECTED_PATH = path.join(__dirname, '../../test/fixtures/lantern-expectations.json');
+
+const TMP_DIR = path.join(process.cwd(), '.tmp');
+const TMP_COMPUTED = path.join(TMP_DIR, 'computed.json');
+const TMP_EXPECTED = path.join(TMP_DIR, 'expected.json');
+
+if (!fs.existsSync(TMP_DIR)) fs.mkdirSync(TMP_DIR);
+
+if (!fs.existsSync(COMPUTED_PATH) || !fs.existsSync(EXPECTED_PATH)) {
+  throw new Error('Usage $0 <computed file>');
+}
+
+try {
+  const computedResults = require(COMPUTED_PATH);
+  const expectedResults = require(EXPECTED_PATH);
+
+  const sites = [];
+  for (const entry of computedResults.sites) {
+    const lanternValues = entry.lantern;
+    Object.keys(lanternValues).forEach(key => lanternValues[key] = Math.round(lanternValues[key]));
+    sites.push({url: entry.url, ...lanternValues});
+  }
+
+  fs.writeFileSync(TMP_COMPUTED, JSON.stringify({sites}, null, 2));
+  fs.writeFileSync(TMP_EXPECTED, JSON.stringify(expectedResults, null, 2));
+
+  try {
+    execSync(`git --no-pager diff --color=always --no-index ${TMP_EXPECTED} ${TMP_COMPUTED}`);
+    console.log('✅  PASS    No changes between expected and computed!');
+  } catch (err) {
+    console.log('❌  FAIL    Changes between expected and computed!\n');
+    console.log(err.stdout.toString());
+  }
+} finally {
+  fs.unlinkSync(TMP_COMPUTED);
+  fs.unlinkSync(TMP_EXPECTED);
+}

lighthouse-core/scripts/lantern/download-traces.sh

@@ -1,13 +1,21 @@
 #!/bin/bash
 
+set -e
+
 DIRNAME="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 LH_ROOT_PATH="$DIRNAME/../../.."
 cd $LH_ROOT_PATH
 
+if [[ -f lantern-data/lantern-expectations.json ]] && ! [[ "$FORCE" ]]; then
+  echo "Lantern data already detected, done."
+  exit 0
+fi
+
 # snapshot of ~100 traces with no throttling recorded 2017-12-06 on a HP z840 workstation
 TAR_URL="https://drive.google.com/a/chromium.org/uc?id=1_w2g6fQVLgHI62FApsyUDejZyHNXMLm0&export=download"
 curl -o lantern-traces.tar.gz -L $TAR_URL
 
+rm -rf lantern-data/
 tar -xzf lantern-traces.tar.gz
-mv lantern-traces-subset lantern-data
+mv lantern-traces-subset/ lantern-data/
 rm lantern-traces.tar.gz

lighthouse-core/scripts/lantern/evaluate-results.js

@@ -8,16 +8,19 @@
 
 /* eslint-disable no-console */
 
+const fs = require('fs');
 const path = require('path');
 
 const GOOD_ABSOLUTE_THRESHOLD = 0.2;
 const OK_ABSOLUTE_THRESHOLD = 0.5;
 
 const GOOD_RANK_THRESHOLD = 0.1;
 
-if (!process.argv[2]) throw new Error('Usage $0 <computed summary file>');
+const COMPUTATIONS_INPUT_ARG = process.argv[2] || './lantern-data/lantern-computed.json';
+const COMPUTATIONS_PATH = path.resolve(process.cwd(), COMPUTATIONS_INPUT_ARG);
+
+if (!fs.existsSync(COMPUTATIONS_PATH)) throw new Error('Usage $0 <computed summary file>');
 
-const COMPUTATIONS_PATH = path.resolve(process.cwd(), process.argv[2]);
 /** @type {{sites: LanternSiteDefinition[]}} */
 const expectations = require(COMPUTATIONS_PATH);
 

lighthouse-core/scripts/lantern/run-all-expectations.js

@@ -12,11 +12,13 @@ const fs = require('fs');
 const path = require('path');
 const execFileSync = require('child_process').execFileSync;
 
-if (!process.argv[2]) throw new Error('Usage $0 <expectations file>');
-
-const RUN_ONCE_PATH = path.join(__dirname, 'run-once.js');
-const EXPECTATIONS_PATH = path.resolve(process.cwd(), process.argv[2]);
+const EXPECTATIONS_INPUT_ARG = process.argv[2] || './lantern-data/lantern-expectations.json';
+const EXPECTATIONS_PATH = path.resolve(process.cwd(), EXPECTATIONS_INPUT_ARG);
 const EXPECTATIONS_DIR = path.dirname(EXPECTATIONS_PATH);
+const RUN_ONCE_PATH = path.join(__dirname, 'run-once.js');
+
+if (!fs.existsSync(EXPECTATIONS_PATH)) throw new Error('Usage $0 <expectations file>');
+
 const expectations = require(EXPECTATIONS_PATH);
 
 for (const site of expectations.sites) {

lighthouse-core/scripts/lantern/update-expectations.js

@@ -0,0 +1,44 @@
+#!/usr/bin/env node
+/**
+ * @license Copyright 2018 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/* eslint-disable no-console */
+
+const fs = require('fs');
+const path = require('path');
+const execFileSync = require('child_process').execFileSync;
+const prettyJSONStringify = require('pretty-json-stringify');
+
+const INPUT_PATH = process.argv[2] || './lantern-data/lantern-expectations.json';
+const EXPECTATIONS_PATH = path.resolve(process.cwd(), INPUT_PATH);
+const EXPECTATIONS_DIR = path.dirname(EXPECTATIONS_PATH);
+const COMPUTED_PATH = path.join(EXPECTATIONS_DIR, 'lantern-computed.json');
+const RUN_ALL_SCRIPT_PATH = path.join(__dirname, 'run-all-expectations.js');
+const OUTPUT_PATH = path.join(__dirname, '../../test/fixtures/lantern-expectations.json');
+
+if (!fs.existsSync(COMPUTED_PATH) || process.env.FORCE) {
+  if (!fs.existsSync(EXPECTATIONS_PATH)) throw new Error('Usage $0 <expectations file>');
+  execFileSync(RUN_ALL_SCRIPT_PATH, [EXPECTATIONS_PATH]);
+}
+
+const computedResults = require(COMPUTED_PATH);
+
+const sites = [];
+for (const entry of computedResults.sites) {
+  const lanternValues = entry.lantern;
+  Object.keys(lanternValues).forEach(key => lanternValues[key] = Math.round(lanternValues[key]));
+  sites.push({url: entry.url, ...lanternValues});
+}
+
+fs.writeFileSync(OUTPUT_PATH, prettyJSONStringify({sites}, {
+  tab: '  ',
+  spaceBeforeColon: '',
+  spaceInsideObject: '',
+  shouldExpand: (_, level) => level < 2,
+}));
+
+

lighthouse-core/scripts/test-lantern.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+DIRNAME="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+LH_ROOT="$DIRNAME/../.."
+cd $LH_ROOT
+
+set -e
+
+# Testing lantern can be expensive, we'll only run the tests if we touched files that affect the simulations.
+CHANGED_FILES=""
+if [[ "$CI" ]]; then
+  CHANGED_FILES=$(git --no-pager diff --name-only $TRAVIS_COMMIT_RANGE)
+else
+  CHANGED_FILES=$(git --no-pager diff --name-only master)
+fi
+
+printf "Determined the following files have been touched:\n\n$CHANGED_FILES\n\n"
+
+if ! echo $CHANGED_FILES | grep -E 'dependency-graph|metrics|lantern' > /dev/null; then
+  echo "No lantern files affected, skipping lantern checks."
+  exit 0
+fi
+
+printf "Lantern files affected!\n\nDownloading test set...\n"
+"$LH_ROOT/lighthouse-core/scripts/lantern/download-traces.sh"
+
+printf "\n\nRunning all expectations...\n"
+"$LH_ROOT/lighthouse-core/scripts/lantern/run-all-expectations.js"
+
+printf "\n\n"
+"$LH_ROOT/lighthouse-core/scripts/lantern/evaluate-results.js"
+
+printf "\n\nComparing to golden expectations...\n"
+"$LH_ROOT/lighthouse-core/scripts/lantern/diff-expectations.js"