Merge pull request #163 from Polymer/bitrot

Fix Firefox/Safari open/resize bugs, housekeeping, release 0.4.19

.travis.yml

@@ -11,7 +11,7 @@ matrix:
   include:
     # https://docs.travis-ci.com/user/reference/linux/
     - os: linux
-      dist: xenial
+      dist: bionic
       env:
         # Note on Linux, non-headless Chrome fails with "Chrome failed to
         # start", and non-headless Firefox fails with "can't kill an exited
@@ -20,9 +20,9 @@ matrix:
 
     # https://docs.travis-ci.com/user/reference/osx/
     - os: osx
-      osx_image: xcode11 # macOS 10.14
+      osx_image: xcode12 # macOS 10.15
       env:
-        TACHOMETER_E2E_TEST_BROWSERS=safari,chrome,chrome-headless,firefox,firefox-headless
+        TACHOMETER_E2E_TEST_BROWSERS=safari
       before_script:
         # Required to enable Safari remote automation.
         - sudo safaridriver --enable

config.schema.json

@@ -266,12 +266,12 @@
             "additionalProperties": false,
             "properties": {
                 "height": {
-                    "description": "Height of the browser window in pixels.",
+                    "description": "Height of the browser window in pixels. Defaults to 768.",
                     "minimum": 0,
                     "type": "integer"
                 },
                 "width": {
-                    "description": "Width of the browser window in pixels.",
+                    "description": "Width of the browser window in pixels. Defaults to 1024.",
                     "minimum": 0,
                     "type": "integer"
                 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "tachometer",
-  "version": "0.4.18",
+  "version": "0.4.19",
   "description": "Web benchmark runner",
   "main": "index.js",
   "directories": {
@@ -16,31 +16,31 @@
     "generate-json-schema": "typescript-json-schema tsconfig.json ConfigFile --include src/configfile.ts --required --noExtraProps > config.schema.json",
     "lint": "tslint --project . --format stylish",
     "format": "clang-format --style=file -i \"--glob=./{client,}/src/**/*.ts\"",
-    "test": "npm run build && mocha"
+    "test": "npm run build && mocha 'lib/test/**/*.js'"
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/PolymerLabs/tachometer.git"
+    "url": "git+https://github.com/Polymer/tachometer.git"
   },
   "author": "The Polymer Project Authors",
   "license": "BSD-3-Clause",
   "bugs": {
-    "url": "https://github.com/PolymerLabs/tachometer/issues"
+    "url": "https://github.com/Polymer/tachometer/issues"
   },
-  "homepage": "https://github.com/PolymerLabs/tachometer#readme",
+  "homepage": "https://github.com/Polymer/tachometer#readme",
   "dependencies": {
     "@types/command-line-usage": "^5.0.1",
     "@types/selenium-webdriver": "^4.0.5",
-    "@types/table": "^4.0.5",
+    "@types/table": "^5.0.0",
     "ansi-escape-sequences": "^5.0.0",
     "chromedriver": ">78.0.1",
     "command-line-args": "^5.0.2",
     "command-line-usage": "^6.1.0",
     "csv-stringify": "^5.3.0",
-    "fs-extra": "^8.0.1",
+    "fs-extra": "^9.0.1",
     "geckodriver": "^1.19.1",
     "get-stream": "^5.1.0",
-    "got": "^10.1.0",
+    "got": "^11.5.0",
     "iedriver": "^3.14.2",
     "jsonschema": "^1.2.4",
     "jsonwebtoken": "^8.5.1",
@@ -62,12 +62,12 @@
   },
   "devDependencies": {
     "@types/ansi-escape-sequences": "^4.0.0",
-    "@types/babel__generator": "~7.0.2",
+    "@types/babel__generator": "^7.6.1",
     "@types/chai": "^4.2.4",
     "@types/chai-as-promised": "^7.1.2",
     "@types/command-line-args": "^5.0.0",
     "@types/csv-stringify": "^3.1.0",
-    "@types/fs-extra": "^8.0.1",
+    "@types/fs-extra": "^9.0.1",
     "@types/get-stream": "^3.0.2",
     "@types/got": "^9.6.8",
     "@types/jsonwebtoken": "^8.3.5",
@@ -76,20 +76,20 @@
     "@types/koa-mount": "^4.0.0",
     "@types/koa-send": "^4.1.2",
     "@types/koa-static": "^4.0.0",
-    "@types/mocha": "^5.2.6",
+    "@types/mocha": "^7.0.2",
     "@types/node-fetch": "^2.5.3",
     "@types/progress": "^2.0.3",
-    "@types/semver": "^6.2.0",
+    "@types/semver": "^7.3.1",
     "@types/systeminformation": "^3.54.1",
     "@types/ua-parser-js": "^0.7.32",
     "chai": "^4.2.0",
     "chai-as-promised": "^7.1.1",
     "clang-format": "^1.3.0",
-    "mocha": "^6.2.2",
+    "mocha": "^8.0.1",
     "node-fetch": "^2.6.0",
     "rimraf": "^3.0.2",
-    "tslint": "^5.12.1",
+    "tslint": "^6.1.2",
     "typescript": "^3.6.4",
-    "typescript-json-schema": "^0.41.0"
+    "typescript-json-schema": "^0.42.0"
   }
-}
+}

src/browser.ts

@@ -163,7 +163,11 @@ export async function makeDriver(config: BrowserConfig):
       config.name === 'ie') {
     // Safari, Edge, and IE don't have flags we can use to launch with a given
     // window size, but webdriver can resize the window after we've started up.
-    await driver.manage().window().setRect(config.windowSize);
+    // Some versions of Safari have a bug where it is required to also provide
+    // an x/y position (see https://github.com/SeleniumHQ/selenium/issues/3796).
+    const rect = config.name === 'safari' ? {...config.windowSize, x: 0, y: 0} :
+                                            config.windowSize;
+    await driver.manage().window().setRect(rect);
   }
   return driver;
 }
@@ -223,23 +227,41 @@ export async function openAndSwitchToNewTab(
   if (tabsBefore.length !== 1) {
     throw new Error(`Expected only 1 open tab, got ${tabsBefore.length}`);
   }
+
   // "noopener=yes" prevents the new window from being able to access the
   // first window. We set that here because in Chrome (and perhaps other
   // browsers) we see a very significant improvement in the reliability of
   // measurements, in particular it appears to eliminate interference between
   // code across runs. It is likely this flag increases process isolation in a
   // way that prevents code caching across tabs.
   await driver.executeScript('window.open("", "", "noopener=yes");');
-  const tabsAfter = await driver.getAllWindowHandles();
-  const newTabs = tabsAfter.filter((tab) => tab !== tabsBefore[0]);
-  if (newTabs.length !== 1) {
-    throw new Error(`Expected to create 1 new tab, got ${newTabs.length}`);
+  // Firefox (and maybe other browsers) won't always report the new tab ID
+  // immediately, so we'll need to poll for it.
+  const maxRetries = 20;
+  const retrySleepMs = 250;
+  let retries = 0;
+  let newTabId;
+  while (true) {
+    const tabsAfter = await driver.getAllWindowHandles();
+    const newTabs = tabsAfter.filter((tab) => tab !== tabsBefore[0]);
+    if (newTabs.length === 1) {
+      newTabId = newTabs[0];
+      break;
+    }
+    retries++;
+    if (newTabs.length > 1 || retries > maxRetries) {
+      throw new Error(`Expected to create 1 new tab, got ${newTabs.length}`);
+    }
+    await new Promise((resolve) => setTimeout(resolve, retrySleepMs));
   }
-  await driver.switchTo().window(newTabs[0]);
-  if (config.name === 'ie') {
-    // For IE we get a new window instead of a new tab, so we need to resize
-    // every time.
-    await driver.manage().window().setRect(config.windowSize);
+  await driver.switchTo().window(newTabId);
+
+  if (config.name === 'ie' || config.name === 'safari') {
+    // For IE and Safari (with rel=noopener) we get a new window instead of a
+    // new tab, so we need to resize every time.
+    const rect = config.name === 'safari' ? {...config.windowSize, x: 0, y: 0} :
+                                            config.windowSize;
+    await driver.manage().window().setRect(rect);
   }
   type WithSendDevToolsCommand = {
     sendDevToolsCommand?: (command: string, config: {}) => Promise<void>,

src/configfile.ts

@@ -159,15 +159,15 @@ interface BrowserConfigBase {
 
 interface WindowSize {
   /**
-   * Width of the browser window in pixels.
+   * Width of the browser window in pixels. Defaults to 1024.
    *
    * @TJS-type integer
    * @TJS-minimum 0
    */
   width: number;
 
   /**
-   * Height of the browser window in pixels.
+   * Height of the browser window in pixels. Defaults to 768.
    *
    * @TJS-type integer
    * @TJS-minimum 0

src/format.ts

@@ -27,7 +27,7 @@ export const spinner = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧',
 interface Dimension {
   label: string;
   format: (r: ResultStats) => string;
-  tableConfig?: table.ColumnConfig;
+  tableConfig?: table.TableColumns;
 }
 
 export interface ResultTable {
@@ -139,9 +139,9 @@ export function verticalTermResultTable({dimensions, results}: ResultTable):
  */
 export function horizontalTermResultTable({dimensions, results}: ResultTable):
     string {
-  const columns: table.ColumnConfig[] = [
+  const columns: table.TableColumns[] = [
     {alignment: 'right'},
-    ...results.map((): table.ColumnConfig => ({alignment: 'left'})),
+    ...results.map((): table.TableColumns => ({alignment: 'left'})),
   ];
   const rows = dimensions.map((d) => {
     return [

src/test/browser_test.ts

@@ -10,6 +10,7 @@
  */
 
 import {assert} from 'chai';
+import {suite, test} from 'mocha';
 
 import {BrowserName, parseBrowserConfigString, validateBrowserConfig} from '../browser';
 import * as defaults from '../defaults';

src/test/config_test.ts

@@ -10,6 +10,7 @@
  */
 
 import {assert} from 'chai';
+import {suite, suiteSetup, suiteTeardown, test} from 'mocha';
 
 import {Config, makeConfig, parseHorizons} from '../config';
 import {parseFlags} from '../flags';

src/test/configfile_test.ts

@@ -11,6 +11,7 @@
 
 import * as chai from 'chai';
 import * as chaiAsPromised from 'chai-as-promised';
+import {suite, suiteSetup, suiteTeardown, test} from 'mocha';
 import * as path from 'path';
 
 chai.use(chaiAsPromised);

src/test/csv_test.ts

@@ -10,6 +10,7 @@
  */
 
 import {assert} from 'chai';
+import {suite, test} from 'mocha';
 
 import {ConfigFile} from '../configfile';
 import {formatCsv} from '../csv';

src/test/e2e_test.ts

@@ -10,17 +10,24 @@
  */
 
 import {assert} from 'chai';
+import {suite, test} from 'mocha';
 import * as path from 'path';
 import {main} from '../cli';
 import {ConfidenceInterval} from '../stats';
 import {testData} from './test_helpers';
 
 // Set this environment variable to change the browsers under test.
-const browsers = (process.env.TACHOMETER_E2E_TEST_BROWSERS ||
-                  'chrome-headless,firefox-headless')
-                     .split(',')
-                     .map((b) => b.trim())
-                     .filter((b) => b.length > 0);
+let browsers = (process.env.TACHOMETER_E2E_TEST_BROWSERS || '')
+                   .split(',')
+                   .map((b) => b.trim())
+                   .filter((b) => b.length > 0);
+
+if (browsers.length === 0) {
+  browsers = ['chrome-headless', 'firefox-headless'];
+  if (process.platform === 'darwin') {
+    browsers.push('safari');
+  }
+}
 
 /**
  * Test function wrapper to suppress tachometer's stdout/stderr output. Note we

src/test/flags_test.ts

@@ -10,6 +10,7 @@
  */
 
 import {assert} from 'chai';
+import {suite, test} from 'mocha';
 
 import {parseFlags} from '../flags';
 

src/test/format_test.ts

@@ -10,6 +10,7 @@
  */
 
 import {assert} from 'chai';
+import {suite, suiteSetup, suiteTeardown, test} from 'mocha';
 import * as path from 'path';
 import stripAnsi = require('strip-ansi');
 

src/test/json-output_test.ts

@@ -10,6 +10,7 @@
  */
 
 import {assert} from 'chai';
+import {suite, test} from 'mocha';
 
 import {ConfigFile} from '../configfile';
 import {jsonOutput, JsonOutputFile} from '../json-output';

src/test/server_test.ts

@@ -11,6 +11,7 @@
 
 import {assert} from 'chai';
 import {readJSONSync} from 'fs-extra';
+import {setup, suite, teardown, test} from 'mocha';
 import fetch from 'node-fetch';
 import * as path from 'path';
 

src/test/specs_test.ts

@@ -11,12 +11,14 @@
 
 import * as chai from 'chai';
 import * as chaiAsPromised from 'chai-as-promised';
+import {suite, suiteSetup, suiteTeardown, test} from 'mocha';
 import * as path from 'path';
 
 import * as defaults from '../defaults';
 import {optDefs, Opts} from '../flags';
 import {specsFromOpts} from '../specs';
 import {BenchmarkSpec} from '../types';
+
 import {testData} from './test_helpers';
 
 import commandLineArgs = require('command-line-args');

src/test/stats_test.ts

@@ -12,6 +12,7 @@
 const jstat = require('jstat');
 
 import {assert} from 'chai';
+import {suite, test} from 'mocha';
 import {summaryStats, computeDifference, intervalContains} from '../stats';
 
 suite('statistics', function() {

src/test/versions_test.ts

@@ -10,6 +10,7 @@
  */
 
 import {assert} from 'chai';
+import {suite, test} from 'mocha';
 import * as path from 'path';
 
 import * as defaults from '../defaults';