Percy visual diffing of AMP pages served from localhost and using a l…

…ocally built runtime (#9833)

This PR adds a ruby script that starts up a local server for the AMP runtime, loads a test page from it, and grabs a snapshot via Percy. This will run on master, like before.

Visual diffing for PRs depends on making encrypted Percy tokens available on for PRs on Travis, and will come up in a separate PR.

#9469
Fixes #9928

.travis.yml

@@ -25,9 +25,6 @@ before_install:
   - sh -e /etc/init.d/xvfb start
   - wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
   - sudo dpkg -i google-chrome*.deb
-before_script:
-  - pip install --user protobuf
-  - gem install percy-cli
 script: node build-system/pr-check.js
 branches:
   only:
@@ -42,7 +39,14 @@ env:
 matrix:
   include:
     - env: BUILD_SHARD="pre_build_checks_and_unit_tests"
+      before_script:
+        - pip install --user protobuf
+        - gem install percy-capybara
+        - gem install phantomjs
+        - gem install poltergeist
     - env: BUILD_SHARD="integration_tests"
+      before_script:
+        - pip install --user protobuf
       addons:
         sauce_connect:
           username: "amphtml"

build-system/pr-check.js

@@ -26,11 +26,11 @@
  */
 const exec = require('./exec.js').exec;
 const execOrDie = require('./exec.js').execOrDie;
+const extensionsVersions = require('./extensions-versions-config');
 const getStdout = require('./exec.js').getStdout;
-const path = require('path');
 const minimist = require('minimist');
+const path = require('path');
 const util = require('gulp-util');
-const extensionsVersions = require('./extensions-versions-config');
 
 const gulp = 'node_modules/gulp/bin/gulp.js';
 const fileLogPrefix = util.colors.yellow.bold('pr-check.js:');
@@ -274,7 +274,7 @@ const command = {
     // This must only be run for push builds, since Travis hides the encrypted
     // environment variables required by Percy during pull request builds.
     // For now, this is warning-only.
-    timedExec(`${gulp} visual-diff`);
+    timedExec(`ruby ${path.resolve('build-system/tasks/visual-diff.rb')}`);
   },
   runPresubmitTests: function() {
     timedExecOrDie(`${gulp} presubmit`);
@@ -293,6 +293,7 @@ function runAllCommands() {
     command.testBuildSystem();
     command.cleanBuild();
     command.buildRuntime();
+    command.runVisualDiffTests();  // Only called during push builds.
     command.runJsonAndLintChecks();
     command.runDepAndTypeChecks();
     command.runUnitTests();
@@ -304,7 +305,6 @@ function runAllCommands() {
     command.cleanBuild();
     command.buildRuntimeMinified();
     command.runPresubmitTests();  // Needs runtime to be built and served.
-    // command.runVisualDiffTests();  // Only called during push builds.
     command.runIntegrationTests();
   }
 }

build-system/tasks/serve.js

@@ -48,8 +48,11 @@ function serve() {
       'SERVE_PORT': port,
       'SERVE_HOST': host,
       'SERVE_USEHTTPS': useHttps},
+  })
+  .once('exit', function () {
+    util.log(util.colors.green('Shutting down server'));
+    process.exit();
   });
-
   util.log(util.colors.yellow('Run `gulp build` then go to '
       + getHost() + '/examples/article.amp.html'
   ));

build-system/tasks/visual-diff.rb

@@ -0,0 +1,149 @@
+#!/usr/bin/ruby
+#
+# Copyright 2017 The AMP HTML Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS-IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+# Visual diff generator for AMP webpages using Percy.
+#
+# Note: This is done in ruby to use Percy's API for snapshotting pages served by
+# a localhost server. See https://percy.io/docs/clients/ruby/percy-anywhere.
+
+
+require 'percy/capybara/anywhere'
+require 'capybara/poltergeist'
+require 'phantomjs'
+require 'json'
+require "net/http"
+
+
+ENV['PERCY_DEBUG'] = '1'  # Enable debugging output.
+DEFAULT_WIDTHS = [375, 411]  # CSS widths: iPhone: 375, Pixel: 411.
+HOST = 'localhost'
+PORT = '8000'
+
+
+# Launches a background AMP webserver for unminified js using gulp.
+#
+# Returns:
+# - Process ID of server process.
+def launchWebServer()
+  webserverCmd = "gulp serve --host #{HOST} --port #{PORT}"
+  webserverUrl = "http://#{HOST}:#{PORT}"
+  @pid = fork do
+    Signal.trap("INT") { exit }
+    exec webserverCmd
+  end
+  Process.detach(@pid)
+  @pid
+end
+
+
+# Checks if a webserver is up and running.
+#
+# Returns:
+# - true if the server returns an OK (200) response code.
+def isWebServerRunning()
+  http = Net::HTTP.start(HOST, PORT)
+  response = http.head("/")
+  response.code == "200"
+rescue SystemCallError
+  false
+end
+
+
+# Waits up to 15 seconds for the webserver to start up.
+#
+# Returns:
+# - true if webserver is running.
+def waitForWebServer()
+  tries = 0
+  until isWebServerRunning()
+    sleep(1)
+    tries += 1
+    break if tries > 15
+  end
+  isWebServerRunning()
+end
+
+
+# Closes the webserver process with the given process ID.
+#
+# Args:
+# - pid: Process ID of the webserver.
+def closeWebServer(pid)
+  Process.kill("INT", pid)
+  Process.wait(pid, Process::WNOHANG)
+end
+
+
+# Loads the list of pages to snapshot from a well-known json config file.
+def loadPagesToSnapshotJson()
+  jsonFile = File.open(
+      File.join(
+          File.dirname(__FILE__),
+          "../../test/visual-diff/visual-tests.json"),
+      "r")
+  jsonContent = jsonFile.read
+end
+
+
+# Generates a percy snapshot for a given webpage.
+#
+# Args:
+# - server: URL of the webserver.
+# - assets_dir: Path to assets (images, etc.) used by the webpage.
+# - assets_base_url: Base URL of assets on webserver.
+# - url: Relative URL of page to be snapshotted.
+# - name: Name of snapshot on Percy.
+def generateSnapshot(server, assets_dir, assets_base_url, url, name)
+  Percy::Capybara::Anywhere.run(
+      server, assets_dir, assets_base_url) do |page|
+    page.driver.options[:phantomjs] = Phantomjs.path
+    page.driver.options[:js_errors] = false
+    page.visit(url)
+    Percy.config.default_widths = DEFAULT_WIDTHS
+    Percy::Capybara.snapshot(page, name: name)
+ end
+end
+
+
+# Launches a webserver, loads test pages, and generates Percy snapshots.
+def main()
+  pid = launchWebServer()
+  if not waitForWebServer()
+    puts "Failed to start webserver"
+    closeWebServer(pid)
+    exit(false)
+  end
+  pagesToSnapshotJson = loadPagesToSnapshotJson()
+  pagesToSnapshot = JSON.parse(pagesToSnapshotJson)
+  server = "http://#{HOST}:#{PORT}"
+  webpages = pagesToSnapshot["webpages"]
+  webpages.each do |webpage|
+    assets_base_url = webpage["assets_base_url"]
+    assets_dir = File.expand_path(
+        "../../../#{webpage["assets_dir"]}",
+        __FILE__)
+    url = webpage["url"]
+    name = webpage["name"]
+    generateSnapshot(server, assets_dir, assets_base_url, url, name)
+  end
+  closeWebServer(pid)
+end
+
+
+if __FILE__ == $0
+  main()
+end

test/visual-diff/visual-tests.json

@@ -1,4 +1,11 @@
 {
   "_comment": "HTML file(s) used for visual diff tests. Paths relative to src.",
-  "webpage": "examples/amp-by-example.html"
+  "webpages": [
+    {
+      "assets_dir": "examples/amp-by-example",
+      "assets_base_url": "/",
+      "url": "examples/amp-by-example/amp-by-example.html",
+      "name": "Amp By Example"
+    }
+  ]
 }