clients(lightrider): always use transferSize from X-TotalFetchedSize …

…header (#7478)
GoogleChrome · Mar 14, 2019 · b548452 · b548452
1 parent bec96a6
commit b548452
Show file tree

Hide file tree

Showing 5 changed files with 270 additions and 9 deletions.
diff --git a/clients/lightrider-entry.js b/clients/lightrider-entry.js
@@ -31,7 +31,7 @@ async function runLighthouseInLR(connection, url, flags, lrOpts) {
   const {lrDevice, categoryIDs, logAssets, keepRawValues} = lrOpts;
 
   // Certain fixes need to kick in under LR, see https://github.com/GoogleChrome/lighthouse/issues/5839
-  global.isLightRider = true;
+  global.isLightrider = true;
 
   // disableStorageReset because it causes render server hang
   flags.disableStorageReset = true;

diff --git a/lighthouse-core/lib/network-request.js b/lighthouse-core/lib/network-request.js
@@ -67,6 +67,7 @@ module.exports = class NetworkRequest {
     /** @type {number} */
     this.responseReceivedTime = -1;
 
+    // Go read the comment on _updateTransferSizeForLightrider.
     this.transferSize = 0;
     this.resourceSize = 0;
     this.fromDiskCache = false;
@@ -191,7 +192,7 @@ module.exports = class NetworkRequest {
     }
 
     this._updateResponseReceivedTimeIfNecessary();
-    this._updateTransferSizeForLightRiderIfNecessary();
+    this._updateTransferSizeForLightrider();
   }
 
   /**
@@ -209,6 +210,7 @@ module.exports = class NetworkRequest {
     this.localizedFailDescription = data.errorText;
 
     this._updateResponseReceivedTimeIfNecessary();
+    this._updateTransferSizeForLightrider();
   }
 
   /**
@@ -261,8 +263,6 @@ module.exports = class NetworkRequest {
 
     if (this.fromMemoryCache) this.timing = undefined;
     if (this.timing) this._recomputeTimesWithResourceTiming(this.timing);
-
-    this._updateTransferSizeForLightRiderIfNecessary();
   }
 
   /**
@@ -297,12 +297,24 @@ module.exports = class NetworkRequest {
 
   /**
    * LR loses transfer size information, but passes it in the 'X-TotalFetchedSize' header.
+   * 'X-TotalFetchedSize' is the canonical transfer size in LR. Nothing should supersede it.
+   *
+   * The total length of the encoded data is spread out among multiple events. The sum of the
+   * values in onResponseReceived and all the onDataReceived events typically equals the value
+   * seen on the onLoadingFinished event. In <1% of cases we see the values differ. As we process
+   * onResonseReceived and onDataReceived we accumulate the total encodedDataLength. When we
+   * process onLoadingFinished, we override the accumulated total. We do this so that if the
+   * request is aborted or fails, we still get a value via the accumulation.
+   *
+   * In Lightrider, due to instrumentation limitations, our values for encodedDataLength are bogus
+   * and not valid. However the resource's true encodedDataLength/transferSize is shared via a
+   * special response header, X-TotalFetchedSize. In this situation, we read this value from
+   * responseReceived, use it for the transferSize and ignore the encodedDataLength values in
+   * both dataReceived and loadingFinished.
    */
-  _updateTransferSizeForLightRiderIfNecessary() {
-    // Bail if we're not in LightRider, this only applies there.
-    if (!global.isLightRider) return;
+  _updateTransferSizeForLightrider() {
     // Bail if we somehow already have transfer size data.
-    if (this.transferSize) return;
+    if (!global.isLightrider) return;
 
     const totalFetchedSize = this.responseHeaders.find(item => item.name === 'X-TotalFetchedSize');
     // Bail if the header was missing.

diff --git a/lighthouse-core/test/fixtures/lr.devtoolslog.json b/lighthouse-core/test/fixtures/lr.devtoolslog.json
@@ -0,0 +1,231 @@
+[
+    {
+        "method": "Network.requestWillBeSent",
+        "params": {
+            "requestId": "B38A026891F997E88601C20D85DC4616",
+            "loaderId": "B38A026891F997E88601C20D85DC4616",
+            "documentURL": "https://www.paulirish.com/",
+            "request": {
+                "url": "https://www.paulirish.com/",
+                "method": "GET",
+                "headers": {
+                    "Upgrade-Insecure-Requests": "1",
+                    "User-Agent": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3559.0 Mobile Safari/537.36",
+                    "Accept-Language": "en-US"
+                },
+                "mixedContentType": "none",
+                "initialPriority": "VeryHigh",
+                "referrerPolicy": "no-referrer-when-downgrade"
+            },
+            "timestamp": 10696044.979453,
+            "wallTime": 1552333431.031513,
+            "initiator": {
+                "type": "other"
+            },
+            "type": "Document",
+            "frameId": "31670FA9D804DF7165CB1197095646F9",
+            "hasUserGesture": false
+        }
+    },
+    {
+        "method": "Network.responseReceived",
+        "params": {
+            "requestId": "B38A026891F997E88601C20D85DC4616",
+            "loaderId": "B38A026891F997E88601C20D85DC4616",
+            "timestamp": 10696045.169156,
+            "type": "Document",
+            "response": {
+                "url": "https://www.paulirish.com/",
+                "status": 200,
+                "statusText": "OK",
+                "headers": {
+                    "Date": "Mon, 11 Mar 2019 19:43:51 GMT",
+                    "Content-Type": "text/html",
+                    "Transfer-Encoding": "chunked",
+                    "Connection": "keep-alive",
+                    "Set-Cookie": "__cfduid=d3c8d1a78f953298e1297c905193115fd1552333431; path=/; domain=.paulirish.com; HttpOnly",
+                    "Last-Modified": "Wed, 28 Feb 2018 20:45:39 GMT",
+                    "Vary": "Accept-Encoding",
+                    "Content-Security-Policy": "default-src 'self' ; script-src 'self' 'unsafe-inline' https://ssl.google-analytics.com https://api.github.com https://disqus.com https://go.disqus.com https://*.disquscdn.com https://www.google-analytics.com https://paulirish.disqus.com https://platform.twitter.com https://cdn.syndication.twimg.com https://ajax.googleapis.com; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com https://platform.twitter.com https://c.disquscdn.com; img-src * 'self' data: ; font-src 'self' data: https://fonts.gstatic.com; object-src http://vimeo.com http://static.slidesharecdn.com; frame-src 'self' https://platform.twitter.com https://accounts.google.com https://player.vimeo.com https://www.youtube.com https://apis.google.com https://disqus.com https://www.youtube.com https://paulirish.wufoo.com http://jsfiddle.net http://player.vimeo.com https://staticxx.facebook.com https://syndication.twitter.com; worker-src 'self' blob:; upgrade-insecure-requests; report-uri https://paulirish.report-uri.com/r/d/csp/enforce;",
+                    "Expect-CT": "max-age=604800, report-uri=\"https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct\"",
+                    "Server": "cloudflare",
+                    "CF-RAY": "4b6005086d532aa3-SEA",
+                    "X-Original-Content-Encoding": "br",
+                    "X-TotalFetchedSize": "22889"
+                },
+                "mimeType": "text/html",
+                "connectionReused": false,
+                "connectionId": 0,
+                "remoteIPAddress": "",
+                "remotePort": 0,
+                "fromDiskCache": false,
+                "fromServiceWorker": false,
+                "encodedDataLength": 1549,
+                "timing": {
+                    "requestTime": 10696044.979916,
+                    "proxyStart": -1,
+                    "proxyEnd": -1,
+                    "dnsStart": -1,
+                    "dnsEnd": -1,
+                    "connectStart": -1,
+                    "connectEnd": -1,
+                    "sslStart": -1,
+                    "sslEnd": -1,
+                    "workerStart": -1,
+                    "workerReady": -1,
+                    "sendStart": -1,
+                    "sendEnd": -1,
+                    "pushStart": 0,
+                    "pushEnd": 0,
+                    "receiveHeadersEnd": 188.235
+                },
+                "protocol": "http/1.1",
+                "securityState": "secure"
+            },
+            "frameId": "31670FA9D804DF7165CB1197095646F9"
+        }
+    },
+    {
+        "method": "Network.dataReceived",
+        "params": {
+            "requestId": "B38A026891F997E88601C20D85DC4616",
+            "timestamp": 10696045.172754,
+            "dataLength": 65536,
+            "encodedDataLength": 0
+        }
+    },
+    {
+        "method": "Network.dataReceived",
+        "params": {
+            "requestId": "B38A026891F997E88601C20D85DC4616",
+            "timestamp": 10696045.182541,
+            "dataLength": 9685,
+            "encodedDataLength": 75221
+        }
+    },
+    {
+        "method": "Network.loadingFinished",
+        "params": {
+            "requestId": "B38A026891F997E88601C20D85DC4616",
+            "timestamp": 10696045.168171,
+            "encodedDataLength": 76770,
+            "shouldReportCorbBlocking": false
+        }
+    },
+    {
+        "method": "Network.requestWillBeSent",
+        "params": {
+            "requestId": "1000170085.13",
+            "loaderId": "B38A026891F997E88601C20D85DC4616",
+            "documentURL": "https://www.paulirish.com/",
+            "request": {
+                "url": "https://www.paulirish.com/javascripts/modernizr-2.0.js",
+                "method": "GET",
+                "headers": {
+                    "Referer": "https://www.paulirish.com/",
+                    "User-Agent": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3559.0 Mobile Safari/537.36"
+                },
+                "mixedContentType": "none",
+                "initialPriority": "Medium",
+                "referrerPolicy": "no-referrer-when-downgrade"
+            },
+            "timestamp": 10696045.190184,
+            "wallTime": 1552333431.24224,
+            "initiator": {
+                "type": "parser",
+                "url": "https://www.paulirish.com/",
+                "lineNumber": 444
+            },
+            "type": "Script",
+            "frameId": "31670FA9D804DF7165CB1197095646F9",
+            "hasUserGesture": false
+        }
+    },
+    {
+        "method": "Network.responseReceived",
+        "params": {
+            "requestId": "1000170085.13",
+            "loaderId": "B38A026891F997E88601C20D85DC4616",
+            "timestamp": 10696045.906526,
+            "type": "Script",
+            "response": {
+                "url": "https://www.paulirish.com/javascripts/modernizr-2.0.js",
+                "status": 200,
+                "statusText": "OK",
+                "headers": {
+                    "Date": "Mon, 11 Mar 2019 19:43:51 GMT",
+                    "CF-Cache-Status": "HIT",
+                    "Cf-Polished": "origSize=9946",
+                    "Transfer-Encoding": "chunked",
+                    "X-TotalFetchedSize": "4730",
+                    "Connection": "keep-alive",
+                    "Last-Modified": "Wed, 28 Feb 2018 20:45:39 GMT",
+                    "Server": "cloudflare",
+                    "ETag": "W/\"26da-5664bd3750cc6\"",
+                    "Expect-CT": "max-age=604800, report-uri=\"https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct\"",
+                    "Vary": "Accept-Encoding",
+                    "Content-Type": "application/javascript",
+                    "Cf-Bgj": "minify",
+                    "Cache-Control": "public, max-age=86400",
+                    "X-Original-Content-Encoding": "br",
+                    "CF-RAY": "4b6005098dce2aa3-SEA",
+                    "Expires": "Tue, 12 Mar 2019 19:43:51 GMT"
+                },
+                "mimeType": "application/javascript",
+                "connectionReused": false,
+                "connectionId": 0,
+                "fromDiskCache": false,
+                "fromServiceWorker": false,
+                "encodedDataLength": 579,
+                "timing": {
+                    "requestTime": 10696045.190569,
+                    "proxyStart": -1,
+                    "proxyEnd": -1,
+                    "dnsStart": -1,
+                    "dnsEnd": -1,
+                    "connectStart": -1,
+                    "connectEnd": -1,
+                    "sslStart": -1,
+                    "sslEnd": -1,
+                    "workerStart": -1,
+                    "workerReady": -1,
+                    "sendStart": -1,
+                    "sendEnd": -1,
+                    "pushStart": 0,
+                    "pushEnd": 0,
+                    "receiveHeadersEnd": 148.784
+                },
+                "protocol": "http/1.1",
+                "securityState": "unknown"
+            },
+            "frameId": "31670FA9D804DF7165CB1197095646F9"
+        }
+    },
+    {
+        "method": "Network.dataReceived",
+        "params": {
+            "requestId": "1000170085.13",
+            "timestamp": 10696045.906583,
+            "dataLength": 9736,
+            "encodedDataLength": 0
+        }
+    },
+    {
+        "method": "Network.dataReceived",
+        "params": {
+            "requestId": "1000170085.13",
+            "timestamp": 10696045.90672,
+            "dataLength": 0,
+            "encodedDataLength": 9736
+        }
+    },
+    {
+        "method": "Network.loadingFinished",
+        "params": {
+            "requestId": "1000170085.13",
+            "timestamp": 10696045.339356,
+            "encodedDataLength": 10315,
+            "shouldReportCorbBlocking": false
+        }
+    }
+]
diff --git a/lighthouse-core/test/lib/network-recorder-test.js b/lighthouse-core/test/lib/network-recorder-test.js
@@ -11,6 +11,7 @@ const assert = require('assert');
 const devtoolsLogItems = require('../fixtures/artifacts/perflog/defaultPass.devtoolslog.json');
 const redirectsDevtoolsLog = require('../fixtures/wikipedia-redirect.devtoolslog.json');
 const redirectsScriptDevtoolsLog = require('../fixtures/redirects-from-script.devtoolslog.json');
+const lrRequestDevtoolsLog = require('../fixtures/lr.devtoolslog.json');
 
 /* eslint-env jest */
 describe('network recorder', function() {
@@ -162,6 +163,23 @@ describe('network recorder', function() {
     expect(records).toMatchObject([{url: 'http://example.com', startTime: 1, endTime: 2}]);
   });
 
+  it('should use X-TotalFetchedSize in Lightrider for transferSize', () => {
+    global.isLightrider = true;
+    const records = NetworkRecorder.recordsFromLogs(lrRequestDevtoolsLog);
+    global.isLightrider = false;
+
+    expect(records.find(r => r.url === 'https://www.paulirish.com/'))
+    .toMatchObject({
+      resourceSize: 75221,
+      transferSize: 22889,
+    });
+    expect(records.find(r => r.url === 'https://www.paulirish.com/javascripts/modernizr-2.0.js'))
+    .toMatchObject({
+      resourceSize: 9736,
+      transferSize: 4730,
+    });
+  });
+
   describe('#findNetworkQuietPeriods', () => {
     function record(data) {
       const url = data.url || 'https://example.com';

diff --git a/types/node.d.ts b/types/node.d.ts
@@ -6,6 +6,6 @@
 
 declare module NodeJS {
   interface Global {
-    isLightRider?: boolean;
+    isLightrider?: boolean;
   }
 }