core(fr): convert response-compression gatherer (#12508)

lighthouse-core/fraggle-rock/config/default-config.js

@@ -34,6 +34,7 @@ const artifacts = {
   NetworkUserAgent: '',
   OptimizedImages: '',
   PasswordInputsWithPreventedPaste: '',
+  ResponseCompression: '',
   RobotsTxt: '',
   SourceMaps: '',
   Stacks: '',
@@ -79,6 +80,7 @@ const defaultConfig = {
     {id: artifacts.NetworkUserAgent, gatherer: 'network-user-agent'},
     {id: artifacts.OptimizedImages, gatherer: 'dobetterweb/optimized-images'},
     {id: artifacts.PasswordInputsWithPreventedPaste, gatherer: 'dobetterweb/password-inputs-with-prevented-paste'},
+    {id: artifacts.ResponseCompression, gatherer: 'dobetterweb/response-compression'},
     {id: artifacts.RobotsTxt, gatherer: 'seo/robots-txt'},
     {id: artifacts.SourceMaps, gatherer: 'source-maps'},
     {id: artifacts.Stacks, gatherer: 'stacks'},
@@ -126,6 +128,7 @@ const defaultConfig = {
         artifacts.NetworkUserAgent,
         artifacts.OptimizedImages,
         artifacts.PasswordInputsWithPreventedPaste,
+        artifacts.ResponseCompression,
         artifacts.RobotsTxt,
         artifacts.SourceMaps,
         artifacts.Stacks,

lighthouse-core/gather/gatherers/dobetterweb/response-compression.js

@@ -10,17 +10,20 @@
   */
 'use strict';
 
-const Gatherer = require('../gatherer.js');
+const FRGatherer = require('../../../fraggle-rock/gather/base-gatherer.js');
 const URL = require('../../../lib/url-shim.js');
 const Sentry = require('../../../lib/sentry.js');
 const NetworkRequest = require('../../../lib/network-request.js');
 const gzip = require('zlib').gzip;
+const DevtoolsLog = require('../devtools-log.js');
+const {fetchResponseBodyFromCache} = require('../../driver/network.js');
+const NetworkRecords = require('../../../computed/network-records.js');
 
 const CHROME_EXTENSION_PROTOCOL = 'chrome-extension:';
 const compressionHeaders = ['content-encoding', 'x-original-content-encoding'];
 const compressionTypes = ['gzip', 'br', 'deflate'];
 const binaryMimeTypes = ['image', 'audio', 'video'];
-/** @type {Array<LH.Crdp.Network.ResourceType>} */
+/** @type {LH.Crdp.Network.ResourceType[]} */
 const textResourceTypes = [
   NetworkRequest.TYPES.Document,
   NetworkRequest.TYPES.Script,
@@ -30,9 +33,15 @@ const textResourceTypes = [
   NetworkRequest.TYPES.EventSource,
 ];
 
-class ResponseCompression extends Gatherer {
+class ResponseCompression extends FRGatherer {
+  /** @type {LH.Gatherer.GathererMeta<'DevtoolsLog'>} */
+  meta = {
+    supportedModes: ['timespan', 'navigation'],
+    dependencies: {DevtoolsLog: DevtoolsLog.symbol},
+  }
+
   /**
-   * @param {Array<LH.Artifacts.NetworkRequest>} networkRecords
+   * @param {LH.Artifacts.NetworkRequest[]} networkRecords
    * @return {LH.Artifacts['ResponseCompression']}
    */
   static filterUnoptimizedResponses(networkRecords) {
@@ -77,17 +86,16 @@ class ResponseCompression extends Gatherer {
   }
 
   /**
-   * @param {LH.Gatherer.PassContext} passContext
-   * @param {LH.Gatherer.LoadData} loadData
+   * @param {LH.Gatherer.FRTransitionalContext} context
+   * @param {LH.Artifacts.NetworkRequest[]} networkRecords
    * @return {Promise<LH.Artifacts['ResponseCompression']>}
    */
-  afterPass(passContext, loadData) {
-    const networkRecords = loadData.networkRecords;
+  async _getArtifact(context, networkRecords) {
+    const session = context.driver.defaultSession;
     const textRecords = ResponseCompression.filterUnoptimizedResponses(networkRecords);
 
-    const driver = passContext.driver;
     return Promise.all(textRecords.map(record => {
-      return driver.getRequestContent(record.requestId).then(content => {
+      return fetchResponseBodyFromCache(session, record.requestId).then(content => {
         // if we don't have any content, gzipSize is already set to 0
         if (!content) {
           return record;
@@ -117,6 +125,25 @@ class ResponseCompression extends Gatherer {
       });
     }));
   }
+
+  /**
+   * @param {LH.Gatherer.FRTransitionalContext<'DevtoolsLog'>} context
+   * @return {Promise<LH.Artifacts['ResponseCompression']>}
+   */
+  async getArtifact(context) {
+    const devtoolsLog = context.dependencies.DevtoolsLog;
+    const networkRecords = await NetworkRecords.request(devtoolsLog, context);
+    return this._getArtifact(context, networkRecords);
+  }
+
+  /**
+   * @param {LH.Gatherer.PassContext} passContext
+   * @param {LH.Gatherer.LoadData} loadData
+   * @return {Promise<LH.Artifacts['ResponseCompression']>}
+   */
+  async afterPass(passContext, loadData) {
+    return this._getArtifact({...passContext, dependencies: {}}, loadData.networkRecords);
+  }
 }
 
 module.exports = ResponseCompression;

lighthouse-core/test/fraggle-rock/api-test-pptr.js

@@ -159,7 +159,7 @@ describe('Fraggle Rock API', () => {
       const {lhr} = result;
       const {auditResults, failedAudits, erroredAudits} = getAuditsBreakdown(lhr);
       // TODO(FR-COMPAT): This assertion can be removed when full compatibility is reached.
-      expect(auditResults.length).toMatchInlineSnapshot(`112`);
+      expect(auditResults.length).toMatchInlineSnapshot(`113`);
       expect(erroredAudits).toHaveLength(0);
 
       const failedAuditIds = failedAudits.map(audit => audit.id);

lighthouse-core/test/fraggle-rock/gather/mock-driver.js

@@ -150,21 +150,24 @@ function createMockContext() {
 }
 
 function mockDriverSubmodules() {
-  const navigationMock = {gotoURL: jest.fn(),
-  };
+  const navigationMock = {gotoURL: jest.fn()};
   const prepareMock = {
     prepareTargetForNavigationMode: jest.fn(),
     prepareTargetForIndividualNavigation: jest.fn(),
   };
   const storageMock = {clearDataForOrigin: jest.fn()};
   const emulationMock = {clearThrottling: jest.fn()};
+  const networkMock = {
+    fetchResponseBodyFromCache: jest.fn(),
+  };
 
   function reset() {
     navigationMock.gotoURL = jest.fn().mockResolvedValue({finalUrl: 'https://example.com', warnings: [], timedOut: false});
     prepareMock.prepareTargetForNavigationMode = jest.fn().mockResolvedValue({warnings: []});
     prepareMock.prepareTargetForIndividualNavigation = jest.fn().mockResolvedValue({warnings: []});
     storageMock.clearDataForOrigin = jest.fn();
     emulationMock.clearThrottling = jest.fn();
+    networkMock.fetchResponseBodyFromCache = jest.fn().mockResolvedValue('');
   }
 
   /**
@@ -178,13 +181,15 @@ function mockDriverSubmodules() {
   jest.mock('../../../gather/driver/navigation.js', () => new Proxy(navigationMock, {get}));
   jest.mock('../../../gather/driver/prepare.js', () => new Proxy(prepareMock, {get}));
   jest.mock('../../../gather/driver/storage.js', () => new Proxy(storageMock, {get}));
+  jest.mock('../../../gather/driver/network.js', () => new Proxy(networkMock, {get}));
   jest.mock('../../../lib/emulation.js', () => new Proxy(emulationMock, {get}));
 
   return {
     navigationMock,
     prepareMock,
     storageMock,
     emulationMock,
+    networkMock,
     reset,
   };
 }