GZIP resources before uploading (#1566)

* gzip resources before uploading * add env var * Add tests for client.js * Add test for discovery * Use pako in test * Use pako in test * unset env var * unset env var * remove test * Add test * delete process.env.PERCY_GZIP before each it block
percy · Apr 3, 2024 · a1fe62c · a1fe62c
1 parent 958a5b3
commit a1fe62c
Show file tree

Hide file tree

Showing 7 changed files with 95 additions and 2 deletions.
diff --git a/packages/client/package.json b/packages/client/package.json
@@ -32,6 +32,7 @@
     "test:coverage": "yarn test --coverage"
   },
   "dependencies": {
+    "pako": "^2.1.0",
     "@percy/env": "1.28.3-beta.1",
     "@percy/logger": "1.28.3-beta.1"
   }

diff --git a/packages/client/src/client.js b/packages/client/src/client.js
@@ -2,6 +2,7 @@ import fs from 'fs';
 import PercyEnv from '@percy/env';
 import { git } from '@percy/env/utils';
 import logger from '@percy/logger';
+import Pako from 'pako';
 
 import {
   pool,
@@ -317,7 +318,12 @@ export class PercyClient {
   // created from `content` if one is not provided.
   async uploadResource(buildId, { url, sha, filepath, content } = {}) {
     validateId('build', buildId);
-    if (filepath) content = await fs.promises.readFile(filepath);
+    if (filepath) {
+      content = await fs.promises.readFile(filepath);
+      if (process.env.PERCY_GZIP) {
+        content = Pako.gzip(content);
+      }
+    }
     let encodedContent = base64encode(content);
 
     this.log.debug(`Uploading ${formatBytes(encodedContent.length)} resource: ${url}...`);

diff --git a/packages/client/test/client.test.js b/packages/client/test/client.test.js
@@ -6,13 +6,15 @@ import PercyClient from '@percy/client';
 import api from './helpers.js';
 import * as CoreConfig from '@percy/core/config';
 import PercyConfig from '@percy/config';
+import Pako from 'pako';
 
 describe('PercyClient', () => {
   let client;
 
   beforeEach(async () => {
     await logger.mock({ level: 'debug' });
     await api.mock();
+    delete process.env.PERCY_GZIP;
 
     client = new PercyClient({
       token: 'PERCY_TOKEN'
@@ -589,6 +591,27 @@ describe('PercyClient', () => {
         }
       });
     });
+
+    it('can upload a resource from a local path in GZIP format', async () => {
+      process.env.PERCY_GZIP = true;
+
+      spyOn(fs.promises, 'readFile').and.callFake(async p => `contents of ${p}`);
+
+      await expectAsync(client.uploadResource(123, {
+        sha: 'foo-sha',
+        filepath: 'foo/bar'
+      })).toBeResolved();
+
+      expect(api.requests['/builds/123/resources'][0].body).toEqual({
+        data: {
+          type: 'resources',
+          id: 'foo-sha',
+          attributes: {
+            'base64-content': base64encode(Pako.gzip('contents of foo/bar'))
+          }
+        }
+      });
+    });
   });
 
   describe('#uploadResources()', () => {

diff --git a/packages/core/package.json b/packages/core/package.json
@@ -56,6 +56,7 @@
     "mime-types": "^2.1.34",
     "path-to-regexp": "^6.2.0",
     "rimraf": "^3.0.2",
-    "ws": "^8.0.0"
+    "ws": "^8.0.0",
+    "pako": "^2.1.0"
   }
 }
diff --git a/packages/core/src/discovery.js b/packages/core/src/discovery.js
@@ -11,6 +11,10 @@ import {
   snapshotLogName,
   withRetries
 } from './utils.js';
+import {
+  sha256hash
+} from '@percy/client/utils';
+import Pako from 'pako';
 
 // Logs verbose debug logs detailing various snapshot options.
 function debugSnapshotOptions(snapshot) {
@@ -136,6 +140,13 @@ function processSnapshotResources({ domSnapshot, resources, ...snapshot }) {
     log.meta.snapshot?.testCase === snapshot.meta.snapshot.testCase && log.meta.snapshot?.name === snapshot.meta.snapshot.name
   ))));
 
+  if (process.env.PERCY_GZIP) {
+    for (let index = 0; index < resources.length; index++) {
+      resources[index].content = Pako.gzip(resources[index].content);
+      resources[index].sha = sha256hash(resources[index].content);
+    }
+  }
+
   return { ...snapshot, resources };
 }
 

diff --git a/packages/core/test/discovery.test.js b/packages/core/test/discovery.test.js
@@ -3,6 +3,7 @@ import { logger, api, setupTest, createTestServer, dedent } from './helpers/inde
 import Percy from '@percy/core';
 import { RESOURCE_CACHE_KEY } from '../src/discovery.js';
 import Session from '../src/session.js';
+import Pako from 'pako';
 
 describe('Discovery', () => {
   let percy, server, captured;
@@ -28,6 +29,7 @@ describe('Discovery', () => {
     captured = [];
     await setupTest();
     delete process.env.PERCY_BROWSER_EXECUTABLE;
+    delete process.env.PERCY_GZIP;
 
     api.reply('/builds/123/snapshots', ({ body }) => {
       // resource order is not important, stabilize it for testing
@@ -57,6 +59,50 @@ describe('Discovery', () => {
     await server.close();
   });
 
+  it('gathers resources for a snapshot in GZIP format', async () => {
+    process.env.PERCY_GZIP = true;
+
+    await percy.snapshot({
+      name: 'test snapshot',
+      url: 'http://localhost:8000',
+      domSnapshot: testDOM
+    });
+
+    await percy.idle();
+
+    let paths = server.requests.map(r => r[0]);
+    // does not request the root url (serves domSnapshot instead)
+    expect(paths).not.toContain('/');
+    expect(paths).toContain('/style.css');
+    expect(paths).toContain('/img.gif');
+
+    expect(captured[0]).toEqual([
+      jasmine.objectContaining({
+        attributes: jasmine.objectContaining({
+          'resource-url': jasmine.stringMatching(/^\/percy\.\d+\.log$/)
+        })
+      }),
+      jasmine.objectContaining({
+        id: sha256hash(Pako.gzip(testDOM)),
+        attributes: jasmine.objectContaining({
+          'resource-url': 'http://localhost:8000/'
+        })
+      }),
+      jasmine.objectContaining({
+        id: sha256hash(Pako.gzip(pixel)),
+        attributes: jasmine.objectContaining({
+          'resource-url': 'http://localhost:8000/img.gif'
+        })
+      }),
+      jasmine.objectContaining({
+        id: sha256hash(Pako.gzip(testCSS)),
+        attributes: jasmine.objectContaining({
+          'resource-url': 'http://localhost:8000/style.css'
+        })
+      })
+    ]);
+  });
+
   it('gathers resources for a snapshot', async () => {
     await percy.snapshot({
       name: 'test snapshot',

diff --git a/yarn.lock b/yarn.lock
@@ -6648,6 +6648,11 @@ pacote@^13.0.3, pacote@^13.6.1:
     ssri "^9.0.0"
     tar "^6.1.11"
 
+pako@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/pako/-/pako-2.1.0.tgz#266cc37f98c7d883545d11335c00fbd4062c9a86"
+  integrity sha512-w+eufiZ1WuJYgPXbV/PO3NCMEc3xqylkKHzp8bxp1uW4qaSNQUkwmLLEc3kKsfz8lpV1F8Ht3U1Cm+9Srog2ug==
+
 parent-module@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/parent-module/-/parent-module-1.0.1.tgz#691d2709e78c79fae3a156622452d00762caaaa2"