From 55b5184391f3f5dc93dafc12a0d5cfb0e3d0e736 Mon Sep 17 00:00:00 2001 From: Pankaj Yadav Date: Tue, 2 Apr 2024 14:41:29 +0530 Subject: [PATCH] GZIP resources before uploading (#1566) * gzip resources before uploading * add env var * Add tests for client.js * Add test for discovery * Use pako in test * Use pako in test * unset env var * unset env var * remove test * Add test * delete process.env.PERCY_GZIP before each it block --- packages/client/package.json | 1 + packages/client/src/client.js | 8 ++++- packages/client/test/client.test.js | 23 ++++++++++++++ packages/core/package.json | 3 +- packages/core/src/discovery.js | 11 +++++++ packages/core/test/discovery.test.js | 46 ++++++++++++++++++++++++++++ yarn.lock | 5 +++ 7 files changed, 95 insertions(+), 2 deletions(-) diff --git a/packages/client/package.json b/packages/client/package.json index a5b26b06a..bc4775d49 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -32,6 +32,7 @@ "test:coverage": "yarn test --coverage" }, "dependencies": { + "pako": "^2.1.0", "@percy/env": "1.28.3-beta.1", "@percy/logger": "1.28.3-beta.1" } diff --git a/packages/client/src/client.js b/packages/client/src/client.js index 863d0bd96..022afa825 100644 --- a/packages/client/src/client.js +++ b/packages/client/src/client.js @@ -2,6 +2,7 @@ import fs from 'fs'; import PercyEnv from '@percy/env'; import { git } from '@percy/env/utils'; import logger from '@percy/logger'; +import Pako from 'pako'; import { pool, @@ -317,7 +318,12 @@ export class PercyClient { // created from `content` if one is not provided. async uploadResource(buildId, { url, sha, filepath, content } = {}) { validateId('build', buildId); - if (filepath) content = await fs.promises.readFile(filepath); + if (filepath) { + content = await fs.promises.readFile(filepath); + if (process.env.PERCY_GZIP) { + content = Pako.gzip(content); + } + } let encodedContent = base64encode(content); this.log.debug(`Uploading ${formatBytes(encodedContent.length)} resource: ${url}...`); diff --git a/packages/client/test/client.test.js b/packages/client/test/client.test.js index bba675d8d..1f186aee4 100644 --- a/packages/client/test/client.test.js +++ b/packages/client/test/client.test.js @@ -6,6 +6,7 @@ import PercyClient from '@percy/client'; import api from './helpers.js'; import * as CoreConfig from '@percy/core/config'; import PercyConfig from '@percy/config'; +import Pako from 'pako'; describe('PercyClient', () => { let client; @@ -13,6 +14,7 @@ describe('PercyClient', () => { beforeEach(async () => { await logger.mock({ level: 'debug' }); await api.mock(); + delete process.env.PERCY_GZIP; client = new PercyClient({ token: 'PERCY_TOKEN' @@ -589,6 +591,27 @@ describe('PercyClient', () => { } }); }); + + it('can upload a resource from a local path in GZIP format', async () => { + process.env.PERCY_GZIP = true; + + spyOn(fs.promises, 'readFile').and.callFake(async p => `contents of ${p}`); + + await expectAsync(client.uploadResource(123, { + sha: 'foo-sha', + filepath: 'foo/bar' + })).toBeResolved(); + + expect(api.requests['/builds/123/resources'][0].body).toEqual({ + data: { + type: 'resources', + id: 'foo-sha', + attributes: { + 'base64-content': base64encode(Pako.gzip('contents of foo/bar')) + } + } + }); + }); }); describe('#uploadResources()', () => { diff --git a/packages/core/package.json b/packages/core/package.json index 056bdad8d..9b211de7f 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -56,6 +56,7 @@ "mime-types": "^2.1.34", "path-to-regexp": "^6.2.0", "rimraf": "^3.0.2", - "ws": "^8.0.0" + "ws": "^8.0.0", + "pako": "^2.1.0" } } diff --git a/packages/core/src/discovery.js b/packages/core/src/discovery.js index fab06ea6f..82dee0502 100644 --- a/packages/core/src/discovery.js +++ b/packages/core/src/discovery.js @@ -11,6 +11,10 @@ import { snapshotLogName, withRetries } from './utils.js'; +import { + sha256hash +} from '@percy/client/utils'; +import Pako from 'pako'; // Logs verbose debug logs detailing various snapshot options. function debugSnapshotOptions(snapshot) { @@ -136,6 +140,13 @@ function processSnapshotResources({ domSnapshot, resources, ...snapshot }) { log.meta.snapshot?.testCase === snapshot.meta.snapshot.testCase && log.meta.snapshot?.name === snapshot.meta.snapshot.name )))); + if (process.env.PERCY_GZIP) { + for (let index = 0; index < resources.length; index++) { + resources[index].content = Pako.gzip(resources[index].content); + resources[index].sha = sha256hash(resources[index].content); + } + } + return { ...snapshot, resources }; } diff --git a/packages/core/test/discovery.test.js b/packages/core/test/discovery.test.js index 8d5245c82..52ffd80e1 100644 --- a/packages/core/test/discovery.test.js +++ b/packages/core/test/discovery.test.js @@ -3,6 +3,7 @@ import { logger, api, setupTest, createTestServer, dedent } from './helpers/inde import Percy from '@percy/core'; import { RESOURCE_CACHE_KEY } from '../src/discovery.js'; import Session from '../src/session.js'; +import Pako from 'pako'; describe('Discovery', () => { let percy, server, captured; @@ -28,6 +29,7 @@ describe('Discovery', () => { captured = []; await setupTest(); delete process.env.PERCY_BROWSER_EXECUTABLE; + delete process.env.PERCY_GZIP; api.reply('/builds/123/snapshots', ({ body }) => { // resource order is not important, stabilize it for testing @@ -57,6 +59,50 @@ describe('Discovery', () => { await server.close(); }); + it('gathers resources for a snapshot in GZIP format', async () => { + process.env.PERCY_GZIP = true; + + await percy.snapshot({ + name: 'test snapshot', + url: 'http://localhost:8000', + domSnapshot: testDOM + }); + + await percy.idle(); + + let paths = server.requests.map(r => r[0]); + // does not request the root url (serves domSnapshot instead) + expect(paths).not.toContain('/'); + expect(paths).toContain('/style.css'); + expect(paths).toContain('/img.gif'); + + expect(captured[0]).toEqual([ + jasmine.objectContaining({ + attributes: jasmine.objectContaining({ + 'resource-url': jasmine.stringMatching(/^\/percy\.\d+\.log$/) + }) + }), + jasmine.objectContaining({ + id: sha256hash(Pako.gzip(testDOM)), + attributes: jasmine.objectContaining({ + 'resource-url': 'http://localhost:8000/' + }) + }), + jasmine.objectContaining({ + id: sha256hash(Pako.gzip(pixel)), + attributes: jasmine.objectContaining({ + 'resource-url': 'http://localhost:8000/img.gif' + }) + }), + jasmine.objectContaining({ + id: sha256hash(Pako.gzip(testCSS)), + attributes: jasmine.objectContaining({ + 'resource-url': 'http://localhost:8000/style.css' + }) + }) + ]); + }); + it('gathers resources for a snapshot', async () => { await percy.snapshot({ name: 'test snapshot', diff --git a/yarn.lock b/yarn.lock index 9cdace299..d368ec79a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6648,6 +6648,11 @@ pacote@^13.0.3, pacote@^13.6.1: ssri "^9.0.0" tar "^6.1.11" +pako@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/pako/-/pako-2.1.0.tgz#266cc37f98c7d883545d11335c00fbd4062c9a86" + integrity sha512-w+eufiZ1WuJYgPXbV/PO3NCMEc3xqylkKHzp8bxp1uW4qaSNQUkwmLLEc3kKsfz8lpV1F8Ht3U1Cm+9Srog2ug== + parent-module@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/parent-module/-/parent-module-1.0.1.tgz#691d2709e78c79fae3a156622452d00762caaaa2"