From c2a324ccf1d2c56e9a57d33fd97f9bb12b8958bf Mon Sep 17 00:00:00 2001 From: Steve Lawrence Date: Tue, 10 Dec 2024 09:13:47 -0500 Subject: [PATCH 1/4] Allow for reproducible .vsix packages Running the same build produces .vsix package that have the same content, but are not bit for bit the same, making it somewhat complicated to verify reproducible builds. Two changes are needed to fix this: 1. The mtime of each file added to the .vsix archive is included in each archive entry, so builds that happen at different times will have different entry timestamps. To fix this, if the SOURCE_DATE_EPOCH environment variable is defined, it it now used as entry timestamp value instead. Builds will now be reproducible as long as they set the same SOURCE_DATE_EPOCH value. If the environment variable is not defined or is not an integer, the current behavior is used. 2. The order that files are collected in preparation for packaging into the .vsix file is non-deterministic, which can lead to archives with the same content but in different orders. To fix this, files are sorted by archive entry name prior to adding. Fixes #906 --- src/package.ts | 22 +++++++++++++++------- src/test/package.test.ts | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 7 deletions(-) diff --git a/src/package.ts b/src/package.ts index 6dac2581..bb60db51 100644 --- a/src/package.ts +++ b/src/package.ts @@ -1796,7 +1796,7 @@ export function collect(manifest: ManifestPackage, options: IPackageOptions = {} }); } -function writeVsix(files: IFile[], packagePath: string): Promise { +export function writeVsix(files: IFile[], packagePath: string): Promise { return fs.promises .unlink(packagePath) .catch(err => (err.code !== 'ENOENT' ? Promise.reject(err) : Promise.resolve(null))) @@ -1804,13 +1804,21 @@ function writeVsix(files: IFile[], packagePath: string): Promise { () => new Promise((c, e) => { const zip = new yazl.ZipFile(); - files.forEach(f => + + const sde = process.env.SOURCE_DATE_EPOCH; + const epoch = sde ? parseInt(sde) : undefined; + const mtime = epoch ? new Date(epoch * 1000) : undefined; + + files.sort((a, b) => a.path.localeCompare(b.path)).forEach(f => { + let options = { + mode: f.mode, + } as any; + if (mtime) options.mtime = mtime; + isInMemoryFile(f) - ? zip.addBuffer(typeof f.contents === 'string' ? Buffer.from(f.contents, 'utf8') : f.contents, f.path, { - mode: f.mode, - }) - : zip.addFile(f.localPath, f.path, { mode: f.mode }) - ); + ? zip.addBuffer(typeof f.contents === 'string' ? Buffer.from(f.contents, 'utf8') : f.contents, f.path, options) + : zip.addFile(f.localPath, f.path, options) + }); zip.end(); const zipStream = fs.createWriteStream(packagePath); diff --git a/src/test/package.test.ts b/src/test/package.test.ts index 467850a7..ab8c74ec 100644 --- a/src/test/package.test.ts +++ b/src/test/package.test.ts @@ -15,6 +15,7 @@ import { VSIX, LicenseProcessor, printAndValidatePackagedFiles, + writeVsix, } from '../package'; import { ManifestPackage } from '../manifest'; import * as path from 'path'; @@ -26,6 +27,7 @@ import { XMLManifest, parseXmlManifest, parseContentTypes } from '../xml'; import { flatten, log } from '../util'; import { validatePublisher } from '../validation'; import * as jsonc from 'jsonc-parser'; +import { globSync } from 'glob'; // don't warn in tests console.warn = () => null; @@ -3196,3 +3198,41 @@ describe('version', function () { assert.strictEqual(newManifest.version, '1.0.0'); }); }); + +describe('writeVsix', () => { + it('should be reproducible', async () => { + const dir = tmp.dirSync({ unsafeCleanup: true }); + const cwd = dir.name + + const srcDir = fixture('manifestFiles'); + fs.cpSync(srcDir, cwd, { recursive: true }); + + const files = globSync("**", { cwd }); + + process.env["SOURCE_DATE_EPOCH"] = '1000000000'; + + const vsix1 = tmp.fileSync(); + const epoch1 = 1000000001 + files.forEach((f) => fs.utimesSync(path.join(cwd, f), epoch1, epoch1)); + const manifest1 = await readManifest(cwd); + const iFiles1 = await collect(manifest1, { cwd }); + await writeVsix(iFiles1, vsix1.name); + + // different timestamp and iFiles are reversed + const vsix2 = tmp.fileSync(); + const epoch2 = 1000000002 + files.forEach((f) => fs.utimesSync(path.join(cwd, f), epoch2, epoch2)); + const manifest2 = await readManifest(cwd); + const iFiles2 = (await collect(manifest2, { cwd })).reverse(); + await writeVsix(iFiles2, vsix2.name); + + const vsix1bytes = fs.readFileSync(vsix1.name); + const vsix2bytes = fs.readFileSync(vsix2.name); + + assert.deepStrictEqual(vsix1bytes, vsix2bytes); + + dir.removeCallback(); + vsix1.removeCallback(); + vsix2.removeCallback(); + }); +}); From 018208d46e5df1ff04df867b0a26463e1ec2a36f Mon Sep 17 00:00:00 2001 From: BeniBenj Date: Tue, 10 Dec 2024 16:24:07 +0100 Subject: [PATCH 2/4] Create tmp folder in fixtures folder --- src/test/package.test.ts | 55 +++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/src/test/package.test.ts b/src/test/package.test.ts index ab8c74ec..2de654bd 100644 --- a/src/test/package.test.ts +++ b/src/test/package.test.ts @@ -3199,40 +3199,43 @@ describe('version', function () { }); }); -describe('writeVsix', () => { +describe('writeVsix', function () { + this.timeout(60_000); + it('should be reproducible', async () => { - const dir = tmp.dirSync({ unsafeCleanup: true }); - const cwd = dir.name + const exampleProject = fixture('manifestFiles'); + const fixtureDir = fixture(''); - const srcDir = fixture('manifestFiles'); - fs.cpSync(srcDir, cwd, { recursive: true }); + const testDir = tmp.dirSync({ unsafeCleanup: true, tmpdir: fixtureDir }); + const cwd = testDir.name - const files = globSync("**", { cwd }); + try { + fs.cpSync(exampleProject, cwd, { recursive: true }); - process.env["SOURCE_DATE_EPOCH"] = '1000000000'; + const files = globSync("**", { cwd }); - const vsix1 = tmp.fileSync(); - const epoch1 = 1000000001 - files.forEach((f) => fs.utimesSync(path.join(cwd, f), epoch1, epoch1)); - const manifest1 = await readManifest(cwd); - const iFiles1 = await collect(manifest1, { cwd }); - await writeVsix(iFiles1, vsix1.name); + process.env["SOURCE_DATE_EPOCH"] = '1000000000'; - // different timestamp and iFiles are reversed - const vsix2 = tmp.fileSync(); - const epoch2 = 1000000002 - files.forEach((f) => fs.utimesSync(path.join(cwd, f), epoch2, epoch2)); - const manifest2 = await readManifest(cwd); - const iFiles2 = (await collect(manifest2, { cwd })).reverse(); - await writeVsix(iFiles2, vsix2.name); + const createVsix = async (vsixPath: string, epoch: number) => { + files.forEach((f) => fs.utimesSync(path.join(cwd, f), epoch, epoch)); + const manifest1 = await readManifest(cwd); + const iFiles1 = await collect(manifest1, { cwd }); + await writeVsix(iFiles1, vsixPath); + } - const vsix1bytes = fs.readFileSync(vsix1.name); - const vsix2bytes = fs.readFileSync(vsix2.name); + const vsix1 = testDir.name + '/vsix1.vsix'; + const vsix2 = testDir.name + '/vsix2.vsix'; - assert.deepStrictEqual(vsix1bytes, vsix2bytes); + await createVsix(vsix1, 1000000001); + await createVsix(vsix2, 1000000002); - dir.removeCallback(); - vsix1.removeCallback(); - vsix2.removeCallback(); + const vsix1bytes = fs.readFileSync(vsix1); + const vsix2bytes = fs.readFileSync(vsix2); + + assert.deepStrictEqual(vsix1bytes, vsix2bytes); + + } finally { + testDir.removeCallback(); + } }); }); From b400da1d079780b3567b958a8032c07fd8f4905f Mon Sep 17 00:00:00 2001 From: BeniBenj Date: Tue, 10 Dec 2024 16:51:39 +0100 Subject: [PATCH 3/4] :lipstick: --- src/package.ts | 22 +++++++++++----------- src/test/package.test.ts | 34 +++++++++++++++------------------- 2 files changed, 26 insertions(+), 30 deletions(-) diff --git a/src/package.ts b/src/package.ts index bb60db51..9f5686e5 100644 --- a/src/package.ts +++ b/src/package.ts @@ -1804,21 +1804,21 @@ export function writeVsix(files: IFile[], packagePath: string): Promise { () => new Promise((c, e) => { const zip = new yazl.ZipFile(); + const zipOptions: Partial = {}; + // reproducible zip files const sde = process.env.SOURCE_DATE_EPOCH; - const epoch = sde ? parseInt(sde) : undefined; - const mtime = epoch ? new Date(epoch * 1000) : undefined; - - files.sort((a, b) => a.path.localeCompare(b.path)).forEach(f => { - let options = { - mode: f.mode, - } as any; - if (mtime) options.mtime = mtime; + if (sde) { + const epoch = parseInt(sde); + zipOptions.mtime = new Date(epoch * 1000); + files = files.sort((a, b) => a.path.localeCompare(b.path)) + } + files.forEach(f => isInMemoryFile(f) - ? zip.addBuffer(typeof f.contents === 'string' ? Buffer.from(f.contents, 'utf8') : f.contents, f.path, options) - : zip.addFile(f.localPath, f.path, options) - }); + ? zip.addBuffer(typeof f.contents === 'string' ? Buffer.from(f.contents, 'utf8') : f.contents, f.path, { ...zipOptions, mode: f.mode }) + : zip.addFile(f.localPath, f.path, { ...zipOptions, mode: f.mode }) + ); zip.end(); const zipStream = fs.createWriteStream(packagePath); diff --git a/src/test/package.test.ts b/src/test/package.test.ts index 2de654bd..6b01a159 100644 --- a/src/test/package.test.ts +++ b/src/test/package.test.ts @@ -14,8 +14,7 @@ import { versionBump, VSIX, LicenseProcessor, - printAndValidatePackagedFiles, - writeVsix, + printAndValidatePackagedFiles, pack } from '../package'; import { ManifestPackage } from '../manifest'; import * as path from 'path'; @@ -27,7 +26,6 @@ import { XMLManifest, parseXmlManifest, parseContentTypes } from '../xml'; import { flatten, log } from '../util'; import { validatePublisher } from '../validation'; import * as jsonc from 'jsonc-parser'; -import { globSync } from 'glob'; // don't warn in tests console.warn = () => null; @@ -2288,13 +2286,13 @@ describe('ManifestProcessor', () => { }); it('should not throw error for engine version with x (e.g. 1.95.x)', async () => { - const root = fixture('uuid'); - const manifest = JSON.parse(await fs.promises.readFile(path.join(root, 'package.json'), 'utf8')); - manifest.engines.vscode = '1.95.x'; // Non-strict semver, but acceptable + const root = fixture('uuid'); + const manifest = JSON.parse(await fs.promises.readFile(path.join(root, 'package.json'), 'utf8')); + manifest.engines.vscode = '1.95.x'; // Non-strict semver, but acceptable - assert.doesNotThrow(() => new ManifestProcessor(manifest, { target: 'web' })); - assert.doesNotThrow(() => new ManifestProcessor(manifest, { preRelease: true })); - }); + assert.doesNotThrow(() => new ManifestProcessor(manifest, { target: 'web' })); + assert.doesNotThrow(() => new ManifestProcessor(manifest, { preRelease: true })); + }); }); describe('MarkdownProcessor', () => { @@ -3212,27 +3210,25 @@ describe('writeVsix', function () { try { fs.cpSync(exampleProject, cwd, { recursive: true }); - const files = globSync("**", { cwd }); - - process.env["SOURCE_DATE_EPOCH"] = '1000000000'; - const createVsix = async (vsixPath: string, epoch: number) => { - files.forEach((f) => fs.utimesSync(path.join(cwd, f), epoch, epoch)); - const manifest1 = await readManifest(cwd); - const iFiles1 = await collect(manifest1, { cwd }); - await writeVsix(iFiles1, vsixPath); + process.env["SOURCE_DATE_EPOCH"] = `${epoch}`; + await pack({ cwd, packagePath: vsixPath }); } const vsix1 = testDir.name + '/vsix1.vsix'; const vsix2 = testDir.name + '/vsix2.vsix'; + const vsix3 = testDir.name + '/vsix3.vsix'; - await createVsix(vsix1, 1000000001); - await createVsix(vsix2, 1000000002); + await createVsix(vsix1, 1000000000); + await createVsix(vsix2, 1000000000); + await createVsix(vsix3, 1000000002); const vsix1bytes = fs.readFileSync(vsix1); const vsix2bytes = fs.readFileSync(vsix2); + const vsix3bytes = fs.readFileSync(vsix3); assert.deepStrictEqual(vsix1bytes, vsix2bytes); + assert.notDeepStrictEqual(vsix1bytes, vsix3bytes); } finally { testDir.removeCallback(); From c36740481e9c8c1eae219273e6a8781428ad990c Mon Sep 17 00:00:00 2001 From: BeniBenj Date: Tue, 10 Dec 2024 16:54:13 +0100 Subject: [PATCH 4/4] :lipstick: --- src/package.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/package.ts b/src/package.ts index 9f5686e5..cb5c0708 100644 --- a/src/package.ts +++ b/src/package.ts @@ -505,7 +505,7 @@ export class ManifestProcessor extends BaseProcessor { if (!minEngineVersion) { throw new Error('Failed to get minVersion of engines.vscode') } - + if (target) { if (engineSemver.version !== 'latest' && !semver.satisfies(minEngineVersion, '>=1.61', { includePrerelease: true })) { throw new Error( @@ -1796,7 +1796,7 @@ export function collect(manifest: ManifestPackage, options: IPackageOptions = {} }); } -export function writeVsix(files: IFile[], packagePath: string): Promise { +function writeVsix(files: IFile[], packagePath: string): Promise { return fs.promises .unlink(packagePath) .catch(err => (err.code !== 'ENOENT' ? Promise.reject(err) : Promise.resolve(null)))