From 121fe1f6ecba08a22d7ae753c19a8bef5e7ded7c Mon Sep 17 00:00:00 2001 From: Connor Peet Date: Mon, 27 Feb 2023 14:54:20 -0800 Subject: [PATCH] feat: automatically use vscode version matching engine Automatically use the latest version of VS Code that satisfies all constraints in `engines.vscode` in `package.json`. (But only use Insiders if no stable build satisfies the constraints.) This also lets users pass `X.Y.Z-insider` to runTests, where previously it was only possible to request the latest Insiders version. Fixes #176 --- lib/download.test.ts | 101 ++++++++++++++++++++++++++++-- lib/download.ts | 142 +++++++++++++++++++++++++++++++++++-------- lib/util.ts | 31 +++++++++- package.json | 4 +- yarn.lock | 12 ++++ 5 files changed, 256 insertions(+), 34 deletions(-) diff --git a/lib/download.test.ts b/lib/download.test.ts index d911b751..6efa9459 100644 --- a/lib/download.test.ts +++ b/lib/download.test.ts @@ -1,19 +1,32 @@ import { spawnSync } from 'child_process'; import { existsSync, promises as fs } from 'fs'; import { tmpdir } from 'os'; -import { join } from 'path'; -import { afterAll, beforeAll, describe, expect, test } from 'vitest'; -import { downloadAndUnzipVSCode } from './download'; +import { dirname, join } from 'path'; +import { afterAll, afterEach, beforeAll, describe, expect, test } from 'vitest'; +import { + downloadAndUnzipVSCode, + fetchInsiderVersions, + fetchStableVersions, + fetchTargetInferredVersion, +} from './download'; import { SilentReporter } from './progress'; import { resolveCliPathFromVSCodeExecutablePath, systemDefaultPlatform } from './util'; -const platforms = ['darwin', 'darwin-arm64', 'win32-archive', 'win32-x64-archive', 'linux-x64', 'linux-arm64', 'linux-armhf']; +const platforms = [ + 'darwin', + 'darwin-arm64', + 'win32-archive', + 'win32-x64-archive', + 'linux-x64', + 'linux-arm64', + 'linux-armhf', +]; describe('sane downloads', () => { const testTempDir = join(tmpdir(), 'vscode-test-download'); beforeAll(async () => { - await fs.mkdir(testTempDir, { recursive: true }) + await fs.mkdir(testTempDir, { recursive: true }); }); for (const platform of platforms) { @@ -39,7 +52,7 @@ describe('sane downloads', () => { expect(version.status).to.equal(0); expect(version.stdout.toString().trim()).to.not.be.empty; } - }) + }); } afterAll(async () => { @@ -50,3 +63,79 @@ describe('sane downloads', () => { } }); }); + +describe('fetchTargetInferredVersion', () => { + let stable: string[]; + let insiders: string[]; + let extensionsDevelopmentPath = join(tmpdir(), 'vscode-test-tmp-workspace'); + + beforeAll(async () => { + [stable, insiders] = await Promise.all([fetchStableVersions(5000), fetchInsiderVersions(5000)]); + }); + + afterEach(async () => { + await fs.rm(extensionsDevelopmentPath, { recursive: true, force: true }); + }); + + const writeJSON = async (path: string, contents: object) => { + const target = join(extensionsDevelopmentPath, path); + await fs.mkdir(dirname(target), { recursive: true }); + await fs.writeFile(target, JSON.stringify(contents)); + }; + + const doFetch = (paths = ['./']) => + fetchTargetInferredVersion({ + cachePath: join(extensionsDevelopmentPath, '.cache'), + platform: 'win32-archive', + timeout: 5000, + extensionsDevelopmentPath: paths.map(p => join(extensionsDevelopmentPath, p)), + }); + + test('matches stable if no workspace', async () => { + const version = await doFetch(); + expect(version).to.equal(stable[0]); + }); + + test('matches stable by default', async () => { + await writeJSON('package.json', {}); + const version = await doFetch(); + expect(version).to.equal(stable[0]); + }); + + test('matches if stable is defined', async () => { + await writeJSON('package.json', { engines: { vscode: '^1.50.0' } }); + const version = await doFetch(); + expect(version).to.equal(stable[0]); + }); + + test('matches best', async () => { + await writeJSON('package.json', { engines: { vscode: '<=1.60.5' } }); + const version = await doFetch(); + expect(version).to.equal('1.60.2'); + }); + + test('matches multiple workspaces', async () => { + await writeJSON('a/package.json', { engines: { vscode: '<=1.60.5' } }); + await writeJSON('b/package.json', { engines: { vscode: '<=1.55.5' } }); + const version = await doFetch(['a', 'b']); + expect(version).to.equal('1.55.2'); + }); + + test('matches insiders to better stable if there is one', async () => { + await writeJSON('package.json', { engines: { vscode: '^1.60.0-insider' } }); + const version = await doFetch(); + expect(version).to.equal(stable[0]); + }); + + test('matches current insiders', async () => { + await writeJSON('package.json', { engines: { vscode: `^${insiders[0]}` } }); + const version = await doFetch(); + expect(version).to.equal(insiders[0]); + }); + + test('matches insiders to exact', async () => { + await writeJSON('package.json', { engines: { vscode: '1.60.0-insider' } }); + const version = await doFetch(); + expect(version).to.equal('1.60.0-insider'); + }); +}); diff --git a/lib/download.ts b/lib/download.ts index 0fd4bcc3..6f9af34f 100644 --- a/lib/download.ts +++ b/lib/download.ts @@ -12,15 +12,19 @@ import { promisify } from 'util'; import * as del from './del'; import { ConsoleReporter, ProgressReporter, ProgressReportStage } from './progress'; import * as request from './request'; +import * as semver from 'semver'; import { downloadDirToExecutablePath, + getInsidersVersionMetadata, getLatestInsidersMetadata, getVSCodeDownloadUrl, insidersDownloadDirMetadata, insidersDownloadDirToExecutablePath, isDefined, + isInsiderVersionIdentifier, isStableVersionIdentifier, isSubdirectory, + onceWithoutRejections, streamToBuffer, systemDefaultPlatform, } from './util'; @@ -29,6 +33,7 @@ const extensionRoot = process.cwd(); const pipelineAsync = promisify(pipeline); const vscodeStableReleasesAPI = `https://update.code.visualstudio.com/api/releases/stable`; +const vscodeInsiderReleasesAPI = `https://update.code.visualstudio.com/api/releases/insider`; const vscodeInsiderCommitsAPI = (platform: string) => `https://update.code.visualstudio.com/api/commits/insider/${platform}`; @@ -37,43 +42,117 @@ const makeDownloadDirName = (platform: string, version: string) => `vscode-${pla const DOWNLOAD_ATTEMPTS = 3; +interface IFetchStableOptions { + timeout: number; + cachePath: string; + platform: string; +} + +interface IFetchInferredOptions extends IFetchStableOptions { + extensionsDevelopmentPath?: string | string[]; +} + +export const fetchStableVersions = onceWithoutRejections((timeout: number) => + request.getJSON(vscodeStableReleasesAPI, timeout) +); +export const fetchInsiderVersions = onceWithoutRejections((timeout: number) => + request.getJSON(vscodeInsiderReleasesAPI, timeout) +); + /** * Returns the stable version to run tests against. Attempts to get the latest * version from the update sverice, but falls back to local installs if * not available (e.g. if the machine is offline). */ -async function fetchTargetStableVersion(timeout: number, cachePath: string, platform: string): Promise { - let versions: string[] = []; +async function fetchTargetStableVersion({ timeout, cachePath, platform }: IFetchStableOptions): Promise { try { - versions = await request.getJSON(vscodeStableReleasesAPI, timeout); + const versions = await fetchStableVersions(timeout); + return versions[0]; } catch (e) { - const entries = await fs.promises.readdir(cachePath).catch(() => [] as string[]); - const [fallbackTo] = entries - .map((e) => downloadDirNameFormat.exec(e)) - .filter(isDefined) - .filter((e) => e.groups!.platform === platform) - .map((e) => e.groups!.version) - .sort((a, b) => Number(b) - Number(a)); - - if (fallbackTo) { - console.warn(`Error retrieving VS Code versions, using already-installed version ${fallbackTo}`, e); - return fallbackTo; + return fallbackToLocalEntries(cachePath, platform, e as Error); + } +} + +export async function fetchTargetInferredVersion(options: IFetchInferredOptions) { + if (!options.extensionsDevelopmentPath) { + return fetchTargetStableVersion(options); + } + + // load all engines versions from all development paths. Then, get the latest + // stable version (or, latest Insiders version) that satisfies all + // `engines.vscode` constraints. + const extPaths = Array.isArray(options.extensionsDevelopmentPath) + ? options.extensionsDevelopmentPath + : [options.extensionsDevelopmentPath]; + const maybeExtVersions = await Promise.all(extPaths.map(getEngineVersionFromExtension)); + const extVersions = maybeExtVersions.filter(isDefined); + const matches = (v: string) => !extVersions.some((range) => !semver.satisfies(v, range, { includePrerelease: true })); + + try { + const stable = await fetchStableVersions(options.timeout); + const found1 = stable.find(matches); + if (found1) { + return found1; } - throw e; + const insiders = await fetchInsiderVersions(options.timeout); + const found2 = insiders.find(matches); + if (found2) { + return found2; + } + + console.warn(`No version of VS Code satisfies all extension engine constraints (${extVersions.join(', ')}). Falling back to stable.`); + + return stable[0]; // 🤷 + } catch (e) { + return fallbackToLocalEntries(options.cachePath, options.platform, e as Error); + } +} + +async function getEngineVersionFromExtension(extensionPath: string): Promise { + try { + const packageContents = await fs.promises.readFile(path.join(extensionPath, 'package.json'), 'utf8'); + const packageJson = JSON.parse(packageContents); + return packageJson?.engines?.vscode; + } catch { + return undefined; + } +} + +async function fallbackToLocalEntries(cachePath: string, platform: string, fromError: Error) { + const entries = await fs.promises.readdir(cachePath).catch(() => [] as string[]); + const [fallbackTo] = entries + .map((e) => downloadDirNameFormat.exec(e)) + .filter(isDefined) + .filter((e) => e.groups!.platform === platform) + .map((e) => e.groups!.version) + .sort((a, b) => Number(b) - Number(a)); + + if (fallbackTo) { + console.warn(`Error retrieving VS Code versions, using already-installed version ${fallbackTo}`, fromError); + return fallbackTo; } - return versions[0]; + throw fromError; } async function isValidVersion(version: string, platform: string, timeout: number) { - if (version === 'insiders') { + if (version === 'insiders' || version === 'stable') { return true; } - const stableVersionNumbers: string[] = await request.getJSON(vscodeStableReleasesAPI, timeout); - if (stableVersionNumbers.includes(version)) { - return true; + if (isStableVersionIdentifier(version)) { + const stableVersionNumbers = await fetchStableVersions(timeout); + if (stableVersionNumbers.includes(version)) { + return true; + } + } + + if (isInsiderVersionIdentifier(version)) { + const insiderVersionNumbers = await fetchInsiderVersions(timeout); + if (insiderVersionNumbers.includes(version)) { + return true; + } } const insiderCommits: string[] = await request.getJSON(vscodeInsiderCommitsAPI(platform), timeout); @@ -97,6 +176,7 @@ export interface DownloadOptions { readonly cachePath: string; readonly version: DownloadVersion; readonly platform: DownloadPlatform; + readonly extensionDevelopmentPath?: string | string[]; readonly reporter?: ProgressReporter; readonly extractSync?: boolean; readonly timeout?: number; @@ -116,6 +196,7 @@ async function downloadVSCodeArchive(options: DownloadOptions) { const timeout = options.timeout!; const downloadUrl = getVSCodeDownloadUrl(options.version, options.platform); + options.reporter?.report({ stage: ProgressReportStage.ResolvingCDNLocation, url: downloadUrl }); const res = await request.getStream(downloadUrl, timeout); if (res.statusCode !== 302) { @@ -248,7 +329,9 @@ export async function download(options: Partial = {}): Promise< timeout = 15_000, } = options; - if (version && version !== 'stable') { + if (version === 'stable') { + version = await fetchTargetStableVersion({ timeout, cachePath, platform }); + } else if (version) { /** * Only validate version against server when no local download that matches version exists */ @@ -258,20 +341,27 @@ export async function download(options: Partial = {}): Promise< } } } else { - version = await fetchTargetStableVersion(timeout, cachePath, platform); + version = await fetchTargetInferredVersion({ + timeout, + cachePath, + platform, + extensionsDevelopmentPath: options.extensionDevelopmentPath, + }); } reporter.report({ stage: ProgressReportStage.ResolvedVersion, version }); const downloadedPath = path.resolve(cachePath, makeDownloadDirName(platform, version)); if (fs.existsSync(downloadedPath)) { - if (version === 'insiders') { + if (isInsiderVersionIdentifier(version)) { reporter.report({ stage: ProgressReportStage.FetchingInsidersMetadata }); const { version: currentHash, date: currentDate } = insidersDownloadDirMetadata(downloadedPath, platform); - const { version: latestHash, timestamp: latestTimestamp } = await getLatestInsidersMetadata( - systemDefaultPlatform - ); + const { version: latestHash, timestamp: latestTimestamp } = + version === 'insiders' + ? await getLatestInsidersMetadata(systemDefaultPlatform) + : await getInsidersVersionMetadata(systemDefaultPlatform, version); + if (currentHash === latestHash) { reporter.report({ stage: ProgressReportStage.FoundMatchingInstall, downloadedPath }); return Promise.resolve(insidersDownloadDirToExecutablePath(downloadedPath, platform)); diff --git a/lib/util.ts b/lib/util.ts index 74794d6a..0b32b58d 100644 --- a/lib/util.ts +++ b/lib/util.ts @@ -37,13 +37,19 @@ switch (process.platform) { : 'linux-x64'; } +export function isInsiderVersionIdentifier(version: string): boolean { + return version === 'insider' || version.endsWith('-insider'); // insider or 1.2.3-insider version string +} + export function isStableVersionIdentifier(version: string): boolean { - return version === 'stable' || version.includes('.'); // stable or 1.2.3 version string + return version === 'stable' || /^[0-9]+\.[0-9]+\.[0-9]$/.test(version); // stable or 1.2.3 version string } export function getVSCodeDownloadUrl(version: string, platform = systemDefaultPlatform) { if (version === 'insiders') { return `https://update.code.visualstudio.com/latest/${platform}/insider`; + } else if (isInsiderVersionIdentifier(version)) { + return `https://update.code.visualstudio.com/${version}/${platform}/insider`; } else if (isStableVersionIdentifier(version)) { return `https://update.code.visualstudio.com/${version}/${platform}/stable`; } else { // insiders commit hash @@ -124,6 +130,11 @@ export interface IUpdateMetadata { supportsFastUpdate: boolean; } +export async function getInsidersVersionMetadata(platform: string, version: string) { + const remoteUrl = `https://update.code.visualstudio.com/api/versions/${version}/${platform}/insider`; + return await request.getJSON(remoteUrl, 30_000); +} + export async function getLatestInsidersMetadata(platform: string) { const remoteUrl = `https://update.code.visualstudio.com/api/update/${platform}/insider/latest`; return await request.getJSON(remoteUrl, 30_000); @@ -196,3 +207,21 @@ export function isSubdirectory(parent: string, child: string) { const relative = path.relative(parent, child); return !relative.startsWith('..') && !path.isAbsolute(relative); } + +/** + * Wraps a function so that it's called once, and never again, memoizing + * the result unless it rejects. + */ +export function onceWithoutRejections(fn: (...args: Args) => Promise) { + let value: Promise | undefined; + return (...args: Args) => { + if (!value) { + value = fn(...args).catch(err => { + value = undefined; + throw err; + }); + } + + return value + } +} diff --git a/package.json b/package.json index 6cc95dea..259a1cd6 100644 --- a/package.json +++ b/package.json @@ -15,11 +15,13 @@ "http-proxy-agent": "^4.0.1", "https-proxy-agent": "^5.0.0", "jszip": "^3.10.1", - "rimraf": "^3.0.2" + "rimraf": "^3.0.2", + "semver": "^7.3.8" }, "devDependencies": { "@types/node": "^18", "@types/rimraf": "^3.0.0", + "@types/semver": "^7.3.13", "@typescript-eslint/eslint-plugin": "^4.13.0", "@typescript-eslint/parser": "^4.13.0", "eslint": "^7.17.0", diff --git a/yarn.lock b/yarn.lock index 49312dad..66061caa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -119,6 +119,11 @@ "@types/glob" "*" "@types/node" "*" +"@types/semver@^7.3.13": + version "7.3.13" + resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.3.13.tgz#da4bfd73f49bd541d28920ab0e2bf0ee80f71c91" + integrity sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw== + "@typescript-eslint/eslint-plugin@^4.13.0": version "4.13.0" resolved "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-4.13.0.tgz" @@ -1187,6 +1192,13 @@ semver@^7.2.1, semver@^7.3.2: dependencies: lru-cache "^6.0.0" +semver@^7.3.8: + version "7.3.8" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.8.tgz#07a78feafb3f7b32347d725e33de7e2a2df67798" + integrity sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A== + dependencies: + lru-cache "^6.0.0" + setimmediate@^1.0.5: version "1.0.5" resolved "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz"