From 78d5ff19d536bd9aefc934c31a7e94e293f1fb33 Mon Sep 17 00:00:00 2001 From: vince-fugnitto Date: Tue, 24 Aug 2021 12:29:09 -0400 Subject: [PATCH 1/3] cli: introduce exclude id list The commit introduces additional functionality when downloading extension-packs so end-users have the ability to declare extensions (by id) they wish to exclude when resolving packs. Previously no mechanism was available to exclude extensions pulled by packs besides using post-processing such as deleting plugins from the plugin directory before building. Extensions referenced by id in `theiaPluginsExcludeIds` will be excluded when we resolve extension-packs. Once `ids` of an extension-pack are resolved, the script will handle removing them from the download folder so the framework does not attempt to re-resolve them at runtime. Signed-off-by: vince-fugnitto --- dev-packages/cli/README.md | 14 +- dev-packages/cli/src/download-plugins.ts | 175 ++++++++++++++--------- 2 files changed, 123 insertions(+), 66 deletions(-) diff --git a/dev-packages/cli/README.md b/dev-packages/cli/README.md index c12f74a347243..dc8a3790d7a07 100644 --- a/dev-packages/cli/README.md +++ b/dev-packages/cli/README.md @@ -313,11 +313,21 @@ The property `theiaPlugins` describes the list of plugins to download, for examp ```json "theiaPlugins": { - "vscode-builtin-bat": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/bat-1.39.1-prel.vsix", - "vscode-builtin-clojure": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/clojure-1.39.1-prel.vsix", + "vscode-builtin-extension-pack": "https://open-vsx.org/api/eclipse-theia/builtin-extension-pack/1.50.0/file/eclipse-theia.builtin-extension-pack-1.50.0.vsix", + "vscode-editorconfig": "https://open-vsx.org/api/EditorConfig/EditorConfig/0.14.4/file/EditorConfig.EditorConfig-0.14.4.vsix", + "vscode-eslint": "https://open-vsx.org/api/dbaeumer/vscode-eslint/2.1.1/file/dbaeumer.vscode-eslint-2.1.1.vsix", } ``` +The property `theiaPluginsExcludeIds` can be used to declare the list of plugin `ids` to exclude when using extension-packs. +The `ids` referenced by the property will not be downloaded when resolving extension-packs, and can be used to omit extensions which are problematic or unwanted. The format of the property is as follows: + +```json +"theiaPluginsExcludeIds": [ + "vscode.cpp" +] +``` + ## Autogenerated Application This package can auto-generate application code for both the backend and frontend, as well as webpack configuration files. diff --git a/dev-packages/cli/src/download-plugins.ts b/dev-packages/cli/src/download-plugins.ts index 8f873d5c7208e..5051d50aca460 100644 --- a/dev-packages/cli/src/download-plugins.ts +++ b/dev-packages/cli/src/download-plugins.ts @@ -19,7 +19,7 @@ import fetch, { Response, RequestInit } from 'node-fetch'; import { HttpsProxyAgent } from 'https-proxy-agent'; import { getProxyForUrl } from 'proxy-from-env'; -import { promises as fs, createWriteStream } from 'fs'; +import { promises as fs, createWriteStream, existsSync } from 'fs'; import * as mkdirp from 'mkdirp'; import * as path from 'path'; import * as process from 'process'; @@ -27,7 +27,7 @@ import * as stream from 'stream'; import * as decompress from 'decompress'; import * as temp from 'temp'; -import { green, red } from 'colors/safe'; +import { green, red, yellow } from 'colors/safe'; import { promisify } from 'util'; import { OVSXClient } from '@theia/ovsx-client/lib/ovsx-client'; @@ -36,6 +36,8 @@ const pipelineAsPromised = promisify(stream.pipeline); temp.track(); +export const extensionPackCacheName = '.packs'; + /** * Available options when downloading. */ @@ -84,49 +86,76 @@ export default async function downloadPlugins(options: DownloadPluginsOptions = // Resolve the directory for which to download the plugins. const pluginsDir = pck.theiaPluginsDir || 'plugins'; + // Excluded extension ids. + const excludedIds = pck.theiaPluginsExcludeIds || []; + await mkdirpAsPromised(pluginsDir); if (!pck.theiaPlugins) { console.log(red('error: missing mandatory \'theiaPlugins\' property.')); return; } + + let extensionPacks; try { - await Promise.all(Object.keys(pck.theiaPlugins).map( - plugin => downloadPluginAsync(failures, plugin, pck.theiaPlugins[plugin], pluginsDir, packed) + // Retrieve the cached extension-packs in order to not re-download them. + const extensionPackCachePath = path.join(process.cwd(), pluginsDir, extensionPackCacheName); + let cachedExtensionPacks: string[] = []; + if (existsSync(extensionPackCachePath)) { + cachedExtensionPacks = await fs.readdir(extensionPackCachePath); + } + + /** Download the raw plugins defined by the `theiaPlugins` property. */ + await Promise.all(Object.entries(pck.theiaPlugins).map( + ([plugin, url]) => downloadPluginAsync(failures, plugin, url as string, pluginsDir, packed, cachedExtensionPacks) + )); + + /** + * Given that the plugins are downloaded on disk, resolve the extension-packs by downloading the `ids` they reference. + */ + extensionPacks = await getExtensionPacks(pluginsDir, excludedIds); + if (extensionPacks.size > 0) { + console.log('--- resolving extension-packs ---'); + const client = new OVSXClient({ apiVersion, apiUrl }); + // De-duplicate the ids. + const ids = new Set(); + for (const idSet of extensionPacks.values()) { + for (const id of idSet) { + ids.add(id); + } + } + await Promise.all(Array.from(ids, async id => { + const extension = await client.getLatestCompatibleExtensionVersion(id); + const downloadUrl = extension?.files.download; + if (downloadUrl) { + await downloadPluginAsync(failures, id, downloadUrl, pluginsDir, packed, cachedExtensionPacks); + } + })); + } } finally { temp.cleanupSync(); + if (extensionPacks) { + cleanupExtensionPacks(pluginsDir, Array.from(extensionPacks.keys())); + } } + failures.forEach(e => { console.error(e); }); if (!ignoreErrors && failures.length > 0) { throw new Error('Errors downloading some plugins. To make these errors non fatal, re-run with --ignore-errors'); } - - // Resolve extension pack plugins. - const ids = await getAllExtensionPackIds(pluginsDir); - if (ids.length) { - const client = new OVSXClient({ apiVersion, apiUrl }); - ids.forEach(async id => { - const extension = await client.getLatestCompatibleExtensionVersion(id); - const downloadUrl = extension?.files.download; - if (downloadUrl) { - await downloadPluginAsync(failures, id, downloadUrl, pluginsDir, packed); - } - }); - } - } /** * Downloads a plugin, will make multiple attempts before actually failing. - * - * @param failures reference to an array storing all failures - * @param plugin plugin short name - * @param pluginUrl url to download the plugin at - * @param pluginsDir where to download the plugin in - * @param packed whether to decompress or not + * @param failures reference to an array storing all failures. + * @param plugin plugin short name. + * @param pluginUrl url to download the plugin at. + * @param pluginsDir where to download the plugin in. + * @param packed whether to decompress or not. + * @param cachedExtensionPacks the list of cached extension packs already downloaded. */ -async function downloadPluginAsync(failures: string[], plugin: string, pluginUrl: string, pluginsDir: string, packed: boolean): Promise { +async function downloadPluginAsync(failures: string[], plugin: string, pluginUrl: string, pluginsDir: string, packed: boolean, cachedExtensionPacks: string[]): Promise { if (!plugin) { return; } @@ -141,7 +170,7 @@ async function downloadPluginAsync(failures: string[], plugin: string, pluginUrl } const targetPath = path.join(process.cwd(), pluginsDir, `${plugin}${packed === true ? fileExt : ''}`); // Skip plugins which have previously been downloaded. - if (await isDownloaded(targetPath)) { + if (cachedExtensionPacks.includes(plugin) || await isDownloaded(targetPath)) { console.warn('- ' + plugin + ': already downloaded - skipping'); return; } @@ -219,58 +248,76 @@ export function xfetch(url: string, options?: RequestInit): Promise { } /** - * Get the list of all available ids referenced by extension packs. + * Walk the plugin directory and collect available extension paths. * @param pluginDir the plugin directory. - * @returns the list of all referenced extension pack ids. - */ -async function getAllExtensionPackIds(pluginDir: string): Promise { - const extensions = await getPackageFiles(pluginDir); - const extensionIds: string[] = []; - const ids = await Promise.all(extensions.map(ext => getExtensionPackIds(ext))); - ids.forEach(id => { - extensionIds.push(...id); - }); - return extensionIds; -} - -/** - * Walk the plugin directory collecting available extension paths. - * @param dirPath the plugin directory - * @returns the list of extension paths. + * @returns the list of all available extension paths. */ -async function getPackageFiles(dirPath: string): Promise { - let fileList: string[] = []; - const files = await fs.readdir(dirPath); +async function getPackageJsonPaths(pluginDir: string): Promise { + let packageJsonPathList: string[] = []; + const files = await fs.readdir(pluginDir); // Recursively fetch the list of extension `package.json` files. for (const file of files) { - const filePath = path.join(dirPath, file); + const filePath = path.join(pluginDir, file); if ((await fs.stat(filePath)).isDirectory()) { - fileList = [...fileList, ...(await getPackageFiles(filePath))]; + // Exclude the `.packs` folder used to store extension-packs after being resolved. + if (filePath.includes(extensionPackCacheName)) { + continue; + } + packageJsonPathList = [...packageJsonPathList, ...(await getPackageJsonPaths(filePath))]; } else if ((path.basename(filePath) === 'package.json' && !path.dirname(filePath).includes('node_modules'))) { - fileList.push(filePath); + packageJsonPathList.push(filePath); } } + return packageJsonPathList; +} - return fileList; +/** + * Get the mapping of extension-pack paths and their included plugin ids. + * - If an extension-pack references an explicitly excluded `id` the `id` will be omitted. + * @param pluginDir the plugin directory. + * @param excludedIds the list of plugin ids to exclude. + * @returns the mapping of extension-pack paths and their included plugin ids. + */ +async function getExtensionPacks(pluginDir: string, excludedIds: string[]): Promise>> { + const extensionPackPaths: Map> = new Map(); + const packageJsonPaths = await getPackageJsonPaths(pluginDir); + for (const packageJsonPath of packageJsonPaths) { + const content = await fs.readFile(packageJsonPath, 'utf-8'); + const json = JSON.parse(content); + const extensionPack = json.extensionPack as string[]; + if (extensionPack) { + const ids = new Set(); + for (const id of extensionPack) { + if (excludedIds.includes(id)) { + console.log(yellow(`'${id}' referenced by the extension-pack is explicitly excluded`)); + continue; + } + ids.add(id); + } + extensionPackPaths.set(packageJsonPath, ids); + } + } + return extensionPackPaths; } /** - * Get the list of extension ids referenced by the extension pack. - * @param extPath the individual extension path. - * @returns the list of individual extension ids. + * Removes the extension-packs downloaded under the plugin directory. + * - Since the `ids` referenced in the extension-packs are resolved, we remove the extension-packs so the framework does not attempt to resolve the packs again at runtime. + * @param extensionPacksPaths the list of extension-pack paths. */ -async function getExtensionPackIds(extPath: string): Promise { - const ids = new Set(); - const content = await fs.readFile(extPath, 'utf-8'); - const json = JSON.parse(content); - - // The `extensionPack` object. - const extensionPack = json.extensionPack as string[]; - for (const ext in extensionPack) { - if (ext !== undefined) { - ids.add(extensionPack[ext]); +async function cleanupExtensionPacks(pluginsDir: string, extensionPacksPaths: string[]): Promise { + const packsFolderPath = path.join(path.resolve(process.cwd(), pluginsDir), extensionPackCacheName); + try { + await fs.mkdir(packsFolderPath, { recursive: true }); + for (const pack of extensionPacksPaths) { + const oldPath = path.join(pack, '../../'); // navigate back up from the `package.json`. + const newPath = path.join(packsFolderPath, path.basename(oldPath)); + if (!existsSync(newPath)) { + await fs.rename(oldPath, newPath); + } } + } catch (e) { + console.log(e); } - return Array.from(ids); } From 8569f70a3ff7910ade12ad137f57c32ddd200cfa Mon Sep 17 00:00:00 2001 From: Paul Marechal Date: Wed, 25 Aug 2021 20:03:41 -0400 Subject: [PATCH 2/3] misc refactoring --- dev-packages/cli/src/download-plugins.ts | 185 +++++++++++++---------- 1 file changed, 104 insertions(+), 81 deletions(-) diff --git a/dev-packages/cli/src/download-plugins.ts b/dev-packages/cli/src/download-plugins.ts index 5051d50aca460..ec0b11c7ad1b2 100644 --- a/dev-packages/cli/src/download-plugins.ts +++ b/dev-packages/cli/src/download-plugins.ts @@ -16,13 +16,18 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ +declare global { + interface Array { + flat(depth?: number): any + } +} + import fetch, { Response, RequestInit } from 'node-fetch'; import { HttpsProxyAgent } from 'https-proxy-agent'; import { getProxyForUrl } from 'proxy-from-env'; import { promises as fs, createWriteStream, existsSync } from 'fs'; import * as mkdirp from 'mkdirp'; import * as path from 'path'; -import * as process from 'process'; import * as stream from 'stream'; import * as decompress from 'decompress'; import * as temp from 'temp'; @@ -67,10 +72,6 @@ export interface DownloadPluginsOptions { } export default async function downloadPlugins(options: DownloadPluginsOptions = {}): Promise { - - // Collect the list of failures to be appended at the end of the script. - const failures: string[] = []; - const { packed = false, ignoreErrors = false, @@ -78,16 +79,17 @@ export default async function downloadPlugins(options: DownloadPluginsOptions = apiUrl = 'https://open-vsx.org/api' } = options; - console.warn('--- downloading plugins ---'); + // Collect the list of failures to be appended at the end of the script. + const failures: string[] = []; // Resolve the `package.json` at the current working directory. - const pck = require(path.resolve(process.cwd(), 'package.json')); + const pck = JSON.parse(await fs.readFile(path.resolve('package.json'), 'utf8')); // Resolve the directory for which to download the plugins. const pluginsDir = pck.theiaPluginsDir || 'plugins'; // Excluded extension ids. - const excludedIds = pck.theiaPluginsExcludeIds || []; + const excludedIds = new Set(pck.theiaPluginsExcludeIds || []); await mkdirpAsPromised(pluginsDir); @@ -95,52 +97,49 @@ export default async function downloadPlugins(options: DownloadPluginsOptions = console.log(red('error: missing mandatory \'theiaPlugins\' property.')); return; } - - let extensionPacks; try { // Retrieve the cached extension-packs in order to not re-download them. - const extensionPackCachePath = path.join(process.cwd(), pluginsDir, extensionPackCacheName); - let cachedExtensionPacks: string[] = []; - if (existsSync(extensionPackCachePath)) { - cachedExtensionPacks = await fs.readdir(extensionPackCachePath); + const extensionPackCachePath = path.resolve(pluginsDir, extensionPackCacheName); + const cachedExtensionPacks = new Set( + existsSync(extensionPackCachePath) + ? await fs.readdir(extensionPackCachePath) + : [] + ); + console.warn('--- downloading plugins ---'); + // Download the raw plugins defined by the `theiaPlugins` property. + // This will include both "normal" plugins as well as "extension packs". + const downloads = []; + for (const [plugin, pluginUrl] of Object.entries(pck.theiaPlugins)) { + // Skip extension packs that were moved to `.packs`: + if (cachedExtensionPacks.has(plugin) || typeof pluginUrl !== 'string') { + continue; + } + downloads.push(downloadPluginAsync(failures, plugin, pluginUrl, pluginsDir, packed)); } - - /** Download the raw plugins defined by the `theiaPlugins` property. */ - await Promise.all(Object.entries(pck.theiaPlugins).map( - ([plugin, url]) => downloadPluginAsync(failures, plugin, url as string, pluginsDir, packed, cachedExtensionPacks) - - )); - - /** - * Given that the plugins are downloaded on disk, resolve the extension-packs by downloading the `ids` they reference. - */ - extensionPacks = await getExtensionPacks(pluginsDir, excludedIds); + await Promise.all(downloads); + console.warn('--- collecting extension-packs ---'); + const extensionPacks = await collectExtensionPacks(pluginsDir, excludedIds); + console.warn(`--- found ${extensionPacks.size} extension-packs ---`); if (extensionPacks.size > 0) { - console.log('--- resolving extension-packs ---'); + await cacheExtensionPacks(pluginsDir, extensionPacks); + console.warn('--- resolving extension-packs ---'); const client = new OVSXClient({ apiVersion, apiUrl }); - // De-duplicate the ids. - const ids = new Set(); - for (const idSet of extensionPacks.values()) { - for (const id of idSet) { - ids.add(id); - } - } + // De-duplicate extension ids to only download each once: + const ids = new Set(Array.from(extensionPacks.values()).flat()); await Promise.all(Array.from(ids, async id => { const extension = await client.getLatestCompatibleExtensionVersion(id); const downloadUrl = extension?.files.download; if (downloadUrl) { - await downloadPluginAsync(failures, id, downloadUrl, pluginsDir, packed, cachedExtensionPacks); + await downloadPluginAsync(failures, id, downloadUrl, pluginsDir, packed); } })); } } finally { temp.cleanupSync(); - if (extensionPacks) { - cleanupExtensionPacks(pluginsDir, Array.from(extensionPacks.keys())); - } } - - failures.forEach(e => { console.error(e); }); + for (const failure of failures) { + console.error(failure); + } if (!ignoreErrors && failures.length > 0) { throw new Error('Errors downloading some plugins. To make these errors non fatal, re-run with --ignore-errors'); } @@ -151,11 +150,11 @@ export default async function downloadPlugins(options: DownloadPluginsOptions = * @param failures reference to an array storing all failures. * @param plugin plugin short name. * @param pluginUrl url to download the plugin at. - * @param pluginsDir where to download the plugin in. + * @param target where to download the plugin in. * @param packed whether to decompress or not. * @param cachedExtensionPacks the list of cached extension packs already downloaded. */ -async function downloadPluginAsync(failures: string[], plugin: string, pluginUrl: string, pluginsDir: string, packed: boolean, cachedExtensionPacks: string[]): Promise { +async function downloadPluginAsync(failures: string[], plugin: string, pluginUrl: string, pluginsDir: string, packed: boolean): Promise { if (!plugin) { return; } @@ -168,9 +167,10 @@ async function downloadPluginAsync(failures: string[], plugin: string, pluginUrl failures.push(red(`error: '${plugin}' has an unsupported file type: '${pluginUrl}'`)); return; } - const targetPath = path.join(process.cwd(), pluginsDir, `${plugin}${packed === true ? fileExt : ''}`); + const targetPath = path.resolve(pluginsDir, `${plugin}${packed === true ? fileExt : ''}`); + // Skip plugins which have previously been downloaded. - if (cachedExtensionPacks.includes(plugin) || await isDownloaded(targetPath)) { + if (await isDownloaded(targetPath)) { console.warn('- ' + plugin + ': already downloaded - skipping'); return; } @@ -252,20 +252,16 @@ export function xfetch(url: string, options?: RequestInit): Promise { * @param pluginDir the plugin directory. * @returns the list of all available extension paths. */ -async function getPackageJsonPaths(pluginDir: string): Promise { - let packageJsonPathList: string[] = []; +async function collectPackageJsonPaths(pluginDir: string): Promise { + const packageJsonPathList: string[] = []; const files = await fs.readdir(pluginDir); - // Recursively fetch the list of extension `package.json` files. for (const file of files) { const filePath = path.join(pluginDir, file); - if ((await fs.stat(filePath)).isDirectory()) { - // Exclude the `.packs` folder used to store extension-packs after being resolved. - if (filePath.includes(extensionPackCacheName)) { - continue; - } - packageJsonPathList = [...packageJsonPathList, ...(await getPackageJsonPaths(filePath))]; - } else if ((path.basename(filePath) === 'package.json' && !path.dirname(filePath).includes('node_modules'))) { + // Exclude the `.packs` folder used to store extension-packs after being resolved. + if (!filePath.startsWith(extensionPackCacheName) && (await fs.stat(filePath)).isDirectory()) { + packageJsonPathList.push(...await collectPackageJsonPaths(filePath)); + } else if (path.basename(filePath) === 'package.json' && !path.dirname(filePath).includes('node_modules')) { packageJsonPathList.push(filePath); } } @@ -279,45 +275,72 @@ async function getPackageJsonPaths(pluginDir: string): Promise { * @param excludedIds the list of plugin ids to exclude. * @returns the mapping of extension-pack paths and their included plugin ids. */ -async function getExtensionPacks(pluginDir: string, excludedIds: string[]): Promise>> { - const extensionPackPaths: Map> = new Map(); - const packageJsonPaths = await getPackageJsonPaths(pluginDir); - for (const packageJsonPath of packageJsonPaths) { - const content = await fs.readFile(packageJsonPath, 'utf-8'); - const json = JSON.parse(content); - const extensionPack = json.extensionPack as string[]; - if (extensionPack) { - const ids = new Set(); - for (const id of extensionPack) { - if (excludedIds.includes(id)) { - console.log(yellow(`'${id}' referenced by the extension-pack is explicitly excluded`)); - continue; +async function collectExtensionPacks(pluginDir: string, excludedIds: Set): Promise> { + const extensionPackPaths = new Map(); + const packageJsonPaths = await collectPackageJsonPaths(pluginDir); + await Promise.all(packageJsonPaths.map(async packageJsonPath => { + const json = JSON.parse(await fs.readFile(packageJsonPath, 'utf8')); + const extensionPack: unknown = json.extensionPack; + if (extensionPack && Array.isArray(extensionPack)) { + extensionPackPaths.set(packageJsonPath, extensionPack.filter(id => { + if (excludedIds.has(id)) { + console.log(yellow(`'${id}' referenced by '${json.name}' (ext pack) is excluded because of 'theiaPluginsExcludeIds'`)); + return false; // remove } - ids.add(id); - } - extensionPackPaths.set(packageJsonPath, ids); + return true; // keep + })); } - } + })); return extensionPackPaths; } /** - * Removes the extension-packs downloaded under the plugin directory. - * - Since the `ids` referenced in the extension-packs are resolved, we remove the extension-packs so the framework does not attempt to resolve the packs again at runtime. + * Move extension-packs downloaded from `pluginsDir/x` to `pluginsDir/.packs/x`. + * + * The issue we are trying to solve is the following: + * We may skip some extensions declared in a pack due to the `theiaPluginsExcludeIds` list. But once we start + * a Theia application the plugin system will detect the pack and install the missing extensions. + * + * By moving the packs to a subdirectory it should make it invisible to the plugin system, only leaving + * the plugins that were installed under `pluginsDir` directly. + * * @param extensionPacksPaths the list of extension-pack paths. */ -async function cleanupExtensionPacks(pluginsDir: string, extensionPacksPaths: string[]): Promise { - const packsFolderPath = path.join(path.resolve(process.cwd(), pluginsDir), extensionPackCacheName); - try { - await fs.mkdir(packsFolderPath, { recursive: true }); - for (const pack of extensionPacksPaths) { - const oldPath = path.join(pack, '../../'); // navigate back up from the `package.json`. - const newPath = path.join(packsFolderPath, path.basename(oldPath)); +async function cacheExtensionPacks(pluginsDir: string, extensionPacks: Map): Promise { + const packsFolderPath = path.resolve(pluginsDir, extensionPackCacheName); + await fs.mkdir(packsFolderPath, { recursive: true }); + await Promise.all(Array.from(extensionPacks.entries(), async ([extensionPackPath, value]) => { + extensionPackPath = path.resolve(extensionPackPath); + if (extensionPackPath.startsWith(packsFolderPath)) { + return; // skip + } + try { + const oldPath = getExtensionRoot(pluginsDir, extensionPackPath); + const newPath = path.resolve(packsFolderPath, path.basename(oldPath)); if (!existsSync(newPath)) { await fs.rename(oldPath, newPath); + // Update the map to reflect the changed paths: + extensionPacks.delete(extensionPackPath); + extensionPacks.set(newPath, value); } + } catch (error) { + console.error(error); } - } catch (e) { - console.log(e); + })); +} + +/** + * Walk back to the root of an extension starting from its `package.json`. e.g. + * + * ```ts + * getExtensionRoot('/a/b/c', '/a/b/c/EXT/d/e/f/package.json') === '/a/b/c/EXT' + * ``` + */ +function getExtensionRoot(root: string, packageJsonPath: string): string { + root = path.resolve(root); + packageJsonPath = path.resolve(packageJsonPath); + if (!packageJsonPath.startsWith(root)) { + throw new Error(`unexpected paths:\n root: ${root}\n package.json: ${packageJsonPath}`); } + return packageJsonPath.substr(0, packageJsonPath.indexOf(path.sep, root.length + 1)); } From 01e536979669b18ca5fdb01d58f503300e928d63 Mon Sep 17 00:00:00 2001 From: Paul Marechal Date: Thu, 26 Aug 2021 11:28:04 -0400 Subject: [PATCH 3/3] cleanup --- dev-packages/cli/package.json | 1 - dev-packages/cli/src/download-plugins.ts | 29 +++++++++++------------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/dev-packages/cli/package.json b/dev-packages/cli/package.json index f3724fe9dc7a1..c743fb78c093d 100644 --- a/dev-packages/cli/package.json +++ b/dev-packages/cli/package.json @@ -43,7 +43,6 @@ "colors": "^1.4.0", "decompress": "^4.2.1", "https-proxy-agent": "^5.0.0", - "mkdirp": "^0.5.0", "mocha": "^7.0.0", "node-fetch": "^2.6.0", "proxy-from-env": "^1.1.0", diff --git a/dev-packages/cli/src/download-plugins.ts b/dev-packages/cli/src/download-plugins.ts index ec0b11c7ad1b2..50c9519490d02 100644 --- a/dev-packages/cli/src/download-plugins.ts +++ b/dev-packages/cli/src/download-plugins.ts @@ -18,25 +18,23 @@ declare global { interface Array { + // Supported since Node >=11.0 flat(depth?: number): any } } -import fetch, { Response, RequestInit } from 'node-fetch'; +import { OVSXClient } from '@theia/ovsx-client/lib/ovsx-client'; +import { green, red, yellow } from 'colors/safe'; +import * as decompress from 'decompress'; +import { createWriteStream, existsSync, promises as fs } from 'fs'; import { HttpsProxyAgent } from 'https-proxy-agent'; -import { getProxyForUrl } from 'proxy-from-env'; -import { promises as fs, createWriteStream, existsSync } from 'fs'; -import * as mkdirp from 'mkdirp'; +import fetch, { RequestInit, Response } from 'node-fetch'; import * as path from 'path'; +import { getProxyForUrl } from 'proxy-from-env'; import * as stream from 'stream'; -import * as decompress from 'decompress'; import * as temp from 'temp'; - -import { green, red, yellow } from 'colors/safe'; - import { promisify } from 'util'; -import { OVSXClient } from '@theia/ovsx-client/lib/ovsx-client'; -const mkdirpAsPromised = promisify(mkdirp); + const pipelineAsPromised = promisify(stream.pipeline); temp.track(); @@ -91,7 +89,7 @@ export default async function downloadPlugins(options: DownloadPluginsOptions = // Excluded extension ids. const excludedIds = new Set(pck.theiaPluginsExcludeIds || []); - await mkdirpAsPromised(pluginsDir); + await fs.mkdir(pluginsDir, { recursive: true }); if (!pck.theiaPlugins) { console.log(red('error: missing mandatory \'theiaPlugins\' property.')); @@ -119,8 +117,9 @@ export default async function downloadPlugins(options: DownloadPluginsOptions = await Promise.all(downloads); console.warn('--- collecting extension-packs ---'); const extensionPacks = await collectExtensionPacks(pluginsDir, excludedIds); - console.warn(`--- found ${extensionPacks.size} extension-packs ---`); if (extensionPacks.size > 0) { + console.warn(`--- found ${extensionPacks.size} extension-packs ---`); + // Move extension-packs to `.packs` await cacheExtensionPacks(pluginsDir, extensionPacks); console.warn('--- resolving extension-packs ---'); const client = new OVSXClient({ apiVersion, apiUrl }); @@ -216,7 +215,7 @@ async function downloadPluginAsync(failures: string[], plugin: string, pluginUrl const file = createWriteStream(targetPath); await pipelineAsPromised(response.body, file); } else { - await mkdirpAsPromised(targetPath); + await fs.mkdir(targetPath, { recursive: true }); const tempFile = temp.createWriteStream('theia-plugin-download'); await pipelineAsPromised(response.body, tempFile); await decompress(tempFile.path, targetPath); @@ -311,6 +310,7 @@ async function cacheExtensionPacks(pluginsDir: string, extensionPacks: Map { extensionPackPath = path.resolve(extensionPackPath); + // Skip entries found in `.packs` if (extensionPackPath.startsWith(packsFolderPath)) { return; // skip } @@ -319,9 +319,6 @@ async function cacheExtensionPacks(pluginsDir: string, extensionPacks: Map