diff --git a/extension/package.json b/extension/package.json index 774b13f594..095e7eee31 100644 --- a/extension/package.json +++ b/extension/package.json @@ -1628,6 +1628,7 @@ "test-vscode": "node ./dist/test/runTest.js", "test-e2e": "wdio run ./src/test/e2e/wdio.conf.ts", "test": "jest --collect-coverage", + "setup-venv": "node ./scripts/virtualenv-install.js", "cover-vscode-run": "node ./scripts/coverIntegrationTests.js", "vscode:prepublish": "" }, @@ -1635,7 +1636,7 @@ "@hediet/std": "0.6.0", "@vscode/extension-telemetry": "0.7.7", "appdirs": "1.1.0", - "execa": "5.1.1", + "execa": "7.1.1", "fs-extra": "11.1.1", "js-yaml": "4.1.0", "json5": "2.2.3", @@ -1646,6 +1647,7 @@ "lodash.merge": "4.6.2", "lodash.omit": "4.5.0", "node-fetch": "2.6.9", + "process-exists": "5.0.0", "tree-kill": "1.2.2", "uuid": "9.0.0", "vega-util": "1.17.2", @@ -1690,7 +1692,6 @@ "lint-staged": "13.2.2", "mocha": "10.2.0", "mock-require": "3.0.3", - "process-exists": "4.1.0", "shx": "0.3.4", "sinon": "15.0.4", "sinon-chai": "3.7.0", diff --git a/extension/scripts/coverIntegrationTests.js b/extension/scripts/coverIntegrationTests.js index 82c41d7d52..9b76126be2 100644 --- a/extension/scripts/coverIntegrationTests.js +++ b/extension/scripts/coverIntegrationTests.js @@ -1,6 +1,10 @@ const { resolve, join } = require('path') const { writeFileSync } = require('fs-extra') -const execa = require('execa') + +const getExeca = async () => { + const { execa } = await import('execa') + return execa +} let activationEvents = [] let failed @@ -18,26 +22,28 @@ activationEvents = packageJson.activationEvents packageJson.activationEvents = ['onStartupFinished'] writeFileSync(packageJsonPath, JSON.stringify(packageJson)) -const tests = execa('node', [join(cwd, 'dist', 'test', 'runTest.js')], { - cwd -}) - -pipe(tests) -tests - .then(() => {}) - .catch(() => { - failed = true +getExeca().then(execa => { + const tests = execa('node', [join(cwd, 'dist', 'test', 'runTest.js')], { + cwd }) - .finally(() => { - packageJson.activationEvents = activationEvents - writeFileSync(packageJsonPath, JSON.stringify(packageJson)) - - const prettier = execa('prettier', ['--write', 'package.json'], { cwd }) - pipe(prettier) - prettier.then(() => { - if (failed) { - process.exit(1) - } + pipe(tests) + tests + .then(() => {}) + .catch(() => { + failed = true }) - }) + .finally(() => { + packageJson.activationEvents = activationEvents + + writeFileSync(packageJsonPath, JSON.stringify(packageJson)) + + const prettier = execa('prettier', ['--write', 'package.json'], { cwd }) + pipe(prettier) + prettier.then(() => { + if (failed) { + process.exit(1) + } + }) + }) +}) diff --git a/extension/scripts/virtualenv-install.js b/extension/scripts/virtualenv-install.js new file mode 100644 index 0000000000..a3399329e0 --- /dev/null +++ b/extension/scripts/virtualenv-install.js @@ -0,0 +1,13 @@ +const { join, resolve } = require('path') +require('../dist/vscode/mockModule') + +const importModuleAfterMockingVsCode = async () => { + const { setupTestVenv } = require('../dist/python') + return setupTestVenv +} + +importModuleAfterMockingVsCode().then(setupTestVenv => { + const cwd = resolve(__dirname, '..', '..', 'demo') + + setupTestVenv(cwd, '.env', '-r', join('.', 'requirements.txt')) +}) diff --git a/extension/src/extension.ts b/extension/src/extension.ts index 66bc19f6c8..f7fa6c56a4 100644 --- a/extension/src/extension.ts +++ b/extension/src/extension.ts @@ -42,7 +42,7 @@ import { GitExecutor } from './cli/git/executor' import { GitReader } from './cli/git/reader' import { Setup } from './setup' import { definedAndNonEmpty } from './util/array' -import { stopProcesses } from './process/execution' +import { esmModulesImported, stopProcesses } from './process/execution' import { Flag } from './cli/dvc/constants' import { LanguageClient } from './languageClient' import { collectRunningExperimentPids } from './experiments/processExecution/collect' @@ -304,8 +304,10 @@ class Extension extends Disposable { let extension: undefined | Extension export function activate(context: ExtensionContext): void { - extension = new Extension(context) - context.subscriptions.push(extension) + void esmModulesImported.then(() => { + extension = new Extension(context) + context.subscriptions.push(extension) + }) } export function deactivate(): void { diff --git a/extension/src/process/execution.test.ts b/extension/src/process/execution.test.ts deleted file mode 100644 index f8fb1e471d..0000000000 --- a/extension/src/process/execution.test.ts +++ /dev/null @@ -1,32 +0,0 @@ -import process from 'process' -import { executeProcess, processExists } from './execution' - -describe('executeProcess', () => { - it('should be able to run a process', async () => { - const output = await executeProcess({ - args: ['some', 'text'], - cwd: __dirname, - executable: 'echo' - }) - expect(output).toMatch(/some.*text/) - }) - - it('should return the stderr if the process throws with stderr', async () => { - await expect( - executeProcess({ - args: ['me', 'outside'], - cwd: __dirname, - executable: 'find' - }) - ).rejects.toBeTruthy() - }) -}) - -describe('processExists', () => { - it('should return true if the process exists', async () => { - expect(await processExists(process.pid)).toBe(true) - }) - it('should return false if it does not', async () => { - expect(await processExists(-123.321)).toBe(false) - }) -}) diff --git a/extension/src/process/execution.ts b/extension/src/process/execution.ts index 6817d5ead7..c53064f9ab 100644 --- a/extension/src/process/execution.ts +++ b/extension/src/process/execution.ts @@ -2,11 +2,36 @@ import { ChildProcess } from 'child_process' import { Readable } from 'stream' import { Event, EventEmitter } from 'vscode' import { Disposable } from '@hediet/std/disposable' -import execa from 'execa' -import doesProcessExists from 'process-exists' +import { Deferred } from '@hediet/std/synchronization' import kill from 'tree-kill' import { getProcessPlatform } from '../env' +const deferred = new Deferred() +export const esmModulesImported = deferred.promise + +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore +type EsmExeca = typeof import('execa').execa +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore +type EsmProcessExists = typeof import('process-exists').processExists + +const envCanImportEsm = process.env.NODE_ENV !== 'test' + +let execa: EsmExeca +let doesProcessExist: EsmProcessExists +const importEsmModules = async () => { + const [{ execa: esmExeca }, { processExists: esmProcessExists }] = + await Promise.all([import('execa'), import('process-exists')]) + execa = esmExeca + doesProcessExist = esmProcessExists + deferred.resolve() +} + +if (envCanImportEsm) { + void importEsmModules() +} + interface RunningProcess extends ChildProcess { all?: Readable } @@ -86,7 +111,7 @@ export const executeProcess = async ( } export const processExists = (pid: number): Promise => - doesProcessExists(pid) + doesProcessExist(pid) export const stopProcesses = async (pids: number[]): Promise => { let allKilled = true diff --git a/extension/src/python/index.test.ts b/extension/src/python/index.test.ts index 18cca17267..c2563cb40a 100644 --- a/extension/src/python/index.test.ts +++ b/extension/src/python/index.test.ts @@ -1,5 +1,5 @@ import { join } from 'path' -import { setupVenv } from '.' +import { setupTestVenv } from '.' import { Process, createProcess } from '../process/execution' import { getProcessPlatform } from '../env' @@ -16,7 +16,7 @@ beforeEach(() => { jest.resetAllMocks() }) -describe('setupVenv', () => { +describe('setupTestVenv', () => { it('should create the correct python processes on sane operating systems', async () => { mockedCreateProcess.mockResolvedValue(mockedProcess) mockedGetProcessPlatform.mockReturnValue('freebsd') @@ -24,7 +24,7 @@ describe('setupVenv', () => { const envDir = '.env' const cwd = __dirname - await setupVenv(__dirname, envDir, 'dvc') + await setupTestVenv(__dirname, envDir, 'dvc') expect(mockedCreateProcess).toHaveBeenCalledTimes(3) expect(mockedCreateProcess).toHaveBeenCalledWith({ @@ -53,7 +53,7 @@ describe('setupVenv', () => { const envDir = '.env' const cwd = __dirname - await setupVenv(__dirname, envDir, '-r', 'requirements.txt') + await setupTestVenv(__dirname, envDir, '-r', 'requirements.txt') expect(mockedCreateProcess).toHaveBeenCalledTimes(3) expect(mockedCreateProcess).toHaveBeenCalledWith({ diff --git a/extension/src/python/index.ts b/extension/src/python/index.ts index 77c9b911b3..7f151f2052 100644 --- a/extension/src/python/index.ts +++ b/extension/src/python/index.ts @@ -3,7 +3,12 @@ import { getVenvBinPath } from './path' import { getProcessPlatform } from '../env' import { exists } from '../fileSystem' import { Logger } from '../common/logger' -import { createProcess, executeProcess, Process } from '../process/execution' +import { + createProcess, + esmModulesImported, + executeProcess, + Process +} from '../process/execution' const sendOutput = (process: Process) => { process.all?.on('data', chunk => @@ -29,11 +34,12 @@ export const installPackages = ( export const getDefaultPython = (): string => getProcessPlatform() === 'win32' ? 'python' : 'python3' -export const setupVenv = async ( +export const setupTestVenv = async ( cwd: string, envDir: string, ...installArgs: string[] ) => { + await esmModulesImported if (!exists(join(cwd, envDir))) { const initVenv = createProcess({ args: ['-m', 'venv', envDir], diff --git a/extension/src/test/cli/index.ts b/extension/src/test/cli/index.ts index d4b425c972..a98c42bcbf 100644 --- a/extension/src/test/cli/index.ts +++ b/extension/src/test/cli/index.ts @@ -5,12 +5,12 @@ require('../../vscode/mockModule') const importModulesAfterMockingVsCode = () => { const { removeDir } = require('../../fileSystem') const { runMocha } = require('../util/mocha') - const { setupVenv } = require('../../python') + const { setupTestVenv } = require('../../python') - return { removeDir, runMocha, setupVenv } + return { removeDir, runMocha, setupTestVenv } } -const { setupVenv, removeDir, runMocha } = importModulesAfterMockingVsCode() +const { setupTestVenv, removeDir, runMocha } = importModulesAfterMockingVsCode() async function main() { try { @@ -19,7 +19,7 @@ async function main() { 'ts', async () => { await mkdirp(TEMP_DIR) - await setupVenv( + await setupTestVenv( TEMP_DIR, ENV_DIR, 'git+https://github.com/iterative/dvc#egg=dvc[s3]' diff --git a/extension/src/test/e2e/util.ts b/extension/src/test/e2e/util.ts index 73f3f7a3bc..32e31f7518 100644 --- a/extension/src/test/e2e/util.ts +++ b/extension/src/test/e2e/util.ts @@ -1,7 +1,7 @@ import { Key } from 'webdriverio' import { $$, browser } from '@wdio/globals' import { ViewControl } from 'wdio-vscode-service' -import { PlotsWebview } from './pageObjects/plotsWebview' +import { PlotsWebview } from './pageObjects/plotsWebview.js' const findProgressBars = () => $$('.monaco-progress-container') diff --git a/extension/src/test/suite/cli/child.ts b/extension/src/test/suite/cli/child.ts index 757d159ae8..387b51cb29 100644 --- a/extension/src/test/suite/cli/child.ts +++ b/extension/src/test/suite/cli/child.ts @@ -6,12 +6,14 @@ require('../../../vscode/mockModule') const importModuleAfterMockingVsCode = () => { const { Cli } = require('../../../cli') - return { Cli } + const { esmModulesImported } = require('../../../process/execution') + return { Cli, esmModulesImported } } const main = async () => { - const { Cli } = importModuleAfterMockingVsCode() + const { Cli, esmModulesImported } = importModuleAfterMockingVsCode() + await esmModulesImported const cli = new Cli() const options = getOptions('background') diff --git a/extension/src/test/suite/process/execution.test.ts b/extension/src/test/suite/process/execution.test.ts new file mode 100644 index 0000000000..ae9a00e3e2 --- /dev/null +++ b/extension/src/test/suite/process/execution.test.ts @@ -0,0 +1,36 @@ +import process from 'process' +import { describe, it, suite } from 'mocha' +import { expect } from 'chai' +import { executeProcess, processExists } from '../../../process/execution' + +suite('Process Manager Test Suite', () => { + describe('executeProcess', () => { + it('should be able to run a process', async () => { + const output = await executeProcess({ + args: ['some', 'text'], + cwd: __dirname, + executable: 'echo' + }) + expect(output).to.match(/some.*text/) + }) + + it('should return the stderr if the process throws with stderr', async () => { + await expect( + executeProcess({ + args: ['me', 'outside'], + cwd: __dirname, + executable: 'find' + }) + ).to.be.eventually.rejected + }) + }) + + describe('processExists', () => { + it('should return true if the process exists', async () => { + expect(await processExists(process.pid)).to.be.true + }) + it('should return false if it does not', async () => { + expect(await processExists(-123.321)).to.be.false + }) + }) +}) diff --git a/extension/tsconfig.json b/extension/tsconfig.json index e72ac8000b..68db4db9e1 100644 --- a/extension/tsconfig.json +++ b/extension/tsconfig.json @@ -1,6 +1,7 @@ { "compilerOptions": { "resolveJsonModule": true, + "moduleResolution": "node16", "module": "commonjs", "target": "es6", "outDir": "dist", diff --git a/package.json b/package.json index 6fa4aba08d..b4d115d1a0 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "postinstall": "husky install && git submodule init && git submodule update", "storybook": "yarn workspace dvc-vscode-webview storybook", "build-storybook": "yarn turbo run build-storybook --filter=dvc-vscode-webview", - "setup:venv": "ts-node ./scripts/virtualenv-install.ts", + "setup:venv": "yarn turbo run lint:build && yarn workspace dvc run setup-venv", "scheduled:cli:test": "ts-node ./extension/src/test/cli/index.ts", "create-svgs": "ts-node ./scripts/create-svgs.ts", "svgr": "yarn workspace dvc-vscode-webview svgr" diff --git a/renovate.json b/renovate.json index 4e6cb10b27..d70aa3c0f5 100644 --- a/renovate.json +++ b/renovate.json @@ -1,5 +1,5 @@ { - "ignoreDeps": ["@types/node", "@types/vscode", "execa", "process-exists"], + "ignoreDeps": ["@types/node", "@types/vscode"], "extends": ["config:base"], "packageRules": [ { diff --git a/scripts/virtualenv-install.ts b/scripts/virtualenv-install.ts deleted file mode 100644 index fe9f5735e4..0000000000 --- a/scripts/virtualenv-install.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { join, resolve } from 'path' -require('dvc/src/vscode/mockModule') - -const importModuleAfterMockingVsCode = () => { - const { setupVenv } = require('dvc/src/python') - - return setupVenv -} - -const setupVenv = importModuleAfterMockingVsCode() - -const cwd = resolve(__dirname, '..', 'demo') - -setupVenv(cwd, '.env', '-r', join('.', 'requirements.txt')) diff --git a/yarn.lock b/yarn.lock index 5a10287c9f..d2679b4b6f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10755,7 +10755,22 @@ evp_bytestokey@^1.0.0, evp_bytestokey@^1.0.3: md5.js "^1.3.4" safe-buffer "^5.1.1" -execa@5.1.1, execa@^5.0.0, execa@^5.1.1: +execa@7.1.1, execa@^7.1.1: + version "7.1.1" + resolved "https://registry.yarnpkg.com/execa/-/execa-7.1.1.tgz#3eb3c83d239488e7b409d48e8813b76bb55c9c43" + integrity sha512-wH0eMf/UXckdUYnO21+HDztteVv05rq2GXksxT4fCGeHkBhw1DROXh40wcjMcRqDOWE7iPJ4n3M7e2+YFP+76Q== + dependencies: + cross-spawn "^7.0.3" + get-stream "^6.0.1" + human-signals "^4.3.0" + is-stream "^3.0.0" + merge-stream "^2.0.0" + npm-run-path "^5.1.0" + onetime "^6.0.0" + signal-exit "^3.0.7" + strip-final-newline "^3.0.0" + +execa@^5.0.0, execa@^5.1.1: version "5.1.1" resolved "https://registry.yarnpkg.com/execa/-/execa-5.1.1.tgz#f80ad9cbf4298f7bd1d4c9555c21e93741c411dd" integrity sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg== @@ -10785,21 +10800,6 @@ execa@^7.0.0: signal-exit "^3.0.7" strip-final-newline "^3.0.0" -execa@^7.1.1: - version "7.1.1" - resolved "https://registry.yarnpkg.com/execa/-/execa-7.1.1.tgz#3eb3c83d239488e7b409d48e8813b76bb55c9c43" - integrity sha512-wH0eMf/UXckdUYnO21+HDztteVv05rq2GXksxT4fCGeHkBhw1DROXh40wcjMcRqDOWE7iPJ4n3M7e2+YFP+76Q== - dependencies: - cross-spawn "^7.0.3" - get-stream "^6.0.1" - human-signals "^4.3.0" - is-stream "^3.0.0" - merge-stream "^2.0.0" - npm-run-path "^5.1.0" - onetime "^6.0.0" - signal-exit "^3.0.7" - strip-final-newline "^3.0.0" - exenv-es6@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/exenv-es6/-/exenv-es6-1.1.1.tgz#80b7a8c5af24d53331f755bac07e84abb1f6de67" @@ -16643,12 +16643,12 @@ prismjs@^1.27.0, prismjs@~1.27.0: resolved "https://registry.yarnpkg.com/prismjs/-/prismjs-1.27.0.tgz#bb6ee3138a0b438a3653dd4d6ce0cc6510a45057" integrity sha512-t13BGPUlFDR7wRB5kQDG4jjl7XeuH6jbJGt11JHPL96qwsEHNX2+68tFXqc1/k+/jALsbSWJKUOT/hcYAZ5LkA== -process-exists@4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/process-exists/-/process-exists-4.1.0.tgz#4132c516324c1da72d65896851cdbd8bbdf5b9d8" - integrity sha512-BBJoiorUKoP2AuM5q/yKwIfT1YWRHsaxjW+Ayu9erLhqKOfnXzzVVML0XTYoQZuI1YvcWKmc1dh06DEy4+KzfA== +process-exists@5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/process-exists/-/process-exists-5.0.0.tgz#0b6dcd3d19e85e1f72c633f56d38e498196e2855" + integrity sha512-6QPRh5fyHD8MaXr4GYML8K/YY0Sq5dKHGIOrAKS3cYpHQdmygFCcijIu1dVoNKAZ0TWAMoeh8KDK9dF8auBkJA== dependencies: - ps-list "^6.3.0" + ps-list "^8.0.0" process-nextick-args@~2.0.0: version "2.0.1" @@ -16745,10 +16745,10 @@ prr@~1.0.1: resolved "https://registry.yarnpkg.com/prr/-/prr-1.0.1.tgz#d3fc114ba06995a45ec6893f484ceb1d78f5f476" integrity sha1-0/wRS6BplaRexok/SEzrHXj19HY= -ps-list@^6.3.0: - version "6.3.0" - resolved "https://registry.yarnpkg.com/ps-list/-/ps-list-6.3.0.tgz#a2b775c2db7d547a28fbaa3a05e4c281771259be" - integrity sha512-qau0czUSB0fzSlBOQt0bo+I2v6R+xiQdj78e1BR/Qjfl5OHWJ/urXi8+ilw1eHe+5hSeDI1wrwVTgDp2wst4oA== +ps-list@^8.0.0: + version "8.1.1" + resolved "https://registry.yarnpkg.com/ps-list/-/ps-list-8.1.1.tgz#9ff1952b26a9a07fcc05270407e60544237ae581" + integrity sha512-OPS9kEJYVmiO48u/B9qneqhkMvgCxT+Tm28VCEJpheTpl8cJ0ffZRRNgS5mrQRTrX5yRTpaJ+hRDeefXYmmorQ== pseudomap@^1.0.2: version "1.0.2"