Investigate using webworker for Python discovery

.vscode/settings.json

@@ -72,6 +72,7 @@
 		"main",
 		"release/*"
 	],
+    "git.branchProtectionPrompt": "alwaysCommitToNewBranch",
     "git.pullBeforeCheckout": true,
     // Open merge editor for resolving conflicts.
     "git.mergeEditor": true,

package.json

@@ -506,7 +506,7 @@
                     ]
                 },
                 "python.condaPath": {
-                    "default": "",
+                    "default": "conda",
                     "description": "%python.condaPath.description%",
                     "scope": "machine",
                     "type": "string"
@@ -2138,7 +2138,8 @@
         "vscode-tas-client": "^0.1.63",
         "which": "^2.0.2",
         "winreg": "^1.2.4",
-        "xml2js": "^0.5.0"
+        "xml2js": "^0.5.0",
+        "vscode-uri":"^3.0.7"
     },
     "devDependencies": {
         "@istanbuljs/nyc-config-typescript": "^1.0.2",
@@ -2221,4 +2222,4 @@
         "webpack-require-from": "^1.8.6",
         "yargs": "^15.3.1"
     }
-}
+}

src/client/common/platform/fileSystemWatcher.ts

@@ -1,8 +1,9 @@
+/* eslint-disable global-require */
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-import { RelativePattern, workspace } from 'vscode';
-import { traceVerbose } from '../../logging';
+import type * as vscodeTypes from 'vscode';
+import { traceError, traceVerbose } from '../../logging';
 import { IDisposable } from '../types';
 import { Disposables } from '../utils/resourceLifecycle';
 
@@ -20,12 +21,20 @@ export function watchLocationForPattern(
     pattern: string,
     callback: (type: FileChangeType, absPath: string) => void,
 ): IDisposable {
-    const globPattern = new RelativePattern(baseDir, pattern);
-    const disposables = new Disposables();
-    traceVerbose(`Start watching: ${baseDir} with pattern ${pattern} using VSCode API`);
-    const watcher = workspace.createFileSystemWatcher(globPattern);
-    disposables.push(watcher.onDidCreate((e) => callback(FileChangeType.Created, e.fsPath)));
-    disposables.push(watcher.onDidChange((e) => callback(FileChangeType.Changed, e.fsPath)));
-    disposables.push(watcher.onDidDelete((e) => callback(FileChangeType.Deleted, e.fsPath)));
-    return disposables;
+    try {
+        const vscode = require('vscode') as typeof vscodeTypes;
+        const globPattern = new vscode.RelativePattern(baseDir, pattern);
+        const disposables = new Disposables();
+        traceVerbose(`Start watching: ${baseDir} with pattern ${pattern} using VSCode API`);
+        const watcher = vscode.workspace.createFileSystemWatcher(globPattern);
+        disposables.push(watcher.onDidCreate((e) => callback(FileChangeType.Created, e.fsPath)));
+        disposables.push(watcher.onDidChange((e) => callback(FileChangeType.Changed, e.fsPath)));
+        disposables.push(watcher.onDidDelete((e) => callback(FileChangeType.Deleted, e.fsPath)));
+        return disposables;
+    } catch (ex) {
+        traceError('Watcher', (ex as Error).message);
+        console.log('Watcher', (ex as Error).message);
+        // eslint-disable-next-line @typescript-eslint/no-empty-function
+        return { dispose: () => {} };
+    }
 }

src/client/common/utils/filesystem.ts

@@ -2,10 +2,14 @@
 // Licensed under the MIT License.
 
 import * as fs from 'fs';
-import * as vscode from 'vscode';
 import { traceError } from '../../logging';
 
-export import FileType = vscode.FileType;
+export enum FileType {
+    Unknown = 0,
+    File = 1,
+    Directory = 2,
+    SymbolicLink = 64,
+}
 
 export type DirEntry = {
     filename: string;

src/client/pythonEnvironments/base/info/env.ts

@@ -4,7 +4,6 @@
 import { cloneDeep, isEqual } from 'lodash';
 import * as path from 'path';
 import { Uri } from 'vscode';
-import { getArchitectureDisplayName } from '../../../common/platform/registry';
 import { Architecture } from '../../../common/utils/platform';
 import { arePathsSame, isParentPath, normCasePath } from '../../common/externalDependencies';
 import { getKindDisplayName } from './envKind';
@@ -77,6 +76,17 @@ export function buildEnvInfo(init?: {
     return env;
 }
 
+export function getArchitectureDisplayName(arch?: Architecture): string {
+    switch (arch) {
+        case Architecture.x64:
+            return '64-bit';
+        case Architecture.x86:
+            return '32-bit';
+        default:
+            return '';
+    }
+}
+
 export function areEnvsDeepEqual(env1: PythonEnvInfo, env2: PythonEnvInfo): boolean {
     const env1Clone = cloneDeep(env1);
     const env2Clone = cloneDeep(env2);

src/client/pythonEnvironments/base/locator.ts

@@ -195,6 +195,24 @@ interface IResolver {
 
 export interface IResolvingLocator<I = PythonEnvInfo> extends IResolver, ICompositeLocator<I> {}
 
+export type EnvIteratorId = number;
+
+export interface IMiddleware {
+    iterInitialize(query?: PythonLocatorQuery): Promise<EnvIteratorId>;
+    iterNext(id: EnvIteratorId): Promise<PythonEnvInfo | undefined>;
+    resolveEnv(path: string): Promise<PythonEnvInfo | undefined>;
+}
+
+export interface IEnvsMiddleware extends IMiddleware {
+    iterOnUpdated(id: EnvIteratorId): Event<PythonEnvUpdatedEvent | ProgressNotificationEvent> | undefined;
+    readonly onChanged: Event<PythonEnvsChangedEvent>;
+}
+
+export interface IWorkerMiddleWare extends IMiddleware {
+    iterOnUpdated(id: EnvIteratorId): Event<PythonEnvUpdatedEvent | ProgressNotificationEvent> | undefined;
+    readonly onChanged: Event<PythonEnvsChangedEvent>;
+}
+
 export interface GetRefreshEnvironmentsOptions {
     /**
      * Get refresh promise which resolves once the following stage has been reached for the list of known environments.

src/client/pythonEnvironments/base/locators/composite/envsCollectionService.ts

@@ -14,7 +14,7 @@ import { getEnvPath } from '../../info/env';
 import {
     GetRefreshEnvironmentsOptions,
     IDiscoveryAPI,
-    IResolvingLocator,
+    IEnvsMiddleware,
     isProgressEvent,
     ProgressNotificationEvent,
     ProgressReportStage,
@@ -54,9 +54,9 @@ export class EnvsCollectionService extends PythonEnvsWatcher<PythonEnvCollection
         return this.progressPromises.get(stage)?.promise;
     }
 
-    constructor(private readonly cache: IEnvsCollectionCache, private readonly locator: IResolvingLocator) {
+    constructor(private readonly cache: IEnvsCollectionCache, private readonly middleware: IEnvsMiddleware) {
         super();
-        this.locator.onChanged((event) => {
+        this.middleware.onChanged((event) => {
             const query: PythonLocatorQuery | undefined = event.providerId
                 ? { providerId: event.providerId, envPath: event.envPath }
                 : undefined; // We can also form a query based on the event, but skip that for simplicity.
@@ -91,7 +91,7 @@ export class EnvsCollectionService extends PythonEnvsWatcher<PythonEnvCollection
             traceVerbose(`Resolved ${path} from cache: ${JSON.stringify(cachedEnv)}`);
             return cachedEnv;
         }
-        const resolved = await this.locator.resolveEnv(path).catch((ex) => {
+        const resolved = await this.middleware.resolveEnv(path).catch((ex) => {
             traceError(`Failed to resolve ${path}`, ex);
             return undefined;
         });
@@ -133,16 +133,16 @@ export class EnvsCollectionService extends PythonEnvsWatcher<PythonEnvCollection
     }
 
     private async addEnvsToCacheForQuery(query: PythonLocatorQuery | undefined) {
-        const iterator = this.locator.iterEnvs(query);
+        const iteratorId = await this.middleware.iterInitialize(query);
         const seen: PythonEnvInfo[] = [];
         const state = {
             done: false,
             pending: 0,
         };
         const updatesDone = createDeferred<void>();
-
-        if (iterator.onUpdated !== undefined) {
-            const listener = iterator.onUpdated(async (event) => {
+        const itOnUpdated = this.middleware.iterOnUpdated(iteratorId);
+        if (itOnUpdated !== undefined) {
+            const listener = itOnUpdated(async (event) => {
                 if (isProgressEvent(event)) {
                     switch (event.stage) {
                         case ProgressReportStage.discoveryFinished:
@@ -175,9 +175,11 @@ export class EnvsCollectionService extends PythonEnvsWatcher<PythonEnvCollection
             updatesDone.resolve();
         }
 
-        for await (const env of iterator) {
+        let env = await this.middleware.iterNext(iteratorId);
+        while (env !== undefined) {
             seen.push(env);
             this.cache.addEnv(env);
+            env = await this.middleware.iterNext(iteratorId);
         }
         await updatesDone.promise;
         // If query for all envs is done, `seen` should contain the list of all envs.

src/client/pythonEnvironments/base/locators/composite/envsMiddleware.ts

@@ -0,0 +1,82 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+import { Event, WorkspaceFolder, WorkspaceFoldersChangeEvent } from 'vscode';
+import { PythonEnvInfo } from '../../info';
+import {
+    EnvIteratorId,
+    IEnvsMiddleware,
+    IPythonEnvsIterator,
+    IResolvingLocator,
+    ProgressNotificationEvent,
+    PythonEnvUpdatedEvent,
+    PythonLocatorQuery,
+} from '../../locator';
+import { PythonEnvsWatcher } from '../../watcher';
+import { IDisposableRegistry } from '../../../../common/types';
+import { WorkspaceLocators } from '../wrappers';
+import { createSubLocators } from '../../../locator';
+import { PythonDiscoverySettings } from '../../../common/settings';
+
+/**
+ * A service acts as a bridge between Env Resolver and Env Collection.
+ */
+export class EnvsMiddleWare extends PythonEnvsWatcher implements IEnvsMiddleware {
+    private idCount = 0;
+
+    private readonly locator: IResolvingLocator;
+
+    private readonly workspaceLocator: WorkspaceLocators;
+
+    private disposables: IDisposableRegistry;
+
+    private iterators = new Map<EnvIteratorId, IPythonEnvsIterator>();
+
+    constructor(folders: readonly WorkspaceFolder[] | undefined, settings: PythonDiscoverySettings) {
+        super();
+        const { locator, disposables, workspaceLocator } = createSubLocators(folders, settings);
+        this.disposables = disposables;
+        this.locator = locator;
+        this.workspaceLocator = workspaceLocator;
+        this.locator.onChanged((e) => {
+            this.fire(e);
+        });
+    }
+
+    public dispose(): void {
+        this.disposables.forEach((d) => d.dispose());
+    }
+
+    public onDidChangeWorkspaceFolders(event: WorkspaceFoldersChangeEvent): void {
+        this.workspaceLocator.onDidChangeWorkspaceFolders(event);
+    }
+
+    public async resolveEnv(path: string): Promise<PythonEnvInfo | undefined> {
+        return this.locator.resolveEnv(path);
+    }
+
+    public async iterInitialize(query?: PythonLocatorQuery | undefined): Promise<EnvIteratorId> {
+        const it = this.locator.iterEnvs(query);
+        const id = this.idCount;
+        this.iterators.set(id, it);
+        return id;
+    }
+
+    public async iterNext(id: EnvIteratorId): Promise<PythonEnvInfo | undefined> {
+        const it = this.getIterator(id);
+        const result = await it?.next();
+        if (result?.done) {
+            return undefined;
+        }
+        return result?.value;
+    }
+
+    public iterOnUpdated(id: EnvIteratorId): Event<PythonEnvUpdatedEvent | ProgressNotificationEvent> | undefined {
+        const it = this.getIterator(id);
+        return it?.onUpdated;
+    }
+
+    private getIterator(id: EnvIteratorId) {
+        return this.iterators.get(id);
+    }
+}

src/client/pythonEnvironments/base/locators/composite/envsReducer.ts

@@ -2,7 +2,7 @@
 // Licensed under the MIT License.
 
 import { cloneDeep, isEqual, uniq } from 'lodash';
-import { Event, EventEmitter } from 'vscode';
+import { Event } from 'vscode';
 import { traceVerbose } from '../../../../logging';
 import { PythonEnvKind } from '../../info';
 import { areSameEnv } from '../../info/env';
@@ -19,6 +19,7 @@ import {
     PythonLocatorQuery,
 } from '../../locator';
 import { PythonEnvsChangedEvent } from '../../watcher';
+import { EventEmitter } from '../../../common/eventEmitter';
 
 /**
  * Combines duplicate environments received from the incoming locator into one and passes on unique environments

src/client/pythonEnvironments/base/locators/composite/envsResolver.ts

@@ -2,7 +2,7 @@
 // Licensed under the MIT License.
 
 import { cloneDeep } from 'lodash';
-import { Event, EventEmitter } from 'vscode';
+import { Event, WorkspaceFolder } from 'vscode';
 import { identifyEnvironment } from '../../../common/environmentIdentifier';
 import { IEnvironmentInfoService } from '../../info/environmentInfoService';
 import { PythonEnvInfo, PythonEnvKind } from '../../info';
@@ -24,6 +24,7 @@ import { resolveBasicEnv } from './resolverUtils';
 import { traceVerbose, traceWarn } from '../../../../logging';
 import { getEnvironmentDirFromPath, getInterpreterPathFromDir, isPythonExecutable } from '../../../common/commonUtils';
 import { getEmptyVersion } from '../../info/pythonVersion';
+import { EventEmitter } from '../../../common/eventEmitter';
 
 /**
  * Calls environment info service which runs `interpreterInfo.py` script on environments received
@@ -37,6 +38,7 @@ export class PythonEnvsResolver implements IResolvingLocator {
     constructor(
         private readonly parentLocator: ICompositeLocator<BasicEnvInfo>,
         private readonly environmentInfoService: IEnvironmentInfoService,
+        private readonly workspaceFolders: readonly WorkspaceFolder[] | undefined,
     ) {
         this.parentLocator.onChanged((event) => {
             if (event.type && event.searchLocation !== undefined) {
@@ -50,7 +52,7 @@ export class PythonEnvsResolver implements IResolvingLocator {
         const [executablePath, envPath] = await getExecutablePathAndEnvPath(path);
         path = executablePath.length ? executablePath : envPath;
         const kind = await identifyEnvironment(path);
-        const environment = await resolveBasicEnv({ kind, executablePath, envPath });
+        const environment = await resolveBasicEnv({ kind, executablePath, envPath }, this.workspaceFolders);
         const info = await this.environmentInfoService.getEnvironmentInfo(environment);
         traceVerbose(
             `Environment resolver resolved ${path} for ${JSON.stringify(environment)} to ${JSON.stringify(info)}`,
@@ -98,7 +100,7 @@ export class PythonEnvsResolver implements IResolvingLocator {
                 } else if (seen[event.index] !== undefined) {
                     const old = seen[event.index];
                     await setKind(event.update, environmentKinds);
-                    seen[event.index] = await resolveBasicEnv(event.update);
+                    seen[event.index] = await resolveBasicEnv(event.update, this.workspaceFolders);
                     didUpdate.fire({ old, index: event.index, update: seen[event.index] });
                     this.resolveInBackground(event.index, state, didUpdate, seen).ignoreErrors();
                 } else {
@@ -116,7 +118,7 @@ export class PythonEnvsResolver implements IResolvingLocator {
         while (!result.done) {
             // Use cache from the current refresh where possible.
             await setKind(result.value, environmentKinds);
-            const currEnv = await resolveBasicEnv(result.value);
+            const currEnv = await resolveBasicEnv(result.value, this.workspaceFolders);
             seen.push(currEnv);
             yield currEnv;
             this.resolveInBackground(seen.indexOf(currEnv), state, didUpdate, seen).ignoreErrors();