diff --git a/src/language/typescript/tsWorker.ts b/src/language/typescript/tsWorker.ts
index c4f7d3eba0..750f4124c8 100644
--- a/src/language/typescript/tsWorker.ts
+++ b/src/language/typescript/tsWorker.ts
@@ -37,6 +37,8 @@ export class TypeScriptWorker implements ts.LanguageServiceHost, ITypeScriptWork
 
 	private _ctx: worker.IWorkerContext;
 	private _extraLibs: IExtraLibs = Object.create(null);
+	/** Extra libs may have been opened as models, so we track previously removed libs to ensure they are not considered */
+	private _removedExtraLibs: { uri: string; version: number }[] = [];
 	private _languageService = ts.createLanguageService(this);
 	private _compilerOptions: ts.CompilerOptions;
 	private _inlayHintsOptions?: ts.UserPreferences;
@@ -63,8 +65,35 @@ export class TypeScriptWorker implements ts.LanguageServiceHost, ITypeScriptWork
 	}
 
 	getScriptFileNames(): string[] {
-		const allModels = this._ctx.getMirrorModels().map((model) => model.uri);
-		const models = allModels.filter((uri) => !fileNameIsLib(uri)).map((uri) => uri.toString());
+		const allModels = this._ctx.getMirrorModels();
+		const models = allModels
+			// remove default libs
+			.filter((model) => !fileNameIsLib(model.uri))
+			// remove extra libs and previously removed libs
+			// Note that is is required for all model names to be unique, so
+			// while the logic here depends on the end user not creating extra
+			// libs with conflicting filenames, that is already a requirement.
+			.filter((model) => {
+				const uriAsString = model.uri.toString();
+				// if the extra lib was not given a name, then it gets an autogenerated name prefixed with ts:extralib-
+				if (uriAsString.startsWith('ts:extralib-')) {
+					return false;
+				}
+				// Otherwise, the prefix will be file:/// and the name will be what the user provided to add/setExtraLibs
+				if (
+					this._removedExtraLibs.some(
+						(removed) =>
+							`file:///${removed.uri}` === uriAsString && removed.version === model.version
+					)
+				) {
+					return false;
+				}
+				if (this._extraLibs[uriAsString.replace(/^file:\/\/\//, '')]?.version === model.version) {
+					return false;
+				}
+				return true;
+			})
+			.map((model) => model.uri.toString());
 		return models.concat(Object.keys(this._extraLibs));
 	}
 
@@ -450,6 +479,9 @@ export class TypeScriptWorker implements ts.LanguageServiceHost, ITypeScriptWork
 	}
 
 	async updateExtraLibs(extraLibs: IExtraLibs): Promise<void> {
+		this._removedExtraLibs.push(
+			...Object.entries(this._extraLibs).map(([uri, lib]) => ({ uri, version: lib.version }))
+		);
 		this._extraLibs = extraLibs;
 	}