Merge pull request #225191 from mjbvz/novel-mink

Avoid using marked renderer just to find tokens

src/vs/workbench/api/common/extHostTypeConverters.ts

@@ -10,7 +10,7 @@ import { createSingleCallFunction } from 'vs/base/common/functional';
 import * as htmlContent from 'vs/base/common/htmlContent';
 import { DisposableStore } from 'vs/base/common/lifecycle';
 import { ResourceMap, ResourceSet } from 'vs/base/common/map';
-import { marked } from 'vs/base/common/marked/marked';
+import * as marked from 'vs/base/common/marked/marked';
 import { parse, revive } from 'vs/base/common/marshalling';
 import { Mimes } from 'vs/base/common/mime';
 import { cloneAndChange } from 'vs/base/common/objects';
@@ -377,11 +377,16 @@ export namespace MarkdownString {
 			}
 			return '';
 		};
-		const renderer = new marked.Renderer();
-		renderer.link = collectUri;
-		renderer.image = href => typeof href === 'string' ? collectUri(htmlContent.parseHrefAndDimensions(href)) : '';
 
-		marked(res.value, { renderer });
+		marked.marked.walkTokens(marked.marked.lexer(res.value), token => {
+			if (token.type === 'link') {
+				collectUri({ href: token.href });
+			} else if (token.type === 'image') {
+				if (typeof token.href === 'string') {
+					collectUri(htmlContent.parseHrefAndDimensions(token.href));
+				}
+			}
+		});
 
 		return res;
 	}

src/vs/workbench/contrib/chat/common/chatViewModel.ts

@@ -306,46 +306,13 @@ export class ChatViewModel extends Disposable implements IChatViewModel {
 		}
 
 		let codeBlockIndex = 0;
-		const renderer = new marked.marked.Renderer();
-		renderer.code = ({ text, lang }: marked.Tokens.Code) => {
-			lang ??= '';
-			this.codeBlockModelCollection.update(this._model.sessionId, model, codeBlockIndex++, { text, languageId: lang });
-			return '';
-		};
-
-		marked.marked.parse(this.ensureFencedCodeBlocksTerminated(content), { renderer });
-	}
-
-	/**
-	 * Marked doesn't consistently render fenced code blocks that aren't terminated.
-	 *
-	 * Try to close them ourselves to workaround this.
-	 */
-	private ensureFencedCodeBlocksTerminated(content: string): string {
-		const lines = content.split('\n');
-
-		let codeBlockState: undefined | { readonly delimiter: string; readonly indent: string };
-		for (let i = 0; i < lines.length; i++) {
-			const line = lines[i];
-
-			if (codeBlockState) {
-				if (new RegExp(`^\\s*${codeBlockState.delimiter}\\s*$`).test(line)) {
-					codeBlockState = undefined;
-				}
-			} else {
-				const match = line.match(/^(\s*)(`{3,}|~{3,})/);
-				if (match) {
-					codeBlockState = { delimiter: match[2], indent: match[1] };
-				}
+		marked.walkTokens(marked.lexer(content), token => {
+			if (token.type === 'code') {
+				const lang = token.lang || '';
+				const text = token.text;
+				this.codeBlockModelCollection.update(this._model.sessionId, model, codeBlockIndex++, { text, languageId: lang });
 			}
-		}
-
-		// If we're still in a code block at the end of the content, add a closing fence
-		if (codeBlockState) {
-			lines.push(codeBlockState.indent + codeBlockState.delimiter);
-		}
-
-		return lines.join('\n');
+		});
 	}
 }